From afa7537aa4e5b5bfbaccea401ef61de807933c50 Mon Sep 17 00:00:00 2001 From: Dusan Milicevic Date: Fri, 31 Jul 2026 03:19:14 -0500 Subject: [PATCH] Fact-lock article sections so thin briefs cannot invent to fill length Add an article system prompt, list BRIEF-allowed names/figures in section directives, scale section word aims down for thin briefs, and omit sections that still invent after retry. Eval invent now uses the visible brief. Co-authored-by: Cursor --- README.md | 4 +- src/personality_protect/eval_write_article.py | 112 ++++++++++++---- src/personality_protect/prompt_write.py | 48 ++++++- src/personality_protect/write_article.py | 123 ++++++++++++++++-- src/personality_protect/writer_guards.py | 13 ++ tests/test_eval_write_article.py | 14 +- tests/test_prompt_write.py | 16 +++ tests/test_write_article.py | 51 ++++++-- 8 files changed, 328 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index e9212dc..bd95b71 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ personality-protect write --topic "…" --points "…" --json `--topic` and `--points` are the only content the draft may use; retrieved pieces supply rhythm, not facts. Every `write` above runs base weights (`adapter=none`). -On `--channel article`, each `--points` bullet becomes a section (2–8), retrieval is restricted to `linkedin_article` pieces so posts cannot become the rhythm reference, and sections that restate each other are dropped before stitching. The channel refuses to draft unless at least five `linkedin_article` pieces are in the corpus *and* five are in the voice index — a large carve that leaves retrieval empty is an error, not a silently thinner draft. +On `--channel article`, each `--points` bullet becomes a section (2–8), retrieval is restricted to `linkedin_article` pieces so posts cannot become the rhythm reference, and sections that restate each other are dropped before stitching. Section prompts use an article-specific system rule set: the BRIEF is the only fact source, allowed names/figures from the BRIEF are listed explicitly, thin briefs lower the per-section word aim, and a section that still invents after retry is omitted from the stitch rather than padded in. The channel refuses to draft unless at least five `linkedin_article` pieces are in the corpus *and* five are in the voice index — a large carve that leaves retrieval empty is an error, not a silently thinner draft. ### Article holdout eval @@ -215,7 +215,7 @@ PP_MLX_ALLOW=1 personality-protect eval-write-article --out receipt.json The carve is deterministic (`blake2b(piece_id)` order), keeps previously carved ids pinned, and never drops the voice index below the five-article floor. Each holdout is reduced to a lossy brief — a topic plus 3–6 section bullets drawn one per segment of the piece, capped at 60 words and 10% of the source — so neither arm is handed the article back to paraphrase. -Two arms then write the same brief with the same outline, per-section budget, and trim. The product arm gets retrieved exemplars and the measured style card; the control arm gets neither. Drafts are scored on distance to the holdout's own cadence axes, and a draft that parrots its exemplars, echoes the brief, or invents entities or figures is disqualified regardless of distance. Receipts carry ids, distances, and flags — never draft or corpus text. +Two arms then write the same brief with the same outline, per-section budget, and trim. The product arm gets retrieved exemplars and the measured style card; the control arm gets neither. Invention is judged against the visible brief both arms saw (not the full source article). Drafts are scored on distance to the holdout's own cadence axes, and a draft that parrots its exemplars, echoes the brief, or invents entities or figures is disqualified regardless of distance. Receipts carry ids, distances, and flags — never draft or corpus text. The verdict needs all three of: the article arm wins the majority, the margin clears `--alpha` (default 0.10) on a one-sided sign test, and it is not disqualified more often than the control. When both arms are disqualified on every holdout, distance never decided anything, and the receipt says so (`distance_ever_decided: false`) rather than reporting it as a cadence loss. diff --git a/src/personality_protect/eval_write_article.py b/src/personality_protect/eval_write_article.py index a6a78da..1b9fb09 100644 --- a/src/personality_protect/eval_write_article.py +++ b/src/personality_protect/eval_write_article.py @@ -40,7 +40,7 @@ from personality_protect.chat_prompt import flatten_chat_messages from personality_protect.config import ProfilePaths, load_config from personality_protect.corpus_text import normalize_corpus_text -from personality_protect.draft_trim import drop_repeated_paragraphs, trim_draft +from personality_protect.draft_trim import drop_repeated_paragraphs, trim_draft, word_count from personality_protect.eval_write_holdout import ( TIE_EPSILON, assert_receipt_contoso_safe, @@ -53,21 +53,28 @@ from personality_protect.prompt_write import build_write_messages from personality_protect.style_profile import ( article_section_words, - article_word_aim, load_style_profile, ) -from personality_protect.write import DEFAULT_WRITE_K, GenerateFn, mlx_generate_no_adapter +from personality_protect.write import ( + DEFAULT_WRITE_K, + GenerateFn, + build_brief, + mlx_generate_no_adapter, +) from personality_protect.write_article import ( DEFAULT_ARTICLE_SECTION_MAX_TOKENS, SECTION_TOKENS_PER_WORD, SECTION_TRIM_HEADROOM, + THIN_SECTION_WORD_FLOOR, + _guard_flags, _section_brief, - article_draft_ceiling, count_indexed_article_pieces, outline_from_brief, run_write_article, + scale_section_words_for_brief, section_structure_directives, ) +from personality_protect.writer_guards import brief_allowed_facts # One-sided bar, matching the writer ship gate. Stated rather than enforced: # with fourteen articles a carve cannot exceed five, and a clean sweep of three @@ -84,18 +91,29 @@ def article_word_budget(paths: ProfilePaths, topic: str, points: str) -> dict[st """ style = load_style_profile(paths) sections = outline_from_brief(topic, points) - section_words = article_section_words(style, sections=len(sections)) + brief = build_brief(topic, points) + brief_words = word_count(brief) + section_words = scale_section_words_for_brief( + article_section_words(style, sections=len(sections)), + brief_words=brief_words, + sections=len(sections), + ) + word_aim = section_words * len(sections) + allowed = brief_allowed_facts(brief) return { "sections": sections, "section_count": len(sections), - "word_aim": article_word_aim(style), + "word_aim": word_aim, "section_words": section_words, "section_trim_words": int(round(section_words * SECTION_TRIM_HEADROOM)), - "word_ceiling": article_draft_ceiling(style, sections=len(sections)), + "word_ceiling": int(round(section_words * SECTION_TRIM_HEADROOM)) * len(sections), "max_tokens": max( DEFAULT_ARTICLE_SECTION_MAX_TOKENS, int(round(section_words * SECTION_TOKENS_PER_WORD)), ), + "allowed_entities": allowed["entities"], + "allowed_numbers": allowed["numbers"], + "visible_brief": brief, } @@ -122,13 +140,16 @@ def run_bare_base_article( exemplars and the measured style profile — which is the only thing the comparison is meant to be about. """ + invent_brief = str(budget.get("visible_brief") or build_brief(topic, points)) section_drafts: list[str] = [] + messages: list[dict[str, str]] = [] for index, section in enumerate(budget["sections"], start=1): section_topic, section_points = _section_brief(topic, section, points) messages = build_write_messages( topic=section_topic, points=section_points, examples=(), + channel="article", style_directives=section_structure_directives( section=section, index=index, @@ -136,17 +157,35 @@ def run_bare_base_article( word_aim=budget["word_aim"], section_words=budget["section_words"], section_trim_words=budget["section_trim_words"], + allowed_entities=budget.get("allowed_entities") or (), + allowed_numbers=budget.get("allowed_numbers") or (), ), ) - raw = str( - generate_fn( - messages, - base_model=base_model, - max_tokens=budget["max_tokens"], - prompt_sink=prompt_sink, - ) - ).strip() - section_drafts.append(trim_draft(raw, max_words=budget["section_trim_words"])) + draft = "" + kept = False + for attempt in range(1, 3): + attempt_trim = budget["section_trim_words"] + attempt_tokens = budget["max_tokens"] + if attempt > 1: + attempt_trim = max(THIN_SECTION_WORD_FLOOR, budget["section_words"] // 2) + attempt_tokens = max( + 256, int(round(attempt_trim * SECTION_TOKENS_PER_WORD)) + ) + raw = str( + generate_fn( + messages, + base_model=base_model, + max_tokens=attempt_tokens, + prompt_sink=prompt_sink, + ) + ).strip() + draft = trim_draft(raw, max_words=attempt_trim) + guards = _guard_flags(invent_brief, draft, ()) + if not guards["parrot_reject"] and not guards["invent_reject"]: + kept = True + break + if kept and draft.strip(): + section_drafts.append(draft) text = drop_repeated_paragraphs( "\n\n".join(part for part in section_drafts if part.strip()) @@ -158,7 +197,7 @@ def run_bare_base_article( "model": base_model, "k": 0, "exemplar_ids": [], - "section_count": len(budget["sections"]), + "section_count": len(section_drafts), "prompt": flatten_chat_messages(messages) if budget["sections"] else "", } @@ -326,19 +365,46 @@ def run_eval_write_article( base_model=config.base_model, prompt_sink=base_prompts, ) + visible = f"{brief['topic']}\n{brief['points']}" score = score_rag_vs_base( holdout_text, article_result["text"], base_result["text"], - brief["guard_facts"], + # Invent against the same visible brief the product arm fact-locks + # to — not the full source article the model never saw. + visible, rag_exemplars=list(article_result.get("exemplar_texts") or []), tie_epsilon=tie_epsilon, - # Invention is judged against the article; echo against the outline - # the model was handed. On a de-voiced brief those are different - # texts, and checking echo against the article would miss a draft - # that simply lists the bullets back. - visible_brief=f"{brief['topic']}\n{brief['points']}", + visible_brief=visible, ) + # Sections dropped for invent never reach the stitch; surface that as + # invent DQ even when the remaining text is empty or brief-clean. + if article_result.get("invent_reject"): + score["rag"]["invent_reject"] = True + score["rag"]["disqualified"] = True + score["rag"]["invented_entities"] = sorted( + set(score["rag"].get("invented_entities") or []) + | set(article_result.get("invented_entities") or []) + ) + score["rag"]["invented_numbers"] = sorted( + set(score["rag"].get("invented_numbers") or []) + | set(article_result.get("invented_numbers") or []) + ) + score["rag"]["invented_entities_count"] = len( + score["rag"]["invented_entities"] + ) + score["rag"]["invented_numbers_count"] = len( + score["rag"]["invented_numbers"] + ) + if score["winner"] == "rag": + score["winner"] = "base" if not score["base"]["disqualified"] else "tie" + if base_result.get("section_count", 1) == 0: + score["base"]["invent_reject"] = True + score["base"]["disqualified"] = True + if score["winner"] == "base": + score["winner"] = ( + "rag" if not score["rag"]["disqualified"] else "tie" + ) item = _item_receipt( holdout_id=piece.id, holdout_text=holdout_text, diff --git a/src/personality_protect/prompt_write.py b/src/personality_protect/prompt_write.py index cd16348..b582c6a 100644 --- a/src/personality_protect/prompt_write.py +++ b/src/personality_protect/prompt_write.py @@ -38,6 +38,30 @@ "lines, no hashtags copied from the EXAMPLES." ) +WRITE_ARTICLE_SYSTEM_PROMPT = ( + "You write one section of a LinkedIn article in the author's voice.\n" + "\n" + "Rules:\n" + "- Write ONLY the current section about the BRIEF. The BRIEF is the only " + "source of facts.\n" + "- Expand claims already in the BRIEF. Prefer a shorter true section over " + "padding that invents companies, people, products, places, or figures.\n" + "- Do not invent companies, people, products, or figures the BRIEF did not " + "give you. If the BRIEF has no number, write no number.\n" + "- Follow the VOICE cadence targets. Treat the section word aim as a soft " + "ceiling: stop early rather than invent.\n" + "- The EXAMPLES are rhythm reference only: match their line lengths, " + "paragraph breaks, and sentence cadence.\n" + "- Never copy, quote, continue, summarize, or list the EXAMPLES. Reuse none " + "of their words, sentences, facts, names, numbers, or links.\n" + "- Names have been removed from the EXAMPLES. Do not guess them, and never " + "output bracketed or capitalized placeholders of any kind.\n" + "- No AI filler (leverage, delve, moreover, tapestry).\n" + "- Output the section text and nothing else: no title, no preamble, " + "no commentary, no markdown headings, no section labels, no separator " + "lines, no hashtags copied from the EXAMPLES." +) + _EXAMPLES_HEADER = ( "EXAMPLES (rhythm reference only — names removed; never reuse their words, " "facts, or names):" @@ -48,6 +72,11 @@ "Write one new post from the BRIEF now, in the rhythm of the EXAMPLES. " "Do not repeat any EXAMPLE." ) +_WRITE_ARTICLE_INSTRUCTION = "Write this article section from the BRIEF now." +_WRITE_ARTICLE_INSTRUCTION_WITH_EXAMPLES = ( + "Write this article section from the BRIEF now, in the rhythm of the " + "EXAMPLES. Do not repeat any EXAMPLE." +) _STYLE_HEADER = "VOICE (cadence targets measured from the author's own posts):" @@ -59,6 +88,7 @@ def build_write_user_content( points: str, examples: Sequence[str], style_directives: Sequence[str] = (), + channel: str = "post", ) -> str: """User turn: voice card, optional exemplars, the brief, then the instruction. @@ -81,7 +111,14 @@ def build_write_user_content( if kept: blocks.append(_EXAMPLES_HEADER + "\n\n" + _EXAMPLE_SEPARATOR.join(kept)) blocks.append(f"BRIEF:\nTopic: {topic}\nPoints:\n{points}") - blocks.append(_WRITE_INSTRUCTION_WITH_EXAMPLES if kept else _WRITE_INSTRUCTION) + if channel == "article": + blocks.append( + _WRITE_ARTICLE_INSTRUCTION_WITH_EXAMPLES + if kept + else _WRITE_ARTICLE_INSTRUCTION + ) + else: + blocks.append(_WRITE_INSTRUCTION_WITH_EXAMPLES if kept else _WRITE_INSTRUCTION) return "\n\n".join(blocks) @@ -91,10 +128,14 @@ def build_write_messages( points: str, examples: Sequence[str], style_directives: Sequence[str] = (), + channel: str = "post", ) -> list[dict[str, str]]: """Locked writing prompt as chat turns (system + user).""" + system = ( + WRITE_ARTICLE_SYSTEM_PROMPT if channel == "article" else WRITE_SYSTEM_PROMPT + ) return [ - {"role": "system", "content": WRITE_SYSTEM_PROMPT}, + {"role": "system", "content": system}, { "role": "user", "content": build_write_user_content( @@ -102,6 +143,7 @@ def build_write_messages( points=points, examples=examples, style_directives=style_directives, + channel=channel, ), }, ] @@ -113,6 +155,7 @@ def build_write_prompt( points: str, examples: Sequence[str], style_directives: Sequence[str] = (), + channel: str = "post", ) -> str: """Flat rendering of the locked prompt (no chat template available).""" return flatten_chat_messages( @@ -121,5 +164,6 @@ def build_write_prompt( points=points, examples=examples, style_directives=style_directives, + channel=channel, ) ) diff --git a/src/personality_protect/write_article.py b/src/personality_protect/write_article.py index 9528ef3..c2a07c5 100644 --- a/src/personality_protect/write_article.py +++ b/src/personality_protect/write_article.py @@ -37,6 +37,7 @@ normalize_sentence_case, ) from personality_protect.writer_guards import ( + brief_allowed_facts, check_invention, mask_exemplar_entities, parrot_reject, @@ -63,6 +64,11 @@ # Longform gives the model more room to copy, not less, so the clip stays short # and voice travels as measured cadence instead. ARTICLE_EXEMPLAR_WORDS = MAX_EXEMPLAR_WORDS +# When the visible brief is thin, demanding a full article-length section is +# what forces invention. Cap the per-section aim from brief richness instead. +THIN_BRIEF_WORDS = 80 +THIN_SECTION_WORD_FLOOR = 80 +THIN_WORDS_PER_BRIEF_WORD = 2.5 _BULLET_RE = re.compile(r"^\s*[-*•]\s+") @@ -165,6 +171,32 @@ def _section_brief(topic: str, section: str, points: str) -> tuple[str, str]: ) +def scale_section_words_for_brief( + section_words: int, + *, + brief_words: int, + sections: int, +) -> int: + """Lower the section aim when the brief cannot support a long expand. + + A 60-word brief asking for 280 words per section is the invent pressure the + holdout measured. The scaled aim still leaves room to write, but stops + treating length as a hard quota over facts. + """ + base = max(1, int(section_words)) + n_sections = max(1, int(sections)) + brief_n = max(0, int(brief_words)) + if brief_n >= THIN_BRIEF_WORDS: + return base + # Total article aim ≈ brief_words * factor, split across sections. + thin_total = max( + THIN_SECTION_WORD_FLOOR * n_sections, + int(round(brief_n * THIN_WORDS_PER_BRIEF_WORD * n_sections)), + ) + thin_per = max(THIN_SECTION_WORD_FLOOR, thin_total // n_sections) + return min(base, thin_per) + + def section_structure_directives( *, section: str, @@ -173,21 +205,41 @@ def section_structure_directives( word_aim: int, section_words: int, section_trim_words: int, + allowed_entities: Sequence[str] = (), + allowed_numbers: Sequence[str] = (), ) -> list[str]: - """Where this section sits and how long it runs. + """Where this section sits, how long it may run, and which facts are allowed. Structure, not voice. Kept separate from the cadence card so the eval's control arm can be asked for an article of the same shape without also being handed the measured style profile — otherwise the comparison would only be establishing that asking for an article produces one. """ - return [ + lines = [ f"This is section {index} of {total} in a longform article of about " f"{word_aim} words; the other sections cover the rest of the brief.", f"Write only the section about: {section}", - f"Write about {section_words} words in this section, and no more " - f"than {section_trim_words}.", + f"Aim for about {section_words} words in this section, and no more " + f"than {section_trim_words}. Prefer stopping early over inventing " + "facts to fill the count.", ] + entities = [str(item).strip() for item in allowed_entities if str(item).strip()] + numbers = [str(item).strip() for item in allowed_numbers if str(item).strip()] + if entities: + lines.append( + "ALLOWED names from the BRIEF only: " + ", ".join(entities) + "." + ) + else: + lines.append( + "The BRIEF names no companies or people. Invent none." + ) + if numbers: + lines.append( + "ALLOWED figures from the BRIEF only: " + ", ".join(numbers) + "." + ) + else: + lines.append("The BRIEF has no figures. Write no numbers.") + return lines def _guard_flags(brief: str, draft: str, exemplars: Sequence[str]) -> dict[str, Any]: @@ -228,15 +280,26 @@ def run_write_article( # not from the post band, whose ceiling is a LinkedIn character limit. word_aim = article_word_aim(style) word_ceiling = article_word_target(style) - section_words = article_section_words(style, sections=len(sections)) + full_brief = build_brief(topic, points) + brief_words = word_count(full_brief) + section_words = scale_section_words_for_brief( + article_section_words(style, sections=len(sections)), + brief_words=brief_words, + sections=len(sections), + ) + # Recompute the stitched aim so receipts and the eval ceiling match what + # sections were actually asked to write. + word_aim = max(word_aim, section_words * len(sections)) + if brief_words < THIN_BRIEF_WORDS: + word_aim = section_words * len(sections) section_trim_words = int(round(section_words * SECTION_TRIM_HEADROOM)) section_max_tokens = max( int(max_tokens), int(round(section_words * SECTION_TOKENS_PER_WORD)) ) + allowed = brief_allowed_facts(full_brief) generator = generate_fn or mlx_generate_no_adapter model_id = config.base_model or DEFAULT_MLX_MODEL - full_brief = build_brief(topic, points) matches = retrieve( full_brief, k=k, @@ -258,8 +321,11 @@ def run_write_article( ] section_drafts: list[str] = [] + dropped_sections: list[str] = [] all_messages: list[list[dict[str, str]]] = [] attempts_total = 0 + dropped_invented_entities: set[str] = set() + dropped_invented_numbers: set[str] = set() last_guards: dict[str, Any] = { "parrot_reject": False, "invent_reject": False, @@ -269,40 +335,58 @@ def run_write_article( for section in sections: section_topic, section_points = _section_brief(topic, section, points) - section_brief = build_brief(section_topic, section_points) + invent_brief = full_brief messages = build_write_messages( topic=section_topic, points=section_points, examples=masked, + channel="article", style_directives=[ *directives, *section_structure_directives( section=section, - index=len(section_drafts) + 1, + index=len(section_drafts) + len(dropped_sections) + 1, total=len(sections), word_aim=word_aim, section_words=section_words, section_trim_words=section_trim_words, + allowed_entities=allowed["entities"], + allowed_numbers=allowed["numbers"], ), ], ) all_messages.append(messages) draft = "" + kept = False for attempt in range(1, 3): attempts_total += 1 + # Second attempt: stricter short budget if the first invents. + attempt_trim = section_trim_words + attempt_tokens = section_max_tokens + if attempt > 1: + attempt_trim = max(THIN_SECTION_WORD_FLOOR, section_words // 2) + attempt_tokens = max( + 256, int(round(attempt_trim * SECTION_TOKENS_PER_WORD)) + ) raw = str( generator( messages, base_model=model_id, - max_tokens=section_max_tokens, + max_tokens=attempt_tokens, prompt_sink=prompt_sink, ) ).strip() - draft = trim_draft(raw, max_words=section_trim_words) - last_guards = _guard_flags(section_brief, draft, exemplars) + draft = trim_draft(raw, max_words=attempt_trim) + last_guards = _guard_flags(invent_brief, draft, exemplars) if not last_guards["parrot_reject"] and not last_guards["invent_reject"]: + kept = True break - section_drafts.append(draft) + if kept and draft.strip(): + section_drafts.append(draft) + else: + dropped_sections.append(section) + dropped_invented_entities.update(last_guards.get("invented_entities") or []) + dropped_invented_numbers.update(last_guards.get("invented_numbers") or []) # Sections are generated independently from the same brief, so two of them # can arrive as the same paragraph. Stitching them unfiltered is what turns @@ -312,6 +396,16 @@ def run_write_article( ).strip() # Final invent check against the full brief the author supplied. final_guards = _guard_flags(full_brief, text, exemplars) + if dropped_sections: + final_guards["invent_reject"] = True + final_guards["invented_entities"] = sorted( + set(final_guards["invented_entities"]) | dropped_invented_entities + ) + final_guards["invented_numbers"] = sorted( + set(final_guards["invented_numbers"]) | dropped_invented_numbers + ) + if not section_drafts: + final_guards["invent_reject"] = True return { "text": text, @@ -330,8 +424,11 @@ def run_write_article( "section_count_hint": article_section_count_hint(style), "article_count": article_count, "sections": sections, - "section_count": len(sections), + "section_count": len(section_drafts), + "dropped_sections": dropped_sections, "draft_words": word_count(text), + "allowed_entities": allowed["entities"], + "allowed_numbers": allowed["numbers"], **final_guards, "exemplar_texts": exemplars, "messages": all_messages[0] if all_messages else [], diff --git a/src/personality_protect/writer_guards.py b/src/personality_protect/writer_guards.py index e5cd34e..220cb01 100644 --- a/src/personality_protect/writer_guards.py +++ b/src/personality_protect/writer_guards.py @@ -504,3 +504,16 @@ def check_invention(brief: str, draft: str) -> InventionResult: invented_entities=frozenset(invented_entities), invented_numbers=frozenset(invented_numbers), ) + + +def brief_allowed_facts(brief: str) -> dict[str, list[str]]: + """Entities and figures the draft may reuse from the visible brief. + + Returned as sorted lists for Contoso-safe prompts and receipts — never as + free prose from the author corpus. + """ + text = brief or "" + return { + "entities": sorted(extract_entity_keys(text)), + "numbers": sorted(extract_evidence_number_keys(text)), + } diff --git a/tests/test_eval_write_article.py b/tests/test_eval_write_article.py index 971f65d..eb07b79 100644 --- a/tests/test_eval_write_article.py +++ b/tests/test_eval_write_article.py @@ -94,7 +94,8 @@ def test_control_arm_writes_an_article_without_the_voice_machinery(tmp_path: Pat def fake_generate(messages, **_kwargs: object) -> str: seen.append(messages[1]["content"]) - return f"Contoso Ledger paragraph for section {len(seen)} of the piece." + # Stay inside the brief — Contoso is allowed; invented vendors are not. + return f"Contoso packaging section {len(seen)} covers the one claim." result = run_bare_base_article( "Contoso packaging", @@ -105,10 +106,11 @@ def fake_generate(messages, **_kwargs: object) -> str: ) assert result["mode"] == "bare_base_article" assert result["section_count"] == 3 - assert len(seen) == 3 + assert len(seen) >= 3 # Same structure as the product arm... assert "section 1 of 3" in seen[0] - assert f"Write about {budget['section_words']} words" in seen[0] + assert f"Aim for about {budget['section_words']} words" in seen[0] + assert "ALLOWED names from the BRIEF only" in seen[0] # ...and none of the voice machinery. assert "EXAMPLES" not in seen[0] assert "Sentence length varies" not in seen[0] @@ -135,7 +137,7 @@ def base_fn(messages, **_kwargs: object) -> str: assert "Never use these words" in article_prompts[0] assert "Never use these words" not in base_prompts[0] assert "EXAMPLES" not in base_prompts[0] - for line in ("section 1 of", "Write about", "Write only the section about:"): + for line in ("section 1 of", "Aim for about", "Write only the section about:"): assert line in article_prompts[0] assert line in base_prompts[0] @@ -234,8 +236,10 @@ def test_invented_entities_disqualify_the_article_arm(tmp_path: Path): generate_fn_base=lambda _m, **_k: BASE_DRAFT, ) item = receipt["items"][0] - assert item["article_invented_entities_count"] > 0 + # Inventing sections are dropped from the stitch; the arm still DQs. assert item["article_disqualified"] + assert item["article_invent_reject"] or item["article_invented_entities_count"] > 0 + assert item["winner"] != "article" def test_both_arms_are_held_to_the_same_length_ceiling(tmp_path: Path): diff --git a/tests/test_prompt_write.py b/tests/test_prompt_write.py index a3e08bd..82f3b1f 100644 --- a/tests/test_prompt_write.py +++ b/tests/test_prompt_write.py @@ -1,6 +1,7 @@ """Tests for the locked RAG writing prompt (chat turns + flat fallback).""" from personality_protect.prompt_write import ( + WRITE_ARTICLE_SYSTEM_PROMPT, WRITE_SYSTEM_PROMPT, build_write_messages, build_write_prompt, @@ -32,6 +33,21 @@ def test_build_write_messages_are_system_plus_user(): assert "POST:" not in user +def test_article_channel_uses_article_system_prompt_not_post_shaped(): + messages = build_write_messages( + topic="Contoso packaging", + points="- Name one owner\n- Cut exceptions", + examples=["Short Contoso lines."], + channel="article", + ) + assert messages[0]["content"] == WRITE_ARTICLE_SYSTEM_PROMPT + assert "You write one section of a LinkedIn article" in messages[0]["content"] + assert "Write ONE new post" not in messages[0]["content"] + assert "Prefer a shorter true section" in messages[0]["content"] + assert "Write this article section from the BRIEF now" in messages[1]["content"] + assert "Write one new post from the BRIEF now" not in messages[1]["content"] + + def test_build_write_prompt_flat_fallback_matches_locked_content(): prompt = build_write_prompt( topic="Contoso's quarterly planning", diff --git a/tests/test_write_article.py b/tests/test_write_article.py index 36a1a91..b8b9dc9 100644 --- a/tests/test_write_article.py +++ b/tests/test_write_article.py @@ -8,24 +8,24 @@ from contoso_articles import contoso_articles, contoso_post from personality_protect.config import init_profile +from personality_protect.draft_trim import word_count from personality_protect.models import save_index from personality_protect.style_profile import ( article_section_words, - article_word_aim, build_style_profile, draft_word_target, save_style_profile, ) from personality_protect.voice_index import build_voice_index -from personality_protect.write import run_write +from personality_protect.write import build_brief, run_write from personality_protect.write_article import ( MIN_ARTICLE_CORPUS, SECTION_TRIM_HEADROOM, - article_draft_ceiling, assert_article_corpus, count_indexed_article_pieces, outline_from_brief, run_write_article, + scale_section_words_for_brief, ) BRIEF_POINTS = "- Name one owner\n- Cut exceptions\n- Keep Ledger boring" @@ -129,12 +129,19 @@ def fake_generate(messages, **_kwargs: object) -> str: prompt_sink=sinks, ) style = build_style_profile([*contoso_articles(6), contoso_post()]) - expected = article_section_words(style, sections=3) + brief_words = word_count(build_brief("Contoso packaging", BRIEF_POINTS)) + expected = scale_section_words_for_brief( + article_section_words(style, sections=3), + brief_words=brief_words, + sections=3, + ) assert result["section_words"] == expected - assert result["word_aim"] == article_word_aim(style) + assert result["word_aim"] == expected * 3 # The post ceiling is a LinkedIn character limit and must not be the target. - assert f"Write about {expected} words in this section" in seen[0] + assert f"Aim for about {expected} words in this section" in seen[0] assert f"Never exceed {draft_word_target(style)} words" not in seen[0] + assert "ALLOWED names from the BRIEF only" in seen[0] + assert "Prefer stopping early over inventing" in seen[0] def test_section_prompt_states_its_place_in_the_article(tmp_path: Path): @@ -169,9 +176,15 @@ def test_section_trim_keeps_a_long_section_from_running_away(tmp_path: Path): generate_fn=lambda _m, **_k: runaway, ) style = build_style_profile([*contoso_articles(6), contoso_post()]) - per_section = int(round(article_section_words(style, sections=3) * SECTION_TRIM_HEADROOM)) + brief_words = word_count(build_brief("Contoso packaging", BRIEF_POINTS)) + expected_words = scale_section_words_for_brief( + article_section_words(style, sections=3), + brief_words=brief_words, + sections=3, + ) + per_section = int(round(expected_words * SECTION_TRIM_HEADROOM)) assert result["section_trim_words"] == per_section - assert result["draft_words"] <= article_draft_ceiling(style, sections=3) + assert result["draft_words"] <= per_section * 3 def test_article_retrieval_never_mixes_in_posts(tmp_path: Path): @@ -189,6 +202,28 @@ def test_article_retrieval_never_mixes_in_posts(tmp_path: Path): assert all(piece_id.startswith("contoso-article") for piece_id in result["exemplar_ids"]) +def test_inventing_section_is_dropped_from_the_stitch(tmp_path: Path): + _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) + paths, _, _ = init_profile("contoso", home=tmp_path) + calls = {"n": 0} + + def fake_generate(_messages, **_kwargs: object) -> str: + calls["n"] += 1 + if calls["n"] <= 2: + return ( + "Fabrikam Northwind invented a packaging vendor Contoso never named." + ) + return "Contoso names one owner and cuts exceptions. Keep Ledger boring." + + result = run_write_article( + "Contoso packaging", BRIEF_POINTS, paths, k=1, generate_fn=fake_generate + ) + assert result["dropped_sections"] + assert "Fabrikam" not in result["text"] + assert result["invent_reject"] + assert result["invented_entities"] + + def test_run_write_channel_article_delegates(tmp_path: Path): _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) paths, _, _ = init_profile("contoso", home=tmp_path)