diff --git a/README.md b/README.md index 555e5d5..b533df0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ > Voluntary, opt-in PR-author comprehension quizzes. Surface the gap between what you think your code does and what it actually does — before you merge. -![Demo: `cognit take` generates a quiz from a PR diff while Claude reads the code, then the author answers four questions — multiple choice, a mermaid-pick diagram, an open question, and true/false — and gets graded scores in the browser.](docs/img/cognit-demo.gif) +![Demo: `cognit take` generates a quiz from a real PR diff while Claude reads the code, then the author answers five questions — three multiple-choice, a mermaid-pick flow diagram, and true/false — and gets graded scores in the browser.](docs/img/cognit-demo.gif) **cognit flips the usual AI coding loop.** Normally you write a prompt and Claude writes the code. cognit has Claude *read* the code and write prompts back — questions only you, the author, can answer about the diff you're about to merge. Same model, arrows reversed, and the loop closes on the one question that matters: *does this code do what you intended?* Each question you wrestle with — especially the ones you get wrong — is comprehension credit banked against [comprehension debt](#why-this-exists). Call it **comprehension-driven development (CDD)**. diff --git a/docs/img/cognit-demo.gif b/docs/img/cognit-demo.gif index 31d8d52..b631e83 100644 Binary files a/docs/img/cognit-demo.gif and b/docs/img/cognit-demo.gif differ diff --git a/scripts/capture_demo_quiz.py b/scripts/capture_demo_quiz.py new file mode 100644 index 0000000..2174ab3 --- /dev/null +++ b/scripts/capture_demo_quiz.py @@ -0,0 +1,65 @@ +"""Capture a REAL cognit-generated quiz for the README demo. + +Runs the actual generation engine (`ClaudeAgentLLM` → the `claude` binary) against +a live PR, recording both the generated quiz and the activity feed Claude emits +while it reads the diff. The demo recorder (`record_demo.py`) replays these so the +GIF shows genuine, model-authored questions — not a hand-written stand-in — while +staying offline and deterministic on re-record. + +Usage: + uv run python scripts/capture_demo_quiz.py [--model claude-sonnet-4-6] + +Writes: + scripts/demo_data/quiz.json — the generated Quiz (post label-neutralization) + scripts/demo_data/feed.json — the activity events (thinking/text/tool_use) +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from cognit.engine.generate import generate_quiz +from cognit.engine.llm_claude_agent import ClaudeAgentLLM +from cognit.ghio.pr import fetch_pr_info + +OUT_DIR = Path(__file__).resolve().parent / "demo_data" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("pr_url") + ap.add_argument("--model", default="claude-sonnet-4-6") + args = ap.parse_args() + + info = fetch_pr_info(args.pr_url) + print(f"PR #{info.number}: {info.title!r} (branch {info.branch})") + + events: list[dict[str, Any]] = [] + llm = ClaudeAgentLLM(model=args.model) + llm.on_event = events.append + + print(f"generating with {args.model} (this can take a few minutes)…") + quiz = generate_quiz( + pr_title=info.title, + pr_body=info.body, + pr_number=info.number, + pr_url=args.pr_url, + branch=info.branch, + llm=llm, + model=args.model, + ) + + OUT_DIR.mkdir(parents=True, exist_ok=True) + (OUT_DIR / "quiz.json").write_text(quiz.model_dump_json(indent=2)) + (OUT_DIR / "feed.json").write_text(json.dumps(events, indent=2)) + + kinds = {q.type: sum(1 for x in quiz.questions if x.type == q.type) for q in quiz.questions} + print(f"wrote {OUT_DIR}/quiz.json — {len(quiz.questions)} questions {kinds}") + print(f"wrote {OUT_DIR}/feed.json — {len(events)} activity events") + + +if __name__ == "__main__": + main() diff --git a/scripts/demo_data/feed.json b/scripts/demo_data/feed.json new file mode 100644 index 0000000..293044d --- /dev/null +++ b/scripts/demo_data/feed.json @@ -0,0 +1,43 @@ +[ + { + "kind": "step", + "tool": "submit_quiz" + }, + { + "kind": "thinking", + "text": "Let me fetch the diffs for both changed files to understand the implementation.", + "tool": "submit_quiz" + }, + { + "kind": "tool_use", + "name": "mcp__cognit__file_diff", + "tool": "submit_quiz", + "detail": "src/cognit/engine/generate.py" + }, + { + "kind": "tool_use", + "name": "mcp__cognit__file_diff", + "tool": "submit_quiz", + "detail": "tests/engine/test_generate.py" + }, + { + "kind": "thinking", + "text": "Now I have a clear picture of the change. Let me plan the quiz shape.\n\n**Understanding the change:**\n\n1. `_is_retryable(exc)` \u2014 classifies exceptions:\n - `ValidationError` \u2192 False (non-retryable)\n - Fatal markers in message \u2192 False\n - Retryable markers in message \u2192 True\n - Otherwise (no markers match) \u2192 False (the final `return any(...)` returns False)\n\n2. `_backoff_delay(attempt, rng)` \u2014 0-based attempt:\n - Exponential backoff with a 30-second cap, doubling the delay with each retry (2s, 4s, 8s, etc.)\n\n3. `_draft_with_retry` \u2014 3 total attempts:\n - Loops through attempts 0, 1, 2, and exits early if the exception is non-retryable or if we've exhausted all retries\n - Sleep happens between retries but not after the final failure\n - The retry message displays the next attempt number (so attempt 0 shows \"retrying (2/3)\")\n\n4. What's interesting about `_is_retryable` for a `RuntimeError` with no markers \u2014 it returns False, so a generic RuntimeError like \"agent did not call submit_quiz\" won't be retried, even though people might expect all RuntimeErrors to be retryable.\n\n5. The backoff delay calculation for attempt 0 caps out at 2.0 seconds since `min(30, 2.0 * 2**0)` equals 2.0.\n\n6. The `last_exc` variable gets assigned in the except block, but the final raise is unreachable because the exception fires during the last attempt before that line executes.\n\n7. The `on_event` check verifies if `sink` is callable, where `sink` comes from `getattr(llm, \"on_event\", None)`.\n\n8. If `on_event` doesn't exist or is None, no notification gets sent. The activity sink shows the next attempt number as `attempt + 2`, so when attempt=0 it displays \"2/3\" to indicate this is retry attempt 2 of 3 total.\n\nNow I'm thinking through what makes effective quiz questions. For the `_is_retryable` function, I want to test whether it correctly identifies unrecognized errors like `RuntimeError(\"agent did not call submit_quiz\")` \u2014 it should return False since there's no matching retryable marker, no ValidationError, and no fatal marker.\n\nThe second question targets the sleep behavior: after exhausting all 3 attempts with transient failures, `sleep` gets called exactly 2 times, not 3, because the final attempt fails immediately without sleeping afterward.\n\nFor the third question, I'm checking the maximum delay from `_backoff_delay(0, rng)` \u2014 it caps at 2.0 seconds since that's the minimum of 30 and 2.0 times 2 to the power of 0.\n\nThe fourth question asks whether the final `raise last_exc` statement is actually reachable at runtime \u2014 it's not, because on the last attempt the exception gets raised inside the except block before the loop completes.\n\nFor the fifth question, I should diagram the control flow of the retry logic itself, since that's where the real complexity lies. The key points to clarify are whether sleeping happens before or after raising on non-retryable errors, whether it sleeps after the final failed attempt, and which raise statement actually exits the loop.\n\nThe sixth question explores why the backoff uses full jitter instead of a fixed delay. The reasoning is that fixed schedules cause thundering herd problems when multiple clients retry simultaneously, while spreading waits uniformly across the window decorrelates them. But this explanation is already in the PR body, so it wouldn't pass the lookup test.\n\nI'm reconsidering what makes a good open question. The `_MAX_ATTEMPTS = 3` choice is interesting because the comment explains the reasoning\u2014three attempts is enough to survive brief rate-limit windows without turning a temporary issue into a prolonged outage.\n\nWhat about why `_draft_with_retry` accepts a `sleep` parameter instead of calling `time.sleep` directly? That enables deterministic testing without actual delays, letting the test capture and verify sleep durations. The design intent isn't explicitly documented, so this could work as an open question.\n\nActually, here's another angle: why does the function re-raise the original exception unchanged rather than wrapping it? The PR body and docstring both explain this is so callers see the real cause, which means it might fail the lookup test too.\n\nLet me look for something more subtle. The `generate_quiz` call site uses `rng=random.Random()` instead of calling `random.uniform` directly. That's interesting because using a separate instance isolates the backoff jitter from affecting reproducibility elsewhere in the code, like in `_neutralize_mermaid_labels` which also takes an `rng` parameter...\n\nActually, I don't need to go down that rabbit hole. Let me focus on crafting better quiz questions that require real inference from the code.\n\nI'm sketching out a revised quiz structure: an MCQ about what `_is_retryable` returns for an unrecognized RuntimeError with no retryable or fatal markers, another about the maximum possible return value from `_backoff_delay(0, rng)`, and a few more that test understanding of the actual logic rather than just surface-level reading.\n\nFor the sleep-call question, I need to verify the exact condition\u2014whether the check happens before or after the sleep call matters for getting the count right. And for the Mermaid diagram, I'm mapping out the control flow through a single attempt in `_draft_with_retry`, starting with the LLM call and branching on success or exception handling. if final attempt OR not retryable \u2192 raise immediately\n - Otherwise \u2192 sleep \u2192 continue loop\n \n Misconceptions to encode:\n A: **Correct**: check `(final attempt OR not retryable)` \u2192 raise; else sleep \u2192 next attempt\n B: Check retryable first, then check final attempt (wrong order \u2014 but actually the code is `attempt + 1 >= _MAX_ATTEMPTS or not _is_retryable(exc)`, so both are checked with OR, order doesn't matter for outcome... but conceptually someone might think it sleeps even on final attempt if retryable)\n C: Sleep before checking if retryable/final\n D: Raise immediately for all exceptions without checking retryability\n\n I'm thinking about what a mermaid diagram would best illustrate here. The most instructive approach would show the full retry loop with all three attempts and when sleep actually occurs, which would directly address the misconception about sleeping after the final failure. But if I ask that as a multiple choice question and also include a mermaid diagram covering the same ground, that creates redundancy.\n\n Instead, I should make the mermaid focus on the per-attempt decision flow\u2014showing what happens within each iteration\u2014while keeping the multiple choice question about the specific count of sleep operations. That way they're complementary rather than overlapping. single caught exception* \u2014 what happens when `draft_quiz` raises:\n - Check: is this the final attempt OR not retryable?\n - Yes \u2192 re-raise immediately\n - No \u2192 compute backoff \u2192 notify sink (if present) \u2192 sleep \u2192 next iteration\n \n Key misconceptions:\n B: Sleeps first, then checks retryability (swap order)\n C: Checks retryability but always sleeps before re-raising on final attempt\n D: Always re-raises, never sleeps (never retries)\n \n Now I'm sketching out the question shapes. First, a multiple choice on `_is_retryable` with a `RuntimeError(\"agent did not call submit_quiz\")` \u2014 the trap is thinking any non-ValidationError runtime error gets retried. Then another MCQ about the maximum backoff delay from `_backoff_delay(0, rng)` \u2014 people often confuse 2^0 with the cap or think it always maxes out at 30 seconds. A third question checks whether sleep actually gets called after all three transient failures or if it skips the final one. I'll add a Mermaid diagram showing the exception flow inside the retry loop, highlighting the decision point between raising and sleeping. Finally, a true/false about whether calling a non-callable `llm.on_event` (like a string) would trigger a TypeError \u2014 the answer is false because the code checks if it's callable first. \u2014 the code checks `if callable(sink)` before calling it, so if `on_event` is a non-callable attribute, the notification is silently skipped. If `on_event` exists but isn't callable (like a string), `getattr` retrieves it, but the `callable()` guard prevents a TypeError from being raised. This is a subtle behavioral detail worth highlighting.\n\nNow I'm considering whether this is the most interesting angle, or if I should focus on something else instead. Looking at the retry notification text, when `attempt=1` triggers the third attempt, the formula `attempt + 2` gives `3`, so it displays \"(3/3)\". Let me frame this as a true/false question about what the activity feed actually shows during the first retry.\n\nThe key insight is that people would naturally assume the notification says \"retry 1\" for the first retry attempt, but it actually displays \"(2/3)\" because `attempt + 2 = 0 + 2 = 2`, representing the next attempt number about to start. This requires understanding that `attempt` is zero-indexed and tracing the arithmetic, which makes it a solid question.\n\nI'm settling on this as my true/false item and now moving toward structuring the quiz with an MCQ about `_is_retryable` handling generic RuntimeError. no markers \u2192 False. (file: generate.py `_is_retryable`)\n2. **MCQ** \u2014 `_backoff_delay(0, rng)` max possible return value = 2.0s. (file: generate.py `_backoff_delay`)\n3. **MCQ** \u2014 Number of `sleep` calls when all 3 attempts fail transiently = 2. (file: generate.py `_draft_with_retry`)\n4. **Mermaid** \u2014 Retry loop per-attempt decision flow.\n5. **TF** \u2014 Activity feed shows \"(2/3)\" not \"(1/3)\" when first attempt (attempt=0) fails.\n\nThat's 5 questions for a ~123-line addition. Reasonable.\n\nLet me check each against the usefulness criteria:\n\n1. MCQ `_is_retryable` for no-marker RuntimeError catches a common misconception \u2014 people often assume any non-fatal RuntimeError gets retried, but the logic requires specific markers. The difficulty isn't trivial since you need to trace through the final `any()` check to see it returns False when nothing matches. This one's worth keeping.\n\n2. MCQ `_backoff_delay(0, rng)` max tests whether students can evaluate the formula correctly. Many would either think `2**0 = 1` so the cap is 1s, or misread it as always 30s. You need to actually compute `min(30, 2.0 * (2**0))` to get the right answer.\n\n3. MCQ sleep count \u2014 people naturally assume there's a sleep after every failed attempt, but the final raise prevents that. Tracing the loop control flow isn't obvious.\n\n4. Mermaid diagram adds visual clarity on the decision order and control flow, which is distinct from the MCQs.\n\n5. TF about the activity feed format \u2014 I'm checking whether the event text shows \"(2/3)\" when attempt=0 fails. The format string is `f\"transient error \u2014 retrying ({attempt + 2}/{_MAX_ATTEMPTS}) in {delay:.0f}s\u2026\"`, so when attempt=0, it shows (0+2)/(3) = (2/3). This is more trivia than conceptual understanding, so I'm less confident it's worth including.\n\nActually, let me replace this with a better true/false: a RuntimeError containing \"service unavailable: upstream timeout\" would be non-retryable because it doesn't match any entry in `_RETRYABLE_MARKERS`. But checking the markers list \u2014 \"service unavailable\" and \"timeout\" are both there, so the error IS retryable. This makes the statement false and tests whether students actually understand the marker matching logic.\n\nThe question works well since it requires tracing through the actual markers, though it's conceptually similar to the first question about `_is_retryable`. I should think of something more distinct that tests a different aspect of the code.\n\nActually, here's a better angle: when `_draft_with_retry` exhausts all attempts, it re-raises the exception from the final attempt, not the first one. The test confirms this by checking for the last error message. So a true/false statement claiming the first exception gets raised would be false \u2014 the code explicitly re-raises the current exception when the attempt limit is reached, meaning the last failure is what propagates to the caller. the first one (especially since the PR body says \"re-raises the original exception unchanged\"). But \"original\" refers to the exception from the current/last attempt, not the first one.\n\nWait: the PR body says \"A non-retryable error \u2014 or the final attempt \u2014 re-raises the original exception unchanged.\" The docstring says \"A non-retryable error \u2014 or the final attempt \u2014 re-raises the original exception unchanged, so the caller's error handling (and the activity feed) sees the real cause, not a wrapper.\" The word \"original\" here means \"the raw exception from that attempt, not wrapped in something else,\" not \"the very first exception.\"\n\nSo the TF: \"When all 3 attempts fail with transient errors, the exception propagated to the caller is the one raised by the first attempt.\"\n- Answer: False \u2014 the exception from the **final** (third) attempt is what propagates, since `raise` re-raises the current exception in scope.\n- The test confirms this: `match=f\"#{_MAX_ATTEMPTS - 1}\"` = \"#2\" (the last queued exception, 0-indexed).\n\nI was confusing \"original exception\" in the docstring with the first exception ever raised, but it actually means the exception from the current (final) attempt. This passes all checks. The mermaid question tests the flow while this tests which exception propagates\u2014different aspects. And this doesn't overlap with the sleep count question either.\n\nNow I'm finalizing the answers: the retryability check for `RuntimeError(\"agent did not call submit_quiz\")` returns False, and I need to work through the backoff calculation next.\n\nDrafting the mermaid diagram for the retry loop control flow... I need to standardize these diagrams so they all have the same structure and node count, with each one representing a single misconception. The correct flow should show that when an exception is caught, we check if it's either the final attempt or non-retryable\u2014if either is true, we raise; otherwise we sleep and continue. My three distractors will represent common mistakes: sleeping before checking, only checking retryability and ignoring the final attempt condition, and always raising immediately without any retry logic.\n\nLet me restructure all four diagrams to include both the success and exception paths so they're comparable in complexity.\n\nGenerating flowchart diagrams... I see that distractor C is actually distinct from A because it removes the final attempt check entirely, creating a scenario where retryable errors would loop indefinitely without ever hitting a maximum attempt limit. Now I'm working through distractor D, which represents the opposite problem\u2014a version that never retries and immediately raises exceptions.\n\nLet me refine these flowchart diagrams to be more consistent and concise, keeping them all around the same size with roughly 5-6 nodes each. I'm working through distractor options for the flowchart, trying different variations that maintain six nodes but introduce logical errors\u2014like skipping the final-attempt check entirely, or raising exceptions without attempting retries. Let me refine distractor D to be distinct from the others. I'm considering a fourth distractor where the check order is reversed\u2014testing retryability before the final attempt check, then sleeping regardless of whether it's the final attempt. This creates a 7-node diagram, which is getting unwieldy compared to the others, so I should probably compress it or find a more concise misconception to represent.\n\nNow I'm refocusing the diagrams to start after the exception is caught, removing the initial \"raises?\" decision point to keep them consistent. I'm laying out the correct flow alongside distractors that each represent a specific ordering mistake\u2014like sleeping before checking conditions, or checking final attempt before retryability. I'm verifying that each distractor changes exactly one element from the correct answer\u2014A combines both checks, B reverses the sleep and check order, C omits the final-attempt check, and D omits the retryability check. Now I need to double-check the sequence in the actual code to make sure the sink notification happens after the sleep.\n\nActually, looking at the code more carefully, the sink gets notified before sleeping, not after\u2014so my diagram has the order backwards. Let me correct diagram A to show: exception caught \u2192 check conditions \u2192 re-raise if needed \u2192 notify sink \u2192 sleep \u2192 next attempt.\n\nNow I'm realizing this ordering difference could itself be a useful distractor. I can create multiple versions: one with the correct sequence (notify then sleep), and another that swaps them to see if people catch the mistake.\n\nI'm also building out distractors that isolate specific misconceptions \u2014 like checking only retryability without considering if it's the final attempt, or vice versa. Each diagram targets a different gap in understanding the retry logic.\n\nThe subtle difference between options A and B (whether notification happens before or after the sleep) actually matters because it affects when the user learns about the delay. Getting notified before waiting is more useful than after, so this distinction tests whether someone carefully traced through the code.\n\nLet me trace through the actual retry loop once more... I'm checking whether these four diagrams are distinct enough from each other. A and B differ in the order of notification versus sleep, while A and C differ in how the retry check is framed. C and D are quite similar though\u2014both have the same structure with only different labels on the decision node. To make them more visually distinct, I could reshape D to show a different misconception, like checking the sink notification before validating whether the error is retryable. attempt checked (D)\n\nBut C and D are structurally very similar. A better D might be: \"always sleeps before raising, even on non-retryable/final\" \u2014 i.e., there's no early raise. This encodes the misconception that you always sleep first, then check if it's the final attempt to decide whether to re-raise, missing both the retryability check and the proper ordering.\n\nSo the four options break down as: A is the correct combined check with proper ordering, B has the sink and sleep reversed, C only checks retryability but skips the final-attempt check, and D always notifies and sleeps first before checking if it should raise, missing the retryability check entirely.\n\nActually, the sleep versus notify order in B might be too subtle. Let me reconsider what would be a more impactful misconception \u2014 maybe checking retryability but not final attempt, combined with sleep before notify. Writing flowchart diagrams... I'm laying out four different flowchart designs to compare how they handle exception catching, retryability checks, and backoff logic\u2014each with six nodes and the same left-to-right flow but different orderings of when to check if an attempt is final or retryable. Now I'm drafting a multiple-choice question about what `_is_retryable` returns when given an unrecognized error like `RuntimeError`, where the correct answer is that it returns False since the error matches neither a retryable pattern nor a fatal marker.\n\nFor the explanation, the key insight is that `_is_retryable` only returns True if it positively detects a retryable marker in the error message\u2014the function doesn't default to retrying unknown errors, so a RuntimeError without a recognized marker gets False. Moving on to the second question about the maximum delay value from `_backoff_delay(attempt=0, rng)`, which involves understanding how the exponential backoff calculation caps out. Now I'm working through question 3 about sleep call counts. When all three attempts fail, sleep only gets called twice\u2014after the first and second failures. The third failure triggers the raise immediately without sleeping, since there's no next attempt to wait for. For question 4, I'm confirming the control flow diagram.\n\nMoving to question 5, I need to clarify which exception actually propagates when all attempts are exhausted. The final raise statement re-raises the current exception from the last attempt, not the first one\u2014the test confirms this by matching against the third error message. The key insight is that \"original exception\" in the PR documentation refers to the raw SDK exception without wrapping, not necessarily the first failure. The mermaid diagram shows the re-raise as a terminal node, which means Q3 and Q4 are testing pretty much the same thing\u2014both about understanding when the loop exits. I should replace Q3 with something that tests a different aspect of the code, maybe around the random number generator behavior or how the quiz content gets generated.\n\nActually, I could make Q3 about what the activity feed notification displays when the first attempt fails and we're about to retry. The notification text uses `f\"transient error \u2014 retrying ({attempt + 2}/{_MAX_ATTEMPTS})\"`, so when attempt=0 fails, it shows \"(2/3)\" for the next attempt. This tests whether someone understands both the 0-based indexing and how the notification message is constructed. So the formula `attempt + 2` correctly calculates the next attempt number when `attempt` is 0-based: a failed first attempt (attempt=0) shows \"(2/3)\" because you're about to start attempt 2, and a failed second attempt (attempt=1) shows \"(3/3)\" for attempt 3. The semantics work out because we're displaying the next attempt number, not the current one.\n\nThis passes the lookup test reasonably well\u2014a reader can see the arithmetic and understand it's simple, though they do need to know that `attempt` is 0-based and that `_MAX_ATTEMPTS = 3`. I think the sleep count question is stronger for Q3 since it requires tracing the control flow and understanding how the `raise` statement affects the loop.\n\nI'm settling on these five questions: identifying when `_is_retryable` returns false for a RuntimeError without a marker, calculating the max value from `_backoff_delay(0)`, counting total sleep calls across all three failures, a mermaid diagram showing the per-exception control flow, and a true/false about whether the first failure's exception propagates. This covers both the lookup-style details and the structural understanding. Now I should consider whether to add something about the `generate_quiz` call site and how `random.Random()` is used there. I'm verifying the `_is_retryable` logic for that RuntimeError\u2014none of the retryable markers match, so it correctly returns False. For Q2, the maximum delay from `_backoff_delay(0, rng)` works out to 2.0 seconds. And for Q3 with three transient failures, I'm tracing through the retry loop to count how many times sleep gets called\u2014it should be invoked once per failed attempt before the final one.\n\nFor Q5, I'm checking that the test properly queues three RuntimeErrors and expects the last one (with \"#2\") to be raised when attempts are exhausted. Now I'm working on the mermaid diagrams...\n\nWriting mermaid diagrams... I'm laying out four different flowchart variations to compare how they handle the exception retry logic\u2014each one adjusts the decision point and order of operations slightly to see which feels clearest for the flow.\n\nNow I'm checking that the mermaid syntax will parse correctly with the basic elements I'm using: flowchart declarations, rectangular and diamond nodes, and labeled edges. Everything looks solid.\n\nFor the quiz submission, I'm assigning unique IDs to each question and making sure the explanation for the mermaid diagram covers all four options\u2014highlighting that the correct answer combines both the final attempt check and retryability check before deciding whether to raise immediately or proceed with notification and backoff, while the other options miss one of these guards.\n\nNow I'm finalizing the multiple choice options for the first two questions, making sure each option is clearly written with its reasoning.\n\nFor Q3, the answer is B because the sleep happens after the exception is raised on the final attempt, so it never executes. Q5 is false. Let me compile everything and submit.", + "tool": "submit_quiz" + }, + { + "kind": "text", + "text": "Now let me plan the quiz shape before drafting:\n\n**Slots:**\n1. **MCQ** \u2014 `_is_retryable` for a `RuntimeError` with *no* matching markers (e.g. `\"agent did not call submit_quiz\"`). Misconception: \"any non-`ValidationError`, non-fatal `RuntimeError` is retried.\" \u2014 `generate.py:_is_retryable`\n2. **MCQ** \u2014 `_backoff_delay(attempt=0, rng)` max possible return value. Misconception: \"`2**0 = 1` so cap = 1 s\" or \"always caps at 30 s.\" \u2014 `generate.py:_backoff_delay`\n3. **MCQ** \u2014 number of `sleep` calls when all 3 attempts fail transiently. Misconception: \"sleep is called after every failure, including the last.\" \u2014 `generate.py:_draft_with_retry` loop\n4. **Mermaid** \u2014 per-exception control flow inside the retry loop (check order, notify-then-sleep ordering). Four structural misconceptions.\n5. **TF** \u2014 after attempt exhaustion, the exception propagated to the caller is from the *first* failed attempt. Answer: False (it's the *last* one, re-raised directly).", + "tool": "submit_quiz" + }, + { + "kind": "tool_use", + "name": "mcp__cognit__submit_quiz", + "tool": "submit_quiz" + }, + { + "kind": "text", + "text": "checking diagrams\u2026", + "tool": "submit_quiz" + } +] \ No newline at end of file diff --git a/scripts/demo_data/quiz.json b/scripts/demo_data/quiz.json new file mode 100644 index 0000000..275f4c1 --- /dev/null +++ b/scripts/demo_data/quiz.json @@ -0,0 +1,62 @@ +{ + "version": "1", + "pr_number": 20, + "questions": [ + { + "type": "mcq", + "id": "q1", + "prompt": "Given `_is_retryable(RuntimeError(\"agent did not call submit_quiz\"))`, what does the function return?", + "options": [ + "True — it is a RuntimeError, not a ValidationError, and its message contains no fatal marker", + "False — no retryable marker appears in the message, so the final `return any(...)` evaluates to False", + "True — only ValidationError is excluded; all other exception types are unconditionally retried" + ], + "answer": "False — no retryable marker appears in the message, so the final `return any(...)` evaluates to False", + "explanation": "`_is_retryable` first short-circuits on `ValidationError`, then on any fatal marker — neither applies here. But it only returns `True` if a *retryable* marker is positively matched. The message 'agent did not call submit_quiz' matches nothing in `_RETRYABLE_MARKERS`, so `any(...)` is `False` and the function returns `False`. The misconception is that the absence of a fatal marker is enough to trigger a retry; it isn't — a retryable marker must match." + }, + { + "type": "mcq", + "id": "q2", + "prompt": "What is the maximum value `_backoff_delay(attempt=0, rng)` can return?", + "options": [ + "1.0 s — `2**0 = 1`, making the cap 1 second before the `min`", + "2.0 s — `min(30.0, 2.0 × 2**0)` = `min(30.0, 2.0)` = 2.0", + "30.0 s — every call saturates at `_MAX_DELAY_S` regardless of attempt" + ], + "answer": "2.0 s — `min(30.0, 2.0 × 2**0)` = `min(30.0, 2.0)` = 2.0", + "explanation": "The formula is `min(_MAX_DELAY_S, _BASE_DELAY_S * (2**attempt))`. At attempt 0: `2.0 × 2**0 = 2.0 × 1 = 2.0`, and `min(30.0, 2.0) = 2.0`. The common slip is dropping the `_BASE_DELAY_S` multiplier (giving 1.0), or assuming the cap always saturates at 30 s (which only happens from attempt 4 onward: `2.0 × 16 = 32 > 30`)." + }, + { + "type": "mcq", + "id": "q3", + "prompt": "When all 3 attempts each fail with `RuntimeError('429 rate limit')`, how many times is the `sleep` callable invoked?", + "options": [ + "3 — once after each of the three failures", + "2 — on the final attempt the `raise` inside the except block fires before `sleep` is reached", + "0 — the last transient failure short-circuits all preceding sleeps" + ], + "answer": "2 — on the final attempt the `raise` inside the except block fires before `sleep` is reached", + "explanation": "On attempt 2 (0-based), `attempt + 1 >= _MAX_ATTEMPTS` is `3 >= 3` — True — so `raise` executes immediately in the except block, before the `sleep` call. Attempts 0 and 1 do sleep (they pass the condition check), giving exactly 2 sleeps for 3 total failures. The test `assert len(slept) == _MAX_ATTEMPTS - 1` directly verifies this." + }, + { + "type": "mermaid", + "id": "q4", + "prompt": "Which diagram correctly models what `_draft_with_retry` does when `llm.draft_quiz` raises an exception on a given attempt?", + "options": { + "A": "flowchart LR\n A[exception caught] --> B{final or non-retryable?}\n B -- yes --> C[re-raise]\n B -- no --> D[sleep backoff]\n D --> E[notify sink]\n E --> F[next attempt]", + "B": "flowchart LR\n A[exception caught] --> B{final or non-retryable?}\n B -- yes --> C[re-raise]\n B -- no --> D[notify sink]\n D --> E[sleep backoff]\n E --> F[next attempt]", + "C": "flowchart LR\n A[exception caught] --> B{final attempt?}\n B -- yes --> C[re-raise]\n B -- no --> D[notify sink]\n D --> E[sleep backoff]\n E --> F[next attempt]", + "D": "flowchart LR\n A[exception caught] --> B{not retryable?}\n B -- yes --> C[re-raise]\n B -- no --> D[notify sink]\n D --> E[sleep backoff]\n E --> F[next attempt]" + }, + "answer": "B", + "explanation": "" + }, + { + "type": "tf", + "id": "q5", + "prompt": "When all 3 attempts fail with transient errors, the exception that ultimately propagates to the caller is the one raised by the **first** failed attempt.", + "answer": false, + "explanation": "On the final attempt (attempt=2), `raise` re-raises the *current* `exc` from that iteration — not the stored `last_exc` from earlier iterations. The test confirms this with `match=f'#{_MAX_ATTEMPTS - 1}'`, which matches '#2' (the third error). The PR doc says 're-raises the original exception unchanged' — 'original' means the raw SDK exception, not wrapped in a new one, not necessarily the chronologically first one." + } + ] +} \ No newline at end of file diff --git a/scripts/record_demo.py b/scripts/record_demo.py index 20a9abd..6f8819f 100644 --- a/scripts/record_demo.py +++ b/scripts/record_demo.py @@ -1,33 +1,29 @@ """Record the README demo GIF of the `cognit take` browser flow. -Boots the FastAPI app with a polished inline demo quiz and a FAKE LLM (no Claude -tokens, no `gh` auth, no network), starts in the "generating" phase so the GIF -tells the full story, then drives Chromium via Playwright through a natural-paced -run: +Boots the FastAPI app with a **real, cognit-generated** quiz (captured from a live +PR by `scripts/capture_demo_quiz.py` into `scripts/demo_data/`) and a FAKE LLM for +grading only — no Claude tokens, no `gh` auth, no network. It starts in the +"generating" phase and replays the actual activity feed Claude emitted while +reading the diff, so the GIF tells the full story: - command → Claude "generating" activity feed → quiz renders → author answers - all four question types → Submit → results / scores + command → Claude's real "generating" activity feed → quiz renders → author + answers every question (answers computed from the quiz, so this adapts to + whatever mix/order the model produced) → Submit → results / scores Playwright records the run to a `.webm`, which ffmpeg converts to an optimized, -looping GIF at `docs/img/cognit-demo.gif` (two-pass palettegen/paletteuse). - -Everything runs offline and deterministically — this is the same fake-server -pattern the test suite uses (see tests/conftest.py and tests/server/). +looping GIF at `docs/img/cognit-demo.gif` (two-pass palettegen/paletteuse). The +content is genuine; the playback is deterministic and offline, so re-recording +after a UI change needs no Claude call. Prerequisites ------------- -The Chromium browser Playwright drives must be installed once: - - uv run playwright install chromium - -ffmpeg must be on PATH (used for the webm → gif conversion). + uv run playwright install chromium # once + ffmpeg on PATH + scripts/demo_data/{quiz.json,feed.json} # via scripts/capture_demo_quiz.py Usage ----- - uv run python scripts/record_demo.py - - # or the thin wrapper: - scripts/record-demo.sh + uv run python scripts/record_demo.py # or scripts/record-demo.sh Re-run after any UI change in src/cognit/server/assets/. Output overwrites docs/img/cognit-demo.gif; temp video files are cleaned up automatically. @@ -35,6 +31,7 @@ from __future__ import annotations +import json import shutil import socket import subprocess @@ -42,11 +39,12 @@ import threading import time from pathlib import Path +from typing import Any import httpx import uvicorn from fastapi import FastAPI -from playwright.sync_api import sync_playwright +from playwright.sync_api import Locator, Page, sync_playwright from cognit.engine.llm_fake import FakeLLM from cognit.engine.models import ( @@ -60,99 +58,108 @@ REPO_ROOT = Path(__file__).resolve().parent.parent OUT_PATH = REPO_ROOT / "docs" / "img" / "cognit-demo.gif" - -# Crisp viewport; device_scale_factor=2 renders at 2x for sharpness, then ffmpeg -# scales the recording down to GIF_WIDTH with lanczos. -VIEWPORT = {"width": 1280, "height": 800} +DEMO_DATA = Path(__file__).resolve().parent / "demo_data" + +# High-resolution capture. Playwright records the page at the *CSS viewport* +# resolution: device_scale_factor enlarges the backing store but NOT the screencast, +# and a record size larger than the viewport just pads the frame with empty canvas. +# So the viewport IS the resolution lever — we render and record 1:1 at 1440×900 and +# emit the GIF at that width with no downscale softening. +VIEWPORT = {"width": 1440, "height": 900} DEVICE_SCALE_FACTOR = 2 +RECORD_SIZE = VIEWPORT -GIF_WIDTH = 1000 -GIF_FPS = 14 - -PR_URL = "https://github.com/jonasbrami/cognit/pull/142" - - -def _demo_quiz() -> Quiz: - """A representative PR quiz: one of each question type, believable prompts, - and a 4-option mermaid-pick (A/B/C/D) to match the real product.""" - return Quiz( - pr_number=142, - questions=[ - MCQQuestion( - id="q1", - prompt="When `rate_limit_exceeded(key)` returns True, what does the middleware do?", - options=[ - "Raises HTTPException(429) and lets the handler catch it", - "Returns JSONResponse(status_code=429) with a Retry-After header", - "Logs a warning and passes the request through", - "Increments a counter and continues to the route", - ], - answer="Returns JSONResponse(status_code=429) with a Retry-After header", - explanation=( - "It returns a JSONResponse directly — the middleware short-circuits " - "and never lets the rate-limited request reach the route handler." - ), - ), - MermaidQuestion( - id="q2", - prompt="Which diagram matches the request path through the new middleware stack?", - options={ - "A": "flowchart LR\n R[request]-->A[auth]-->L[rate limit]-->H[route]", - "B": "flowchart LR\n R[request]-->L[rate limit]-->A[auth]-->H[route]", - "C": "flowchart LR\n R[request]-->H[route]-->A[auth]-->L[rate limit]", - "D": "flowchart LR\n R[request]-->A[auth]-->H[route]-->L[rate limit]", - }, - answer="A", - explanation="Auth runs first, then the rate limiter, then the route.", - ), - OpenQuestion( - id="q3", - prompt="Why does the counter store use Redis instead of an in-process dict?", - rubric="must mention shared state across worker processes", - ), - TrueFalseQuestion( - id="q4", - prompt="The `@skip_rate_limit` decorator bypasses the middleware entirely.", - answer=False, - explanation=( - "It only sets a flag the middleware reads — the request still passes " - "through the middleware, which then chooses not to count it." - ), - ), - ], - ) +GIF_WIDTH = 1440 +GIF_FPS = 16 + +PR_URL = "https://github.com/jonasbrami/cognit/pull/20" + +# Believable open-answer text, keyed by the captured question id. Filled in to +# match whatever open question the model wrote; a generic fallback is used for any +# id not listed so the recorder never stalls on an empty textarea. +OPEN_ANSWERS: dict[str, str] = {} +_OPEN_FALLBACK = ( + "It's a transient upstream failure — a rate limit or a brief overload — so a " + "short backed-off retry is likely to succeed, whereas a malformed submission " + "would just fail again." +) + +# Generating-phase feed: keep it short and legible. Cap the number of lines and +# truncate long thinking/text so the activity feed reads as a quick montage. +MAX_FEED_LINES = 9 +MAX_TEXT_CHARS = 130 +_DELAY_BY_KIND = {"step": 0.5, "tool_use": 0.55, "text": 0.85, "thinking": 0.9} + + +# ── data loading ──────────────────────────────────────────────────────── -def _stream_generation(broker, quiz: Quiz) -> None: - """Emit a realistic 'Claude is generating' activity feed into the broker, - then flip it to ready. Mirrors the event kinds quiz.js renders (step / text / - tool_use). Runs on a background thread so the page polls /progress live.""" - feed = [ - ({"kind": "step", "tool": "submit_quiz"}, 0.5), - ({"kind": "text", "text": "Reading the diff for PR #142…", "tool": "submit_quiz"}, 0.7), - ( - {"kind": "tool_use", "name": "read_file", "detail": "src/middleware/rate_limit.py"}, - 0.6, - ), - ({"kind": "tool_use", "name": "read_file", "detail": "src/app.py"}, 0.5), - ( - { - "kind": "text", - "text": "Auth runs before the limiter; counters live in Redis. Drafting questions…", - "tool": "submit_quiz", - }, - 0.9, - ), - ({"kind": "tool_use", "name": "grep", "detail": "skip_rate_limit"}, 0.5), - ( - { - "kind": "text", - "text": "Writing 4 questions across the changed paths.", - "tool": "submit_quiz", - }, - 0.8, - ), - ] +def _load_quiz() -> Quiz: + path = DEMO_DATA / "quiz.json" + if not path.exists(): + raise SystemExit( + f"{path} not found — capture a real quiz first:\n" + f" uv run python scripts/capture_demo_quiz.py {PR_URL}" + ) + return Quiz.model_validate_json(path.read_text()) + + +def _curate_feed() -> list[tuple[dict[str, Any], float]]: + """Load the captured activity feed and trim it to a short, legible montage. + + Keeps the leading `step`, then a mix of tool calls, thinking, and prose in + original order up to `MAX_FEED_LINES`, truncating long text. Returns + (event, delay) pairs. Falls back to a tiny synthetic feed if none was captured. + """ + path = DEMO_DATA / "feed.json" + if not path.exists(): + return [({"kind": "step", "tool": "submit_quiz"}, 0.5)] + events: list[dict[str, Any]] = json.loads(path.read_text()) + + curated: list[dict[str, Any]] = [] + for ev in events: + if len(curated) >= MAX_FEED_LINES: + break + kind = ev.get("kind") + if kind in ("text", "thinking"): + text = (ev.get("text") or "").strip() + if not text: + continue + if len(text) > MAX_TEXT_CHARS: + text = text[:MAX_TEXT_CHARS].rstrip() + "…" + curated.append({**ev, "text": text}) + elif kind in ("step", "tool_use"): + curated.append(ev) + return [(ev, _DELAY_BY_KIND.get(ev.get("kind", ""), 0.6)) for ev in curated] + + +# ── correct-answer helpers (data-driven playback) ─────────────────────── + + +def _answer_question(page: Page, file_loc: Locator, q: Any) -> None: + """Select the correct answer for one question, whatever its type/position.""" + file_loc.scroll_into_view_if_needed() + page.wait_for_timeout(550) + if isinstance(q, MCQQuestion): + idx = q.options.index(q.answer) + file_loc.locator(".option").nth(idx).click() + elif isinstance(q, MermaidQuestion): + idx = list(q.options).index(q.answer) + file_loc.locator(".diagram").nth(idx).click() + elif isinstance(q, TrueFalseQuestion): + file_loc.locator(".tf__cell").nth(0 if q.answer else 1).click() + elif isinstance(q, OpenQuestion): + ta = file_loc.locator("textarea") + ta.click() + ta.type(OPEN_ANSWERS.get(q.id, _OPEN_FALLBACK), delay=16) + page.wait_for_timeout(850) + + +# ── server / streaming plumbing ───────────────────────────────────────── + + +def _stream_generation(broker: Any, quiz: Quiz, feed: list[tuple[dict[str, Any], float]]) -> None: + """Replay the curated activity feed into the broker, then flip it to ready.""" for event, delay in feed: broker.emit(event) time.sleep(delay) @@ -187,7 +194,7 @@ def _serve(app: FastAPI, port: int) -> uvicorn.Server: return server -def _record_webm(base_url: str, video_dir: Path) -> Path: +def _record_webm(base_url: str, video_dir: Path, quiz: Quiz) -> Path: """Drive the browser through the demo and return the recorded .webm path.""" with sync_playwright() as p: browser = p.chromium.launch() @@ -195,7 +202,7 @@ def _record_webm(base_url: str, video_dir: Path) -> Path: viewport=VIEWPORT, device_scale_factor=DEVICE_SCALE_FACTOR, record_video_dir=str(video_dir), - record_video_size=VIEWPORT, + record_video_size=RECORD_SIZE, ) page = ctx.new_page() @@ -206,39 +213,16 @@ def _record_webm(base_url: str, video_dir: Path) -> Path: page.wait_for_timeout(2500) # let a few feed lines stream in # --- Quiz renders once the broker flips to ready. - page.wait_for_selector("#questions-root .file .option", timeout=8000) + page.wait_for_selector( + "#questions-root .file .option, #questions-root .diagram", timeout=8000 + ) page.wait_for_selector("#questions-root .diagram svg", timeout=10000) page.wait_for_timeout(1200) - # --- Q1 MCQ: pick the correct option (index 1). - page.locator("#questions-root .file").nth(0).locator(".option").nth(1).click() - page.wait_for_timeout(900) - - # --- Q2 mermaid: scroll into view, pick diagram A (correct, first card). - q2 = page.locator("#questions-root .file").nth(1) - q2.scroll_into_view_if_needed() - page.wait_for_timeout(700) - q2.locator(".diagram").first.click() - page.wait_for_timeout(900) - - # --- Q3 open: type a believable answer at a natural pace. - q3 = page.locator("#questions-root .file").nth(2) - q3.scroll_into_view_if_needed() - page.wait_for_timeout(500) - q3.locator("textarea").click() - q3.locator("textarea").type( - "Each worker is a separate process, so an in-process dict wouldn't share " - "counters. Redis is a single source of truth across all workers.", - delay=18, - ) - page.wait_for_timeout(800) - - # --- Q4 true/false: pick False (index 1, correct). - q4 = page.locator("#questions-root .file").nth(3) - q4.scroll_into_view_if_needed() - page.wait_for_timeout(600) - q4.locator(".tf__cell").nth(1).click() - page.wait_for_timeout(900) + # --- Answer every question correctly, in order (data-driven). + files = page.locator("#questions-root .file") + for i, q in enumerate(quiz.questions): + _answer_question(page, files.nth(i), q) # --- Submit and land on results; pause so scores are readable. page.locator("#reviewbar button.btn--primary").click() @@ -299,18 +283,21 @@ def main() -> None: if shutil.which("ffmpeg") is None: raise SystemExit("ffmpeg not found on PATH — install it to build the GIF.") - quiz = _demo_quiz() + quiz = _load_quiz() + feed = _curate_feed() # Start in the "generating" phase (quiz=None) so the GIF captures the streaming - # activity feed before the quiz appears. A background thread streams events and - # then flips the broker to ready. + # activity feed before the quiz appears. A background thread replays the captured + # feed and then flips the broker to ready. app = build_app( quiz=None, - pr_number=142, + pr_number=quiz.pr_number, pr_url=PR_URL, - llm=FakeLLM(canned_open_score=85, canned_open_feedback="Captures the key idea."), + llm=FakeLLM(canned_open_score=88, canned_open_feedback="Captures the key idea."), post_comment=lambda body: f"{PR_URL}#issuecomment-9999", ) - threading.Thread(target=_stream_generation, args=(app.state.broker, quiz), daemon=True).start() + threading.Thread( + target=_stream_generation, args=(app.state.broker, quiz, feed), daemon=True + ).start() port = _free_port() server = _serve(app, port) @@ -318,7 +305,7 @@ def main() -> None: video_dir = Path(tempfile.mkdtemp(prefix="cognit-demo-")) try: - webm = _record_webm(base, video_dir) + webm = _record_webm(base, video_dir, quiz) _webm_to_gif(webm, OUT_PATH) size_mb = OUT_PATH.stat().st_size / 1_000_000 print(f"wrote {OUT_PATH} ({size_mb:.2f} MB)")