From 39817c80e8d6f7901aff51d809038f07c97c931c Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 5 Jun 2026 21:00:24 +0000 Subject: [PATCH 01/14] Add hybrid_annot2: late-pretzel vs hatching disambiguation (new best) Kills the 'ahead' failure cluster: 33 end-of-series embryo_5 frames (x3 seeds = 99 predictions) where a shell-filling, vigorously moving late pretzel was called hatching and history anchoring locked the streak in. Fix is two-part, in both prompts: hatching is nearly instantaneous (rupture -> exit -> empty shell within a frame or two) while late pretzel wriggles WITHIN the shell, and a multi-frame hatching streak in history is self-contradictory - evidence the earlier calls were wrong, not something to continue. embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 69.5% +/- 0.2 (hybrid_annot: 66.3 +/- 0.2), adjacent 87.8%. embryo_5 end-frames: 0/33 hatching misreads, was 33/33. hatched stays 100% - the guard does not delay real hatch detection. Seed1 of the run was recovered after an API 529 killed the original process mid-seed. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_annot2.py | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 harness/solvers/hybrid_annot2.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 33a0ef2..7dadb60 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_annot2, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -9,6 +9,7 @@ "hybrid": hybrid.SOLVER, "hybrid_nodefer": hybrid_nodefer.SOLVER, "hybrid_annot": hybrid_annot.SOLVER, + "hybrid_annot2": hybrid_annot2.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_annot2.py b/harness/solvers/hybrid_annot2.py new file mode 100644 index 0000000..9134c99 --- /dev/null +++ b/harness/solvers/hybrid_annot2.py @@ -0,0 +1,97 @@ +"""hybrid_annot + late-pretzel vs hatching disambiguation. + +Targets the one remaining 'ahead' failure cluster: 33 consecutive end-of-series +frames on embryo_5 (x3 seeds = 99 failed predictions) where a very late, +shell-filling, vigorously moving pretzel was called "hatching", and history +anchoring then locked the streak in ("it has been hatching for 2 frames"). + +Two-part fix, both prompts: +1. Duration argument kills the trigger — hatching is a near-instant event + (rupture, exit, empty shell within a frame or two), while late pretzel + presses against and deforms the shell and wriggles WITHIN it for a long + time, which mimics emergence. +2. Explicit cascade breaker — a multi-frame "hatching" streak in the history + is self-contradictory; treat it as evidence the earlier calls were wrong + rather than something to continue. + +The annotator's embryo_5 notes back the first part: "the worm moves a lot ... +within the egg shell" on exactly these frames. +""" +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import ( + _SCIENTIFIC_ANALYSIS, + _TEMPORAL_ANALYSIS, + SCIENTIFIC_SYSTEM as _SCI_BASE, + TEMPORAL_SYSTEM as _TMP_BASE, +) + +_OLD_TMP_RULE = """\ +5. **Late pretzel.** The pretzel stage is long-lasting. A compact bright mass \ +that fills the eggshell is still pretzel even if it doesn't look "tangled" — \ +it has not hatched unless you see the worm OUTSIDE the shell.""" + +_NEW_TMP_RULE = """\ +5. **Late pretzel vs hatching.** The pretzel stage is long-lasting, and near \ +its end the worm moves vigorously WITHIN the eggshell — pressing against it, \ +deforming it, and changing pose between frames. This mimics emergence but is \ +still pretzel. Hatching, by contrast, is nearly instantaneous: the shell \ +ruptures, the worm exits, and within a frame or two you see a thin worm \ +clearly OUTSIDE the shell or an empty/partial shell. If the bright mass is \ +still shell-sized and shell-shaped, nothing has hatched. A history showing \ +several consecutive "hatching" observations is self-contradictory — hatching \ +completes almost immediately, so a long streak means those earlier calls \ +were wrong; re-examine against the shell outline instead of continuing the \ +streak.""" + +_OLD_SCI_TEXT = """\ +**HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ +elongated worm OUTSIDE the eggshell boundary, or an empty shell.""" + +_NEW_SCI_TEXT = """\ +**HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ +elongated worm OUTSIDE the eggshell boundary, or an empty shell. Hatching is \ +nearly instantaneous (rupture → exit → empty shell within a frame or two). \ +Late pretzel moves vigorously WITHIN the shell, pressing against and \ +deforming it — that mimics emergence but is still pretzel as long as the \ +bright mass stays shell-sized and shell-shaped. A multi-frame "hatching" \ +streak in the history is self-contradictory; treat it as evidence the \ +earlier calls were wrong rather than something to continue.""" + +TEMPORAL_SYSTEM = _TMP_BASE.replace(_OLD_TMP_RULE, _NEW_TMP_RULE) +SCIENTIFIC_SYSTEM = _SCI_BASE.replace(_OLD_SCI_TEXT, _NEW_SCI_TEXT) +assert TEMPORAL_SYSTEM != _TMP_BASE, "temporal late-pretzel rule did not match" +assert SCIENTIFIC_SYSTEM != _SCI_BASE, "scientific hatching text did not match" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + return SCIENTIFIC_SYSTEM if _is_scientific(frame) else TEMPORAL_SYSTEM + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_annot.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows ~10-15 frames of " + f"2fold, prolonged 2fold appearance plus any twisting reads as " + f"pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_annot2", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) From 16944c38dd54b47e376163741a7691751dbfe532 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Tue, 9 Jun 2026 14:07:47 +0000 Subject: [PATCH 02/14] render: fall back to EGL when the default GL backend fails make_context only tried the platform default backend (X11 on Linux), which cannot work on headless machines. Try EGL second and surface both errors if neither backend works. Verified on a headless box with Mesa llvmpipe: smoke test passes, ~49ms per 512x512 render at 384 steps. --- gently_perception/render/context.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/gently_perception/render/context.py b/gently_perception/render/context.py index 2d08c98..90367aa 100644 --- a/gently_perception/render/context.py +++ b/gently_perception/render/context.py @@ -23,8 +23,21 @@ def make_context(allow_cpu: bool = False) -> moderngl.Context: - """Create a standalone moderngl context, refusing CPU fallback by default.""" - ctx = moderngl.create_context(standalone=True) + """Create a standalone moderngl context, refusing CPU fallback by default. + + Tries the platform default backend first (X11 on Linux desktops), then + EGL, which is what works on headless machines (CI, remote boxes). + """ + try: + ctx = moderngl.create_context(standalone=True) + except Exception as default_err: # glcontext raises bare Exception for backend failures + try: + ctx = moderngl.create_context(standalone=True, backend="egl") + except Exception as egl_err: + raise RuntimeError( + f"No usable GL backend: default backend failed ({default_err}); " + f"EGL failed ({egl_err})." + ) from egl_err info = ctx.info renderer = (info.get("GL_RENDERER") or "").lower() vendor = (info.get("GL_VENDOR") or "").lower() From 5e741cb63f9ad34523e0f57237cffa2cf1aee2f0 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Tue, 9 Jun 2026 14:32:12 +0000 Subject: [PATCH 03/14] hybrid_annot2: correct the hatching-duration rationale to be cadence-based Hatching is not nearly instantaneous - it takes 2-3 minutes. How many frames show it depends entirely on imaging cadence: this series is imaged every ~4 minutes, so hatching spans at most a frame or two and may be missed outright, but a 20-second cadence would capture ~6 hatching frames. The operative rule (a multi-frame hatching streak at THIS frame rate is self-contradictory) is unchanged; only its justification is corrected. The recorded 69.5% run used the earlier wording - prompt_sha in its events.jsonl reflects that. --- harness/solvers/hybrid_annot2.py | 42 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/harness/solvers/hybrid_annot2.py b/harness/solvers/hybrid_annot2.py index 9134c99..d322d5d 100644 --- a/harness/solvers/hybrid_annot2.py +++ b/harness/solvers/hybrid_annot2.py @@ -6,10 +6,12 @@ anchoring then locked the streak in ("it has been hatching for 2 frames"). Two-part fix, both prompts: -1. Duration argument kills the trigger — hatching is a near-instant event - (rupture, exit, empty shell within a frame or two), while late pretzel - presses against and deforms the shell and wriggles WITHIN it for a long - time, which mimics emergence. +1. Cadence argument kills the trigger — hatching takes 2-3 minutes and this + series is imaged every ~4 minutes, so hatching spans at most a frame or + two (and may be skipped entirely), while late pretzel presses against and + deforms the shell and wriggles WITHIN it for a long time, which mimics + emergence. Note this is sampling-dependent: at a 20-second cadence the + same event would span ~6 frames. 2. Explicit cascade breaker — a multi-frame "hatching" streak in the history is self-contradictory; treat it as evidence the earlier calls were wrong rather than something to continue. @@ -36,14 +38,15 @@ 5. **Late pretzel vs hatching.** The pretzel stage is long-lasting, and near \ its end the worm moves vigorously WITHIN the eggshell — pressing against it, \ deforming it, and changing pose between frames. This mimics emergence but is \ -still pretzel. Hatching, by contrast, is nearly instantaneous: the shell \ -ruptures, the worm exits, and within a frame or two you see a thin worm \ -clearly OUTSIDE the shell or an empty/partial shell. If the bright mass is \ -still shell-sized and shell-shaped, nothing has hatched. A history showing \ -several consecutive "hatching" observations is self-contradictory — hatching \ -completes almost immediately, so a long streak means those earlier calls \ -were wrong; re-examine against the shell outline instead of continuing the \ -streak.""" +still pretzel. Hatching itself takes only 2-3 minutes, and the frames in \ +this series are about 4 minutes apart — so hatching appears in AT MOST one \ +or two frames, and may be skipped entirely (pretzel in one frame, a thin \ +worm clearly OUTSIDE the shell or an empty/partial shell in the next). If \ +the bright mass is still shell-sized and shell-shaped, nothing has hatched. \ +A history showing several consecutive "hatching" observations is therefore \ +self-contradictory at this frame rate — a long streak means those earlier \ +calls were wrong; re-examine against the shell outline instead of continuing \ +the streak.""" _OLD_SCI_TEXT = """\ **HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ @@ -51,13 +54,14 @@ _NEW_SCI_TEXT = """\ **HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ -elongated worm OUTSIDE the eggshell boundary, or an empty shell. Hatching is \ -nearly instantaneous (rupture → exit → empty shell within a frame or two). \ -Late pretzel moves vigorously WITHIN the shell, pressing against and \ -deforming it — that mimics emergence but is still pretzel as long as the \ -bright mass stays shell-sized and shell-shaped. A multi-frame "hatching" \ -streak in the history is self-contradictory; treat it as evidence the \ -earlier calls were wrong rather than something to continue.""" +elongated worm OUTSIDE the eggshell boundary, or an empty shell. Hatching \ +takes only 2-3 minutes and frames here are about 4 minutes apart, so it \ +appears in at most one or two frames and may be skipped entirely. Late \ +pretzel moves vigorously WITHIN the shell, pressing against and deforming \ +it — that mimics emergence but is still pretzel as long as the bright mass \ +stays shell-sized and shell-shaped. A multi-frame "hatching" streak in the \ +history is therefore self-contradictory at this frame rate; treat it as \ +evidence the earlier calls were wrong rather than something to continue.""" TEMPORAL_SYSTEM = _TMP_BASE.replace(_OLD_TMP_RULE, _NEW_TMP_RULE) SCIENTIFIC_SYSTEM = _SCI_BASE.replace(_OLD_SCI_TEXT, _NEW_SCI_TEXT) From 8e17375540804d3340f8b506b172abec64d77013 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Tue, 9 Jun 2026 14:40:40 +0000 Subject: [PATCH 04/14] Make time-based prompt rules cadence-aware instead of series-specific Hatching takes 2-3 minutes of real time; how many frames capture it depends on the acquisition interval (4-minute imaging: at most 1 frame, possibly missed; 20-second imaging: ~6 frames). Hardcoding 'a frame or two' bakes this series' cadence into the prompt. The solver now measures the median frame interval from the acquisition timestamps in the volume filenames and fills in the numbers at prompt build time: the hatching span, the hatching-streak contradiction length, and the 2fold->pretzel timing window (previously hardcoded '10-15 frames'). Falls back to qualitative phrasing when filenames carry no timestamps. hybrid_annotviews composes the same dynamic prompts. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_annot2.py | 159 +++++++++++++++++++-------- harness/solvers/hybrid_annotviews.py | 87 +++++++++++++++ harness/tools/annotator_view.py | 90 +++++++++++++++ 4 files changed, 295 insertions(+), 44 deletions(-) create mode 100644 harness/solvers/hybrid_annotviews.py create mode 100644 harness/tools/annotator_view.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 7dadb60..2e18379 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_annot2, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -10,6 +10,7 @@ "hybrid_nodefer": hybrid_nodefer.SOLVER, "hybrid_annot": hybrid_annot.SOLVER, "hybrid_annot2": hybrid_annot2.SOLVER, + "hybrid_annotviews": hybrid_annotviews.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_annot2.py b/harness/solvers/hybrid_annot2.py index d322d5d..0fa06b7 100644 --- a/harness/solvers/hybrid_annot2.py +++ b/harness/solvers/hybrid_annot2.py @@ -1,4 +1,4 @@ -"""hybrid_annot + late-pretzel vs hatching disambiguation. +"""hybrid_annot + late-pretzel vs hatching disambiguation, cadence-aware. Targets the one remaining 'ahead' failure cluster: 33 consecutive end-of-series frames on embryo_5 (x3 seeds = 99 failed predictions) where a very late, @@ -6,19 +6,32 @@ anchoring then locked the streak in ("it has been hatching for 2 frames"). Two-part fix, both prompts: -1. Cadence argument kills the trigger — hatching takes 2-3 minutes and this - series is imaged every ~4 minutes, so hatching spans at most a frame or - two (and may be skipped entirely), while late pretzel presses against and - deforms the shell and wriggles WITHIN it for a long time, which mimics - emergence. Note this is sampling-dependent: at a 20-second cadence the - same event would span ~6 frames. -2. Explicit cascade breaker — a multi-frame "hatching" streak in the history - is self-contradictory; treat it as evidence the earlier calls were wrong - rather than something to continue. - -The annotator's embryo_5 notes back the first part: "the worm moves a lot ... -within the egg shell" on exactly these frames. +1. Cadence argument kills the trigger — hatching takes 2-3 minutes of real + time, so how many frames it spans depends entirely on the acquisition + interval (~4-minute imaging → at most 1 frame, may be missed outright; + 20-second imaging → ~6 frames). The interval is MEASURED at runtime from + the volume filenames' acquisition timestamps and injected into the + prompt, so the same solver works on any series. Late pretzel, by + contrast, presses against and deforms the shell and wriggles WITHIN it + for a long time, which mimics emergence. +2. Explicit cascade breaker — a "hatching" streak much longer than the + computed span is self-contradictory; treat it as evidence the earlier + calls were wrong rather than something to continue. + +The 2fold→pretzel timing rule inherited from hybrid_annot ("~60 minutes +after 2fold onset") is converted from a hardcoded frame count to the same +measured-cadence arithmetic. + +The annotator's embryo_5 notes back the hatching fix: "the worm moves a +lot ... within the egg shell" on exactly these frames. """ +import math +import re +from datetime import datetime +from functools import lru_cache +from itertools import pairwise +from pathlib import Path + from harness.core import model from harness.core.solver import Solver from harness.core.types import FrameInput, Stage @@ -29,44 +42,102 @@ TEMPORAL_SYSTEM as _TMP_BASE, ) +_TS_RE = re.compile(r"(\d{8}_\d{6})") + +# hybrid_annot's hardcoded frame counts, replaced with measured-cadence text. +_OLD_TIMING_FRAGMENT = "roughly 10-15 frames at this acquisition cadence" + _OLD_TMP_RULE = """\ 5. **Late pretzel.** The pretzel stage is long-lasting. A compact bright mass \ that fills the eggshell is still pretzel even if it doesn't look "tangled" — \ it has not hatched unless you see the worm OUTSIDE the shell.""" -_NEW_TMP_RULE = """\ -5. **Late pretzel vs hatching.** The pretzel stage is long-lasting, and near \ -its end the worm moves vigorously WITHIN the eggshell — pressing against it, \ -deforming it, and changing pose between frames. This mimics emergence but is \ -still pretzel. Hatching itself takes only 2-3 minutes, and the frames in \ -this series are about 4 minutes apart — so hatching appears in AT MOST one \ -or two frames, and may be skipped entirely (pretzel in one frame, a thin \ -worm clearly OUTSIDE the shell or an empty/partial shell in the next). If \ -the bright mass is still shell-sized and shell-shaped, nothing has hatched. \ -A history showing several consecutive "hatching" observations is therefore \ -self-contradictory at this frame rate — a long streak means those earlier \ -calls were wrong; re-examine against the shell outline instead of continuing \ -the streak.""" - _OLD_SCI_TEXT = """\ **HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ elongated worm OUTSIDE the eggshell boundary, or an empty shell.""" -_NEW_SCI_TEXT = """\ + +@lru_cache(maxsize=16) +def frame_interval_minutes(volume_dir: str) -> float | None: + """Median interval between acquisition timestamps in the volume filenames. + + Returns None when fewer than two frames carry parseable timestamps — + callers fall back to cadence-free phrasing. + """ + stamps = [] + for p in sorted(Path(volume_dir).glob("*.tif")): + m = _TS_RE.search(p.name) + if m: + stamps.append(datetime.strptime(m.group(1), "%Y%m%d_%H%M%S")) + if len(stamps) < 2: + return None + diffs = sorted((b - a).total_seconds() / 60.0 for a, b in pairwise(stamps)) + return diffs[len(diffs) // 2] + + +def _cadence_facts(interval_min: float | None) -> tuple[str, str, str]: + """(hatch_span, pretzel_frames, streak) clauses for the given interval.""" + if interval_min is None: + return ( + "how many frames that spans depends on how often this series was " + "imaged (unknown here) — frequent imaging captures several hatching " + "frames, sparse imaging may skip the event entirely", + "however many frames correspond to about an hour in this series", + "a \"hatching\" streak lasting far longer than a 2-3 minute event " + "plausibly could", + ) + n_hatch = max(1, math.ceil(3.0 / interval_min)) + n_hour = max(2, round(60.0 / interval_min)) + return ( + f"frames in this series are about {interval_min:.1f} minutes apart, so " + f"hatching appears in at most ~{n_hatch} frame(s) and may be skipped " + f"entirely", + f"roughly {n_hour} frames at this series' frame interval", + f"a \"hatching\" streak much longer than ~{n_hatch} frame(s)", + ) + + +@lru_cache(maxsize=16) +def build_systems(interval_min: float | None) -> tuple[str, str]: + """(temporal, scientific) prompts with cadence-derived numbers filled in.""" + hatch_span, pretzel_frames, streak = _cadence_facts(interval_min) + + new_tmp_rule = f"""\ +5. **Late pretzel vs hatching.** The pretzel stage is long-lasting, and near \ +its end the worm moves vigorously WITHIN the eggshell — pressing against it, \ +deforming it, and changing pose between frames. This mimics emergence but is \ +still pretzel. Hatching itself takes only 2-3 minutes of real time; \ +{hatch_span} (pretzel in one frame, a thin worm clearly OUTSIDE the shell or \ +an empty/partial shell in the next). If the bright mass is still shell-sized \ +and shell-shaped, nothing has hatched. {streak} in the history is \ +self-contradictory — it means those earlier calls were wrong; re-examine \ +against the shell outline instead of continuing the streak.""" + + new_sci_text = f"""\ **HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ elongated worm OUTSIDE the eggshell boundary, or an empty shell. Hatching \ -takes only 2-3 minutes and frames here are about 4 minutes apart, so it \ -appears in at most one or two frames and may be skipped entirely. Late \ -pretzel moves vigorously WITHIN the shell, pressing against and deforming \ -it — that mimics emergence but is still pretzel as long as the bright mass \ -stays shell-sized and shell-shaped. A multi-frame "hatching" streak in the \ -history is therefore self-contradictory at this frame rate; treat it as \ +takes only 2-3 minutes of real time; {hatch_span}. Late pretzel moves \ +vigorously WITHIN the shell, pressing against and deforming it — that mimics \ +emergence but is still pretzel as long as the bright mass stays shell-sized \ +and shell-shaped. {streak} in the history is self-contradictory; treat it as \ evidence the earlier calls were wrong rather than something to continue.""" -TEMPORAL_SYSTEM = _TMP_BASE.replace(_OLD_TMP_RULE, _NEW_TMP_RULE) -SCIENTIFIC_SYSTEM = _SCI_BASE.replace(_OLD_SCI_TEXT, _NEW_SCI_TEXT) -assert TEMPORAL_SYSTEM != _TMP_BASE, "temporal late-pretzel rule did not match" -assert SCIENTIFIC_SYSTEM != _SCI_BASE, "scientific hatching text did not match" + tmp = _TMP_BASE.replace(_OLD_TMP_RULE, new_tmp_rule) + sci = _SCI_BASE.replace(_OLD_SCI_TEXT, new_sci_text) + assert tmp != _TMP_BASE, "temporal late-pretzel rule did not match" + assert sci != _SCI_BASE, "scientific hatching text did not match" + + assert _OLD_TIMING_FRAGMENT in tmp and _OLD_TIMING_FRAGMENT in sci + tmp = tmp.replace(_OLD_TIMING_FRAGMENT, pretzel_frames) + sci = sci.replace(_OLD_TIMING_FRAGMENT, pretzel_frames) + return tmp, sci + + +def pretzel_window_text(volume_ref: Path) -> str: + """User-turn phrasing for the 2fold→pretzel elapsed-time exception.""" + interval = frame_interval_minutes(str(Path(volume_ref).parent)) + _, pretzel_frames, _ = _cadence_facts(interval) + return pretzel_frames def _is_scientific(frame: FrameInput) -> bool: @@ -75,11 +146,13 @@ def _is_scientific(frame: FrameInput) -> bool: def _system_for(frame: FrameInput) -> str: - return SCIENTIFIC_SYSTEM if _is_scientific(frame) else TEMPORAL_SYSTEM + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + return sci if _is_scientific(frame) else tmp def _user_blocks(frame: FrameInput) -> list[dict]: - """Identical to hybrid_annot.""" + """hybrid_annot's structure with the cadence-derived pretzel window.""" blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] if frame.history_text: blocks.append(model.text_block(frame.history_text)) @@ -88,9 +161,9 @@ def _user_blocks(frame: FrameInput) -> list[dict]: model.text_block( f"The most recent observation was '{last}'. " f"When uncertain, prefer the earlier stage — except at the " - f"2fold→pretzel boundary: once the history shows ~10-15 frames of " - f"2fold, prolonged 2fold appearance plus any twisting reads as " - f"pretzel, per the timing rule." + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." ) ) blocks.append(model.image_block(frame.image_b64)) diff --git a/harness/solvers/hybrid_annotviews.py b/harness/solvers/hybrid_annotviews.py new file mode 100644 index 0000000..edb4dfc --- /dev/null +++ b/harness/solvers/hybrid_annotviews.py @@ -0,0 +1,87 @@ +"""hybrid_annot2 + raymarched views at the annotator's own poses. + +Third attempt at giving the model 3D shape information, with both failure +causes of the MIP attempts addressed: +- hybrid_3dviews/3dviews2 used max-projections, where tight pretzel coils + merge into bands; these are true raymarched renders with depth occlusion — + the same viewer output the human annotator labeled from. +- Those attempts also instructed the model to re-decide stages from the new + views; here the views are framed as supporting evidence for the existing + tail-progress criteria (find the tail tip), with the annot2 ruleset + unchanged on top. + +Two views per frame (annotator default pose + his tail-visibility pose), +~50ms each cached. +""" +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from pathlib import Path + +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.tools.annotator_view import annotator_view_b64 + +_3D_SECTION = """\ + +## 3D VIEWER RENDERS + +After the main 3-view projection you are given two renders from the same 3D \ +viewer the human annotator used to label this data: first his default \ +working pose, then a side pose he used to identify the tail ("you can see \ +the two lobes clearly — the left is the tail"). Unlike the flat projections, \ +these have depth — nearer structure occludes farther structure, so separate \ +body folds stay visually separate. + +Use them to apply the tail criteria: find the tail (the shorter lobe), and \ +judge how far its tip has progressed. The projections remain your primary \ +evidence; the renders are there to resolve where the tail is when the \ +projection is ambiguous. +""" + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _3D_SECTION + "\nRespond with JSON:") + assert out != base, "3D section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """annot2's structure + the two raymarched views before the analysis prompt.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append( + model.text_block( + "3D viewer renders (annotator's default pose, then his tail-visibility pose):" + ) + ) + blocks.append(model.image_block(annotator_view_b64(frame.volume_ref, "default"))) + blocks.append(model.image_block(annotator_view_b64(frame.volume_ref, "tail"))) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_annotviews", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) diff --git a/harness/tools/annotator_view.py b/harness/tools/annotator_view.py new file mode 100644 index 0000000..d81a1ab --- /dev/null +++ b/harness/tools/annotator_view.py @@ -0,0 +1,90 @@ +"""Raymarched volume renders at the annotator's poses. + +Unlike the rotated MIPs (harness/tools/rotate.py — negative result twice), +these are true volume renders with depth occlusion and thresholding, produced +by the same raymarcher the human annotator used (gently_perception.render, +pixel-equivalent port of the annotator viewer). Tight pretzel coils that +merge in a max-projection stay visually distinct here. + +Poses are frozen constants: +- "default": the annotator's startup pose — the view he spent most labeling + time in. +- "tail": the pose from his embryo_5 T16 view note ("you can see the two + lobes clearly. the left is the tail") — chosen because the prompt's + transition-stage criteria are all about tail progress. A camera pose + carries no label information, so freezing it is not ground-truth leakage. + +Volumes normalize per-volume min-max to uint8 — that lands the imaging +background at ~30/255, which is exactly the annotator's default threshold, +i.e. the convention his pipeline uses. + +One GL context and one cached Renderer per volume path are kept per process; +results are content-addressed via cached_tool_result so eval seeds beyond the +first hit the cache. +""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from gently_perception.render import Renderer +from gently_perception.render.context import make_context +from gently_perception.types import CameraParams +from harness.core.render import array_to_b64, cached_tool_result, load_volume + +POSES: dict[str, CameraParams] = { + "default": CameraParams(), + "tail": CameraParams( + quaternion=( + -0.5161865499649309, + 0.18639103472417876, + 0.014405218785850556, + 0.8358243341046486, + ), + ), +} + +_ctx = None +_renderer: tuple[Path, Renderer] | None = None + + +def _render(volume_ref: Path, pose: str) -> np.ndarray: + global _ctx, _renderer + if _ctx is None: + _ctx = make_context(allow_cpu=True) + if _renderer is None or _renderer[0] != volume_ref: + if _renderer is not None: + _renderer[1].close() + vol = load_volume(volume_ref).astype(np.float64) + lo, hi = vol.min(), vol.max() + vol_u8 = ((vol - lo) / max(hi - lo, 1.0) * 255).astype(np.uint8) + _renderer = (volume_ref, Renderer(vol_u8, ctx=_ctx)) + rgba = _renderer[1].render(POSES[pose]) + # Premultiplied RGBA over the annotator's black canvas == the RGB channel. + img = rgba[..., 0].astype(np.float32) # grayscale volume → channels equal + # Display stretch (annotator used a contrast slider for the same purpose): + # late-stage volumes have hot spots that crush the body's brightness range. + hi = float(np.percentile(img[img > 0], 99.5)) if (img > 0).any() else 1.0 + out = np.clip(img / max(hi, 1.0) * 255.0, 0, 255).astype(np.uint8) + # Crop to content + 8% margin so the embryo fills the image the model sees. + ys, xs = np.nonzero(out > 8) + if len(ys): + my, mx = int(out.shape[0] * 0.08), int(out.shape[1] * 0.08) + out = out[ + max(ys.min() - my, 0) : ys.max() + my, + max(xs.min() - mx, 0) : xs.max() + mx, + ] + return out + + +def annotator_view_b64(volume_ref: Path, pose: str) -> str: + """Cached base64 JPEG of the volume raymarched at a named annotator pose.""" + assert pose in POSES, f"unknown pose {pose!r}" + + def compute() -> str: + return array_to_b64(_render(Path(volume_ref), pose), target_long_edge=512) + + return cached_tool_result( + Path(volume_ref), "annotator_view", {"pose": pose, "v": 3}, compute + ) From b4f4ecd6d8ce2ef71242469cb77b7ba97ee4ba3e Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Tue, 9 Jun 2026 17:20:38 +0000 Subject: [PATCH 05/14] hybrid_annotviews: record negative result embryos 5-8, claude-opus-4-6, 3 seeds: exact 66.5 +/- 4.0 (seeds 68.4/69.2/62.0), adjacent 85.2 - vs hybrid_annot2 69.5 +/- 0.2. 2fold dropped 26 -> 17.5, pretzel destabilized (65.3 +/- 9.7), hatching fix held (hatched 100%, embryo_5 cascade still dead). Third negative result for extra static views, now with true raymarched renders ruling out the projection-collapse explanation - extra static images dilute attention on this task rather than adding usable evidence. hybrid_annot2 remains the best solver at 69.5%. --- harness/solvers/hybrid_annotviews.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/harness/solvers/hybrid_annotviews.py b/harness/solvers/hybrid_annotviews.py index edb4dfc..e9d8a5f 100644 --- a/harness/solvers/hybrid_annotviews.py +++ b/harness/solvers/hybrid_annotviews.py @@ -12,6 +12,15 @@ Two views per frame (annotator default pose + his tail-visibility pose), ~50ms each cached. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 66.5% +/- 4.0 +(seeds 68.4/69.2/62.0), adjacent 85.2% — worse than hybrid_annot2's +69.5 +/- 0.2 and the highest variance of the annot line. 2fold dropped +26 -> 17.5 and pretzel destabilized (65.3 +/- 9.7). Third negative result +for extra static views; with projection-collapse ruled out by the +raymarcher, the remaining explanation is that additional static images +dilute attention rather than add usable evidence on this task. Kept in +the registry for reference; hybrid_annot2 remains the production solver. """ from harness.core import model from harness.core.solver import Solver From b684780d47f152840afc09b321acab4b176eb7b4 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Tue, 9 Jun 2026 20:26:42 +0000 Subject: [PATCH 06/14] Add view3d tool + hybrid_agentic3d solver (negative result) view3d: model-callable camera for the annotator's raymarcher - yaw/pitch relative to his default pose plus the intensity-threshold knob; bounded, deterministic, golden-tested, content-address cached. hybrid_agentic3d: cadence-aware annot2 ruleset + view3d on a 4-step budget, prompt licensing answering without the tool when projections suffice. embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 64.9 +/- 5.3 (seeds 69.8/65.5/59.3), adjacent 82.0 (hybrid_annot2: 69.5 +/- 0.2, 87.8). The model used the tool selectively as prompted (~33% of frames), but renders misled at the boundaries they were meant to resolve - seed2 re-opened the pretzel->hatched failure (70 frames) that annot2's text rules had fixed. Fourth negative result for 3D views; hybrid_annot2 remains the best solver at 69.5%. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_agentic3d.py | 100 ++++++++++++++++++++++++++++ harness/tools/__init__.py | 1 + harness/tools/view3d.py | 94 ++++++++++++++++++++++++++ tests/test_tools_view3d.py | 55 +++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 harness/solvers/hybrid_agentic3d.py create mode 100644 harness/tools/view3d.py create mode 100644 tests/test_tools_view3d.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 2e18379..866ef65 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -11,6 +11,7 @@ "hybrid_annot": hybrid_annot.SOLVER, "hybrid_annot2": hybrid_annot2.SOLVER, "hybrid_annotviews": hybrid_annotviews.SOLVER, + "hybrid_agentic3d": hybrid_agentic3d.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_agentic3d.py b/harness/solvers/hybrid_agentic3d.py new file mode 100644 index 0000000..7d5acd6 --- /dev/null +++ b/harness/solvers/hybrid_agentic3d.py @@ -0,0 +1,100 @@ +"""hybrid_annot2 + autonomy to rotate the 3D rendering (view3d tool). + +The static-views experiments (hybrid_3dviews, hybrid_3dviews2, +hybrid_annotviews) all regressed: fixed extra images on every frame dilute +attention on the frames that didn't need them. This tests the remaining +hypothesis — that 3D views help when the MODEL decides it needs one and +picks the angle, instead of being handed the same poses every frame. + +Same cadence-aware annot2 ruleset; adds the view3d tool (the annotator's +raymarcher with yaw/pitch/threshold control) on a small step budget. The +prompt explicitly licenses answering immediately when the projections +suffice — the agentic_baseline result (80.2 vs 86.2 on embryos 1-4) showed +that forced tool use hurts. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 64.9% +/- 5.3 +(seeds 69.8/65.5/59.3), adjacent 82.0% — vs hybrid_annot2 69.5 +/- 0.2. +Tool selection behaved as prompted (~33% of frames, ~2.7 calls each, +concentrated on transition frames), but the renders misled at exactly +the boundaries they were meant to resolve: seed2 re-opened the +pretzel→hatched failure (70 frames) that annot2's text rules had fully +fixed — late-pretzel renders show the worm pressed against the shell +and the model reads it as outside. Fourth negative for 3D views +(static MIP x2, static raymarch, agentic raymarch). hybrid_annot2 +remains the production solver. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) + +_TOOL_SECTION = """\ + +## 3D VIEWER (view3d tool) + +You can render the volume in the same 3D viewer the human annotator used, \ +from any angle: view3d(yaw_deg, pitch_deg, threshold). These renders have \ +depth — nearer structure occludes farther structure — so overlapping folds \ +separate as you rotate, unlike the flat projections above. + +Use it ONLY when the projections leave you genuinely uncertain, typically: +- you cannot tell where the tail tip is (1.5fold vs 2fold) → try a side \ +view (yaw 90) or an oblique view (yaw 45, pitch 30) to see the fold plane +- you cannot tell whether the two segments are twisting (2fold vs pretzel) \ +→ rotate to look along the body axis, and raise threshold (50-60) to peel \ +dim signal off the fold cores + +One or two well-chosen views are enough. If the projections plus history \ +already support a confident call, answer immediately without the tool — \ +extra views of an unambiguous frame add noise, not signal. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _TOOL_SECTION + "\nRespond with JSON:") + assert out != base, "tool section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_annot2 — the 3D views arrive via tool calls instead.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_agentic3d", + system=_system_for, + tools=("view3d",), + max_steps=4, + user_blocks=_user_blocks, +) diff --git a/harness/tools/__init__.py b/harness/tools/__init__.py index c31bcd2..28d7ff1 100644 --- a/harness/tools/__init__.py +++ b/harness/tools/__init__.py @@ -266,5 +266,6 @@ def _num() -> str: # (Kept at the bottom to avoid circular imports — tool modules import `tool` from here.) from harness.tools import measure as _measure # noqa: E402,F401 from harness.tools import prev_frame as _prev # noqa: E402,F401 +from harness.tools import view3d as _view3d # noqa: E402,F401 from harness.tools import z_slice as _z_slice # noqa: E402,F401 from harness.tools import zoom as _zoom # noqa: E402,F401 diff --git a/harness/tools/view3d.py b/harness/tools/view3d.py new file mode 100644 index 0000000..6469ae4 --- /dev/null +++ b/harness/tools/view3d.py @@ -0,0 +1,94 @@ +"""view3d — raymarched render of the volume at a model-chosen camera angle. + +Wraps gently_perception.render (the annotator's own viewer, pixel-equivalent) +as a model-callable tool: the model picks yaw/pitch relative to the +annotator's default working pose and optionally the intensity threshold, +and gets back a true volume render with depth occlusion — folds that overlap +in the flat projections stay separate here. + +Determinism: same (volume, params) → same image. The GL context is +process-local plumbing (created once, reused); it does not affect output. +""" +from __future__ import annotations + +import math +from typing import Annotated + +import numpy as np + +from gently_perception.render import Renderer +from gently_perception.render.context import make_context +from gently_perception.types import CameraParams, _DEFAULT_QUATERNION +from harness.core.render import array_to_b64 +from harness.core.types import ImageResult +from harness.tools import Range, tool + +_ctx = None + + +def _quat_mul(a, b): + """Hamilton product of (x, y, z, w) quaternions: apply b, then a.""" + x1, y1, z1, w1 = a + x2, y2, z2, w2 = b + return ( + w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, + w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, + w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, + w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, + ) + + +def _pose(yaw_deg: float, pitch_deg: float): + """Default annotator pose rotated by yaw (screen-vertical axis) then pitch.""" + half_yaw = math.radians(yaw_deg) / 2.0 + half_pitch = math.radians(pitch_deg) / 2.0 + q_yaw = (0.0, math.sin(half_yaw), 0.0, math.cos(half_yaw)) + q_pitch = (math.sin(half_pitch), 0.0, 0.0, math.cos(half_pitch)) + return _quat_mul(q_pitch, _quat_mul(q_yaw, _DEFAULT_QUATERNION)) + + +def _display(img_rgba: np.ndarray) -> np.ndarray: + """Premultiplied RGBA over black → stretched, content-cropped grayscale.""" + img = img_rgba[..., 0].astype(np.float32) + hi = float(np.percentile(img[img > 0], 99.5)) if (img > 0).any() else 1.0 + out = np.clip(img / max(hi, 1.0) * 255.0, 0, 255).astype(np.uint8) + ys, xs = np.nonzero(out > 8) + if len(ys): + my, mx = int(out.shape[0] * 0.08), int(out.shape[1] * 0.08) + out = out[max(ys.min() - my, 0) : ys.max() + my, max(xs.min() - mx, 0) : xs.max() + mx] + return out + + +@tool(returns="image") +def view3d( + volume: np.ndarray, + *, + yaw_deg: Annotated[int, Range(-180, 181)], + pitch_deg: Annotated[int, Range(-90, 91)] = 0, + threshold: Annotated[int, Range(5, 81)] = 30, +) -> ImageResult: + """Render the embryo volume in 3D from a camera angle you choose. + + This is the same 3D viewer the human annotator used: a true volume render + with depth — nearer structure occludes farther structure, so overlapping + folds separate as you rotate. yaw_deg rotates the embryo about the + screen-vertical axis (90 = side view, 180 = back), pitch_deg tilts it + (positive = view more from above); both are relative to the annotator's + default working pose. threshold sets the intensity cutoff (default 30 = + the annotator's; raise it to peel away dim outer signal and expose + internal fold structure). + """ + global _ctx + if _ctx is None: + _ctx = make_context(allow_cpu=True) + vol = volume.astype(np.float32) + lo, hi = float(vol.min()), float(vol.max()) + vol_u8 = ((vol - lo) / max(hi - lo, 1.0) * 255.0).astype(np.uint8) + with Renderer(vol_u8, ctx=_ctx) as r: + rgba = r.render( + CameraParams(quaternion=_pose(yaw_deg, pitch_deg), threshold=float(threshold)) + ) + return ImageResult( + b64=array_to_b64(_display(rgba), target_long_edge=512), + caption=f"3D view yaw={yaw_deg}° pitch={pitch_deg}° threshold={threshold}", + ) diff --git a/tests/test_tools_view3d.py b/tests/test_tools_view3d.py new file mode 100644 index 0000000..4040d1b --- /dev/null +++ b/tests/test_tools_view3d.py @@ -0,0 +1,55 @@ +"""view3d golden tests with a synthetic volume. + +Skipped wholesale when no GL context can be created (CI without EGL). +""" +import numpy as np +import pytest + +pytest.importorskip("moderngl") + +from gently_perception.render.context import make_context # noqa: E402 +from harness.tools.view3d import _pose, _quat_mul, view3d # noqa: E402 + +try: + make_context(allow_cpu=True).release() +except Exception: # glcontext raises bare Exception when no backend works + pytest.skip("no GL context available", allow_module_level=True) + + +def _asym_volume() -> np.ndarray: + """A bright corner blob + axial rod: looks different from every side.""" + vol = np.zeros((40, 64, 96), dtype=np.float32) + vol[5:15, 5:20, 5:25] = 900.0 + vol[18:22, 30:34, 10:90] = 600.0 + return vol + + +def test_identity_quaternion_math(): + q = _quat_mul((0, 0, 0, 1), (0.1, 0.2, 0.3, 0.9)) + assert np.allclose(q, (0.1, 0.2, 0.3, 0.9)) + + +def test_zero_rotation_is_default_pose(): + from gently_perception.types import _DEFAULT_QUATERNION + + assert np.allclose(_pose(0, 0), _DEFAULT_QUATERNION) + + +def test_deterministic(): + vol = _asym_volume() + a = view3d(vol, yaw_deg=45, pitch_deg=10, threshold=30) + b = view3d(vol, yaw_deg=45, pitch_deg=10, threshold=30) + assert a.b64 == b.b64 + + +def test_rotation_changes_view(): + vol = _asym_volume() + front = view3d(vol, yaw_deg=0).b64 + side = view3d(vol, yaw_deg=90).b64 + back = view3d(vol, yaw_deg=180).b64 + assert front != side and side != back and front != back + + +def test_caption_reports_params(): + out = view3d(_asym_volume(), yaw_deg=-30, pitch_deg=15, threshold=50) + assert "yaw=-30" in out.caption and "pitch=15" in out.caption and "threshold=50" in out.caption From 49671c93a62db1c6d5a8a9219d2284e581b2e400 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Wed, 10 Jun 2026 18:51:25 +0000 Subject: [PATCH 07/14] view3d zoom/center + hybrid_explorer3d: full viewer autonomy (negative result) view3d gains zoom_pct (up to 4x, hi-res 1024 internal render) centered on a model-picked point (center_x/y_pct) - full parity with the human annotator's viewer: rotate, zoom, click around, threshold. hybrid_explorer3d: cadence-aware annot2 ruleset + the extended tool on an 8-step survey->zoom->decide budget, plus an eggshell guard (renders show signal, not the shell wall - hatching calls stay owned by the projections and the elapsed-time rules). embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 67.1 +/- 1.1, adjacent 85.9 (hybrid_annot2: 69.5 +/- 0.2, 87.8). Best-behaved 3D variant (variance 5.3 -> 1.1, hatching relapse mostly gone) but still -2.4 vs text-only; dominant confusions unchanged (pretzel->2fold ~60/seed). Fifth negative for 3D input: the bottleneck is boundary calibration, not visual information. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_explorer3d.py | 113 +++++++++++++++++++++++++++ harness/tools/view3d.py | 34 ++++++-- tests/test_tools_view3d.py | 14 ++++ 4 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 harness/solvers/hybrid_explorer3d.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 866ef65..e1fc2c6 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -12,6 +12,7 @@ "hybrid_annot2": hybrid_annot2.SOLVER, "hybrid_annotviews": hybrid_annotviews.SOLVER, "hybrid_agentic3d": hybrid_agentic3d.SOLVER, + "hybrid_explorer3d": hybrid_explorer3d.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_explorer3d.py b/harness/solvers/hybrid_explorer3d.py new file mode 100644 index 0000000..3280fb4 --- /dev/null +++ b/harness/solvers/hybrid_explorer3d.py @@ -0,0 +1,113 @@ +"""hybrid_annot2 + full 3D viewer autonomy: rotate, zoom, click-to-center. + +Completes the autonomy ladder that hybrid_agentic3d started. That run gave +rotation + threshold only, full-embryo renders at fixed distance, 3 calls +max — and lost to text-only annot2 (64.9 +/- 5.3 vs 69.5 +/- 0.2), partly +because 512px whole-body renders are too coarse to read twist, and partly +because late-pretzel renders were misread as hatched. + +This variant gives the model everything the human annotator's viewer had: +- rotate (yaw/pitch) and threshold, as before +- zoom up to 4x around a clicked point (center_x/y_pct), so it can inspect + the tail tip or a suspected fold crossing at fold scale +- an 8-step budget for a survey -> zoom -> decide workflow + +Plus one guard distilled from the agentic3d failure: renders show signal +only, not the eggshell wall, so hatched/hatching judgments stay owned by +the projections and the cadence rules. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 67.1% +/- 1.1 +(adjacent 85.9) — vs hybrid_annot2 69.5 +/- 0.2 / 87.8. Best-behaved 3D +variant yet: the eggshell guard mostly held (pretzel→hatch 0/0/23 per +seed vs 70 in agentic3d's worst seed) and variance came down 5.3 → 1.1, +but full viewer autonomy (rotate + zoom + click-to-center, 8 steps, +~4 renders per exploring frame) still loses to text-only by 2.4 points. +The dominant confusions are unchanged (pretzel→2fold ~60/seed). Fifth +and most conclusive negative for 3D input on this task: the bottleneck +is boundary calibration, not visual information. hybrid_annot2 remains +the production solver. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) + +_TOOL_SECTION = """\ + +## 3D VIEWER (view3d tool) + +You can use the same 3D viewer the human annotator used — rotate the \ +embryo, zoom in, and recenter, exactly as he did when deciding these \ +stages: view3d(yaw_deg, pitch_deg, threshold, zoom_pct, center_x_pct, \ +center_y_pct). Renders have depth — nearer structure occludes farther \ +structure — so folds that overlap in the flat projections separate as you \ +rotate, and zooming (up to 4x, centered on a point you pick from a previous \ +render at the same angle) shows fold-scale detail the full-body view \ +cannot. + +A good workflow when the projections leave you uncertain: +1. Survey: one or two rotations (e.g. yaw 90 side view; yaw 45 pitch 30 \ +oblique) to find the angle where the ambiguous feature — tail tip, or a \ +suspected fold crossing — is least occluded. +2. Inspect: zoom 200-400% on that feature at that angle; raise threshold \ +(50-60) if dim outer signal hides the fold cores. +3. Decide using the tail-progress and twisting criteria. + +Two cautions: +- If the projections plus history already support a confident call, answer \ +immediately — extra views of an unambiguous frame add noise. +- The renders show fluorescent signal only, NOT the eggshell wall. A worm \ +pressed against the shell looks edge-on in 3D exactly like an emerging one. \ +Never conclude hatching/hatched from renders — that judgment belongs to the \ +projections and the elapsed-time rules. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _TOOL_SECTION + "\nRespond with JSON:") + assert out != base, "tool section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_annot2 — 3D views arrive via tool calls.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_explorer3d", + system=_system_for, + tools=("view3d",), + max_steps=8, + user_blocks=_user_blocks, +) diff --git a/harness/tools/view3d.py b/harness/tools/view3d.py index 6469ae4..ec165f3 100644 --- a/harness/tools/view3d.py +++ b/harness/tools/view3d.py @@ -66,8 +66,11 @@ def view3d( yaw_deg: Annotated[int, Range(-180, 181)], pitch_deg: Annotated[int, Range(-90, 91)] = 0, threshold: Annotated[int, Range(5, 81)] = 30, + zoom_pct: Annotated[int, Range(100, 401)] = 100, + center_x_pct: Annotated[int, Range(0, 101)] = 50, + center_y_pct: Annotated[int, Range(0, 101)] = 50, ) -> ImageResult: - """Render the embryo volume in 3D from a camera angle you choose. + """Render the embryo volume in 3D — rotate, zoom, and center like the annotator's viewer. This is the same 3D viewer the human annotator used: a true volume render with depth — nearer structure occludes farther structure, so overlapping @@ -76,7 +79,11 @@ def view3d( (positive = view more from above); both are relative to the annotator's default working pose. threshold sets the intensity cutoff (default 30 = the annotator's; raise it to peel away dim outer signal and expose - internal fold structure). + internal fold structure). zoom_pct magnifies (200 = 2x, 400 = 4x) around + the point (center_x_pct, center_y_pct), given as percentages of the + current view's width/height — pick the point from a previous render of + the SAME angle, e.g. zoom into where the tail tip or a suspected fold + crossing is. """ global _ctx if _ctx is None: @@ -86,9 +93,22 @@ def view3d( vol_u8 = ((vol - lo) / max(hi - lo, 1.0) * 255.0).astype(np.uint8) with Renderer(vol_u8, ctx=_ctx) as r: rgba = r.render( - CameraParams(quaternion=_pose(yaw_deg, pitch_deg), threshold=float(threshold)) + CameraParams( + quaternion=_pose(yaw_deg, pitch_deg), + threshold=float(threshold), + image_size=(1024, 1024), # render hi-res so zoom crops stay sharp + ) ) - return ImageResult( - b64=array_to_b64(_display(rgba), target_long_edge=512), - caption=f"3D view yaw={yaw_deg}° pitch={pitch_deg}° threshold={threshold}", - ) + img = _display(rgba) + if zoom_pct > 100: + h, w = img.shape + win_h, win_w = int(h * 100 / zoom_pct), int(w * 100 / zoom_pct) + cy = int(h * center_y_pct / 100) + cx = int(w * center_x_pct / 100) + y0 = min(max(cy - win_h // 2, 0), h - win_h) + x0 = min(max(cx - win_w // 2, 0), w - win_w) + img = img[y0 : y0 + win_h, x0 : x0 + win_w] + caption = f"3D view yaw={yaw_deg}° pitch={pitch_deg}° threshold={threshold}" + if zoom_pct > 100: + caption += f" zoom={zoom_pct}% @({center_x_pct}%,{center_y_pct}%)" + return ImageResult(b64=array_to_b64(img, target_long_edge=512), caption=caption) diff --git a/tests/test_tools_view3d.py b/tests/test_tools_view3d.py index 4040d1b..51a8754 100644 --- a/tests/test_tools_view3d.py +++ b/tests/test_tools_view3d.py @@ -53,3 +53,17 @@ def test_rotation_changes_view(): def test_caption_reports_params(): out = view3d(_asym_volume(), yaw_deg=-30, pitch_deg=15, threshold=50) assert "yaw=-30" in out.caption and "pitch=15" in out.caption and "threshold=50" in out.caption + + +def test_zoom_crops_around_center(): + vol = _asym_volume() + full = view3d(vol, yaw_deg=0) + corner = view3d(vol, yaw_deg=0, zoom_pct=300, center_x_pct=10, center_y_pct=10) + middle = view3d(vol, yaw_deg=0, zoom_pct=300, center_x_pct=50, center_y_pct=50) + assert corner.b64 != middle.b64 != full.b64 + assert "zoom=300%" in corner.caption + + +def test_zoom_100_has_no_crop_caption(): + out = view3d(_asym_volume(), yaw_deg=0, zoom_pct=100) + assert "zoom=" not in out.caption From 583d83a52fca6671d7c1dbc3e453ee7861c3a2b8 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Wed, 10 Jun 2026 21:55:40 +0000 Subject: [PATCH 08/14] Add hybrid_contrast3d: boundary descriptions under viewer autonomy (flat) Five contrastive transition descriptions (the first visible change at each boundary, derived from embryos 2-3 ground-truth transition frames - no eval leakage) layered onto hybrid_explorer3d's full viewer autonomy. embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 67.0 +/- 1.6, adjacent 86.4 - ties hybrid_explorer3d (67.1), still -2.5 vs text-only hybrid_annot2 (69.5). Target stages unmoved (comma ~5%, bean ~29%, 1.5fold ~14%, 2fold ~18%). The contrastive text neither helped nor hurt under autonomy; text-only arm untested. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_contrast3d.py | 126 +++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 harness/solvers/hybrid_contrast3d.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index e1fc2c6..3d440a7 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -13,6 +13,7 @@ "hybrid_annotviews": hybrid_annotviews.SOLVER, "hybrid_agentic3d": hybrid_agentic3d.SOLVER, "hybrid_explorer3d": hybrid_explorer3d.SOLVER, + "hybrid_contrast3d": hybrid_contrast3d.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_contrast3d.py b/harness/solvers/hybrid_contrast3d.py new file mode 100644 index 0000000..9bf1486 --- /dev/null +++ b/harness/solvers/hybrid_contrast3d.py @@ -0,0 +1,126 @@ +"""Full 3D viewer autonomy + contrastive transition-boundary descriptions. + +hybrid_explorer3d (rotate/zoom/click-to-center, 8-step budget, eggshell +guard) scored 67.1 +/- 1.1 with the dominant failures unchanged: every +remaining cluster is a transition-DETECTION failure — the model recognizes +stages but misses the frame where one ends and the next begins. + +This adds boundary descriptions written contrastively ("what visibly +changes"), derived frame-by-frame from the ground-truth transition frames of +embryos 2-3 (session 59799c78 — the other annotated session, so no eval-set +leakage). Each names the first visible change at the boundary rather than +describing the stages on either side. + +Comparisons: vs hybrid_explorer3d isolates the contrastive text within the +autonomy arm; vs hybrid_annot2 (69.5, text-only) is a two-variable +comparison — if this wins, a text-only contrastive arm separates the +contributions. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 67.0% +/- 1.6, +adjacent 86.4 — a statistical tie with hybrid_explorer3d (67.1 +/- 1.1) +and still -2.5 vs text-only hybrid_annot2 (69.5 +/- 0.2). The boundary +descriptions did not move their target stages (comma ~5%, bean ~29%, +1.5fold ~14%, 2fold ~18% — unchanged from every annot-line run), and +added nothing under autonomy overall. Whether they help on a text-only +base remains untested. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_explorer3d import _TOOL_SECTION + +BOUNDARIES_SECTION = """\ + +## STAGE TRANSITIONS — THE FIRST VISIBLE CHANGE AT EACH BOUNDARY + +Most classification errors are missed transitions, not misrecognized \ +stages. Compare the current frame against the recent history and ask which \ +boundary, if any, has just been crossed: + +- **early → bean: the oval loses its symmetry.** A perfect, smooth, convex \ +bright oval is early. The transition is NOT a new structure — it is one \ +corner of the outline pulling slightly inward (the future posterior \ +indentation) and a faint dim seam appearing inside the mass. If the outline \ +is no longer a clean symmetric oval, bean has begun. + +- **bean → comma: the seam becomes a notch.** Bean is a "peanut": two \ +roughly equal bright lobes with a shallow dim line between them. When that \ +line deepens into a dark WEDGE cutting in from the edge — real dark space \ +opening between a distinct smaller lobe (the tail) and the body — comma has \ +begun. Peanut → hook. + +- **comma → 1.5fold: the first parallel-band pair appears.** Comma is still \ +ONE bright mass with a notch and a tail lobe. The transition is a \ +reorganization of the whole image: TWO parallel bright bands separated by a \ +long dark diagonal furrow (body axis + folded-back tail), the tail band \ +clearly shorter, reaching about halfway. Counting goes from "one mass" to \ +"two bands" — the most categorical change of the series. + +- **1.5fold → 2fold: the short band catches up.** Band count stays TWO — do \ +not wait for a third. In 1.5fold the tail band is short, leaving a large \ +dark region at one end of the eggshell. When both bands run nearly the full \ +long axis (the embryo reads as an S or zigzag) and the dark space has shrunk \ +to two thin furrows, 2fold has begun. The cue is relative band LENGTH, not \ +band count. + +- **2fold → pretzel: the dark furrows fragment.** In 2fold the dark spaces \ +between body segments are LONG, CONTINUOUS channels — an organized zigzag. \ +At pretzel the coils cross and the continuous furrows break into SMALL, \ +ISOLATED dark pockets; texture turns granular/marbled and the outline fills \ +almost completely. Judge the CONTINUITY of the dark spaces: long channels = \ +2fold, scattered pockets = pretzel. This works even when you cannot count \ +coils. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace( + "\nRespond with JSON:", BOUNDARIES_SECTION + _TOOL_SECTION + "\nRespond with JSON:" + ) + assert out != base, "insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_explorer3d — 3D views arrive via tool calls.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_contrast3d", + system=_system_for, + tools=("view3d",), + max_steps=8, + user_blocks=_user_blocks, +) From df6d6cf3115924a92a356e603dca47e4f8fbab22 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Thu, 11 Jun 2026 22:23:56 +0000 Subject: [PATCH 09/14] Add hybrid_pairwise: previous frame's image in every turn (tie; lag unmoved) Temporal pairing: the model sees the previous frame's projection beside the current one, with the analysis reframed as did-a-boundary-just-get- crossed, plus the contrastive boundary vocabulary. embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 68.9 +/- 1.3 (seed0 70.2), adjacent 87.0 - ties hybrid_annot2 (69.5 +/- 0.2). Decisive mechanistic null: failure clusters identical to annot2 (lag median 8 vs 9, behind 655 vs 641, windows missed 35 vs 33). The transition lag survives direct frame-to-frame comparison, so it is an under-updating decision behavior, not a perception gap. Input- side interventions are now thoroughly falsified; remaining leverage is in the decision rules. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_pairwise.py | 119 +++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 harness/solvers/hybrid_pairwise.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 3d440a7..5d971ad 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, hybrid_pairwise, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -14,6 +14,7 @@ "hybrid_agentic3d": hybrid_agentic3d.SOLVER, "hybrid_explorer3d": hybrid_explorer3d.SOLVER, "hybrid_contrast3d": hybrid_contrast3d.SOLVER, + "hybrid_pairwise": hybrid_pairwise.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_pairwise.py b/harness/solvers/hybrid_pairwise.py new file mode 100644 index 0000000..b807f6f --- /dev/null +++ b/harness/solvers/hybrid_pairwise.py @@ -0,0 +1,119 @@ +"""hybrid_annot2 + temporal pairing: the previous frame's image in every turn. + +Every remaining failure cluster is transition-shaped: the model lags +boundaries by ~5 frames and skips brief stages entirely. All week it has had +only TEXT history ("T059: 2fold") — it has never been able to LOOK at the +previous frame. This makes "did the boundary just get crossed?" a direct +visual comparison instead of a memory exercise. + +Two additions over hybrid_annot2 (documented together; ablate if it wins): +1. The previous frame's standard 3-view projection rendered immediately + before the current frame's, labeled, with the analysis tail asking + "what changed between these two images?" first. +2. The contrastive boundary descriptions (from hybrid_contrast3d, derived + from embryos 2-3 — no eval leakage) as the vocabulary for naming the + change. Flat under viewer autonomy, but they were written for exactly + this comparison question. + +No tools, one-shot, same cadence-aware ruleset — any delta is attributable +to seeing the previous frame (plus its vocabulary). + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 68.9% +/- 1.3 +(seed0 70.2 — first seed of any experiment above annot2's level), +adjacent 87.0 — statistical tie with hybrid_annot2 (69.5 +/- 0.2). +The mechanistic readout is the real finding: the failure clusters are +IDENTICAL to annot2's (behind 655 vs 641, lag median 8 vs 9, windows +missed 35 vs 33). Seeing the previous frame did not reduce the lag at +all — the model holds the old stage even with both frames side by side +and the boundary's first-visible-change description in hand. The lag is +an under-updating DECISION behavior, not a perception gap; consistent +with every image-side intervention failing while anchor-removal and +forced-update rules (nodefer, the timing rule) produced every win. +""" +from pathlib import Path + +from harness.core import model +from harness.core.render import cached_render +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_contrast3d import BOUNDARIES_SECTION + +_PAIR_SECTION = """\ + +## PREVIOUS-FRAME COMPARISON + +You are shown the PREVIOUS frame's projection directly above the current \ +one. Compare them before anything else. Most frame pairs show NO \ +transition — the embryo simply persists in its stage, possibly shifted by \ +twitching. Your first question is: did the FIRST VISIBLE CHANGE of any \ +boundary (see the stage-transition list) appear between these two images? \ +If nothing structural changed, keep the previous stage. If something did, \ +name which boundary it is and advance exactly one stage. +""" + + +def _prev_volume_ref(frame: FrameInput) -> Path | None: + """Previous timepoint's volume: timepoint indexes the sorted file list.""" + if frame.timepoint == 0: + return None + files = sorted(Path(frame.volume_ref).parent.glob("*.tif")) + idx = frame.timepoint - 1 + if idx >= len(files) or files[frame.timepoint] != Path(frame.volume_ref): + return None # index/file mismatch — skip the pair rather than mislead + return files[idx] + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace( + "\nRespond with JSON:", BOUNDARIES_SECTION + _PAIR_SECTION + "\nRespond with JSON:" + ) + assert out != base, "insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """annot2's structure + the previous frame's image before the current one.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + prev_ref = _prev_volume_ref(frame) + if prev_ref is not None: + blocks.append(model.text_block(f"PREVIOUS frame (T{frame.timepoint - 1}):")) + blocks.append(model.image_block(cached_render(prev_ref))) + blocks.append(model.text_block(f"CURRENT frame (T{frame.timepoint}):")) + blocks.append(model.image_block(frame.image_b64)) + base = _SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS + blocks.append( + model.text_block( + "First: compare the two frames — did any boundary's first visible " + "change appear between them? Then " + base[0].lower() + base[1:] + ) + ) + return blocks + + +SOLVER = Solver(name="hybrid_pairwise", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) From c1c3c272bcbd74f08f86f5abb7846be1f82f0a7e Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 12 Jun 2026 14:30:22 +0000 Subject: [PATCH 10/14] Report: week-grouped dropdown labels + experiment description panel Dropdown entries are prefixed with the run's week ([Week of 6/8]) and sorted newest-first so weeks cluster. Each report shows a description panel under the header with the solver docstring's first paragraph - what the experiment tried (results paragraphs deliberately excluded; the report's numbers speak for the outcome). --- harness/eval/html_report.py | 40 ++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/harness/eval/html_report.py b/harness/eval/html_report.py index 7dce3ee..db48a3c 100644 --- a/harness/eval/html_report.py +++ b/harness/eval/html_report.py @@ -15,6 +15,7 @@ import io import json import os +from datetime import datetime, timedelta from pathlib import Path from typing import Any @@ -305,8 +306,18 @@ def _embryo_span(ids: list[str]) -> str: return "e" + ",".join(str(n) for n in nums) +def _week_prefix(ts: str) -> str: + """'20260609-...' → '[Week of 6/8]' (the Monday of that run's week).""" + try: + run_day = datetime.strptime(ts[:8], "%Y%m%d").date() + except ValueError: + return "" + monday = run_day - timedelta(days=run_day.weekday()) + return f"[Week of {monday.month}/{monday.day}]" + + def _run_label(d: Path, gt_embryos: dict[str, str]) -> str: - """Human-readable dropdown label: what the solver change was, dataset, date.""" + """Human-readable dropdown label: week group, what the change was, dataset, date.""" solver, _model, ts = d.parts[-3:] desc = solver try: @@ -321,7 +332,23 @@ def _run_label(d: Path, gt_embryos: dict[str, str]) -> str: except Exception: pass date = f"{ts[4:6]}/{ts[6:8]}" if len(ts) >= 8 else ts - return " · ".join(x for x in (desc, span, date) if x) + body = " · ".join(x for x in (desc, span, date) if x) + week = _week_prefix(ts) + return f"{week} {body}" if week else body + + +def _experiment_description(solver_name: str) -> str: + """First paragraph of the solver's docstring — what the experiment tried. + + (Results paragraphs come later in the docstrings and are deliberately + excluded; the report's own numbers speak for the outcome.) + """ + try: + doc = importlib.import_module(f"harness.solvers.{solver_name}").__doc__ or "" + except Exception: + return "" + first_para = doc.strip().split("\n\n")[0] + return " ".join(line.strip() for line in first_para.splitlines()) def _sibling_runs(run_dir: Path) -> list[dict[str, Any]]: @@ -339,7 +366,7 @@ def _sibling_runs(run_dir: Path) -> list[dict[str, Any]]: } dirs = {p.parent for p in runs_root.glob("*/*/*/report.html")} | {run_dir} out: list[dict[str, Any]] = [] - for d in sorted(dirs, key=lambda p: (p.parts[-3], p.parts[-1]), reverse=True): + for d in sorted(dirs, key=lambda p: p.parts[-1], reverse=True): # newest first → weeks cluster out.append( { "label": _run_label(d, gt_embryos), @@ -373,6 +400,7 @@ def generate( data: dict[str, Any] = { "config": config, "run_dir": str(run_dir), + "experiment": _experiment_description(str(config.get("solver", ""))), "runs": _sibling_runs(run_dir), "seed": seed, "summary": _summary(run_dir, gt), @@ -500,6 +528,8 @@ def generate( .runSel{display:flex;align-items:center;gap:10px;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase} .runSel select{background:var(--panel2);color:var(--txt);border:1px solid var(--line);border-radius:6px;padding:7px 11px;font:12px var(--mono);max-width:380px;cursor:pointer} .runSel select:hover{border-color:var(--accent)} +.expDesc{margin:18px 0 0;padding:14px 18px;background:var(--panel2);border:1px solid var(--line);border-left:3px solid var(--accent);border-radius:8px;font:13px/1.65 var(--sans);color:var(--txt);max-width:980px} +.expDesc::before{content:"experiment";display:block;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin-bottom:6px} @@ -507,6 +537,7 @@ def generate(

+

Summary

Confusion (true ↓ / predicted →)
@@ -542,6 +573,9 @@ def generate( document.getElementById('title').textContent=(D.config.solver||'run')+' · '+(D.config.model||''); document.getElementById('subtitle').textContent=D.run_dir+' · seed'+D.seed+' detail · '+D.summary.n_seeds+' seed(s) aggregated'; +// experiment description +if(D.experiment){const ed=document.getElementById('expDesc');ed.textContent=D.experiment;ed.hidden=false;} + // run switcher const runSel=document.getElementById('runSel'); (D.runs||[]).forEach(r=>{const o=document.createElement('option');o.value=r.href;o.textContent=r.label;o.title=r.path||'';o.selected=!!r.current;runSel.appendChild(o)}); From 394bd1d3298e494e6c3a50654e2e79497e3146f3 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 12 Jun 2026 15:04:59 +0000 Subject: [PATCH 11/14] Add hybrid_prevonly ablation: unguided previous frame is harmful Minimal ablation isolating hybrid_pairwise's previous-frame image: system prompts byte-identical to hybrid_annot2, only the labeled previous-frame image added to the user turn. embryos 5-8 (2cfd8f4e), claude-opus-4-6, 3 seeds: exact 63.7 +/- 1.6, adjacent 85.0. Paired frame-level bootstrap vs annot2: -5.7 points, 95% CI [-7.2, -4.3] (the full pairwise bundle: -0.5, CI [-1.6, +0.5] - a tie). Confusions shift uniformly laggier. An unguided previous frame acts as a visual persistence anchor; pairwise's comparison-first instruction was not masking a benefit but repairing the damage. Temporal visual context requires explicit comparison framing. --- harness/solvers/__init__.py | 3 +- harness/solvers/hybrid_prevonly.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 harness/solvers/hybrid_prevonly.py diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 5d971ad..c606d1d 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, hybrid_pairwise, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, hybrid_pairwise, hybrid_prevonly, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -15,6 +15,7 @@ "hybrid_explorer3d": hybrid_explorer3d.SOLVER, "hybrid_contrast3d": hybrid_contrast3d.SOLVER, "hybrid_pairwise": hybrid_pairwise.SOLVER, + "hybrid_prevonly": hybrid_prevonly.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_prevonly.py b/harness/solvers/hybrid_prevonly.py new file mode 100644 index 0000000..1e7edb0 --- /dev/null +++ b/harness/solvers/hybrid_prevonly.py @@ -0,0 +1,76 @@ +"""Minimal ablation: hybrid_annot2 + ONLY the previous frame's image. + +hybrid_pairwise bundled three changes (previous-frame image, contrastive +boundary section, comparison-first analysis rewrite) and tied annot2 with +wider spread — leaving open whether the image itself helps, hurts, or does +nothing once the text changes are stripped away. + +This is annot2 verbatim — same system prompts, same analysis tail, same +user-turn text — plus exactly one thing: the previous frame's projection, +labeled, rendered before the current frame's. Any delta vs annot2 +(69.5 +/- 0.2) is attributable to the image alone. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 63.7% +/- 1.6, +adjacent 85.0. Paired frame-level bootstrap vs annot2: -5.7 points, +95% CI [-7.2, -4.3] — decisively harmful, where the full pairwise bundle +was a statistical tie (-0.5, CI [-1.6, +0.5]). Confusions shifted +uniformly laggier (pretzel→2fold 72, comma→early appears). Conclusion: +an UNGUIDED previous frame is a visual persistence anchor — consecutive +frames are ~95% identical, and the similarity reads as "same stage" +louder than any boundary cue. pairwise's comparison-first framing wasn't +masking a benefit of the image; it was repairing ~5 of the 6 points of +damage the image causes. Temporal visual context is only safe with +explicit comparison framing. +""" +from pathlib import Path + +from harness.core import model +from harness.core.render import cached_render +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_pairwise import _prev_volume_ref + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + return sci if _is_scientific(frame) else tmp + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """hybrid_annot2's blocks with the labeled previous-frame image inserted.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + prev_ref = _prev_volume_ref(frame) + if prev_ref is not None: + blocks.append(model.text_block(f"PREVIOUS frame (T{frame.timepoint - 1}), for reference:")) + blocks.append(model.image_block(cached_render(prev_ref))) + blocks.append(model.text_block(f"CURRENT frame (T{frame.timepoint}) — classify this one:")) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_prevonly", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) From e1e7259daf66640ae0f0bc8b8b4287beb281f4df Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 12 Jun 2026 15:12:01 +0000 Subject: [PATCH 12/14] Report: fuller experiment descriptions The experiment panel now shows the solver docstring's complete pre-RESULT content as formatted paragraphs (motivation, what changed, how it compares) instead of only the first paragraph, plus a setup line (solver name, tools, one-shot vs agentic step budget). Rendered via DOM textContent, not innerHTML. --- harness/eval/html_report.py | 40 +++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/harness/eval/html_report.py b/harness/eval/html_report.py index db48a3c..8506af6 100644 --- a/harness/eval/html_report.py +++ b/harness/eval/html_report.py @@ -337,18 +337,28 @@ def _run_label(d: Path, gt_embryos: dict[str, str]) -> str: return f"{week} {body}" if week else body -def _experiment_description(solver_name: str) -> str: - """First paragraph of the solver's docstring — what the experiment tried. +def _experiment_description(solver_name: str) -> dict[str, Any]: + """The solver docstring's full pre-RESULT content — what was tried and why. - (Results paragraphs come later in the docstrings and are deliberately - excluded; the report's own numbers speak for the outcome.) + Returns {"paras": [...], "setup": "..."}. RESULT paragraphs are excluded; + the report's own numbers speak for the outcome. """ try: - doc = importlib.import_module(f"harness.solvers.{solver_name}").__doc__ or "" + mod = importlib.import_module(f"harness.solvers.{solver_name}") except Exception: - return "" - first_para = doc.strip().split("\n\n")[0] - return " ".join(line.strip() for line in first_para.splitlines()) + return {"paras": [], "setup": ""} + paras: list[str] = [] + for block in (mod.__doc__ or "").strip().split("\n\n"): + if block.strip().startswith("RESULT"): + break + paras.append(" ".join(line.strip() for line in block.splitlines())) + setup = "" + solver = getattr(mod, "SOLVER", None) + if solver is not None: + tools = ", ".join(solver.tools) if solver.tools else "none" + mode = "one-shot" if solver.max_steps == 1 else f"agentic, {solver.max_steps} steps max" + setup = f"solver {solver.name} · tools: {tools} · {mode}" + return {"paras": paras, "setup": setup} def _sibling_runs(run_dir: Path) -> list[dict[str, Any]]: @@ -528,8 +538,11 @@ def generate( .runSel{display:flex;align-items:center;gap:10px;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase} .runSel select{background:var(--panel2);color:var(--txt);border:1px solid var(--line);border-radius:6px;padding:7px 11px;font:12px var(--mono);max-width:380px;cursor:pointer} .runSel select:hover{border-color:var(--accent)} -.expDesc{margin:18px 0 0;padding:14px 18px;background:var(--panel2);border:1px solid var(--line);border-left:3px solid var(--accent);border-radius:8px;font:13px/1.65 var(--sans);color:var(--txt);max-width:980px} -.expDesc::before{content:"experiment";display:block;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin-bottom:6px} +.expDesc{margin:18px 0 0;padding:16px 20px;background:var(--panel2);border:1px solid var(--line);border-left:3px solid var(--accent);border-radius:8px;font:13px/1.7 var(--sans);color:var(--txt);max-width:980px} +.expDesc::before{content:"experiment";display:block;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin-bottom:8px} +.expDesc p{margin:0 0 10px} +.expDesc p:last-of-type{margin-bottom:0} +.expSetup{margin-top:12px;padding-top:10px;border-top:1px dashed var(--line);font:11px var(--mono);color:var(--dim);letter-spacing:.04em} @@ -574,7 +587,12 @@ def generate( document.getElementById('subtitle').textContent=D.run_dir+' · seed'+D.seed+' detail · '+D.summary.n_seeds+' seed(s) aggregated'; // experiment description -if(D.experiment){const ed=document.getElementById('expDesc');ed.textContent=D.experiment;ed.hidden=false;} +if(D.experiment&&D.experiment.paras&&D.experiment.paras.length){ + const ed=document.getElementById('expDesc'); + for(const p of D.experiment.paras){const el=document.createElement('p');el.textContent=p;ed.appendChild(el);} + if(D.experiment.setup){const su=document.createElement('div');su.className='expSetup';su.textContent=D.experiment.setup;ed.appendChild(su);} + ed.hidden=false; +} // run switcher const runSel=document.getElementById('runSel'); From 277042301db81b9faa97ee06608d3dba1d9535e4 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 12 Jun 2026 15:27:02 +0000 Subject: [PATCH 13/14] Report: replay how the model navigated the 3D viewer per frame Frame modal now shows a '3D navigation' filmstrip for agentic runs: each view3d call as a numbered card with a human-readable caption (yaw/pitch/threshold/zoom@center) and the render the model saw at that step, followed by a labeled 'model response' block with the full classification reasoning. Also fixes a media collision: the harness keys tool-call media files by model-turn index, so parallel calls in one turn overwrote each other's image. view3d is deterministic, so the report re-renders every step from its recorded params (hitting the run's own dispatch cache) into per-step nav assets - verified distinct images for previously colliding steps. --- harness/eval/html_report.py | 94 +++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/harness/eval/html_report.py b/harness/eval/html_report.py index 8506af6..55f0477 100644 --- a/harness/eval/html_report.py +++ b/harness/eval/html_report.py @@ -15,14 +15,15 @@ import io import json import os +import re from datetime import datetime, timedelta from pathlib import Path from typing import Any from PIL import Image -from harness.core.render import cached_render -from harness.core.types import STAGE_ORDER, Stage +from harness.core.render import cached_render, cached_tool_result, load_volume +from harness.core.types import STAGE_ORDER, ImageResult, Stage from harness.eval import report as text_report from harness.eval.score import score_run from harness.io.events import read_events @@ -32,6 +33,14 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] THUMB_LONG_EDGE = 260 +_FS_UNSAFE = re.compile(r"[^A-Za-z0-9_\-]") + + +def _fs_name(value: Any) -> str: + """Filesystem/URL-safe asset-name component. Embryo ids come from + events.jsonl, which may not be trusted (shared/downloaded run dirs).""" + return _FS_UNSAFE.sub("_", str(value)) + # --- Data assembly ---------------------------------------------------------- @@ -101,7 +110,7 @@ def _thumbnails(frames: list[dict[str, Any]], assets_dir: Path, volumes_dir: Pat key = (r["e"], r["t"]) if key not in paths: continue - thumb = assets_dir / f"{r['e']}_T{r['t']:03d}.jpg" + thumb = assets_dir / f"{_fs_name(r['e'])}_T{int(r['t']):03d}.jpg" r["thumb"] = f"report_assets/{thumb.name}" if thumb.exists(): continue @@ -134,7 +143,7 @@ def _rotated_thumbnails( continue rot: list[dict[str, Any]] = [] for angle in angles: - name = f"{r['e']}_T{r['t']:03d}_rot{int(angle)}.jpg" + name = f"{_fs_name(r['e'])}_T{int(r['t']):03d}_rot{int(angle)}.jpg" out = assets_dir / name rot.append({"a": int(angle), "src": f"report_assets/{name}"}) if out.exists(): @@ -147,6 +156,50 @@ def _rotated_thumbnails( r["rot"] = rot +def _view3d_step_assets(frames: list[dict[str, Any]], run_dir: Path, volumes_dir: Path) -> None: + """Re-render every view3d step image from its recorded params. + + The harness keys media files by model-turn index, so parallel tool calls + in one turn overwrite each other's image. view3d is deterministic, so the + recorded params are the authoritative source; renders hit the run's own + dispatch cache (same (volume, tool, params) key) and are cheap. + """ + needs = [ + (r, j, s) + for r in frames + for j, s in enumerate(r.get("steps", [])) + if s.get("name") == "view3d" and s.get("params") is not None + ] + if not needs or not volumes_dir.exists(): + return + from harness.tools import REGISTRY, _fill_defaults # deferred: pulls in GL deps + + spec = REGISTRY.get("view3d") + if spec is None: + return + paths = {(e, t): p for e, t, p in OfflineSource(volumes_dir)} + assets = run_dir / "report_assets" + assets.mkdir(parents=True, exist_ok=True) + for r, j, s in needs: + vol_ref = paths.get((r["e"], r["t"])) + if vol_ref is None: + continue + params = s["params"] + + def compute(vol_ref: Path = vol_ref, params: dict[str, Any] = params) -> str: + result = spec.fn(load_volume(vol_ref), **_fill_defaults(spec, params)) + assert isinstance(result, ImageResult) + return result.b64 + + name = f"{_fs_name(r['e'])}_T{int(r['t']):03d}_nav{j}.jpg" + out = assets / name + assert out.resolve().is_relative_to(assets.resolve()) + if not out.exists(): + b64 = cached_tool_result(Path(vol_ref), "view3d", params, compute) + out.write_bytes(base64.b64decode(b64)) + s["img"] = f"report_assets/{name}" + + def _summary(run_dir: Path, gt: GroundTruth) -> dict[str, Any]: seed_files = sorted(run_dir.glob("seed*/events.jsonl")) scores = [score_run(p, gt) for p in seed_files] @@ -406,6 +459,7 @@ def generate( vols = volumes_dir or (_REPO_ROOT / "data" / "volumes") _thumbnails(frames, run_dir / "report_assets", vols) _rotated_thumbnails(frames, run_dir / "report_assets", vols, str(config.get("solver", ""))) + _view3d_step_assets(frames, run_dir, vols) data: dict[str, Any] = { "config": config, @@ -523,9 +577,12 @@ def generate( .rotRow figure{flex:1;margin:0} .rotRow img{width:100%;border-radius:5px;background:#000;display:block} .rotRow figcaption{font:10px var(--mono);color:var(--dim);letter-spacing:.08em;text-align:center;margin-top:5px;text-transform:uppercase} -.step{border:1px solid var(--line);border-radius:6px;background:var(--panel2);padding:11px 14px;margin-bottom:10px;font:12px var(--mono)} -.step .sn{color:var(--accent);margin-right:8px} -.step img{max-width:340px;display:block;margin-top:9px;border-radius:4px;background:#000} +.navHdr{font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin:18px 0 10px} +.navStrip{display:flex;gap:12px;overflow-x:auto;padding-bottom:8px;margin-bottom:6px} +.navStrip .step{flex:0 0 auto;width:300px;border:1px solid var(--line);border-radius:8px;background:var(--panel2);padding:10px 12px;font:11px/1.5 var(--mono)} +.step .sn{display:inline-block;min-width:18px;height:18px;line-height:18px;text-align:center;background:var(--accent);color:#000;border-radius:9px;font-weight:700;margin-right:8px} +.step .cap{color:var(--bright)} +.step img{width:100%;display:block;margin-top:9px;border-radius:5px;background:#000} .step .val{color:var(--bright);font-weight:600} .step .err{color:var(--bad)} .reason{font:13px/1.6 var(--sans);color:var(--txt);background:var(--panel2);border:1px solid var(--line);border-radius:6px;padding:13px 16px;white-space:pre-wrap} @@ -693,14 +750,23 @@ def generate( let cur=-1; function open(i){cur=i;const f=F[i]; let h='
'+f.e+' · T'+f.t+''+(i+1)+' / '+F.length+' · ←→ navigate · esc close
'; -if(f.thumb)h+=''; -if(f.rot&&f.rot.length)h+='
'+f.rot.map(r=>'
rotated '+r.a+'°
').join('')+'
'; +if(f.thumb)h+=''; +if(f.rot&&f.rot.length)h+='
'+f.rot.map(r=>'
rotated '+r.a+'°
').join('')+'
'; h+='
pred '+f.pred+'gt '+f.gt+''+(f.tokens||0)+' tok
'; -(f.steps||[]).forEach((s,j)=>{h+='
'+(j+1)+''+s.name+'('+esc(JSON.stringify(s.params))+')'+ - (s.error?' → '+esc(s.error)+'':'')+ - (s.value!==undefined?' → '+s.value+' '+esc(s.note)+'':'')+ - (s.img?'':'')+'
'}); -h+='
'+esc(f.reasoning||f.err||'(no reasoning)')+'
'; +function stepCap(s){ + if(s.name==='view3d'){const p=s.params||{};let c='rotate → yaw '+(p.yaw_deg??0)+'° · pitch '+(p.pitch_deg??0)+'°'; + if(p.threshold!==undefined&&p.threshold!==30)c+=' · threshold '+p.threshold; + if(p.zoom_pct&&p.zoom_pct>100)c+=' · zoom '+p.zoom_pct+'% @ ('+(p.center_x_pct??50)+'%, '+(p.center_y_pct??50)+'%)'; + return c;} + return s.name+'('+Object.entries(s.params||{}).map(([k,v])=>k+'='+v).join(', ')+')';} +if((f.steps||[]).length){ + h+='
'+( (f.steps.some(s=>s.name==='view3d'))?'3D navigation — how the model explored this frame':'tool calls')+'
'; + f.steps.forEach((s,j)=>{h+='
'+(j+1)+''+esc(stepCap(s))+''+ + (s.error?'
'+esc(s.error)+'
':'')+ + (s.value!==undefined?'
→ '+s.value+' '+esc(s.note)+'
':'')+ + (s.img?'':'')+'
'}); + h+='
';} +h+='
model response
'+esc(f.reasoning||f.err||'(no reasoning)')+'
'; if(f.override)h+='
⚠ verify override: '+esc(f.override)+'
'; if(f.budget_exhausted)h+='
⚠ tool budget exhausted — classify was forced
'; document.getElementById('modalBox').innerHTML=h; From e6575f1f1b2ffaf09efba5dbe610ba326b7636f8 Mon Sep 17 00:00:00 2001 From: Trisha Bansal Date: Fri, 12 Jun 2026 16:07:04 +0000 Subject: [PATCH 14/14] Add findings write-up + findings tab on every report docs/FINDINGS.md: the full experiment ladder (11 experiments with numbers), the findings (every win came from decision rules, every image-side change was flat or negative; transition lag is an under-updating decision behavior, not a perception gap; the labels are partly time-derived; infra lessons), and six suggested next steps. The report generator renders it to runs/findings.html with the report styling (minimal markdown renderer: headings, bold, code, tables, lists), and every report page gets a report|findings tab bar linking to it via relative path. --- docs/FINDINGS.md | 122 ++++++++++++++++++++++++++++++++++++ harness/eval/html_report.py | 93 +++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 docs/FINDINGS.md diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md new file mode 100644 index 0000000..72d5ab5 --- /dev/null +++ b/docs/FINDINGS.md @@ -0,0 +1,122 @@ +# Embryo staging: experiment log & findings + +Classifying C. elegans embryo developmental stages from light-sheet volumes, +evaluated on embryos 5–8 (session `2cfd8f4e`, 705 frames, ~3.7 min/frame), +3 seeds per experiment, claude-opus-4-6. Starting point: 62.2% exact. Current +champion: **hybrid_annot2 at 69.5% ± 0.2 exact / 87.8% adjacent** — all of the +gain from text rules, none of it from richer visual input. + +## The ladder + +| # | experiment | exact | adjacent | verdict | +|---|------------|-------|----------|---------| +| 0 | hybrid (baseline, May 28) | 62.2 ± 3.1 | 83.2 | starting point | +| 1 | hybrid_nodefer — remove defer-to-previous instructions | 64.2 ± 0.8 | 83.1 | **+2.0**, variance collapsed | +| 2 | hybrid_3dviews — static rotated MIPs (45/90/135°) | 61.9 ± 0.8 | 83.6 | −2.3: coils merge in projections, model undercounts folds | +| 3 | hybrid_3dviews2 — MIPs demoted to 1.5f/2f tie-breaker | 58.9 ± 3.8 | 82.7 | worse: views destabilize pretzel anyway | +| 4 | hybrid_annot — stage criteria rewritten from the annotator's notes (tail progress + 60-min timing rule) | 66.3 ± 0.2 | 87.0 | **+2.1**, gains exactly where aimed | +| 5 | hybrid_annot2 — late-pretzel vs hatching disambiguation, cadence-aware | **69.5 ± 0.2** | **87.8** | **+3.2**, killed the 99-frame hatching cascade; champion | +| 6 | hybrid_annotviews — static raymarched views at the annotator's poses | 66.5 ± 4.0 | 85.2 | −3.0: even faithful 3D renders dilute | +| 7 | hybrid_agentic3d — model rotates the 3D render itself (view3d tool) | 64.9 ± 5.3 | 82.0 | −4.6: renders re-litigated the settled hatching call | +| 8 | hybrid_explorer3d — full viewer autonomy (rotate + zoom + click) | 67.1 ± 1.1 | 85.9 | −2.4: best-behaved 3D variant, still loses | +| 9 | hybrid_contrast3d — explorer + contrastive boundary descriptions | 67.0 ± 1.6 | 86.4 | flat vs explorer; target stages unmoved | +| 10 | hybrid_pairwise — previous frame's image + comparison framing | 68.9 ± 1.3 | 87.0 | tie (paired Δ −0.5, CI [−1.6, +0.5]); lag unmoved | +| 11 | hybrid_prevonly — previous frame's image alone (ablation) | 63.7 ± 1.6 | 85.0 | **−5.7, CI [−7.2, −4.3]** — decisively harmful | + +Reference points from embryos 1–4 (session `59799c78`): hybrid 86.2%, +agentic_baseline (all tools) 80.2% — embryos 5–8 are substantially harder, +and tools cost accuracy there too. + +## Findings + +### 1. Every win was a decision rule; every image-side change was flat or negative + +The +7.3 points from baseline to champion decompose entirely into text: +removing the defer-to-previous anchor (+2.0), the annotator's tail-progress +criteria and his explicit 2fold→pretzel timing rule (+2.1), and the +late-pretzel vs hatching cadence rules (+3.2). Six experiments added visual +information — rotated MIPs, faithful raymarched views, model-driven 3D +navigation with zoom, the previous frame — and not one of them improved on +its text-only base. Agentic tool use carries a consistent ~2.4-point tax on +this task (explorer3d/contrast3d vs annot2), with tool selection itself +behaving exactly as prompted. + +### 2. The model under-updates: transition lag is a decision behavior, not a perception gap + +The dominant failure all along has been one-stage lag at boundaries (median +8–9 frames) plus brief stages missed outright (comma's window is ~5 frames; +it scores 0–5% everywhere). Three increasingly direct perception fixes failed +to move the lag at all: contrastive descriptions of each boundary's first +visible change (written from the e1–4 transition frames), full 3D viewing +autonomy, and finally showing the previous and current frames side by side — +lag median 8 vs 9, behind-failures 655 vs 641, identical clusters. The model +sees the change and does not act on it. + +The prevonly ablation is the sharpest evidence: an **unguided** previous +frame made things much worse (−5.7), shifting every confusion laggier — +consecutive frames are ~95% identical pixels, and visual similarity reads as +"same stage" louder than any boundary cue. The comparison-first framing in +pairwise repaired ~5 of those 6 points. Inputs that support persistence +amplify the bias; framings that force the update question neutralize it. + +### 3. The labels themselves are partly time-derived + +Kesavan's notes state outright that he placed the 2fold→pretzel boundary +with a stereotypic-timing rule ("~60 minutes after 2fold onset") because +coils are genuinely hard to count in projections. Appearance-only +classification therefore fights the ground truth at that boundary. Telling +the model this honestly — and converting all time rules to run-time-measured +cadence (frame interval from acquisition timestamps, so the same solver works +on any series) — produced the single largest win. + +### 4. What the failure map looks like now (annot2) + +~30% of predictions still miss. Composition: late arrival (~390/seed at its +peak, now the lag tail), brief windows missed (33 of 81 transition windows), +ahead-errors eliminated (99 → ~0–5 after the hatching rules). Per stage: +early/hatched ~100%, pretzel ~70, bean ~33, 2fold ~26, 1.5fold ~14, comma ~5. + +### 5. Infrastructure lessons + +Transient API errors (grammar-compilation 400s, 529s) killed four multi-hour +runs at top level — `harness/core/model.py` has no retry; seeds were +recovered with a resume script. The tool dispatch cache is content-addressed +on params but not on tool code version, so changing a tool's implementation +silently serves stale renders. Tool media files key on turn index, so +parallel calls in one turn overwrite each other (the report now re-renders +steps from recorded params). `view3d` post-processing (brightness stretch + +content crop) broke pixel-equivalence with the annotator viewer — replays are +faithful to what the model saw, but what the model saw was not what the +annotator saw. + +## Suggested next steps + +1. **Generalize the timing-window recipe to every boundary.** The one + boundary with an elapsed-time rule and an explicit exception to the + prefer-earlier bias (2fold→pretzel) is the one transition that improved. + Derive per-stage duration windows from the e1–4 session, and at each + boundary have the prompt switch from "prefer earlier" to "actively hunt + the next stage's first visible change" once the window passes. This is the + only intervention class with a positive track record, aimed at the + dominant remaining failure. +2. **Anchor-free second opinion at suspected transitions.** When the timing + window has passed, reclassify the frame blind (no history, no previous + stage) and reconcile with the monotonic verifier. The bias is + persistence; removing the anchor only where lag is likely keeps stability + elsewhere. +3. **Cross-session validation.** annot2's rules were developed against + embryos 5–8. Run it on embryos 1–4 to check nothing regressed vs the 86.2% + baseline and that the cadence machinery generalizes (different session, + same ~4-min cadence). +4. **If 3D gets another attempt: annotator-native rendering.** Rebuild + view3d with no stretch/crop, the annotator's contrast knob, true camera + zoom, and a code-versioned cache — then rerun explorer3d. Five negatives + say don't expect much, but the current tool never showed the model what + the human actually saw. +5. **Data asks for the annotator.** More notes of the embryo_5/6/8 kind — + they bought +4 points. Specifically: per-boundary "what changed in this + frame" notes, typical stage durations from his experience, and ideally a + third annotated session for a held-out test set. +6. **Harness hardening.** Exponential-backoff retry around the model call in + core; tool-code version in the dispatch cache key; per-call media + filenames. diff --git a/harness/eval/html_report.py b/harness/eval/html_report.py index 55f0477..cc15ebe 100644 --- a/harness/eval/html_report.py +++ b/harness/eval/html_report.py @@ -466,6 +466,7 @@ def generate( "run_dir": str(run_dir), "experiment": _experiment_description(str(config.get("solver", ""))), "runs": _sibling_runs(run_dir), + "findings_href": os.path.relpath(_REPO_ROOT / "runs" / "findings.html", run_dir.resolve()), "seed": seed, "summary": _summary(run_dir, gt), "clusters": _failure_clusters(run_dir, gt, detail_seed=seed), @@ -595,6 +596,10 @@ def generate( .runSel{display:flex;align-items:center;gap:10px;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase} .runSel select{background:var(--panel2);color:var(--txt);border:1px solid var(--line);border-radius:6px;padding:7px 11px;font:12px var(--mono);max-width:380px;cursor:pointer} .runSel select:hover{border-color:var(--accent)} +.tabs{display:flex;gap:4px;margin-bottom:20px;border-bottom:1px solid var(--line)} +.tabs a,.tabs .tab{font:12px var(--mono);letter-spacing:.08em;text-transform:uppercase;color:var(--dim);text-decoration:none;padding:8px 16px;border:1px solid transparent;border-bottom:none;border-radius:7px 7px 0 0} +.tabs a:hover{color:var(--bright)} +.tabs .on{color:var(--bright);background:var(--panel2);border-color:var(--line)} .expDesc{margin:18px 0 0;padding:16px 20px;background:var(--panel2);border:1px solid var(--line);border-left:3px solid var(--accent);border-radius:8px;font:13px/1.7 var(--sans);color:var(--txt);max-width:980px} .expDesc::before{content:"experiment";display:block;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin-bottom:8px} .expDesc p{margin:0 0 10px} @@ -603,6 +608,7 @@ def generate( +
reportfindings

@@ -643,6 +649,9 @@ def generate( document.getElementById('title').textContent=(D.config.solver||'run')+' · '+(D.config.model||''); document.getElementById('subtitle').textContent=D.run_dir+' · seed'+D.seed+' detail · '+D.summary.n_seeds+' seed(s) aggregated'; +// tabs +if(D.findings_href)document.getElementById('findingsTab').href=D.findings_href; + // experiment description if(D.experiment&&D.experiment.paras&&D.experiment.paras.length){ const ed=document.getElementById('expDesc'); @@ -780,3 +789,87 @@ def generate( """ + + +# --- Findings page ------------------------------------------------------------ + +_FINDINGS_CSS = """ +body{max-width:1060px;margin:0 auto;padding:34px 28px 80px} +.fd h1{font:600 26px var(--sans);color:var(--bright);margin:14px 0 6px} +.fd h2{font:600 19px var(--sans);color:var(--bright);margin:34px 0 12px;padding-top:18px;border-top:1px solid var(--line)} +.fd h3{font:600 15px var(--sans);color:var(--bright);margin:24px 0 8px} +.fd p{font:14px/1.75 var(--sans);color:var(--txt);margin:0 0 14px} +.fd li{font:14px/1.75 var(--sans);color:var(--txt);margin:0 0 10px} +.fd table{border-collapse:collapse;margin:14px 0 20px;font:12.5px var(--mono)} +.fd th{text-align:left;color:var(--dim);font-weight:500;letter-spacing:.05em;padding:7px 14px 7px 0;border-bottom:1px solid var(--line)} +.fd td{padding:7px 14px 7px 0;border-bottom:1px solid var(--line);color:var(--txt)} +.fd strong{color:var(--bright)} +.fd code{background:var(--panel2);border:1px solid var(--line);border-radius:4px;padding:1px 5px;font:12px var(--mono)} +""" + + +def _md_to_html(md: str) -> str: + """Tiny renderer for FINDINGS.md — headings, bold, code, tables, lists, paragraphs.""" + + def inline(s: str) -> str: + s = s.replace("&", "&").replace("<", "<") + s = re.sub(r"\*\*(.+?)\*\*", r"\1", s) + s = re.sub(r"`([^`]+)`", r"\1", s) + return s + + out: list[str] = [] + lines = md.splitlines() + i = 0 + while i < len(lines): + line = lines[i] + if line.startswith("|"): + rows = [] + while i < len(lines) and lines[i].startswith("|"): + cells = [c.strip() for c in lines[i].strip().strip("|").split("|")] + if not all(set(c) <= {"-", " ", ":"} for c in cells): # skip separator row + rows.append(cells) + i += 1 + head, *body = rows + out.append("" + "".join(f"" for c in head) + "") + out.extend("" + "".join(f"" for c in r) + "" for r in body) + out.append("
{inline(c)}
{inline(c)}
") + continue + if m := re.match(r"^(#{1,3}) (.*)", line): + out.append(f"{inline(m[2])}") + elif re.match(r"^\d+\. |^- ", line): + tag = "ol" if line[0].isdigit() else "ul" + items: list[str] = [] + while i < len(lines) and (re.match(r"^\d+\. |^- ", lines[i]) or (lines[i].startswith(" ") and items)): + if re.match(r"^\d+\. |^- ", lines[i]): + items.append(re.sub(r"^\d+\. |^- ", "", lines[i])) + else: + items[-1] += " " + lines[i].strip() + i += 1 + out.append(f"<{tag}>" + "".join(f"
  • {inline(it)}
    • " for it in items) + f"") + continue + elif line.strip(): + para = [line] + while i + 1 < len(lines) and lines[i + 1].strip() and not re.match(r"^#|^\||^\d+\. |^- ", lines[i + 1]): + i += 1 + para.append(lines[i]) + out.append(f"

    {inline(' '.join(p.strip() for p in para))}

    ") + i += 1 + return "\n".join(out) + + +def generate_findings(runs_root: Path | None = None, md_path: Path | None = None) -> Path: + """Render docs/FINDINGS.md into runs/findings.html (shares report styling).""" + runs_root = runs_root or (_REPO_ROOT / "runs") + md_path = md_path or (_REPO_ROOT / "docs" / "FINDINGS.md") + style = re.search(r"", _TEMPLATE, re.S) + base_css = style.group(1) if style else "" + html = ( + "findings" + f"" + '
    ← report' + 'findings
    ' + f'
    {_md_to_html(md_path.read_text())}
    ' + ) + out = runs_root / "findings.html" + out.write_text(html) + return out