diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md new file mode 100644 index 0000000..72d5ab5 --- /dev/null +++ b/docs/FINDINGS.md @@ -0,0 +1,122 @@ +# Embryo staging: experiment log & findings + +Classifying C. elegans embryo developmental stages from light-sheet volumes, +evaluated on embryos 5–8 (session `2cfd8f4e`, 705 frames, ~3.7 min/frame), +3 seeds per experiment, claude-opus-4-6. Starting point: 62.2% exact. Current +champion: **hybrid_annot2 at 69.5% ± 0.2 exact / 87.8% adjacent** — all of the +gain from text rules, none of it from richer visual input. + +## The ladder + +| # | experiment | exact | adjacent | verdict | +|---|------------|-------|----------|---------| +| 0 | hybrid (baseline, May 28) | 62.2 ± 3.1 | 83.2 | starting point | +| 1 | hybrid_nodefer — remove defer-to-previous instructions | 64.2 ± 0.8 | 83.1 | **+2.0**, variance collapsed | +| 2 | hybrid_3dviews — static rotated MIPs (45/90/135°) | 61.9 ± 0.8 | 83.6 | −2.3: coils merge in projections, model undercounts folds | +| 3 | hybrid_3dviews2 — MIPs demoted to 1.5f/2f tie-breaker | 58.9 ± 3.8 | 82.7 | worse: views destabilize pretzel anyway | +| 4 | hybrid_annot — stage criteria rewritten from the annotator's notes (tail progress + 60-min timing rule) | 66.3 ± 0.2 | 87.0 | **+2.1**, gains exactly where aimed | +| 5 | hybrid_annot2 — late-pretzel vs hatching disambiguation, cadence-aware | **69.5 ± 0.2** | **87.8** | **+3.2**, killed the 99-frame hatching cascade; champion | +| 6 | hybrid_annotviews — static raymarched views at the annotator's poses | 66.5 ± 4.0 | 85.2 | −3.0: even faithful 3D renders dilute | +| 7 | hybrid_agentic3d — model rotates the 3D render itself (view3d tool) | 64.9 ± 5.3 | 82.0 | −4.6: renders re-litigated the settled hatching call | +| 8 | hybrid_explorer3d — full viewer autonomy (rotate + zoom + click) | 67.1 ± 1.1 | 85.9 | −2.4: best-behaved 3D variant, still loses | +| 9 | hybrid_contrast3d — explorer + contrastive boundary descriptions | 67.0 ± 1.6 | 86.4 | flat vs explorer; target stages unmoved | +| 10 | hybrid_pairwise — previous frame's image + comparison framing | 68.9 ± 1.3 | 87.0 | tie (paired Δ −0.5, CI [−1.6, +0.5]); lag unmoved | +| 11 | hybrid_prevonly — previous frame's image alone (ablation) | 63.7 ± 1.6 | 85.0 | **−5.7, CI [−7.2, −4.3]** — decisively harmful | + +Reference points from embryos 1–4 (session `59799c78`): hybrid 86.2%, +agentic_baseline (all tools) 80.2% — embryos 5–8 are substantially harder, +and tools cost accuracy there too. + +## Findings + +### 1. Every win was a decision rule; every image-side change was flat or negative + +The +7.3 points from baseline to champion decompose entirely into text: +removing the defer-to-previous anchor (+2.0), the annotator's tail-progress +criteria and his explicit 2fold→pretzel timing rule (+2.1), and the +late-pretzel vs hatching cadence rules (+3.2). Six experiments added visual +information — rotated MIPs, faithful raymarched views, model-driven 3D +navigation with zoom, the previous frame — and not one of them improved on +its text-only base. Agentic tool use carries a consistent ~2.4-point tax on +this task (explorer3d/contrast3d vs annot2), with tool selection itself +behaving exactly as prompted. + +### 2. The model under-updates: transition lag is a decision behavior, not a perception gap + +The dominant failure all along has been one-stage lag at boundaries (median +8–9 frames) plus brief stages missed outright (comma's window is ~5 frames; +it scores 0–5% everywhere). Three increasingly direct perception fixes failed +to move the lag at all: contrastive descriptions of each boundary's first +visible change (written from the e1–4 transition frames), full 3D viewing +autonomy, and finally showing the previous and current frames side by side — +lag median 8 vs 9, behind-failures 655 vs 641, identical clusters. The model +sees the change and does not act on it. + +The prevonly ablation is the sharpest evidence: an **unguided** previous +frame made things much worse (−5.7), shifting every confusion laggier — +consecutive frames are ~95% identical pixels, and visual similarity reads as +"same stage" louder than any boundary cue. The comparison-first framing in +pairwise repaired ~5 of those 6 points. Inputs that support persistence +amplify the bias; framings that force the update question neutralize it. + +### 3. The labels themselves are partly time-derived + +Kesavan's notes state outright that he placed the 2fold→pretzel boundary +with a stereotypic-timing rule ("~60 minutes after 2fold onset") because +coils are genuinely hard to count in projections. Appearance-only +classification therefore fights the ground truth at that boundary. Telling +the model this honestly — and converting all time rules to run-time-measured +cadence (frame interval from acquisition timestamps, so the same solver works +on any series) — produced the single largest win. + +### 4. What the failure map looks like now (annot2) + +~30% of predictions still miss. Composition: late arrival (~390/seed at its +peak, now the lag tail), brief windows missed (33 of 81 transition windows), +ahead-errors eliminated (99 → ~0–5 after the hatching rules). Per stage: +early/hatched ~100%, pretzel ~70, bean ~33, 2fold ~26, 1.5fold ~14, comma ~5. + +### 5. Infrastructure lessons + +Transient API errors (grammar-compilation 400s, 529s) killed four multi-hour +runs at top level — `harness/core/model.py` has no retry; seeds were +recovered with a resume script. The tool dispatch cache is content-addressed +on params but not on tool code version, so changing a tool's implementation +silently serves stale renders. Tool media files key on turn index, so +parallel calls in one turn overwrite each other (the report now re-renders +steps from recorded params). `view3d` post-processing (brightness stretch + +content crop) broke pixel-equivalence with the annotator viewer — replays are +faithful to what the model saw, but what the model saw was not what the +annotator saw. + +## Suggested next steps + +1. **Generalize the timing-window recipe to every boundary.** The one + boundary with an elapsed-time rule and an explicit exception to the + prefer-earlier bias (2fold→pretzel) is the one transition that improved. + Derive per-stage duration windows from the e1–4 session, and at each + boundary have the prompt switch from "prefer earlier" to "actively hunt + the next stage's first visible change" once the window passes. This is the + only intervention class with a positive track record, aimed at the + dominant remaining failure. +2. **Anchor-free second opinion at suspected transitions.** When the timing + window has passed, reclassify the frame blind (no history, no previous + stage) and reconcile with the monotonic verifier. The bias is + persistence; removing the anchor only where lag is likely keeps stability + elsewhere. +3. **Cross-session validation.** annot2's rules were developed against + embryos 5–8. Run it on embryos 1–4 to check nothing regressed vs the 86.2% + baseline and that the cadence machinery generalizes (different session, + same ~4-min cadence). +4. **If 3D gets another attempt: annotator-native rendering.** Rebuild + view3d with no stretch/crop, the annotator's contrast knob, true camera + zoom, and a code-versioned cache — then rerun explorer3d. Five negatives + say don't expect much, but the current tool never showed the model what + the human actually saw. +5. **Data asks for the annotator.** More notes of the embryo_5/6/8 kind — + they bought +4 points. Specifically: per-boundary "what changed in this + frame" notes, typical stage durations from his experience, and ideally a + third annotated session for a held-out test set. +6. **Harness hardening.** Exponential-backoff retry around the model call in + core; tool-code version in the dispatch cache key; per-call media + filenames. diff --git a/gently_perception/render/context.py b/gently_perception/render/context.py index 2d08c98..90367aa 100644 --- a/gently_perception/render/context.py +++ b/gently_perception/render/context.py @@ -23,8 +23,21 @@ def make_context(allow_cpu: bool = False) -> moderngl.Context: - """Create a standalone moderngl context, refusing CPU fallback by default.""" - ctx = moderngl.create_context(standalone=True) + """Create a standalone moderngl context, refusing CPU fallback by default. + + Tries the platform default backend first (X11 on Linux desktops), then + EGL, which is what works on headless machines (CI, remote boxes). + """ + try: + ctx = moderngl.create_context(standalone=True) + except Exception as default_err: # glcontext raises bare Exception for backend failures + try: + ctx = moderngl.create_context(standalone=True, backend="egl") + except Exception as egl_err: + raise RuntimeError( + f"No usable GL backend: default backend failed ({default_err}); " + f"EGL failed ({egl_err})." + ) from egl_err info = ctx.info renderer = (info.get("GL_RENDERER") or "").lower() vendor = (info.get("GL_VENDOR") or "").lower() diff --git a/harness/eval/html_report.py b/harness/eval/html_report.py index 7dce3ee..cc15ebe 100644 --- a/harness/eval/html_report.py +++ b/harness/eval/html_report.py @@ -15,13 +15,15 @@ import io import json import os +import re +from datetime import datetime, timedelta from pathlib import Path from typing import Any from PIL import Image -from harness.core.render import cached_render -from harness.core.types import STAGE_ORDER, Stage +from harness.core.render import cached_render, cached_tool_result, load_volume +from harness.core.types import STAGE_ORDER, ImageResult, Stage from harness.eval import report as text_report from harness.eval.score import score_run from harness.io.events import read_events @@ -31,6 +33,14 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] THUMB_LONG_EDGE = 260 +_FS_UNSAFE = re.compile(r"[^A-Za-z0-9_\-]") + + +def _fs_name(value: Any) -> str: + """Filesystem/URL-safe asset-name component. Embryo ids come from + events.jsonl, which may not be trusted (shared/downloaded run dirs).""" + return _FS_UNSAFE.sub("_", str(value)) + # --- Data assembly ---------------------------------------------------------- @@ -100,7 +110,7 @@ def _thumbnails(frames: list[dict[str, Any]], assets_dir: Path, volumes_dir: Pat key = (r["e"], r["t"]) if key not in paths: continue - thumb = assets_dir / f"{r['e']}_T{r['t']:03d}.jpg" + thumb = assets_dir / f"{_fs_name(r['e'])}_T{int(r['t']):03d}.jpg" r["thumb"] = f"report_assets/{thumb.name}" if thumb.exists(): continue @@ -133,7 +143,7 @@ def _rotated_thumbnails( continue rot: list[dict[str, Any]] = [] for angle in angles: - name = f"{r['e']}_T{r['t']:03d}_rot{int(angle)}.jpg" + name = f"{_fs_name(r['e'])}_T{int(r['t']):03d}_rot{int(angle)}.jpg" out = assets_dir / name rot.append({"a": int(angle), "src": f"report_assets/{name}"}) if out.exists(): @@ -146,6 +156,50 @@ def _rotated_thumbnails( r["rot"] = rot +def _view3d_step_assets(frames: list[dict[str, Any]], run_dir: Path, volumes_dir: Path) -> None: + """Re-render every view3d step image from its recorded params. + + The harness keys media files by model-turn index, so parallel tool calls + in one turn overwrite each other's image. view3d is deterministic, so the + recorded params are the authoritative source; renders hit the run's own + dispatch cache (same (volume, tool, params) key) and are cheap. + """ + needs = [ + (r, j, s) + for r in frames + for j, s in enumerate(r.get("steps", [])) + if s.get("name") == "view3d" and s.get("params") is not None + ] + if not needs or not volumes_dir.exists(): + return + from harness.tools import REGISTRY, _fill_defaults # deferred: pulls in GL deps + + spec = REGISTRY.get("view3d") + if spec is None: + return + paths = {(e, t): p for e, t, p in OfflineSource(volumes_dir)} + assets = run_dir / "report_assets" + assets.mkdir(parents=True, exist_ok=True) + for r, j, s in needs: + vol_ref = paths.get((r["e"], r["t"])) + if vol_ref is None: + continue + params = s["params"] + + def compute(vol_ref: Path = vol_ref, params: dict[str, Any] = params) -> str: + result = spec.fn(load_volume(vol_ref), **_fill_defaults(spec, params)) + assert isinstance(result, ImageResult) + return result.b64 + + name = f"{_fs_name(r['e'])}_T{int(r['t']):03d}_nav{j}.jpg" + out = assets / name + assert out.resolve().is_relative_to(assets.resolve()) + if not out.exists(): + b64 = cached_tool_result(Path(vol_ref), "view3d", params, compute) + out.write_bytes(base64.b64decode(b64)) + s["img"] = f"report_assets/{name}" + + def _summary(run_dir: Path, gt: GroundTruth) -> dict[str, Any]: seed_files = sorted(run_dir.glob("seed*/events.jsonl")) scores = [score_run(p, gt) for p in seed_files] @@ -305,8 +359,18 @@ def _embryo_span(ids: list[str]) -> str: return "e" + ",".join(str(n) for n in nums) +def _week_prefix(ts: str) -> str: + """'20260609-...' → '[Week of 6/8]' (the Monday of that run's week).""" + try: + run_day = datetime.strptime(ts[:8], "%Y%m%d").date() + except ValueError: + return "" + monday = run_day - timedelta(days=run_day.weekday()) + return f"[Week of {monday.month}/{monday.day}]" + + def _run_label(d: Path, gt_embryos: dict[str, str]) -> str: - """Human-readable dropdown label: what the solver change was, dataset, date.""" + """Human-readable dropdown label: week group, what the change was, dataset, date.""" solver, _model, ts = d.parts[-3:] desc = solver try: @@ -321,7 +385,33 @@ def _run_label(d: Path, gt_embryos: dict[str, str]) -> str: except Exception: pass date = f"{ts[4:6]}/{ts[6:8]}" if len(ts) >= 8 else ts - return " · ".join(x for x in (desc, span, date) if x) + body = " · ".join(x for x in (desc, span, date) if x) + week = _week_prefix(ts) + return f"{week} {body}" if week else body + + +def _experiment_description(solver_name: str) -> dict[str, Any]: + """The solver docstring's full pre-RESULT content — what was tried and why. + + Returns {"paras": [...], "setup": "..."}. RESULT paragraphs are excluded; + the report's own numbers speak for the outcome. + """ + try: + mod = importlib.import_module(f"harness.solvers.{solver_name}") + except Exception: + return {"paras": [], "setup": ""} + paras: list[str] = [] + for block in (mod.__doc__ or "").strip().split("\n\n"): + if block.strip().startswith("RESULT"): + break + paras.append(" ".join(line.strip() for line in block.splitlines())) + setup = "" + solver = getattr(mod, "SOLVER", None) + if solver is not None: + tools = ", ".join(solver.tools) if solver.tools else "none" + mode = "one-shot" if solver.max_steps == 1 else f"agentic, {solver.max_steps} steps max" + setup = f"solver {solver.name} · tools: {tools} · {mode}" + return {"paras": paras, "setup": setup} def _sibling_runs(run_dir: Path) -> list[dict[str, Any]]: @@ -339,7 +429,7 @@ def _sibling_runs(run_dir: Path) -> list[dict[str, Any]]: } dirs = {p.parent for p in runs_root.glob("*/*/*/report.html")} | {run_dir} out: list[dict[str, Any]] = [] - for d in sorted(dirs, key=lambda p: (p.parts[-3], p.parts[-1]), reverse=True): + for d in sorted(dirs, key=lambda p: p.parts[-1], reverse=True): # newest first → weeks cluster out.append( { "label": _run_label(d, gt_embryos), @@ -369,11 +459,14 @@ def generate( vols = volumes_dir or (_REPO_ROOT / "data" / "volumes") _thumbnails(frames, run_dir / "report_assets", vols) _rotated_thumbnails(frames, run_dir / "report_assets", vols, str(config.get("solver", ""))) + _view3d_step_assets(frames, run_dir, vols) data: dict[str, Any] = { "config": config, "run_dir": str(run_dir), + "experiment": _experiment_description(str(config.get("solver", ""))), "runs": _sibling_runs(run_dir), + "findings_href": os.path.relpath(_REPO_ROOT / "runs" / "findings.html", run_dir.resolve()), "seed": seed, "summary": _summary(run_dir, gt), "clusters": _failure_clusters(run_dir, gt, detail_seed=seed), @@ -485,9 +578,12 @@ def generate( .rotRow figure{flex:1;margin:0} .rotRow img{width:100%;border-radius:5px;background:#000;display:block} .rotRow figcaption{font:10px var(--mono);color:var(--dim);letter-spacing:.08em;text-align:center;margin-top:5px;text-transform:uppercase} -.step{border:1px solid var(--line);border-radius:6px;background:var(--panel2);padding:11px 14px;margin-bottom:10px;font:12px var(--mono)} -.step .sn{color:var(--accent);margin-right:8px} -.step img{max-width:340px;display:block;margin-top:9px;border-radius:4px;background:#000} +.navHdr{font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin:18px 0 10px} +.navStrip{display:flex;gap:12px;overflow-x:auto;padding-bottom:8px;margin-bottom:6px} +.navStrip .step{flex:0 0 auto;width:300px;border:1px solid var(--line);border-radius:8px;background:var(--panel2);padding:10px 12px;font:11px/1.5 var(--mono)} +.step .sn{display:inline-block;min-width:18px;height:18px;line-height:18px;text-align:center;background:var(--accent);color:#000;border-radius:9px;font-weight:700;margin-right:8px} +.step .cap{color:var(--bright)} +.step img{width:100%;display:block;margin-top:9px;border-radius:5px;background:#000} .step .val{color:var(--bright);font-weight:600} .step .err{color:var(--bad)} .reason{font:13px/1.6 var(--sans);color:var(--txt);background:var(--panel2);border:1px solid var(--line);border-radius:6px;padding:13px 16px;white-space:pre-wrap} @@ -500,13 +596,24 @@ def generate( .runSel{display:flex;align-items:center;gap:10px;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase} .runSel select{background:var(--panel2);color:var(--txt);border:1px solid var(--line);border-radius:6px;padding:7px 11px;font:12px var(--mono);max-width:380px;cursor:pointer} .runSel select:hover{border-color:var(--accent)} +.tabs{display:flex;gap:4px;margin-bottom:20px;border-bottom:1px solid var(--line)} +.tabs a,.tabs .tab{font:12px var(--mono);letter-spacing:.08em;text-transform:uppercase;color:var(--dim);text-decoration:none;padding:8px 16px;border:1px solid transparent;border-bottom:none;border-radius:7px 7px 0 0} +.tabs a:hover{color:var(--bright)} +.tabs .on{color:var(--bright);background:var(--panel2);border-color:var(--line)} +.expDesc{margin:18px 0 0;padding:16px 20px;background:var(--panel2);border:1px solid var(--line);border-left:3px solid var(--accent);border-radius:8px;font:13px/1.7 var(--sans);color:var(--txt);max-width:980px} +.expDesc::before{content:"experiment";display:block;font:11px var(--mono);color:var(--dim);letter-spacing:.14em;text-transform:uppercase;margin-bottom:8px} +.expDesc p{margin:0 0 10px} +.expDesc p:last-of-type{margin-bottom:0} +.expSetup{margin-top:12px;padding-top:10px;border-top:1px dashed var(--line);font:11px var(--mono);color:var(--dim);letter-spacing:.04em} +
reportfindings

+

Summary

Confusion (true ↓ / predicted →)
@@ -542,6 +649,17 @@ def generate( document.getElementById('title').textContent=(D.config.solver||'run')+' · '+(D.config.model||''); document.getElementById('subtitle').textContent=D.run_dir+' · seed'+D.seed+' detail · '+D.summary.n_seeds+' seed(s) aggregated'; +// tabs +if(D.findings_href)document.getElementById('findingsTab').href=D.findings_href; + +// experiment description +if(D.experiment&&D.experiment.paras&&D.experiment.paras.length){ + const ed=document.getElementById('expDesc'); + for(const p of D.experiment.paras){const el=document.createElement('p');el.textContent=p;ed.appendChild(el);} + if(D.experiment.setup){const su=document.createElement('div');su.className='expSetup';su.textContent=D.experiment.setup;ed.appendChild(su);} + ed.hidden=false; +} + // run switcher const runSel=document.getElementById('runSel'); (D.runs||[]).forEach(r=>{const o=document.createElement('option');o.value=r.href;o.textContent=r.label;o.title=r.path||'';o.selected=!!r.current;runSel.appendChild(o)}); @@ -641,14 +759,23 @@ def generate( let cur=-1; function open(i){cur=i;const f=F[i]; let h='
'+f.e+' · T'+f.t+''+(i+1)+' / '+F.length+' · ←→ navigate · esc close
'; -if(f.thumb)h+=''; -if(f.rot&&f.rot.length)h+='
'+f.rot.map(r=>'
rotated '+r.a+'°
').join('')+'
'; +if(f.thumb)h+=''; +if(f.rot&&f.rot.length)h+='
'+f.rot.map(r=>'
rotated '+r.a+'°
').join('')+'
'; h+='
pred '+f.pred+'gt '+f.gt+''+(f.tokens||0)+' tok
'; -(f.steps||[]).forEach((s,j)=>{h+='
'+(j+1)+''+s.name+'('+esc(JSON.stringify(s.params))+')'+ - (s.error?' → '+esc(s.error)+'':'')+ - (s.value!==undefined?' → '+s.value+' '+esc(s.note)+'':'')+ - (s.img?'':'')+'
'}); -h+='
'+esc(f.reasoning||f.err||'(no reasoning)')+'
'; +function stepCap(s){ + if(s.name==='view3d'){const p=s.params||{};let c='rotate → yaw '+(p.yaw_deg??0)+'° · pitch '+(p.pitch_deg??0)+'°'; + if(p.threshold!==undefined&&p.threshold!==30)c+=' · threshold '+p.threshold; + if(p.zoom_pct&&p.zoom_pct>100)c+=' · zoom '+p.zoom_pct+'% @ ('+(p.center_x_pct??50)+'%, '+(p.center_y_pct??50)+'%)'; + return c;} + return s.name+'('+Object.entries(s.params||{}).map(([k,v])=>k+'='+v).join(', ')+')';} +if((f.steps||[]).length){ + h+='
'+( (f.steps.some(s=>s.name==='view3d'))?'3D navigation — how the model explored this frame':'tool calls')+'
'; + f.steps.forEach((s,j)=>{h+='
'+(j+1)+''+esc(stepCap(s))+''+ + (s.error?'
'+esc(s.error)+'
':'')+ + (s.value!==undefined?'
→ '+s.value+' '+esc(s.note)+'
':'')+ + (s.img?'':'')+'
'}); + h+='
';} +h+='
model response
'+esc(f.reasoning||f.err||'(no reasoning)')+'
'; if(f.override)h+='
⚠ verify override: '+esc(f.override)+'
'; if(f.budget_exhausted)h+='
⚠ tool budget exhausted — classify was forced
'; document.getElementById('modalBox').innerHTML=h; @@ -662,3 +789,87 @@ def generate( """ + + +# --- Findings page ------------------------------------------------------------ + +_FINDINGS_CSS = """ +body{max-width:1060px;margin:0 auto;padding:34px 28px 80px} +.fd h1{font:600 26px var(--sans);color:var(--bright);margin:14px 0 6px} +.fd h2{font:600 19px var(--sans);color:var(--bright);margin:34px 0 12px;padding-top:18px;border-top:1px solid var(--line)} +.fd h3{font:600 15px var(--sans);color:var(--bright);margin:24px 0 8px} +.fd p{font:14px/1.75 var(--sans);color:var(--txt);margin:0 0 14px} +.fd li{font:14px/1.75 var(--sans);color:var(--txt);margin:0 0 10px} +.fd table{border-collapse:collapse;margin:14px 0 20px;font:12.5px var(--mono)} +.fd th{text-align:left;color:var(--dim);font-weight:500;letter-spacing:.05em;padding:7px 14px 7px 0;border-bottom:1px solid var(--line)} +.fd td{padding:7px 14px 7px 0;border-bottom:1px solid var(--line);color:var(--txt)} +.fd strong{color:var(--bright)} +.fd code{background:var(--panel2);border:1px solid var(--line);border-radius:4px;padding:1px 5px;font:12px var(--mono)} +""" + + +def _md_to_html(md: str) -> str: + """Tiny renderer for FINDINGS.md — headings, bold, code, tables, lists, paragraphs.""" + + def inline(s: str) -> str: + s = s.replace("&", "&").replace("<", "<") + s = re.sub(r"\*\*(.+?)\*\*", r"\1", s) + s = re.sub(r"`([^`]+)`", r"\1", s) + return s + + out: list[str] = [] + lines = md.splitlines() + i = 0 + while i < len(lines): + line = lines[i] + if line.startswith("|"): + rows = [] + while i < len(lines) and lines[i].startswith("|"): + cells = [c.strip() for c in lines[i].strip().strip("|").split("|")] + if not all(set(c) <= {"-", " ", ":"} for c in cells): # skip separator row + rows.append(cells) + i += 1 + head, *body = rows + out.append("" + "".join(f"" for c in head) + "") + out.extend("" + "".join(f"" for c in r) + "" for r in body) + out.append("
{inline(c)}
{inline(c)}
") + continue + if m := re.match(r"^(#{1,3}) (.*)", line): + out.append(f"{inline(m[2])}") + elif re.match(r"^\d+\. |^- ", line): + tag = "ol" if line[0].isdigit() else "ul" + items: list[str] = [] + while i < len(lines) and (re.match(r"^\d+\. |^- ", lines[i]) or (lines[i].startswith(" ") and items)): + if re.match(r"^\d+\. |^- ", lines[i]): + items.append(re.sub(r"^\d+\. |^- ", "", lines[i])) + else: + items[-1] += " " + lines[i].strip() + i += 1 + out.append(f"<{tag}>" + "".join(f"
  • {inline(it)}
    • " for it in items) + f"") + continue + elif line.strip(): + para = [line] + while i + 1 < len(lines) and lines[i + 1].strip() and not re.match(r"^#|^\||^\d+\. |^- ", lines[i + 1]): + i += 1 + para.append(lines[i]) + out.append(f"

    {inline(' '.join(p.strip() for p in para))}

    ") + i += 1 + return "\n".join(out) + + +def generate_findings(runs_root: Path | None = None, md_path: Path | None = None) -> Path: + """Render docs/FINDINGS.md into runs/findings.html (shares report styling).""" + runs_root = runs_root or (_REPO_ROOT / "runs") + md_path = md_path or (_REPO_ROOT / "docs" / "FINDINGS.md") + style = re.search(r"", _TEMPLATE, re.S) + base_css = style.group(1) if style else "" + html = ( + "findings" + f"" + '
    ← report' + 'findings
    ' + f'
    {_md_to_html(md_path.read_text())}
    ' + ) + out = runs_root / "findings.html" + out.write_text(html) + return out diff --git a/harness/solvers/__init__.py b/harness/solvers/__init__.py index 33a0ef2..c606d1d 100644 --- a/harness/solvers/__init__.py +++ b/harness/solvers/__init__.py @@ -1,6 +1,6 @@ """Solver registry — explicit dict, not auto-discovery.""" from harness.core.solver import Solver -from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_annot, hybrid_nodefer, hybrid_noprior, minimal, scientific, temporal +from harness.solvers import agentic_baseline, hybrid, hybrid_3dviews, hybrid_3dviews2, hybrid_agentic3d, hybrid_contrast3d, hybrid_explorer3d, hybrid_annot, hybrid_annot2, hybrid_annotviews, hybrid_nodefer, hybrid_noprior, hybrid_pairwise, hybrid_prevonly, minimal, scientific, temporal REGISTRY: dict[str, Solver] = { "minimal": minimal.SOLVER, @@ -9,6 +9,13 @@ "hybrid": hybrid.SOLVER, "hybrid_nodefer": hybrid_nodefer.SOLVER, "hybrid_annot": hybrid_annot.SOLVER, + "hybrid_annot2": hybrid_annot2.SOLVER, + "hybrid_annotviews": hybrid_annotviews.SOLVER, + "hybrid_agentic3d": hybrid_agentic3d.SOLVER, + "hybrid_explorer3d": hybrid_explorer3d.SOLVER, + "hybrid_contrast3d": hybrid_contrast3d.SOLVER, + "hybrid_pairwise": hybrid_pairwise.SOLVER, + "hybrid_prevonly": hybrid_prevonly.SOLVER, "hybrid_3dviews": hybrid_3dviews.SOLVER, "hybrid_3dviews2": hybrid_3dviews2.SOLVER, "hybrid_noprior": hybrid_noprior.SOLVER, diff --git a/harness/solvers/hybrid_agentic3d.py b/harness/solvers/hybrid_agentic3d.py new file mode 100644 index 0000000..7d5acd6 --- /dev/null +++ b/harness/solvers/hybrid_agentic3d.py @@ -0,0 +1,100 @@ +"""hybrid_annot2 + autonomy to rotate the 3D rendering (view3d tool). + +The static-views experiments (hybrid_3dviews, hybrid_3dviews2, +hybrid_annotviews) all regressed: fixed extra images on every frame dilute +attention on the frames that didn't need them. This tests the remaining +hypothesis — that 3D views help when the MODEL decides it needs one and +picks the angle, instead of being handed the same poses every frame. + +Same cadence-aware annot2 ruleset; adds the view3d tool (the annotator's +raymarcher with yaw/pitch/threshold control) on a small step budget. The +prompt explicitly licenses answering immediately when the projections +suffice — the agentic_baseline result (80.2 vs 86.2 on embryos 1-4) showed +that forced tool use hurts. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 64.9% +/- 5.3 +(seeds 69.8/65.5/59.3), adjacent 82.0% — vs hybrid_annot2 69.5 +/- 0.2. +Tool selection behaved as prompted (~33% of frames, ~2.7 calls each, +concentrated on transition frames), but the renders misled at exactly +the boundaries they were meant to resolve: seed2 re-opened the +pretzel→hatched failure (70 frames) that annot2's text rules had fully +fixed — late-pretzel renders show the worm pressed against the shell +and the model reads it as outside. Fourth negative for 3D views +(static MIP x2, static raymarch, agentic raymarch). hybrid_annot2 +remains the production solver. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) + +_TOOL_SECTION = """\ + +## 3D VIEWER (view3d tool) + +You can render the volume in the same 3D viewer the human annotator used, \ +from any angle: view3d(yaw_deg, pitch_deg, threshold). These renders have \ +depth — nearer structure occludes farther structure — so overlapping folds \ +separate as you rotate, unlike the flat projections above. + +Use it ONLY when the projections leave you genuinely uncertain, typically: +- you cannot tell where the tail tip is (1.5fold vs 2fold) → try a side \ +view (yaw 90) or an oblique view (yaw 45, pitch 30) to see the fold plane +- you cannot tell whether the two segments are twisting (2fold vs pretzel) \ +→ rotate to look along the body axis, and raise threshold (50-60) to peel \ +dim signal off the fold cores + +One or two well-chosen views are enough. If the projections plus history \ +already support a confident call, answer immediately without the tool — \ +extra views of an unambiguous frame add noise, not signal. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _TOOL_SECTION + "\nRespond with JSON:") + assert out != base, "tool section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_annot2 — the 3D views arrive via tool calls instead.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_agentic3d", + system=_system_for, + tools=("view3d",), + max_steps=4, + user_blocks=_user_blocks, +) diff --git a/harness/solvers/hybrid_annot2.py b/harness/solvers/hybrid_annot2.py new file mode 100644 index 0000000..0fa06b7 --- /dev/null +++ b/harness/solvers/hybrid_annot2.py @@ -0,0 +1,174 @@ +"""hybrid_annot + late-pretzel vs hatching disambiguation, cadence-aware. + +Targets the one remaining 'ahead' failure cluster: 33 consecutive end-of-series +frames on embryo_5 (x3 seeds = 99 failed predictions) where a very late, +shell-filling, vigorously moving pretzel was called "hatching", and history +anchoring then locked the streak in ("it has been hatching for 2 frames"). + +Two-part fix, both prompts: +1. Cadence argument kills the trigger — hatching takes 2-3 minutes of real + time, so how many frames it spans depends entirely on the acquisition + interval (~4-minute imaging → at most 1 frame, may be missed outright; + 20-second imaging → ~6 frames). The interval is MEASURED at runtime from + the volume filenames' acquisition timestamps and injected into the + prompt, so the same solver works on any series. Late pretzel, by + contrast, presses against and deforms the shell and wriggles WITHIN it + for a long time, which mimics emergence. +2. Explicit cascade breaker — a "hatching" streak much longer than the + computed span is self-contradictory; treat it as evidence the earlier + calls were wrong rather than something to continue. + +The 2fold→pretzel timing rule inherited from hybrid_annot ("~60 minutes +after 2fold onset") is converted from a hardcoded frame count to the same +measured-cadence arithmetic. + +The annotator's embryo_5 notes back the hatching fix: "the worm moves a +lot ... within the egg shell" on exactly these frames. +""" +import math +import re +from datetime import datetime +from functools import lru_cache +from itertools import pairwise +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import ( + _SCIENTIFIC_ANALYSIS, + _TEMPORAL_ANALYSIS, + SCIENTIFIC_SYSTEM as _SCI_BASE, + TEMPORAL_SYSTEM as _TMP_BASE, +) + +_TS_RE = re.compile(r"(\d{8}_\d{6})") + +# hybrid_annot's hardcoded frame counts, replaced with measured-cadence text. +_OLD_TIMING_FRAGMENT = "roughly 10-15 frames at this acquisition cadence" + +_OLD_TMP_RULE = """\ +5. **Late pretzel.** The pretzel stage is long-lasting. A compact bright mass \ +that fills the eggshell is still pretzel even if it doesn't look "tangled" — \ +it has not hatched unless you see the worm OUTSIDE the shell.""" + +_OLD_SCI_TEXT = """\ +**HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ +elongated worm OUTSIDE the eggshell boundary, or an empty shell.""" + + +@lru_cache(maxsize=16) +def frame_interval_minutes(volume_dir: str) -> float | None: + """Median interval between acquisition timestamps in the volume filenames. + + Returns None when fewer than two frames carry parseable timestamps — + callers fall back to cadence-free phrasing. + """ + stamps = [] + for p in sorted(Path(volume_dir).glob("*.tif")): + m = _TS_RE.search(p.name) + if m: + stamps.append(datetime.strptime(m.group(1), "%Y%m%d_%H%M%S")) + if len(stamps) < 2: + return None + diffs = sorted((b - a).total_seconds() / 60.0 for a, b in pairwise(stamps)) + return diffs[len(diffs) // 2] + + +def _cadence_facts(interval_min: float | None) -> tuple[str, str, str]: + """(hatch_span, pretzel_frames, streak) clauses for the given interval.""" + if interval_min is None: + return ( + "how many frames that spans depends on how often this series was " + "imaged (unknown here) — frequent imaging captures several hatching " + "frames, sparse imaging may skip the event entirely", + "however many frames correspond to about an hour in this series", + "a \"hatching\" streak lasting far longer than a 2-3 minute event " + "plausibly could", + ) + n_hatch = max(1, math.ceil(3.0 / interval_min)) + n_hour = max(2, round(60.0 / interval_min)) + return ( + f"frames in this series are about {interval_min:.1f} minutes apart, so " + f"hatching appears in at most ~{n_hatch} frame(s) and may be skipped " + f"entirely", + f"roughly {n_hour} frames at this series' frame interval", + f"a \"hatching\" streak much longer than ~{n_hatch} frame(s)", + ) + + +@lru_cache(maxsize=16) +def build_systems(interval_min: float | None) -> tuple[str, str]: + """(temporal, scientific) prompts with cadence-derived numbers filled in.""" + hatch_span, pretzel_frames, streak = _cadence_facts(interval_min) + + new_tmp_rule = f"""\ +5. **Late pretzel vs hatching.** The pretzel stage is long-lasting, and near \ +its end the worm moves vigorously WITHIN the eggshell — pressing against it, \ +deforming it, and changing pose between frames. This mimics emergence but is \ +still pretzel. Hatching itself takes only 2-3 minutes of real time; \ +{hatch_span} (pretzel in one frame, a thin worm clearly OUTSIDE the shell or \ +an empty/partial shell in the next). If the bright mass is still shell-sized \ +and shell-shaped, nothing has hatched. {streak} in the history is \ +self-contradictory — it means those earlier calls were wrong; re-examine \ +against the shell outline instead of continuing the streak.""" + + new_sci_text = f"""\ +**HATCHING/HATCHED**: The worm is emerging or has left the eggshell — a thin \ +elongated worm OUTSIDE the eggshell boundary, or an empty shell. Hatching \ +takes only 2-3 minutes of real time; {hatch_span}. Late pretzel moves \ +vigorously WITHIN the shell, pressing against and deforming it — that mimics \ +emergence but is still pretzel as long as the bright mass stays shell-sized \ +and shell-shaped. {streak} in the history is self-contradictory; treat it as \ +evidence the earlier calls were wrong rather than something to continue.""" + + tmp = _TMP_BASE.replace(_OLD_TMP_RULE, new_tmp_rule) + sci = _SCI_BASE.replace(_OLD_SCI_TEXT, new_sci_text) + assert tmp != _TMP_BASE, "temporal late-pretzel rule did not match" + assert sci != _SCI_BASE, "scientific hatching text did not match" + + assert _OLD_TIMING_FRAGMENT in tmp and _OLD_TIMING_FRAGMENT in sci + tmp = tmp.replace(_OLD_TIMING_FRAGMENT, pretzel_frames) + sci = sci.replace(_OLD_TIMING_FRAGMENT, pretzel_frames) + return tmp, sci + + +def pretzel_window_text(volume_ref: Path) -> str: + """User-turn phrasing for the 2fold→pretzel elapsed-time exception.""" + interval = frame_interval_minutes(str(Path(volume_ref).parent)) + _, pretzel_frames, _ = _cadence_facts(interval) + return pretzel_frames + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + return sci if _is_scientific(frame) else tmp + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """hybrid_annot's structure with the cadence-derived pretzel window.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_annot2", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) diff --git a/harness/solvers/hybrid_annotviews.py b/harness/solvers/hybrid_annotviews.py new file mode 100644 index 0000000..e9d8a5f --- /dev/null +++ b/harness/solvers/hybrid_annotviews.py @@ -0,0 +1,96 @@ +"""hybrid_annot2 + raymarched views at the annotator's own poses. + +Third attempt at giving the model 3D shape information, with both failure +causes of the MIP attempts addressed: +- hybrid_3dviews/3dviews2 used max-projections, where tight pretzel coils + merge into bands; these are true raymarched renders with depth occlusion — + the same viewer output the human annotator labeled from. +- Those attempts also instructed the model to re-decide stages from the new + views; here the views are framed as supporting evidence for the existing + tail-progress criteria (find the tail tip), with the annot2 ruleset + unchanged on top. + +Two views per frame (annotator default pose + his tail-visibility pose), +~50ms each cached. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 66.5% +/- 4.0 +(seeds 68.4/69.2/62.0), adjacent 85.2% — worse than hybrid_annot2's +69.5 +/- 0.2 and the highest variance of the annot line. 2fold dropped +26 -> 17.5 and pretzel destabilized (65.3 +/- 9.7). Third negative result +for extra static views; with projection-collapse ruled out by the +raymarcher, the remaining explanation is that additional static images +dilute attention rather than add usable evidence on this task. Kept in +the registry for reference; hybrid_annot2 remains the production solver. +""" +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from pathlib import Path + +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.tools.annotator_view import annotator_view_b64 + +_3D_SECTION = """\ + +## 3D VIEWER RENDERS + +After the main 3-view projection you are given two renders from the same 3D \ +viewer the human annotator used to label this data: first his default \ +working pose, then a side pose he used to identify the tail ("you can see \ +the two lobes clearly — the left is the tail"). Unlike the flat projections, \ +these have depth — nearer structure occludes farther structure, so separate \ +body folds stay visually separate. + +Use them to apply the tail criteria: find the tail (the shorter lobe), and \ +judge how far its tip has progressed. The projections remain your primary \ +evidence; the renders are there to resolve where the tail is when the \ +projection is ambiguous. +""" + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _3D_SECTION + "\nRespond with JSON:") + assert out != base, "3D section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """annot2's structure + the two raymarched views before the analysis prompt.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append( + model.text_block( + "3D viewer renders (annotator's default pose, then his tail-visibility pose):" + ) + ) + blocks.append(model.image_block(annotator_view_b64(frame.volume_ref, "default"))) + blocks.append(model.image_block(annotator_view_b64(frame.volume_ref, "tail"))) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_annotviews", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) diff --git a/harness/solvers/hybrid_contrast3d.py b/harness/solvers/hybrid_contrast3d.py new file mode 100644 index 0000000..9bf1486 --- /dev/null +++ b/harness/solvers/hybrid_contrast3d.py @@ -0,0 +1,126 @@ +"""Full 3D viewer autonomy + contrastive transition-boundary descriptions. + +hybrid_explorer3d (rotate/zoom/click-to-center, 8-step budget, eggshell +guard) scored 67.1 +/- 1.1 with the dominant failures unchanged: every +remaining cluster is a transition-DETECTION failure — the model recognizes +stages but misses the frame where one ends and the next begins. + +This adds boundary descriptions written contrastively ("what visibly +changes"), derived frame-by-frame from the ground-truth transition frames of +embryos 2-3 (session 59799c78 — the other annotated session, so no eval-set +leakage). Each names the first visible change at the boundary rather than +describing the stages on either side. + +Comparisons: vs hybrid_explorer3d isolates the contrastive text within the +autonomy arm; vs hybrid_annot2 (69.5, text-only) is a two-variable +comparison — if this wins, a text-only contrastive arm separates the +contributions. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 67.0% +/- 1.6, +adjacent 86.4 — a statistical tie with hybrid_explorer3d (67.1 +/- 1.1) +and still -2.5 vs text-only hybrid_annot2 (69.5 +/- 0.2). The boundary +descriptions did not move their target stages (comma ~5%, bean ~29%, +1.5fold ~14%, 2fold ~18% — unchanged from every annot-line run), and +added nothing under autonomy overall. Whether they help on a text-only +base remains untested. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_explorer3d import _TOOL_SECTION + +BOUNDARIES_SECTION = """\ + +## STAGE TRANSITIONS — THE FIRST VISIBLE CHANGE AT EACH BOUNDARY + +Most classification errors are missed transitions, not misrecognized \ +stages. Compare the current frame against the recent history and ask which \ +boundary, if any, has just been crossed: + +- **early → bean: the oval loses its symmetry.** A perfect, smooth, convex \ +bright oval is early. The transition is NOT a new structure — it is one \ +corner of the outline pulling slightly inward (the future posterior \ +indentation) and a faint dim seam appearing inside the mass. If the outline \ +is no longer a clean symmetric oval, bean has begun. + +- **bean → comma: the seam becomes a notch.** Bean is a "peanut": two \ +roughly equal bright lobes with a shallow dim line between them. When that \ +line deepens into a dark WEDGE cutting in from the edge — real dark space \ +opening between a distinct smaller lobe (the tail) and the body — comma has \ +begun. Peanut → hook. + +- **comma → 1.5fold: the first parallel-band pair appears.** Comma is still \ +ONE bright mass with a notch and a tail lobe. The transition is a \ +reorganization of the whole image: TWO parallel bright bands separated by a \ +long dark diagonal furrow (body axis + folded-back tail), the tail band \ +clearly shorter, reaching about halfway. Counting goes from "one mass" to \ +"two bands" — the most categorical change of the series. + +- **1.5fold → 2fold: the short band catches up.** Band count stays TWO — do \ +not wait for a third. In 1.5fold the tail band is short, leaving a large \ +dark region at one end of the eggshell. When both bands run nearly the full \ +long axis (the embryo reads as an S or zigzag) and the dark space has shrunk \ +to two thin furrows, 2fold has begun. The cue is relative band LENGTH, not \ +band count. + +- **2fold → pretzel: the dark furrows fragment.** In 2fold the dark spaces \ +between body segments are LONG, CONTINUOUS channels — an organized zigzag. \ +At pretzel the coils cross and the continuous furrows break into SMALL, \ +ISOLATED dark pockets; texture turns granular/marbled and the outline fills \ +almost completely. Judge the CONTINUITY of the dark spaces: long channels = \ +2fold, scattered pockets = pretzel. This works even when you cannot count \ +coils. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace( + "\nRespond with JSON:", BOUNDARIES_SECTION + _TOOL_SECTION + "\nRespond with JSON:" + ) + assert out != base, "insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_explorer3d — 3D views arrive via tool calls.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_contrast3d", + system=_system_for, + tools=("view3d",), + max_steps=8, + user_blocks=_user_blocks, +) diff --git a/harness/solvers/hybrid_explorer3d.py b/harness/solvers/hybrid_explorer3d.py new file mode 100644 index 0000000..3280fb4 --- /dev/null +++ b/harness/solvers/hybrid_explorer3d.py @@ -0,0 +1,113 @@ +"""hybrid_annot2 + full 3D viewer autonomy: rotate, zoom, click-to-center. + +Completes the autonomy ladder that hybrid_agentic3d started. That run gave +rotation + threshold only, full-embryo renders at fixed distance, 3 calls +max — and lost to text-only annot2 (64.9 +/- 5.3 vs 69.5 +/- 0.2), partly +because 512px whole-body renders are too coarse to read twist, and partly +because late-pretzel renders were misread as hatched. + +This variant gives the model everything the human annotator's viewer had: +- rotate (yaw/pitch) and threshold, as before +- zoom up to 4x around a clicked point (center_x/y_pct), so it can inspect + the tail tip or a suspected fold crossing at fold scale +- an 8-step budget for a survey -> zoom -> decide workflow + +Plus one guard distilled from the agentic3d failure: renders show signal +only, not the eggshell wall, so hatched/hatching judgments stay owned by +the projections and the cadence rules. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 67.1% +/- 1.1 +(adjacent 85.9) — vs hybrid_annot2 69.5 +/- 0.2 / 87.8. Best-behaved 3D +variant yet: the eggshell guard mostly held (pretzel→hatch 0/0/23 per +seed vs 70 in agentic3d's worst seed) and variance came down 5.3 → 1.1, +but full viewer autonomy (rotate + zoom + click-to-center, 8 steps, +~4 renders per exploring frame) still loses to text-only by 2.4 points. +The dominant confusions are unchanged (pretzel→2fold ~60/seed). Fifth +and most conclusive negative for 3D input on this task: the bottleneck +is boundary calibration, not visual information. hybrid_annot2 remains +the production solver. +""" +from pathlib import Path + +from harness.core import model +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) + +_TOOL_SECTION = """\ + +## 3D VIEWER (view3d tool) + +You can use the same 3D viewer the human annotator used — rotate the \ +embryo, zoom in, and recenter, exactly as he did when deciding these \ +stages: view3d(yaw_deg, pitch_deg, threshold, zoom_pct, center_x_pct, \ +center_y_pct). Renders have depth — nearer structure occludes farther \ +structure — so folds that overlap in the flat projections separate as you \ +rotate, and zooming (up to 4x, centered on a point you pick from a previous \ +render at the same angle) shows fold-scale detail the full-body view \ +cannot. + +A good workflow when the projections leave you uncertain: +1. Survey: one or two rotations (e.g. yaw 90 side view; yaw 45 pitch 30 \ +oblique) to find the angle where the ambiguous feature — tail tip, or a \ +suspected fold crossing — is least occluded. +2. Inspect: zoom 200-400% on that feature at that angle; raise threshold \ +(50-60) if dim outer signal hides the fold cores. +3. Decide using the tail-progress and twisting criteria. + +Two cautions: +- If the projections plus history already support a confident call, answer \ +immediately — extra views of an unambiguous frame add noise. +- The renders show fluorescent signal only, NOT the eggshell wall. A worm \ +pressed against the shell looks edge-on in 3D exactly like an emerging one. \ +Never conclude hatching/hatched from renders — that judgment belongs to the \ +projections and the elapsed-time rules. +""" + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace("\nRespond with JSON:", _TOOL_SECTION + "\nRespond with JSON:") + assert out != base, "tool section insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """Identical to hybrid_annot2 — 3D views arrive via tool calls.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver( + name="hybrid_explorer3d", + system=_system_for, + tools=("view3d",), + max_steps=8, + user_blocks=_user_blocks, +) diff --git a/harness/solvers/hybrid_pairwise.py b/harness/solvers/hybrid_pairwise.py new file mode 100644 index 0000000..b807f6f --- /dev/null +++ b/harness/solvers/hybrid_pairwise.py @@ -0,0 +1,119 @@ +"""hybrid_annot2 + temporal pairing: the previous frame's image in every turn. + +Every remaining failure cluster is transition-shaped: the model lags +boundaries by ~5 frames and skips brief stages entirely. All week it has had +only TEXT history ("T059: 2fold") — it has never been able to LOOK at the +previous frame. This makes "did the boundary just get crossed?" a direct +visual comparison instead of a memory exercise. + +Two additions over hybrid_annot2 (documented together; ablate if it wins): +1. The previous frame's standard 3-view projection rendered immediately + before the current frame's, labeled, with the analysis tail asking + "what changed between these two images?" first. +2. The contrastive boundary descriptions (from hybrid_contrast3d, derived + from embryos 2-3 — no eval leakage) as the vocabulary for naming the + change. Flat under viewer autonomy, but they were written for exactly + this comparison question. + +No tools, one-shot, same cadence-aware ruleset — any delta is attributable +to seeing the previous frame (plus its vocabulary). + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 68.9% +/- 1.3 +(seed0 70.2 — first seed of any experiment above annot2's level), +adjacent 87.0 — statistical tie with hybrid_annot2 (69.5 +/- 0.2). +The mechanistic readout is the real finding: the failure clusters are +IDENTICAL to annot2's (behind 655 vs 641, lag median 8 vs 9, windows +missed 35 vs 33). Seeing the previous frame did not reduce the lag at +all — the model holds the old stage even with both frames side by side +and the boundary's first-visible-change description in hand. The lag is +an under-updating DECISION behavior, not a perception gap; consistent +with every image-side intervention failing while anchor-removal and +forced-update rules (nodefer, the timing rule) produced every win. +""" +from pathlib import Path + +from harness.core import model +from harness.core.render import cached_render +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_contrast3d import BOUNDARIES_SECTION + +_PAIR_SECTION = """\ + +## PREVIOUS-FRAME COMPARISON + +You are shown the PREVIOUS frame's projection directly above the current \ +one. Compare them before anything else. Most frame pairs show NO \ +transition — the embryo simply persists in its stage, possibly shifted by \ +twitching. Your first question is: did the FIRST VISIBLE CHANGE of any \ +boundary (see the stage-transition list) appear between these two images? \ +If nothing structural changed, keep the previous stage. If something did, \ +name which boundary it is and advance exactly one stage. +""" + + +def _prev_volume_ref(frame: FrameInput) -> Path | None: + """Previous timepoint's volume: timepoint indexes the sorted file list.""" + if frame.timepoint == 0: + return None + files = sorted(Path(frame.volume_ref).parent.glob("*.tif")) + idx = frame.timepoint - 1 + if idx >= len(files) or files[frame.timepoint] != Path(frame.volume_ref): + return None # index/file mismatch — skip the pair rather than mislead + return files[idx] + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + base = sci if _is_scientific(frame) else tmp + out = base.replace( + "\nRespond with JSON:", BOUNDARIES_SECTION + _PAIR_SECTION + "\nRespond with JSON:" + ) + assert out != base, "insertion point missing" + return out + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """annot2's structure + the previous frame's image before the current one.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + prev_ref = _prev_volume_ref(frame) + if prev_ref is not None: + blocks.append(model.text_block(f"PREVIOUS frame (T{frame.timepoint - 1}):")) + blocks.append(model.image_block(cached_render(prev_ref))) + blocks.append(model.text_block(f"CURRENT frame (T{frame.timepoint}):")) + blocks.append(model.image_block(frame.image_b64)) + base = _SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS + blocks.append( + model.text_block( + "First: compare the two frames — did any boundary's first visible " + "change appear between them? Then " + base[0].lower() + base[1:] + ) + ) + return blocks + + +SOLVER = Solver(name="hybrid_pairwise", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) diff --git a/harness/solvers/hybrid_prevonly.py b/harness/solvers/hybrid_prevonly.py new file mode 100644 index 0000000..1e7edb0 --- /dev/null +++ b/harness/solvers/hybrid_prevonly.py @@ -0,0 +1,76 @@ +"""Minimal ablation: hybrid_annot2 + ONLY the previous frame's image. + +hybrid_pairwise bundled three changes (previous-frame image, contrastive +boundary section, comparison-first analysis rewrite) and tied annot2 with +wider spread — leaving open whether the image itself helps, hurts, or does +nothing once the text changes are stripped away. + +This is annot2 verbatim — same system prompts, same analysis tail, same +user-turn text — plus exactly one thing: the previous frame's projection, +labeled, rendered before the current frame's. Any delta vs annot2 +(69.5 +/- 0.2) is attributable to the image alone. + +RESULT (embryos 5-8, claude-opus-4-6, 3 seeds): exact 63.7% +/- 1.6, +adjacent 85.0. Paired frame-level bootstrap vs annot2: -5.7 points, +95% CI [-7.2, -4.3] — decisively harmful, where the full pairwise bundle +was a statistical tie (-0.5, CI [-1.6, +0.5]). Confusions shifted +uniformly laggier (pretzel→2fold 72, comma→early appears). Conclusion: +an UNGUIDED previous frame is a visual persistence anchor — consecutive +frames are ~95% identical, and the similarity reads as "same stage" +louder than any boundary cue. pairwise's comparison-first framing wasn't +masking a benefit of the image; it was repairing ~5 of the 6 points of +damage the image causes. Temporal visual context is only safe with +explicit comparison framing. +""" +from pathlib import Path + +from harness.core import model +from harness.core.render import cached_render +from harness.core.solver import Solver +from harness.core.types import FrameInput, Stage +from harness.solvers.hybrid_annot import _SCIENTIFIC_ANALYSIS, _TEMPORAL_ANALYSIS +from harness.solvers.hybrid_annot2 import ( + build_systems, + frame_interval_minutes, + pretzel_window_text, +) +from harness.solvers.hybrid_pairwise import _prev_volume_ref + + +def _is_scientific(frame: FrameInput) -> bool: + """Scientific prompt for 2fold/pretzel, temporal otherwise. Matches harness/solvers/hybrid.py.""" + return frame.last_stage in {Stage.TWO_FOLD, Stage.PRETZEL} + + +def _system_for(frame: FrameInput) -> str: + interval = frame_interval_minutes(str(Path(frame.volume_ref).parent)) + tmp, sci = build_systems(interval) + return sci if _is_scientific(frame) else tmp + + +def _user_blocks(frame: FrameInput) -> list[dict]: + """hybrid_annot2's blocks with the labeled previous-frame image inserted.""" + blocks: list[dict] = [model.text_block(f"\n=== CLASSIFY EMBRYO AT T{frame.timepoint} ===")] + if frame.history_text: + blocks.append(model.text_block(frame.history_text)) + last = frame.last_stage.value if frame.last_stage else "early" + blocks.append( + model.text_block( + f"The most recent observation was '{last}'. " + f"When uncertain, prefer the earlier stage — except at the " + f"2fold→pretzel boundary: once the history shows 2fold for " + f"{pretzel_window_text(frame.volume_ref)}, prolonged 2fold " + f"appearance plus any twisting reads as pretzel, per the timing rule." + ) + ) + prev_ref = _prev_volume_ref(frame) + if prev_ref is not None: + blocks.append(model.text_block(f"PREVIOUS frame (T{frame.timepoint - 1}), for reference:")) + blocks.append(model.image_block(cached_render(prev_ref))) + blocks.append(model.text_block(f"CURRENT frame (T{frame.timepoint}) — classify this one:")) + blocks.append(model.image_block(frame.image_b64)) + blocks.append(model.text_block(_SCIENTIFIC_ANALYSIS if _is_scientific(frame) else _TEMPORAL_ANALYSIS)) + return blocks + + +SOLVER = Solver(name="hybrid_prevonly", system=_system_for, tools=(), max_steps=1, user_blocks=_user_blocks) diff --git a/harness/tools/__init__.py b/harness/tools/__init__.py index c31bcd2..28d7ff1 100644 --- a/harness/tools/__init__.py +++ b/harness/tools/__init__.py @@ -266,5 +266,6 @@ def _num() -> str: # (Kept at the bottom to avoid circular imports — tool modules import `tool` from here.) from harness.tools import measure as _measure # noqa: E402,F401 from harness.tools import prev_frame as _prev # noqa: E402,F401 +from harness.tools import view3d as _view3d # noqa: E402,F401 from harness.tools import z_slice as _z_slice # noqa: E402,F401 from harness.tools import zoom as _zoom # noqa: E402,F401 diff --git a/harness/tools/annotator_view.py b/harness/tools/annotator_view.py new file mode 100644 index 0000000..d81a1ab --- /dev/null +++ b/harness/tools/annotator_view.py @@ -0,0 +1,90 @@ +"""Raymarched volume renders at the annotator's poses. + +Unlike the rotated MIPs (harness/tools/rotate.py — negative result twice), +these are true volume renders with depth occlusion and thresholding, produced +by the same raymarcher the human annotator used (gently_perception.render, +pixel-equivalent port of the annotator viewer). Tight pretzel coils that +merge in a max-projection stay visually distinct here. + +Poses are frozen constants: +- "default": the annotator's startup pose — the view he spent most labeling + time in. +- "tail": the pose from his embryo_5 T16 view note ("you can see the two + lobes clearly. the left is the tail") — chosen because the prompt's + transition-stage criteria are all about tail progress. A camera pose + carries no label information, so freezing it is not ground-truth leakage. + +Volumes normalize per-volume min-max to uint8 — that lands the imaging +background at ~30/255, which is exactly the annotator's default threshold, +i.e. the convention his pipeline uses. + +One GL context and one cached Renderer per volume path are kept per process; +results are content-addressed via cached_tool_result so eval seeds beyond the +first hit the cache. +""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from gently_perception.render import Renderer +from gently_perception.render.context import make_context +from gently_perception.types import CameraParams +from harness.core.render import array_to_b64, cached_tool_result, load_volume + +POSES: dict[str, CameraParams] = { + "default": CameraParams(), + "tail": CameraParams( + quaternion=( + -0.5161865499649309, + 0.18639103472417876, + 0.014405218785850556, + 0.8358243341046486, + ), + ), +} + +_ctx = None +_renderer: tuple[Path, Renderer] | None = None + + +def _render(volume_ref: Path, pose: str) -> np.ndarray: + global _ctx, _renderer + if _ctx is None: + _ctx = make_context(allow_cpu=True) + if _renderer is None or _renderer[0] != volume_ref: + if _renderer is not None: + _renderer[1].close() + vol = load_volume(volume_ref).astype(np.float64) + lo, hi = vol.min(), vol.max() + vol_u8 = ((vol - lo) / max(hi - lo, 1.0) * 255).astype(np.uint8) + _renderer = (volume_ref, Renderer(vol_u8, ctx=_ctx)) + rgba = _renderer[1].render(POSES[pose]) + # Premultiplied RGBA over the annotator's black canvas == the RGB channel. + img = rgba[..., 0].astype(np.float32) # grayscale volume → channels equal + # Display stretch (annotator used a contrast slider for the same purpose): + # late-stage volumes have hot spots that crush the body's brightness range. + hi = float(np.percentile(img[img > 0], 99.5)) if (img > 0).any() else 1.0 + out = np.clip(img / max(hi, 1.0) * 255.0, 0, 255).astype(np.uint8) + # Crop to content + 8% margin so the embryo fills the image the model sees. + ys, xs = np.nonzero(out > 8) + if len(ys): + my, mx = int(out.shape[0] * 0.08), int(out.shape[1] * 0.08) + out = out[ + max(ys.min() - my, 0) : ys.max() + my, + max(xs.min() - mx, 0) : xs.max() + mx, + ] + return out + + +def annotator_view_b64(volume_ref: Path, pose: str) -> str: + """Cached base64 JPEG of the volume raymarched at a named annotator pose.""" + assert pose in POSES, f"unknown pose {pose!r}" + + def compute() -> str: + return array_to_b64(_render(Path(volume_ref), pose), target_long_edge=512) + + return cached_tool_result( + Path(volume_ref), "annotator_view", {"pose": pose, "v": 3}, compute + ) diff --git a/harness/tools/view3d.py b/harness/tools/view3d.py new file mode 100644 index 0000000..ec165f3 --- /dev/null +++ b/harness/tools/view3d.py @@ -0,0 +1,114 @@ +"""view3d — raymarched render of the volume at a model-chosen camera angle. + +Wraps gently_perception.render (the annotator's own viewer, pixel-equivalent) +as a model-callable tool: the model picks yaw/pitch relative to the +annotator's default working pose and optionally the intensity threshold, +and gets back a true volume render with depth occlusion — folds that overlap +in the flat projections stay separate here. + +Determinism: same (volume, params) → same image. The GL context is +process-local plumbing (created once, reused); it does not affect output. +""" +from __future__ import annotations + +import math +from typing import Annotated + +import numpy as np + +from gently_perception.render import Renderer +from gently_perception.render.context import make_context +from gently_perception.types import CameraParams, _DEFAULT_QUATERNION +from harness.core.render import array_to_b64 +from harness.core.types import ImageResult +from harness.tools import Range, tool + +_ctx = None + + +def _quat_mul(a, b): + """Hamilton product of (x, y, z, w) quaternions: apply b, then a.""" + x1, y1, z1, w1 = a + x2, y2, z2, w2 = b + return ( + w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, + w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, + w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, + w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, + ) + + +def _pose(yaw_deg: float, pitch_deg: float): + """Default annotator pose rotated by yaw (screen-vertical axis) then pitch.""" + half_yaw = math.radians(yaw_deg) / 2.0 + half_pitch = math.radians(pitch_deg) / 2.0 + q_yaw = (0.0, math.sin(half_yaw), 0.0, math.cos(half_yaw)) + q_pitch = (math.sin(half_pitch), 0.0, 0.0, math.cos(half_pitch)) + return _quat_mul(q_pitch, _quat_mul(q_yaw, _DEFAULT_QUATERNION)) + + +def _display(img_rgba: np.ndarray) -> np.ndarray: + """Premultiplied RGBA over black → stretched, content-cropped grayscale.""" + img = img_rgba[..., 0].astype(np.float32) + hi = float(np.percentile(img[img > 0], 99.5)) if (img > 0).any() else 1.0 + out = np.clip(img / max(hi, 1.0) * 255.0, 0, 255).astype(np.uint8) + ys, xs = np.nonzero(out > 8) + if len(ys): + my, mx = int(out.shape[0] * 0.08), int(out.shape[1] * 0.08) + out = out[max(ys.min() - my, 0) : ys.max() + my, max(xs.min() - mx, 0) : xs.max() + mx] + return out + + +@tool(returns="image") +def view3d( + volume: np.ndarray, + *, + yaw_deg: Annotated[int, Range(-180, 181)], + pitch_deg: Annotated[int, Range(-90, 91)] = 0, + threshold: Annotated[int, Range(5, 81)] = 30, + zoom_pct: Annotated[int, Range(100, 401)] = 100, + center_x_pct: Annotated[int, Range(0, 101)] = 50, + center_y_pct: Annotated[int, Range(0, 101)] = 50, +) -> ImageResult: + """Render the embryo volume in 3D — rotate, zoom, and center like the annotator's viewer. + + This is the same 3D viewer the human annotator used: a true volume render + with depth — nearer structure occludes farther structure, so overlapping + folds separate as you rotate. yaw_deg rotates the embryo about the + screen-vertical axis (90 = side view, 180 = back), pitch_deg tilts it + (positive = view more from above); both are relative to the annotator's + default working pose. threshold sets the intensity cutoff (default 30 = + the annotator's; raise it to peel away dim outer signal and expose + internal fold structure). zoom_pct magnifies (200 = 2x, 400 = 4x) around + the point (center_x_pct, center_y_pct), given as percentages of the + current view's width/height — pick the point from a previous render of + the SAME angle, e.g. zoom into where the tail tip or a suspected fold + crossing is. + """ + global _ctx + if _ctx is None: + _ctx = make_context(allow_cpu=True) + vol = volume.astype(np.float32) + lo, hi = float(vol.min()), float(vol.max()) + vol_u8 = ((vol - lo) / max(hi - lo, 1.0) * 255.0).astype(np.uint8) + with Renderer(vol_u8, ctx=_ctx) as r: + rgba = r.render( + CameraParams( + quaternion=_pose(yaw_deg, pitch_deg), + threshold=float(threshold), + image_size=(1024, 1024), # render hi-res so zoom crops stay sharp + ) + ) + img = _display(rgba) + if zoom_pct > 100: + h, w = img.shape + win_h, win_w = int(h * 100 / zoom_pct), int(w * 100 / zoom_pct) + cy = int(h * center_y_pct / 100) + cx = int(w * center_x_pct / 100) + y0 = min(max(cy - win_h // 2, 0), h - win_h) + x0 = min(max(cx - win_w // 2, 0), w - win_w) + img = img[y0 : y0 + win_h, x0 : x0 + win_w] + caption = f"3D view yaw={yaw_deg}° pitch={pitch_deg}° threshold={threshold}" + if zoom_pct > 100: + caption += f" zoom={zoom_pct}% @({center_x_pct}%,{center_y_pct}%)" + return ImageResult(b64=array_to_b64(img, target_long_edge=512), caption=caption) diff --git a/tests/test_tools_view3d.py b/tests/test_tools_view3d.py new file mode 100644 index 0000000..51a8754 --- /dev/null +++ b/tests/test_tools_view3d.py @@ -0,0 +1,69 @@ +"""view3d golden tests with a synthetic volume. + +Skipped wholesale when no GL context can be created (CI without EGL). +""" +import numpy as np +import pytest + +pytest.importorskip("moderngl") + +from gently_perception.render.context import make_context # noqa: E402 +from harness.tools.view3d import _pose, _quat_mul, view3d # noqa: E402 + +try: + make_context(allow_cpu=True).release() +except Exception: # glcontext raises bare Exception when no backend works + pytest.skip("no GL context available", allow_module_level=True) + + +def _asym_volume() -> np.ndarray: + """A bright corner blob + axial rod: looks different from every side.""" + vol = np.zeros((40, 64, 96), dtype=np.float32) + vol[5:15, 5:20, 5:25] = 900.0 + vol[18:22, 30:34, 10:90] = 600.0 + return vol + + +def test_identity_quaternion_math(): + q = _quat_mul((0, 0, 0, 1), (0.1, 0.2, 0.3, 0.9)) + assert np.allclose(q, (0.1, 0.2, 0.3, 0.9)) + + +def test_zero_rotation_is_default_pose(): + from gently_perception.types import _DEFAULT_QUATERNION + + assert np.allclose(_pose(0, 0), _DEFAULT_QUATERNION) + + +def test_deterministic(): + vol = _asym_volume() + a = view3d(vol, yaw_deg=45, pitch_deg=10, threshold=30) + b = view3d(vol, yaw_deg=45, pitch_deg=10, threshold=30) + assert a.b64 == b.b64 + + +def test_rotation_changes_view(): + vol = _asym_volume() + front = view3d(vol, yaw_deg=0).b64 + side = view3d(vol, yaw_deg=90).b64 + back = view3d(vol, yaw_deg=180).b64 + assert front != side and side != back and front != back + + +def test_caption_reports_params(): + out = view3d(_asym_volume(), yaw_deg=-30, pitch_deg=15, threshold=50) + assert "yaw=-30" in out.caption and "pitch=15" in out.caption and "threshold=50" in out.caption + + +def test_zoom_crops_around_center(): + vol = _asym_volume() + full = view3d(vol, yaw_deg=0) + corner = view3d(vol, yaw_deg=0, zoom_pct=300, center_x_pct=10, center_y_pct=10) + middle = view3d(vol, yaw_deg=0, zoom_pct=300, center_x_pct=50, center_y_pct=50) + assert corner.b64 != middle.b64 != full.b64 + assert "zoom=300%" in corner.caption + + +def test_zoom_100_has_no_crop_caption(): + out = view3d(_asym_volume(), yaw_deg=0, zoom_pct=100) + assert "zoom=" not in out.caption