diff --git a/scripts/review_viz/README.md b/scripts/review_viz/README.md new file mode 100644 index 0000000..b7d902d --- /dev/null +++ b/scripts/review_viz/README.md @@ -0,0 +1,55 @@ +# review_viz — rollout review / trajectory visualization + +Static-HTML tooling to human-review WeaveBench rollouts: browse per-task +score cards, deliverable previews, and the full agent trajectory timeline +in a browser. No server required — everything renders to self-contained +HTML files. + +## Input layout + +Each script walks one or more *run roots*, expecting the standard rollout +layout produced by the evaluator: + +``` +/<...>/gui//// + score.json # judge output (scores, dimensions, hack flags) + chat.jsonl # agent trajectory + results.tar.gz # deliverables (screenshots, files) +``` + +Edit the `RUNS` dict at the top of `gen_viz.py` / `export_traj.py` to point +at your own run roots (paths are resolved relative to the script by default). + +## Scripts + +| Script | Output | Purpose | +|---|---|---| +| `gen_viz.py` | `viz/index.html` + `viz/tasks/*.html` | Overview dashboard (per-run PassRate @ τ=0.8, per-category table, per-task rows) plus one detailed viewer per task: score card (8 dimensions), deliverable image previews, and the reconstructed trajectory timeline. | +| `export_traj.py` | `traj_txt/*.txt` | Export each `chat.jsonl` to a compact plain-text trajectory (base64 images stripped) — handy for feeding trajectories to a review agent. | +| `gen_quality_html.py` | `viz/quality.html` | Render a quality-assessment report from `viz/quality_result.json` (independent per-trajectory quality review vs. the official judge). | + +## Usage + +```bash +# generate the dashboard + per-task viewers for all configured runs +python3 gen_viz.py + +# or a single run +python3 gen_viz.py pro + +# export plain-text trajectories +python3 export_traj.py + +# quality report (needs viz/quality_result.json) +python3 gen_quality_html.py +``` + +Open `viz/index.html` in a browser to start reviewing. + +## Notes + +- Pure standard library — no third-party dependencies. +- Deliverable images are inlined as base64, so the generated `viz/` can get + large (hundreds of MB to several GB depending on rollout count). It is a + local review artifact; do not commit the generated HTML. +- Pass threshold τ defaults to `0.80` (`TAU` in `gen_viz.py`). diff --git a/scripts/review_viz/export_traj.py b/scripts/review_viz/export_traj.py new file mode 100644 index 0000000..20583e6 --- /dev/null +++ b/scripts/review_viz/export_traj.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""把每条 rollout 的 chat.jsonl 导出为紧凑纯文本轨迹(剥离 base64 图), +供 workflow agent 评估解题质量。输出到 traj_txt/____.txt +""" +import json, os, glob, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +RUNS = {"pro":"seed2.1_pro","turbo":"seed_2.1_turbo"} +OUT = os.path.join(HERE,"traj_txt") +os.makedirs(OUT, exist_ok=True) + +def fmt_action(name, inp): + if name=="Bash": return "$ "+str(inp.get("command",""))[:500] + if name=="Read": return "READ "+str(inp.get("file_path","")) + if name=="Write": return "WRITE "+str(inp.get("file_path",""))+" ("+str(len(str(inp.get("content",""))))+"b)" + if name=="Edit": return "EDIT "+str(inp.get("file_path","")) + if name=="Grep": return "GREP "+str(inp.get("pattern","")) + if name=="Glob": return "GLOB "+str(inp.get("pattern","")) + if "screenshot" in name: return "[screenshot]" + if name=="TodoWrite": + return "TODO: "+" | ".join(t.get("text","")[:60] for t in inp.get("todos",[])[:8]) + if name=="web_fetch": return "WEB_FETCH "+str(inp.get("url",""))[:150] + return name+" "+json.dumps(inp,ensure_ascii=False)[:150] + +def export(chat_path, out_path): + lines=[]; step=0; pending=None + for line in open(chat_path, errors="ignore"): + line=line.strip() + if not line: continue + try: o=json.loads(line) + except: continue + if o.get("type")!="message": continue + m=o.get("message",{}); role=m.get("role"); content=m.get("content") + if not isinstance(content,list): + if role=="assistant" and isinstance(content,str) and content.strip(): + lines.append(f"[think] {content.strip()[:600]}") + continue + think=[] + for b in content: + if not isinstance(b,dict): continue + t=b.get("type") + if t=="text" and role=="assistant": + tx=b.get("text","").strip() + if tx: think.append(tx) + elif t=="tool_use": + name=b.get("name",""); inp=b.get("input",{}) or {} + if think: + lines.append(f"[think] {' '.join(think)[:600]}"); think=[] + step+=1 + lines.append(f"[{step}] {fmt_action(name,inp)}") + pending=len(lines)-1 + elif t=="tool_result" and role=="user": + cont=b.get("content"); txt="" + if isinstance(cont,list): + for x in cont: + if isinstance(x,dict) and x.get("type")=="text": + txt+=x.get("text","") + elif isinstance(x,dict) and x.get("type")=="image": + txt+=" [img]" + elif isinstance(cont,str): txt=cont + if txt.strip(): + lines.append(f" -> {txt.strip()[:400]}") + if think: + lines.append(f"[think] {' '.join(think)[:600]}") + open(out_path,"w").write("\n".join(lines)) + return step + +def main(): + which = sys.argv[1:] if len(sys.argv)>1 else list(RUNS) + tot=0 + for run in which: + root=RUNS[run] + for sc in sorted(glob.glob(f"{HERE}/{root}/**/score.json",recursive=True)): + d=os.path.dirname(sc); parts=d.split(os.sep) + cat=parts[-2]; task=parts[-1] + outp=os.path.join(OUT,f"{run}__{cat}__{task}.txt") + try: + n=export(os.path.join(d,"chat.jsonl"), outp) + tot+=1 + except Exception as e: + print(f"[err] {task}: {e}") + print(f"导出 {tot} 条轨迹 → {OUT}") + +if __name__=="__main__": main() diff --git a/scripts/review_viz/gen_quality_html.py b/scripts/review_viz/gen_quality_html.py new file mode 100644 index 0000000..2473b05 --- /dev/null +++ b/scripts/review_viz/gen_quality_html.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""读 viz/quality_result.json,生成质量评估 HTML 报告 viz/quality.html +(228条轨迹的模型独立解题质量评估 vs 官方judge)""" +import json, os, html + +HERE=os.path.dirname(os.path.abspath(__file__)) +r=json.load(open(f"{HERE}/viz/quality_result.json")) +def esc(s): return html.escape(str(s)) +br=r['by_run']; allr=r['all'] + +def pct(a,b): return f"{100*a/b:.0f}%" if b else "-" + +# 汇总卡 +cards="" +for run in ['pro','turbo']: + s=br[run] + cards+=f"""
{run}
+
{s['quality_mean']}
质量均分 (1-5)
+
真解决 {pct(s['solved'],s['n'])}方法合理 {pct(s['method_sound'],s['n'])}
+
走捷径/造假 {pct(s['shortcut_hack'],s['n'])}高效 {pct(s['efficient'],s['n'])}
+
与judge一致 {pct(s['agree'],s['n'])}
""" + +# 分布条 +def distbar(d): + tot=sum(d); + cols=['#ff5b6e','#f5a623','#c9a23a','#5b9dff','#37c871'] + seg="".join(f'' for i,x in enumerate(d)) + return f'
{seg}
' + +# 类别对比表 +cat_rows="" +for c,v in r['by_cat'].items(): + p,t=v['pro'],v['turbo'] + cat_rows+=f"{c}{p['quality_mean']}{pct(p['solved'],p['n'])}{pct(p['shortcut_hack'],p['n'])}{t['quality_mean']}{pct(t['solved'],t['n'])}{pct(t['shortcut_hack'],t['n'])}" + +# 分歧case +over=[x for x in allr if x.get('agree')=='judge_too_high'] +under=[x for x in allr if x.get('agree')=='judge_too_low'] +def caserows(lst,rev): + lst=sorted(lst,key=lambda a:a['jf'],reverse=rev) + return "".join(f'{x["run"]}{x["cat"]}{esc(x["task"])}{x["quality"]}{x["jf"]}{esc(x["verdict"])}' for x in lst) + +# 全部明细(带质量+judge, 可筛选) +def qcls(q): return "q5" if q>=5 else "q4" if q==4 else "q3" if q==3 else "q12" +det_rows="" +for x in sorted(allr,key=lambda a:(a['run'],a['cat'],a['task'])): + sc='HACK' if x.get('shortcut') else '' + ag={'agree':'','judge_too_high':'⬆偏高','judge_too_low':'⬇偏低','unclear':'?'}.get(x.get('agree'),'') + det_rows+=(f'' + f'{x["run"]}{x["cat"]}{esc(x["task"])}' + f'{x["quality"]}{x["jf"]}' + f'{sc}{ag}{esc(x["verdict"])}') + +page=f""" +seed2.1 解题质量评估 +
+ +

seed2.1 解题质量评估 · 228条轨迹逐条独立评审 vs 官方judge

+
方法: 每条轨迹一个独立agent读完整轨迹文本, 判断是否真解决/方法是否合理/有无走捷径造假/是否高效, 给质量分1-5, 并与judge分对比
+
{cards}
+
质量分布 (红=1差 → 绿=5优):
+
+
pro
{distbar(br['pro']['quality_dist'])}
+
turbo
{distbar(br['turbo']['quality_dist'])}
+
+ +

按类别质量均分对比

+{cat_rows}
类别pro 质量pro 真解决pro 造假turbo 质量turbo 真解决turbo 造假
+ +

⚠ judge 疑似偏高 ({len(over)}) judge给高分但实际走捷径/造假/未真解决

+{caserows(over,True)}
run类别task质量judge评审意见
+ +

⬇ judge 疑似偏低 ({len(under)}) 实际做得不错但judge给0/低分(多为hack误判或解析失败假0)

+{caserows(under,False)}
run类别task质量judge评审意见
+ +

全部 228 条明细

+
+ + + + + + +
+ +{det_rows}
run类别task质量judge造假vs judge评审意见
+
WeaveBench seed2.1 · 解题质量评估
+
+""" +open(f"{HERE}/viz/quality.html","w").write(page) +print("wrote viz/quality.html") +PY_MARKER=1 diff --git a/scripts/review_viz/gen_viz.py b/scripts/review_viz/gen_viz.py new file mode 100644 index 0000000..a19ae63 --- /dev/null +++ b/scripts/review_viz/gen_viz.py @@ -0,0 +1,464 @@ +#!/usr/bin/env python3 +"""seed2.1 (pro / turbo) review 可视化生成器. + +对每个 run 根目录 (seed2.1_pro / seed_2.1_turbo) 遍历所有 rollout: + /seed_60B_baseline/full_gui_as_code/gui/default///{score.json, chat.jsonl, results.tar.gz, ...} + +产出到 /: + index.html 总览 dashboard (pro vs turbo 对比 + 按类别汇总 + 每 task 一行) + tasks/__.html 逐条 viewer (评分卡 + 交付物图 + 轨迹时间线) + +用法: + python3 gen_viz.py # 两个 run 都在则都跑 + python3 gen_viz.py pro # 只跑 pro +""" +import json, os, sys, tarfile, base64, html, glob +from collections import defaultdict + +HERE = os.path.dirname(os.path.abspath(__file__)) +RUNS = { + "pro": os.path.join(HERE, "seed2.1_pro"), + "turbo": os.path.join(HERE, "seed_2.1_turbo"), +} +OUT = os.path.join(HERE, "viz") +TAU = 0.80 # pass 阈值 + +DIMS = ["task_completion","deliverable_correctness","deliverable_quality", + "evidence_authenticity","tool_use_correctness","final_state_correctness", + "efficiency_robustness","instruction_following"] +DIM_ZH = { + "task_completion":"任务完成","deliverable_correctness":"交付正确","deliverable_quality":"交付质量", + "evidence_authenticity":"证据真实","tool_use_correctness":"工具使用","final_state_correctness":"终态正确", + "efficiency_robustness":"效率鲁棒","instruction_following":"指令遵循", +} + +def esc(s): return html.escape(str(s)) + +def find_rollouts(root): + """返回 [(cat, task, dir)]""" + out = [] + for sc in glob.glob(os.path.join(root, "**", "score.json"), recursive=True): + d = os.path.dirname(sc) + parts = d.split(os.sep) + try: + cat = parts[-2]; task = parts[-1] + except Exception: + continue + out.append((cat, task, d)) + return sorted(out) + +def load_score(d): + try: + j = json.load(open(os.path.join(d, "score.json"))) + except Exception: + return None + sc = j.get("scores") or {} + final = j.get("score", sc.get("final_score")) + return { + "task_id": j.get("task_id"), + "category": j.get("category"), + "final": final, + "is_hack": bool(sc.get("is_hack")), + "hack_conf": sc.get("hack_confidence"), + "hack_patterns": sc.get("hack_patterns") or [], + "hack_quotes": sc.get("hack_evidence_quotes") or [], + "dimensions": sc.get("dimensions") or {}, + "artifact_checks": sc.get("artifact_checks") or [], + "summary": sc.get("summary",""), + "judge_model": sc.get("judge_model",""), + "elapsed": j.get("elapsed_seconds"), + "tokens": (j.get("agent_token_usage") or {}).get("total_tokens"), + "n_calls": (j.get("agent_token_usage") or {}).get("n_calls"), + "agent_done": j.get("agent_done"), + "error": j.get("error"), + } + +# ---------- 轨迹解析 ---------- +def fmt_action(name, inp): + if name == "Bash": return "$ " + str(inp.get("command",""))[:300] + if name == "Read": return "read " + str(inp.get("file_path","")) + if name == "Write": return "write " + str(inp.get("file_path","")) + if name == "Edit": return "edit " + str(inp.get("file_path","")) + if name == "Grep": return "grep " + str(inp.get("pattern","")) + if name == "Glob": return "glob " + str(inp.get("pattern","")) + if "screenshot" in name: return "📷 take_screenshot" + if name == "TodoWrite": + n = len(inp.get("todos",[])) + return f"todo ({n} items)" + if name == "web_fetch": return "web_fetch " + str(inp.get("url",""))[:120] + return name + " " + json.dumps(inp, ensure_ascii=False)[:120] + +def tool_kind(name): + if "screenshot" in name: return "gui" + if name in ("Bash",): return "cli" + if name in ("Read","Write","Edit","Grep","Glob"): return "file" + return "misc" + +def build_steps(chat_path): + steps = [] + if not os.path.exists(chat_path): return steps + pending = None + for line in open(chat_path, errors="ignore"): + line = line.strip() + if not line: continue + try: o = json.loads(line) + except Exception: continue + if o.get("type") != "message": continue + m = o.get("message", {}) + role = m.get("role"); content = m.get("content") + if isinstance(content, str): + if role == "assistant" and content.strip(): + steps.append({"kind":"note","text":content.strip()}) + continue + if not isinstance(content, list): continue + think = [] + for b in content: + if not isinstance(b, dict): continue + t = b.get("type") + if t == "text" and role == "assistant": + tx = b.get("text","").strip() + if tx: think.append(tx) + elif t == "tool_use": + name = b.get("name",""); inp = b.get("input",{}) or {} + st = {"kind":tool_kind(name),"tool":name, + "thinking":"\n".join(think),"action":fmt_action(name,inp), + "img":None,"output":None} + think = [] + steps.append(st); pending = st + elif t == "tool_result" and role == "user" and pending is not None: + cont = b.get("content") + if isinstance(cont, list): + for x in cont: + if not isinstance(x, dict): continue + if x.get("type") == "image": + src = x.get("source",{}) + pending["img"] = (src.get("media_type","image/png"), src.get("data","")) + elif x.get("type") == "text": + pending["output"] = (pending.get("output") or "") + x.get("text","") + elif isinstance(cont, str): + pending["output"] = cont + pending = None + if think: + steps.append({"kind":"note","text":"\n".join(think)}) + return steps + +def extract_deliverables(tar_path, max_imgs=8): + imgs, files = [], [] + if not os.path.exists(tar_path): return imgs, files + try: tf = tarfile.open(tar_path, "r:gz") + except Exception: return imgs, files + try: + for mem in tf.getmembers(): + if not mem.isfile(): continue + name = mem.name + rel = name.split("results/",1)[-1] + if "_screenshots/" in name: continue + files.append((rel, mem.size)) + if rel.lower().endswith((".png",".jpg",".jpeg")) and len(imgs) < max_imgs and mem.size < 6_000_000: + try: + data = tf.extractfile(mem).read() + b64 = base64.b64encode(data).decode() + mt = "image/png" if rel.lower().endswith(".png") else "image/jpeg" + imgs.append((rel, mt, b64)) + except Exception: pass + finally: + tf.close() + files.sort() + return imgs, files + +# ---------- 单条 viewer ---------- +VIEWER_CSS = """ +:root{--bg:#0f1115;--card:#1a1d24;--fg:#e6e8ec;--mut:#8b909a;--acc:#5b9dff;--ok:#37c871;--warn:#f5a623;--bad:#ff5b6e;--line:#2a2e37} +*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.6 -apple-system,Segoe UI,Roboto,Arial,sans-serif} +.wrap{max-width:1120px;margin:0 auto;padding:24px} +a{color:var(--acc)} +.hdr{display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap} +h1{font-size:19px;margin:0 0 4px}.sub{color:var(--mut);font-size:13px} +.badge{display:inline-block;padding:2px 8px;border-radius:6px;font-size:12px;background:#2a2e37;color:var(--mut);margin:2px 6px 2px 0} +.scorecard{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:16px;min-width:300px} +.bigscore{font-size:34px;font-weight:700}.pass{color:var(--ok)}.fail{color:var(--bad)} +.dim{display:flex;justify-content:space-between;border-top:1px solid var(--line);padding:6px 0;font-size:13px;gap:10px} +.dim .r{color:var(--mut);font-size:12px;max-width:62%;text-align:right} +.bar{height:6px;border-radius:3px;background:#252a33;margin-top:3px;overflow:hidden}.bar>i{display:block;height:100%} +.sec{margin-top:26px}.sec h2{font-size:15px;color:var(--acc);border-bottom:1px solid var(--line);padding-bottom:6px} +.card{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px} +.deliv{display:flex;gap:12px;flex-wrap:wrap} +.deliv figure{margin:0;background:var(--card);border:1px solid var(--line);border-radius:10px;padding:8px;max-width:330px} +.deliv img{max-width:310px;max-height:230px;border-radius:6px;display:block;cursor:zoom-in} +.deliv figcaption{color:var(--mut);font-size:12px;margin-top:6px;word-break:break-all} +.files{columns:2;font-size:12px;color:var(--mut)} +.chk{border-top:1px solid var(--line);padding:8px 0;font-size:13px} +.chk .ev{color:var(--mut);font-size:12px} +.ok2{color:var(--ok)}.bad2{color:var(--bad)} +.step{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px;margin:12px 0;display:grid;grid-template-columns:auto 1fr;gap:12px;align-items:start} +.num{width:30px;height:30px;border-radius:8px;background:#252a33;display:flex;align-items:center;justify-content:center;font-weight:700;color:var(--acc)} +.tag{font-size:11px;padding:1px 7px;border-radius:5px;margin-right:8px} +.tag.gui{background:#1f3a5f;color:#8fc0ff}.tag.cli{background:#2f3a22;color:#c8e08f}.tag.file{background:#2f2a3f;color:#c0a8ff}.tag.note{background:#3a2f22;color:#e0c88f}.tag.misc{background:#2a2e37;color:#aab} +.think{color:#c9cdd6;white-space:pre-wrap;margin:4px 0} +.act{font-family:ui-monospace,Menlo,Consolas,monospace;font-size:12.5px;background:#0c0e12;border:1px solid var(--line);border-radius:6px;padding:6px 8px;white-space:pre-wrap;word-break:break-all} +.shot img{max-width:100%;border-radius:8px;border:1px solid var(--line);margin-top:8px;cursor:zoom-in} +.out{margin-top:8px;font-family:ui-monospace,monospace;font-size:12px;background:#0c0e12;border:1px solid var(--line);border-radius:6px;padding:8px;color:#9fb0c8;white-space:pre-wrap;max-height:220px;overflow:auto} +details summary{cursor:pointer;color:var(--mut);font-size:12px} +#lb{position:fixed;inset:0;background:rgba(0,0,0,.9);display:none;align-items:center;justify-content:center;z-index:99;cursor:zoom-out} +#lb img{max-width:96vw;max-height:96vh} +""" + +def bar_color(v): + if v is None: return "var(--mut)" + return "var(--ok)" if v>=0.8 else ("var(--warn)" if v>=0.5 else "var(--bad)") + +def render_viewer(run, cat, task, d, s, out_path, back_href): + steps = build_steps(os.path.join(d,"chat.jsonl")) + imgs, files = extract_deliverables(os.path.join(d,"results.tar.gz")) + final = s["final"] + passcls = "pass" if isinstance(final,(int,float)) and final>=TAU else "fail" + score_s = f"{final:.3f}" if isinstance(final,(int,float)) else "N/A" + + dims_html="" + for k in DIMS: + v = s["dimensions"].get(k) or {} + sv = v.get("score"); reason = str(v.get("reason","")) + w = int((sv or 0)*100) + dims_html += (f'
{DIM_ZH.get(k,k)} · {sv}' + f'
' + f'{esc(reason[:180])}
') + + hackbadge="" + if s["is_hack"]: + hackbadge = f'⚠ reward-hack (conf {s["hack_conf"]})' + for p in s["hack_patterns"][:6]: + hackbadge += f'{esc(p)}' + quotes="" + if s["hack_quotes"]: + quotes = "

Hack 证据引用

"+ \ + "".join(f'
“{esc(q[:400])}”
' for q in s["hack_quotes"][:8]) +"
" + + # artifact checks + chk_html="" + for c in s["artifact_checks"]: + if not isinstance(c, dict): continue + corr = c.get("correctness") + exists = c.get("exists"); fok = c.get("format_ok") + mark = f'=0.8 else "bad2"}">corr {corr}' + ex = 'exists' if exists else 'missing' + fs2 = '·fmt✓' if fok else '·fmt✗' + fake = ' [fake_signal]' if c.get("fake_signal") else '' + mw = c.get("missing_or_wrong") or "" + chk_html += (f'
{esc(c.get("id"))} — {mark} · {ex}{fs2}{fake}' + f'
{esc((c.get("evidence_quote") or "")[:260])}
' + + (f'
缺陷: {esc(mw[:220])}
' if mw else '') + + '
') + + deliv="" + for rel,mt,b64 in imgs: + deliv += f'
{esc(rel)}
' + if not deliv: deliv='
无图片交付物（文本/文件类，见清单）
' + files_html="".join(f'
{esc(r)} ({s2}B)
' for r,s2 in files[:60]) + + tl=""; i=0 + for st in steps: + k=st["kind"] + if k=="note": + tx=st["text"] + if not tx.strip(): continue + tl += f'
·
think
{esc(tx[:1500])}
' + continue + i+=1 + think=f'
{esc(st["thinking"][:900])}
' if st.get("thinking") else "" + act=f'
{esc(st["action"])}
' + shot="" + if st.get("img"): + mt,data=st["img"] + if data: shot=f'
' + out="" + if st.get("output"): + out=f'
{esc(st["output"][:1800])}
' + tl += f'
{i}
{esc(st.get("tool",k))}{think}{act}{shot}{out}
' + + meta = (f'{run} · judge={esc(s["judge_model"])} · {i} 步 · ' + f'{s["n_calls"]} calls · {s["tokens"]} tok · {s["elapsed"]}s' + + ('' if s["agent_done"] else ' · agent未完成') + + (f' · err:{esc(str(s["error"])[:60])}' if s["error"] else '')) + + page = f""" +{esc(cat)} {esc(task)} · {run} +
+ +
+

{esc(cat)} · {esc(task)}

{meta}
+
{hackbadge}
+
{score_s}
+
final score (τ={TAU} pass) — {'✅ PASS' if passcls=='pass' else '❌ FAIL'}
{dims_html}
+
+

Judge 总评

{esc(s["summary"])}
+{quotes} +

交付物核查 ({len(s["artifact_checks"])})

{chk_html or '(无)'}
+

交付物预览

{deliv}
+
产出文件清单 ({len(files)})
{files_html}
+

轨迹时间线 ({i} 步)

{tl or '(无轨迹)'}
+
WeaveBench seed2.1 review · {run}
+
+
+""" + os.makedirs(os.path.dirname(out_path), exist_ok=True) + open(out_path,"w").write(page) + return i, len(imgs) + +# ---------- 总览 dashboard ---------- +def render_index(all_data, out_path): + # all_data: {run: [rows]}, row=dict(cat,task,s,viewer_rel) + runs = list(all_data.keys()) + + # 汇总统计 + def agg(rows): + vals=[r["s"]["final"] for r in rows if isinstance(r["s"]["final"],(int,float))] + n=len(vals) + passed=sum(1 for v in vals if v>=TAU) + hack=sum(1 for r in rows if r["s"]["is_hack"]) + mean=sum(vals)/n if n else 0 + return {"n":n,"pass":passed,"pr":100*passed/n if n else 0,"mean":mean,"hack":hack} + + # 按类别 + cats = sorted({r["cat"] for run in runs for r in all_data[run]}) + + summary_cards="" + for run in runs: + a=agg(all_data[run]) + summary_cards += f"""
+
{run}
+
{a['pr']:.1f}%
PassRate (τ={TAU})
+
{a['pass']}/{a['n']} pass均分 {a['mean']:.3f}hack {a['hack']}
+
""" + + # 类别对比表 + thead = "类别" + "".join(f"{run} PR{run} 均分{run} hack" for run in runs) + "" + tbody="" + for cat in cats: + row=f"{cat}" + for run in runs: + rows=[r for r in all_data[run] if r["cat"]==cat] + a=agg(rows) + prc = "pass" if a['pr']>=50 else ("mid" if a['pr']>=30 else "fail") + row += f'{a["pr"]:.0f}% ({a["pass"]}/{a["n"]}){a["mean"]:.2f}{a["hack"]}' + row+="" + tbody+=row + # 合计 + row="合计" + for run in runs: + a=agg(all_data[run]) + row+=f'{a["pr"]:.1f}% ({a["pass"]}/{a["n"]}){a["mean"]:.3f}{a["hack"]}' + row+=""; tbody+=row + + # 每 task 明细(按 run 分栏, 可切换) + task_rows="" + for run in runs: + for r in sorted(all_data[run], key=lambda x:(x["cat"],x["task"])): + s=r["s"]; final=s["final"] + fv = f"{final:.3f}" if isinstance(final,(int,float)) else "—" + pc = "pass" if isinstance(final,(int,float)) and final>=TAU else "fail" + hk = 'HACK' if s["is_hack"] else '' + done = '' if s["agent_done"] else '未完成' + task_rows += (f'' + f'{run}{r["cat"]}' + f'{esc(r["task"])}' + f'{fv}{hk}{done}' + f'{esc(s["summary"][:120])}') + + run_opts = "".join(f'' for run in runs) + cat_opts = "".join(f'' for c in cats) + + page = f""" +seed2.1 review 总览 +
+

seed 2.1 review 总览 · pro vs turbo · τ={TAU}

+
judge: agent_as_judge (gpt-5 via openclaw) · 8 维度评分 · reward-hack 检测
+
{summary_cards}
+ +

按类别对比

+{thead}{tbody}
+ +

逐 task 明细 （点 task 名进单条 viewer 人工 check）

+
+ + + + + +
+ +{task_rows}
run类别task分数标记Judge 摘要
+
WeaveBench seed2.1 · 生成于本地 review
+
+""" + open(out_path,"w").write(page) + +def main(): + which = sys.argv[1:] if len(sys.argv)>1 else list(RUNS.keys()) + os.makedirs(os.path.join(OUT,"tasks"), exist_ok=True) + all_data={} + for run in which: + root = RUNS.get(run) + if not root or not os.path.isdir(root): + print(f"[skip] {run}: 目录不存在 {root}"); continue + rollouts = find_rollouts(root) + print(f"[{run}] {len(rollouts)} rollouts") + rows=[] + for idx,(cat,task,d) in enumerate(rollouts): + s = load_score(d) + if not s: continue + vname = f"{run}__{cat}__{task}.html" + vpath = os.path.join(OUT,"tasks",vname) + try: + nsteps,nimg = render_viewer(run,cat,task,d,s,vpath,"../index.html") + except Exception as e: + print(f" [err] {task}: {e}"); nsteps=nimg=0 + rows.append({"cat":cat,"task":task,"s":s,"viewer_rel":f"tasks/{vname}"}) + if (idx+1)%20==0: print(f" {idx+1}/{len(rollouts)}") + all_data[run]=rows + if not all_data: + print("无数据"); return + render_index(all_data, os.path.join(OUT,"index.html")) + print(f"\n完成 → {os.path.join(OUT,'index.html')}") + for run,rows in all_data.items(): + vals=[r['s']['final'] for r in rows if isinstance(r['s']['final'],(int,float))] + pr=100*sum(1 for v in vals if v>=TAU)/len(vals) if vals else 0 + print(f" {run}: {len(rows)} tasks, PR={pr:.1f}%, hack={sum(1 for r in rows if r['s']['is_hack'])}") + +if __name__=="__main__": + main()