From f455bfe5243a56cd3f1256a9e63870eab2c2e85b Mon Sep 17 00:00:00 2001 From: JoshuaBearup Date: Thu, 11 Jun 2026 22:22:14 +1000 Subject: [PATCH] feat(deploy): record generating model(s) per cell in run manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `generator_models` column to runs//manifest.csv and a `generation` block (token usage by model, from the existing getUsageReport()) to the per-cell local manifest, so a run records which model(s) actually produced each deployed cell. Local manifest only — deliberately NOT baked into the container. RCE/LFI classes can read the baked manifest, and the generating model is metadata the model under test must not see. Mirrors how solvabilityProof is attached post-deploy. Additive column appended at the end of the CSV, so existing manifest.csv consumers are unaffected. Co-Authored-By: Claude Fable 5 --- generator/deploy.mjs | 5 +++++ lib/deploy-screen.mjs | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/generator/deploy.mjs b/generator/deploy.mjs index 85b1447..175fb7f 100644 --- a/generator/deploy.mjs +++ b/generator/deploy.mjs @@ -288,6 +288,11 @@ async function deploy() { // evidence that the target WAS solvable by the reference exploit, surfaced in // eval output. (Local manifest only; not rebuilt into the container.) manifest.solvabilityProof = proof + // Record which model(s) generated this cell + token usage. Local manifest + // only — deliberately NOT baked into the container (RCE/LFI classes can read + // the baked manifest, and the generating model is metadata the model under + // test must not see). + manifest.generation = getUsageReport() try { await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2)) } catch {} const elapsed = ((Date.now() - startTime) / 1000).toFixed(1) diff --git a/lib/deploy-screen.mjs b/lib/deploy-screen.mjs index da4fc47..3b453b8 100644 --- a/lib/deploy-screen.mjs +++ b/lib/deploy-screen.mjs @@ -76,7 +76,7 @@ export async function runDeployScreen({ await fs.mkdir(logsDir, { recursive: true }) - const csvHeader = 'class_id,tier,status,app_name,url,canary,control_key,cost_usd,duration_s,deployed_at,attempts,failure_category,error\n' + const csvHeader = 'class_id,tier,status,app_name,url,canary,control_key,cost_usd,duration_s,deployed_at,attempts,failure_category,error,generator_models\n' await fs.writeFile(csvPath, csvHeader) // Build the work queue: every (class, tier) pair. @@ -334,6 +334,11 @@ async function runOneDeploy({ repoRoot, cell, logsDir, target, onChunk }) { const localManifestPath = path.join(repoRoot, `manifest.${classId}.t${tier}.json`) const m = JSON.parse(await fs.readFile(localManifestPath, 'utf-8')) if (m.controlKey) row.control_key = m.controlKey + // Which model(s) generated this cell, e.g. "claude-haiku-4-5:3|claude-opus-4-8:1". + if (m.generation?.byModel) { + row.generator_models = Object.entries(m.generation.byModel) + .map(([model, v]) => `${model}:${v.calls}`).join('|') + } await fs.unlink(localManifestPath).catch(() => {}) } catch { /* manifest may not exist if deploy failed early */ } @@ -396,6 +401,7 @@ async function appendRow(csvPath, row) { row.class_id, row.tier, row.status, row.app_name, row.url, row.canary, row.control_key, row.cost_usd, row.duration_s, row.deployed_at, row.attempt || row.attempts || 1, row.category || '', row.error, + row.generator_models || '', ] await fs.appendFile(csvPath, cols.map(csvVal).join(',') + '\n') }