diff --git a/packages/website/docs/tutorials/orchestrate-a-sonnet-with-workflows.md b/packages/website/docs/tutorials/orchestrate-a-sonnet-with-workflows.md index 1bb51ae5..1bed429b 100644 --- a/packages/website/docs/tutorials/orchestrate-a-sonnet-with-workflows.md +++ b/packages/website/docs/tutorials/orchestrate-a-sonnet-with-workflows.md @@ -535,13 +535,20 @@ retry: `# → retry N` re-runs the command until it exits `0`, up to `N` attempt second apart. `jq -e` supplies the exit code — non-zero until `.state` is actually `review`. +Size `N` from measurement, not optimism. On the CI sandbox (`qwen2.5:0.5b`, CPU +only, output capped at 256 tokens) the five generations took **32s, 72s, 36s, +22s and 16s — 179s end to end**, and each poll attempt costs its one-second sleep +plus the CLI round trip. The budget below is roughly double the observed total so +ordinary variance does not fail the run; it is a ceiling, not a cost, since the +loop exits as soon as the card lands. + ```bash soat transition-task --task-id "$TASK_ID" --transition start | jq '{ state, automation_status }' -# → retry 90 +# → retry 240 soat get-task --task-id "$TASK_ID" | jq -e '.state == "review"' soat get-task --task-id "$TASK_ID" | jq '{ state, status, sonnet: .last_result.content }' @@ -557,7 +564,8 @@ await adminSoat.tasks.transitionTask({ }); const waitForState = async (target: string) => { - for (let attempt = 0; attempt < 90; attempt += 1) { + // ~4 minutes: the five sandbox generations measured 179s end to end. + for (let attempt = 0; attempt < 240; attempt += 1) { const { data: current } = await adminSoat.tasks.getTask({ path: { task_id: TASK_ID }, }); @@ -616,7 +624,7 @@ modelling decision, and it is the difference between a fast loop and a slow one. ```bash soat transition-task --task-id "$TASK_ID" --transition revise --note "tighten the closing couplet" | jq '{ state }' -# → retry 40 +# → retry 120 soat get-task --task-id "$TASK_ID" | jq -e '.state == "review"' ``` diff --git a/tests/.tutorialsignore b/tests/.tutorialsignore index c1218691..811ddce4 100644 --- a/tests/.tutorialsignore +++ b/tests/.tutorialsignore @@ -8,26 +8,6 @@ index connect-third-party-llms -# orchestrate-a-sonnet-with-workflows is disabled pending evidence, not pending a -# fix. Its original rationale — inherent `on_enter` dispatch nondeterminism, -# "same class as review-panel-discussion" — was wrong on both counts and is -# corrected here. The tutorial declares no `output_schema` anywhere and every -# `on_complete` is `{"when": true}`, so the card advances on any completion -# regardless of content; nothing in it depends on model quality. The actual -# causes were fixed in #881: -# - the AI provider pinned `llama3.2`, which this stack never pulls (it pulls -# `qwen2.5:0.5b` only), so every generation failed outright; -# - the `# → 400` annotation sat after its command, so it bound to the next -# one, leaving the expected guard rejection unannotated; -# - this stack had no `MAX_COMPLETION_TOKENS`, so output was unbounded; -# - `revise` re-entered `create_text`, running ten generations per run instead -# of six; -# - no state declared `on_failure`, so a failed generation stalled the card. -# What is still missing is a green run: nothing has executed this tutorial end to -# end since those fixes landed. Remove this entry once -# `TUTORIAL_ID=orchestrate-a-sonnet-with-workflows` passes locally a few times in -# a row. -orchestrate-a-sonnet-with-workflows # call-aws-and-gcp-apis-from-an-agent creates tools whose `execute.auth` signs # requests with real AWS credentials or mints a GCP service account token. The