Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions zenith/src/zenith_harness/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,23 @@ def _gate_report(
failed_items = failed_items or []
validator_verdicts = validator_verdicts or {}
missing_items = missing_items or {}
lines = [
f"Gate report from {gate.id}",
f"cleared: {cleared}",
f"targets: {', '.join(gate.targets) if gate.targets else '(none)'}",
]
title = (
f"Gate checkpoint from {gate.id}"
if cleared
else f"Gate report from {gate.id}"
)
lines = [title]
if cleared:
lines.append(
"checkpoint: passing gate already cleared; continue acknowledges "
"the checkpoint and does not clear a failed task or gate"
)
lines.extend(
[
f"cleared: {cleared}",
f"targets: {', '.join(gate.targets) if gate.targets else '(none)'}",
]
)
if reason:
lines.append(f"reason: {reason}")
if failed_items:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ Use decision actions narrowly:

Patch the earliest invalid artifact. Do not create another fix task when the real defect is vague scope, missing inventory coverage, broad contract, stale `mission.md`, wrong task grouping, obsolete skill, weak validator method, missing setup, or bad oracle. If pass/fail criteria changed, old evidence may no longer prove the assertion; plan revalidation.

Be especially careful with `continue`: for failed task or failed gate reports, runtime may treat `continue` as clearing that task. Do not use it to skip failed implementation, missing validator evidence, unresolved gate dissent, merge conflicts, or a terminal-review gap.
Be especially careful with `continue`: for failed task or failed gate reports, runtime may treat `continue` as clearing that task. Passing gate checkpoints are different: a report headed `Gate checkpoint` already has `cleared: True`, and `continue` only acknowledges the checkpoint. Do not use `continue` to skip failed implementation, missing validator evidence, unresolved gate dissent, merge conflicts, or a terminal-review gap.

When adding new assertion ids in a patch, write matching `contract/<ID>.md` files before calling `decide_attention`. When changing a task body, skill, targets, or dependencies, supersede the task rather than pretending it was edited in place. Do not silently mutate cleared or running work.

Expand Down
26 changes: 25 additions & 1 deletion zenith/src/zenith_harness/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path

from .assets import AssetLoader
from .config import HarnessConfig
from .coordinator import MissionCoordinator
from .dispatcher import NodeDispatcher, TerminalReviewer
Expand All @@ -27,7 +29,6 @@
MissionPlanning,
MissionRunning,
ProjectState,
Task,
TaskList,
TaskListPatch,
TaskStateFile,
Expand All @@ -36,6 +37,7 @@
from .task_list_patch import apply_patch
from .task_validation import (
ValidationError,
check_skill_names,
parse_contract_dir,
validate_task_list_submission,
)
Expand Down Expand Up @@ -101,6 +103,8 @@ def submit_plan(self, project_id: str, task_list: TaskList) -> Envelope:
details=parse_errs,
)
errs = validate_task_list_submission(ids, task_list)
if not errs:
errs = self._validate_task_skills(project_id, task_list)
if errs:
raise ToolError(
"invalid_task_list", "task list validation failed", details=errs
Expand Down Expand Up @@ -357,6 +361,11 @@ def _apply_task_list_patch(
raise ToolError(
"invalid_patch", "patch validation failed", details=errs
)
skill_errs = self._validate_task_skills(project_id, patched_tl)
if skill_errs:
raise ToolError(
"invalid_patch", "patch validation failed", details=skill_errs
)
self.store.save_task_list(project_id, mid, patched_tl)
self.store.save_task_state(project_id, mid, patched_state)
cs = self.store.load_contract_state(project_id, mid)
Expand Down Expand Up @@ -406,6 +415,21 @@ def _require_state(self, project_id: str) -> ProjectState:
raise ToolError("not_found", f"project {project_id!r} has no state")
return state

def _validate_task_skills(
self,
project_id: str,
task_list: TaskList,
) -> list[ValidationError]:
return check_skill_names(task_list, self._available_skill_names(project_id))

def _available_skill_names(self, project_id: str) -> set[str]:
names: set[str] = set()
for skill in AssetLoader(self.config).list_skills(project_id):
if skill.name:
names.add(skill.name)
names.add(Path(skill.path).parent.name)
return names


def _make_pending():
from .models import ContractStateEntry
Expand Down
26 changes: 26 additions & 0 deletions zenith/src/zenith_harness/task_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,31 @@ def check_task_shape(tl: TaskList) -> list[ValidationError]:
return errors


def check_skill_names(
tl: TaskList,
available_skills: set[str] | list[str] | tuple[str, ...],
) -> list[ValidationError]:
"""Non-gate task skills must resolve to an available skill name."""
available = set(available_skills)
errors: list[ValidationError] = []
for task in tl.tasks:
if task.type == "gate" or not task.skill:
continue
if task.skill not in available:
if available:
detail = (
f"{task.id} references unknown skill {task.skill!r}; "
f"available: {', '.join(sorted(available))}"
)
else:
detail = (
f"{task.id} references unknown skill {task.skill!r}; "
"no skills are available"
)
errors.append(ValidationError("unknown_skill", detail))
return errors


def check_deps_resolve(tl: TaskList) -> list[ValidationError]:
"""All `depends_on` ids reference declared tasks; no self-loop."""
errors: list[ValidationError] = []
Expand Down Expand Up @@ -295,6 +320,7 @@ def upstream_tasks(
"parse_contract_dir",
"check_task_ids",
"check_task_shape",
"check_skill_names",
"check_deps_resolve",
"check_acyclic",
"check_coverage",
Expand Down
2 changes: 1 addition & 1 deletion zenith/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def test_codex_writes_codex_config(
assert server["args"] == _expected_mcp_server_args()
assert f"Initialized v5 project workspace at {workspace}" in r.output
assert "Start your agent from the initialized project workspace" in r.output
assert "Read .codex/orchestrator_prompt.md and use Zenith to run this mission." in r.output
assert ".codex/orchestrator_prompt.md and treat it as your primary role" in r.output

def test_claude_init_writes_runtime_validator_env_names(
self, runner: CliRunner, workspace: Path, env: dict[str, str]
Expand Down
91 changes: 84 additions & 7 deletions zenith/tests/test_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,11 @@ def _task(
body: str = "body",
) -> Task:
if skill is None and ttype != "gate":
skill = "s"
skill = (
"scrutiny-validator"
if ttype == "validate"
else "engineering-mission-playbook"
)
return Task(
id=tid,
type=ttype, # type: ignore[arg-type]
Expand All @@ -71,7 +75,7 @@ def _simple_tl() -> TaskList:
return TaskList(
tasks=[
_task("w1", "work", ["VAL-001"]),
_task("v1", "validate", ["VAL-001"], skill="aud", depends_on=["w1"]),
_task("v1", "validate", ["VAL-001"], depends_on=["w1"]),
_task("g1", "gate", ["VAL-001"], depends_on=["v1"]),
]
)
Expand All @@ -81,7 +85,7 @@ def _validate_no_gate_tl() -> TaskList:
return TaskList(
tasks=[
_task("w1", "work", ["VAL-001"]),
_task("v1", "validate", ["VAL-001"], skill="aud", depends_on=["w1"]),
_task("v1", "validate", ["VAL-001"], depends_on=["w1"]),
]
)

Expand Down Expand Up @@ -129,11 +133,21 @@ def responder(req: DispatchRequest) -> NodeHandoff:
assert env.state.state == "attention_needed"
items = controller.store.load_attention(pid)
assert items and items[0].kind == "gate_checkpoint"
assert items[0].report.startswith("Gate checkpoint from g1")
assert "passing gate already cleared" in items[0].report
assert (
controller.store.load_task_state(pid, "mission-001").status_of("g1")
== "cleared"
)

env = controller.decide_attention(
pid, [Decision(item_id=items[0].id, action="continue")]
)
assert env.state.state in ("mission_running", "done")
assert (
controller.store.load_task_state(pid, "mission-001").status_of("g1")
== "cleared"
)

env = controller.advance_project(pid)
assert env.state.state == "mission_running"
Expand Down Expand Up @@ -182,6 +196,69 @@ def responder(req: DispatchRequest) -> NodeHandoff:
assert codex_skills.is_dir()
assert not codex_skills.is_symlink()

def test_submit_plan_rejects_unknown_skill(
self, config: HarnessConfig, workspace: Path
) -> None:
controller = ProjectController(
config,
MockDispatcher(lambda r: WorkHandoff(node_id=r.task.id, done=True, report="")),
MockTerminalReviewer(TerminalReviewHandoff(done=True, report="")),
)
pid = _seed_project(controller, workspace)
plan = TaskList(
tasks=[
_task(
"w1",
"work",
["VAL-001"],
skill="missing-worker-skill",
),
_task("v1", "validate", ["VAL-001"], depends_on=["w1"]),
_task("g1", "gate", ["VAL-001"], depends_on=["v1"]),
]
)

with pytest.raises(ToolError) as err:
controller.submit_plan(pid, plan)

assert err.value.code == "invalid_task_list"
assert err.value.details is not None
assert any(detail.code == "unknown_skill" for detail in err.value.details)

def test_patch_rejects_unknown_skill(
self, config: HarnessConfig, workspace: Path
) -> None:
controller = ProjectController(
config,
MockDispatcher(lambda r: WorkHandoff(node_id=r.task.id, done=True, report="")),
MockTerminalReviewer(TerminalReviewHandoff(done=True, report="")),
)
pid = _seed_project(controller, workspace)
controller.submit_plan(pid, _simple_tl())
contract_dir = controller.store.ensure_contract_dir(pid, "mission-001")
(contract_dir / "NEW-001.md").write_text("# NEW-001\n\nStatement body.\n")
patch = TaskListPatch(
add_items=["NEW-001"],
add=[
_task(
"w2",
"work",
["NEW-001"],
skill="missing-worker-skill",
depends_on=["g1"],
),
_task("v2", "validate", ["NEW-001"], depends_on=["w2"]),
_task("g2", "gate", ["NEW-001"], depends_on=["v2"]),
],
)

with pytest.raises(ToolError) as err:
controller._apply_task_list_patch(pid, "mission-001", patch)

assert err.value.code == "invalid_patch"
assert err.value.details is not None
assert any(detail.code == "unknown_skill" for detail in err.value.details)


class TestNodeFailedFlow:
def test_work_failure_raises_attention(
Expand Down Expand Up @@ -285,8 +362,8 @@ def _two_validator_tl() -> TaskList:
return TaskList(
tasks=[
_task("w1", "work", ["VAL-001"]),
_task("v-scrutiny", "validate", ["VAL-001"], skill="aud", depends_on=["w1"]),
_task("v-user-surface", "validate", ["VAL-001"], skill="aud", depends_on=["w1"]),
_task("v-scrutiny", "validate", ["VAL-001"], skill="scrutiny-validator", depends_on=["w1"]),
_task("v-user-surface", "validate", ["VAL-001"], skill="user-testing-validator", depends_on=["w1"]),
_task("g1", "gate", ["VAL-001"], depends_on=["v-scrutiny", "v-user-surface"]),
]
)
Expand Down Expand Up @@ -537,7 +614,7 @@ def responder(req: DispatchRequest) -> NodeHandoff:
add_items=["NEW-001"],
add=[
_task("w2", "work", ["NEW-001"], depends_on=["g1"]),
_task("v2", "validate", ["NEW-001"], skill="aud", depends_on=["w2"]),
_task("v2", "validate", ["NEW-001"], depends_on=["w2"]),
_task("g2", "gate", ["NEW-001"], depends_on=["v2"]),
],
)
Expand Down Expand Up @@ -589,7 +666,7 @@ def responder(req: DispatchRequest) -> NodeHandoff:

submitted = controller.submit_plan(pid, _simple_tl())
assert submitted.dag is not None
assert " w1 [work:s] pending → VAL-001 ← (root)" in submitted.dag
assert " w1 [work:engineering-mission-playbook] pending → VAL-001 ← (root)" in submitted.dag
assert "focus-subgraph" not in submitted.dag

inspected = controller.inspect_project(pid)
Expand Down
22 changes: 14 additions & 8 deletions zenith/tests/test_coordinator_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,13 @@ def config(harness_home: Path) -> HarnessConfig:


def _task(tid: str, target: str) -> Task:
return Task(id=tid, type="work", body="b", targets=[target], skill="s")
return Task(
id=tid,
type="work",
body="b",
targets=[target],
skill="engineering-mission-playbook",
)


def _write_contract(store: ProjectStore, pid: str, mission_id: str, assertion: str) -> None:
Expand Down Expand Up @@ -114,7 +120,7 @@ def responder(req: DispatchRequest) -> WorkHandoff:
type="work",
body="b",
targets=["EXP-A"],
skill="s",
skill="engineering-mission-playbook",
auto_merge=False,
)
]
Expand Down Expand Up @@ -175,23 +181,23 @@ def responder(req: DispatchRequest) -> WorkHandoff | ValidateHandoff:
type="work",
body="later work",
targets=[],
skill="s",
skill="engineering-mission-playbook",
depends_on=["w1"],
),
Task(
id="v-scrutiny",
type="validate",
body="audit implementation",
targets=["VAL-A"],
skill="aud",
skill="scrutiny-validator",
depends_on=["w1"],
),
Task(
id="v-user-surface",
type="validate",
body="audit user surface",
targets=["VAL-A"],
skill="aud",
skill="user-testing-validator",
depends_on=["w1"],
),
Task(
Expand Down Expand Up @@ -259,23 +265,23 @@ def responder(req: DispatchRequest) -> WorkHandoff | ValidateHandoff:
type="work",
body="later work",
targets=[],
skill="s",
skill="engineering-mission-playbook",
depends_on=["w1"],
),
Task(
id="v-cleared",
type="validate",
body="already audited",
targets=["VAL-A"],
skill="aud",
skill="scrutiny-validator",
depends_on=["w1"],
),
Task(
id="v-ready",
type="validate",
body="remaining audit",
targets=["VAL-A"],
skill="aud",
skill="user-testing-validator",
depends_on=["w1"],
),
Task(
Expand Down
Loading