Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .harness/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ standards_dir: .harness/standards
architecture_docs:
- .harness/docs/ARCHITECTURE.md
test_command: .venv/bin/python -m pytest tests/ -q
# Where this repository's tests live. It is what the workflow's {{tests_dir}}
# token resolves to, so the stage restricted from creating tests is restricted
# here and the stage that writes them is told to write them here. There is no
# default: leave it unset and the target declares no test directory at all, and
# the restriction resolves out of the workflow entirely.
tests_dir: tests/
# The executable the clean-clone check runs the suite under. Deliberately an
# older environment than the one the developer works in, and the one CI
# exercises, so an incompatibility is found before CI rather than by it. It
Expand Down
48 changes: 38 additions & 10 deletions .harness/docs/ARCHITECTURE.md

Large diffs are not rendered by default.

20 changes: 15 additions & 5 deletions orchestration/context_assembler.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,20 @@ def workflow_context(workflow: dict, rules: dict) -> dict[str, str | None]:


def config_context(config: dict) -> dict[str, str | None]:
"""Map the target config's grants to their injectable placeholder name.
"""Map the target config's own facts to their injectable placeholder names.

The granted Bash commands are rendered from the target's own configuration
rather than restated in prose, so what a stage is told it may run cannot
drift from what is actually permitted. A config declaring no allowed_tools
renders as None, the optional-placeholder convention.
drift from what is actually permitted. The same reasoning puts the test
location here: the stage that writes tests is told where they go by the
configuration the restriction is resolved from, so changing the config
changes the rendered prompt with no prompt edit. A config declaring
neither renders as None, the optional-placeholder convention.
"""
return {"allowed_tools": _dashed_lines(config.get("allowed_tools"))}
return {
"allowed_tools": _dashed_lines(config.get("allowed_tools")),
"tests_dir": config.get("tests_dir"),
}


def _read(path: Path) -> str | None:
Expand Down Expand Up @@ -258,7 +264,11 @@ def build_context(
# a stage rendered with no categories in it would be a defect, while a
# stage rendered with no granted list is exactly what every call site
# rendered before this existed, so omitting it must change nothing.
context.update(config_context({"allowed_tools": allowed_tools}))
# `allowed_tools` still arrives as its own argument rather than off the
# config, so a call that omits it renders exactly what it rendered before
# the argument existed; every other configured fact this renders comes off
# the config the caller already passed.
context.update(config_context({**config, "allowed_tools": allowed_tools}))

# Two-pass render: resolve the shared harness-layer partial (including its
# own {{blocked_paths}} placeholder) against the assembled context before
Expand Down
90 changes: 88 additions & 2 deletions orchestration/harness_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from __future__ import annotations

import json
import re
import sys
from pathlib import Path

Expand Down Expand Up @@ -117,9 +118,94 @@ def declared_config_keys(harness_root: Path | None = None) -> tuple[str, ...]:
return tuple(properties)


def load_workflow(harness_root: Path, name: str) -> dict:
#: A workflow declaration references configuration as `{{key}}`, and only as
#: a whole list entry. A general mechanism -- any declaration reaching any
#: config key -- was considered and rejected: one key needs this, and a
#: narrow token leaves every existing reader of a loaded workflow untouched.
_WORKFLOW_TOKEN = re.compile(r"\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}")


def workflow_token_values(config: dict) -> dict[str, str | None]:
"""The configuration a workflow declaration may reference, by token name.

Written as a literal read per key rather than a lookup by whatever name
the definition happens to carry, so the keys this resolution reads are
visible to a reader -- and to the scan that holds the declared set equal
to the set the harness reads -- exactly like every other configured key.
That the mapping has one entry is the narrowness, stated in code.
"""
return {"tests_dir": config.get("tests_dir")}


class UnresolvedWorkflowToken(ValueError):
"""A loaded workflow carries a reference the configuration cannot answer.

Carries `problems` in the shape every pre-flight refusal enumerates, so
the coordinator turns it into a refusal rather than composing its own
wording for it.
"""

def __init__(self, workflow: str, tokens: list[str]):
self.workflow = workflow
self.tokens = tokens
referable = ", ".join(f"{{{{{name}}}}}" for name in workflow_token_values({}))
self.problems = [
f"'{{{{{token}}}}}' is not a configuration reference the harness "
f"resolves; a workflow declaration may reference {referable}, and "
f"only as a whole list entry"
for token in tokens
]
super().__init__("; ".join(self.problems))


def _resolve_tokens(value, values: dict[str, str | None], unresolved: list[str]):
"""Substitute every `{{key}}` list entry, dropping the ones with no value.

An unset key resolves the entry *out of the list* rather than to an empty
string: a restriction whose prefix is "" is a prefix every path is under,
which is the opposite of the "this target declares none" the absence
means. Every other token-shaped string -- one naming a key outside the
narrow set, or a resolvable one somewhere a list entry cannot be dropped
from -- is collected as unresolved for the caller to refuse on.
"""
if isinstance(value, dict):
return {key: _resolve_tokens(item, values, unresolved)
for key, item in value.items()}
if isinstance(value, list):
resolved = []
for item in value:
match = _WORKFLOW_TOKEN.fullmatch(item) if isinstance(item, str) else None
if match is None:
resolved.append(_resolve_tokens(item, values, unresolved))
continue
name = match.group(1)
if name not in values:
unresolved.append(name)
elif values[name]:
resolved.append(values[name])
return resolved
if isinstance(value, str):
unresolved.extend(_WORKFLOW_TOKEN.findall(value))
return value


def load_workflow(harness_root: Path, name: str, config: dict) -> dict:
"""The workflow definition, with its configuration references resolved.

Resolution happens once, when the definition loads, so `stage_restrictions`
and every reader of a loaded workflow reads exactly what it read when the
declaration was a literal directory. `config` is required rather than
defaulted: a caller that omitted it would silently load a definition with
the configured entries missing, which is a quieter wrong answer than a
TypeError.
"""
path = harness_root / "workflows" / f"{name}.json"
return json.loads(path.read_text(encoding="utf-8"))
definition = json.loads(path.read_text(encoding="utf-8"))
unresolved: list[str] = []
resolved = _resolve_tokens(definition, workflow_token_values(config), unresolved)
if unresolved:
raise UnresolvedWorkflowToken(name, unresolved)
return resolved


def load_rules(harness_root: Path) -> dict:
Expand Down
30 changes: 29 additions & 1 deletion orchestration/story_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2330,6 +2330,24 @@ def _refuse_bad_self_routes(workflow: dict, problems: list[str]) -> int:
)


def _refuse_unresolved_workflow_token(
unresolved: harness_config.UnresolvedWorkflowToken,
) -> int:
"""Refuse a workflow that references configuration the harness cannot answer.

Thin, like every other caller of `refuse`. The header names the workflow
and each problem names the token, so the pair a developer needs is in the
message rather than one of them being left to inference.
"""
return refuse(
f"Workflow '{unresolved.workflow}' references configuration the harness "
f"cannot resolve:",
unresolved.problems,
"Fix the workflow definition's configuration references before running "
"a story under it.",
)


def _refuse_undeclared_config_keys(target_root: Path, problems: list[str]) -> int:
"""Refuse a run whose configuration carries a key the harness does not read.

Expand Down Expand Up @@ -2553,7 +2571,17 @@ def run_story(
if undeclared:
return _refuse_undeclared_config_keys(target_root, undeclared)

workflow = harness_config.load_workflow(harness_root, config.get("workflow", "story-workflow"))
# The definition may reference the target's configuration, so it is loaded
# against the config that has just been read. A reference the config
# cannot answer is a defect in the definition that every run under it
# carries, so it is refused here, beside the other pre-flight refusals and
# above everything a run creates: no run directory, no state.json, no log,
# no branch, and no agent invoked.
workflow_name = config.get("workflow", "story-workflow")
try:
workflow = harness_config.load_workflow(harness_root, workflow_name, config)
except harness_config.UnresolvedWorkflowToken as unresolved_token:
return _refuse_unresolved_workflow_token(unresolved_token)
rules = harness_config.load_rules(harness_root)
stages = workflow["stages"]
stage_names = [s["name"] for s in stages]
Expand Down
5 changes: 3 additions & 2 deletions prompts/tester.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Do not:
- weaken, skip, or delete existing tests, or
- decide whether the workflow may continue (the verifier owns that decision).

New tests belong in tests/ and become permanent repository assets.
New tests belong in {{tests_dir}} and become permanent repository assets.

Name a validation module for the behaviour it validates, so that a reader
looking for that behaviour finds the module by its name rather than by
Expand Down Expand Up @@ -44,7 +44,8 @@ Baselines resolved out of git are the recurring instance of this. Do not
resolve one as `HEAD` or as the working tree against the repository root:
the coordinator commits the working tree at the end of a successful run, so
those comparisons go vacuously green the moment the story commits. Use the
shared resolution in `tests/conftest.py`.
shared baseline resolution the existing validation already provides rather
than writing a second one beside it.

When you finish, write these files to the run directory at {{run_dir}}:

Expand Down
4 changes: 4 additions & 0 deletions schemas/harness-config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@
"type": "string",
"description": "Directory, relative to the target root, holding approved story artifacts. Defaults to .harness/stories. Also the directory l5-plan snapshots to decide what a planning session produced."
},
"tests_dir": {
"type": "string",
"description": "Where a target's tests live, as a repository-relative path prefix ending in a slash. Set, it is the prefix the workflow's {{tests_dir}} token resolves to, so the stage restricted from creating tests is restricted there and the stage that writes them is told to write them there. There is no default: unset, the target declares no test directory at all, the token resolves out of the list it appears in entirely rather than becoming an empty prefix that would match every path, and the restriction does not exist for that target."
},
"test_command": {
"type": "string",
"description": "The command the clean-clone and revert checks run inside a scratch clone. Read without a fallback: a target that omits it cannot run either check."
Expand Down
2 changes: 1 addition & 1 deletion scripts/l5-plan
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def main() -> None:
target_root = harness_config.find_target_root(Path.cwd())
config = harness_config.load_config(target_root)
workflow = harness_config.load_workflow(
HARNESS_ROOT, config.get("workflow", "story-workflow")
HARNESS_ROOT, config.get("workflow", "story-workflow"), config
)
rules = harness_config.load_rules(HARNESS_ROOT)
# The planner is not a workflow stage, so no coordinator renders its
Expand Down
8 changes: 8 additions & 0 deletions templates/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ standards_dir: .harness/standards
architecture_docs:
- .harness/docs/ARCHITECTURE.md
test_command: {test_command}
# Where this repository's tests live, as a path prefix ending in a slash. It is
# what the workflow's {{tests_dir}} token resolves to: the stage restricted from
# creating tests is restricted here, and the stage that writes them is told to
# write them here. A starter value like every other line in this file — change
# it to wherever this repository actually keeps its tests. There is no default:
# delete the key and this repository declares no test directory at all, and the
# restriction resolves out of the workflow entirely.
tests_dir: tests/
# Bash commands stage agents may run without prompting. Headless agents
# cannot answer permission prompts. Read-only search and inspection is
# granted broadly: a denial costs a turn and buys nothing, because the
Expand Down
68 changes: 68 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,73 @@
HARNESS_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(HARNESS_ROOT / "orchestration"))

import harness_config # noqa: E402


# --------------------------------------------------------------------------
# Loading the shipped workflow the way a run loads it.
#
# A workflow declaration may reference the target's configuration -- the
# implementer's create restriction is the token `{{tests_dir}}` -- and the
# reference is resolved when the definition loads. A module that wants the
# definition a run of *this* repository executes therefore has to load it
# against *this* repository's configuration, which is what these two do. A
# module that learns the restricted prefix must learn it this way rather than
# by reading `workflows/story-workflow.json` as text, where it would find the
# token rather than the value.
# --------------------------------------------------------------------------


def repository_config(root: Path = HARNESS_ROOT) -> dict:
"""This repository's own `.harness/config.yaml`, loaded."""
return harness_config.load_config(root)


#: `load_workflow` gained a required `config` argument when a workflow
#: declaration became able to reference configuration. A module that recovers
#: an entry point out of git to compare its behaviour against today's is
#: comparing the change that story made, not the arity of a call that story
#: never touched, so the recovered call site is repointed — minimally, at the
#: one line, keeping the recovered code otherwise byte for byte what it was.
#: Without it the recovered script raises TypeError and the comparison stops
#: being about its own subject.
HISTORICAL_WORKFLOW_LOADS = (
# scripts/l5-plan
('harness_config.load_workflow(\n'
' HARNESS_ROOT, config.get("workflow", "story-workflow")\n'
' )',
'harness_config.load_workflow(\n'
' HARNESS_ROOT, config.get("workflow", "story-workflow"), config\n'
' )'),
# orchestration/story_coordinator.py
('harness_config.load_workflow(harness_root, '
'config.get("workflow", "story-workflow"))',
'harness_config.load_workflow(harness_root, '
'config.get("workflow", "story-workflow"), config)'),
)


def repointed_at_todays_signature(source: str) -> str:
"""Recovered source, with each historical `load_workflow` call repointed.

Only that call is touched, and only where it appears; everything else the
revision carried is byte for byte what it was.
"""
for old, new in HISTORICAL_WORKFLOW_LOADS:
source = source.replace(old, new)
return source


def shipped_workflow(root: Path = HARNESS_ROOT,
name: str = "story-workflow") -> dict:
"""The named workflow under `root`, resolved against this repository's config.

`root` is a harness root, which is this repository unless a test has
mirrored one; the configuration stays this repository's, because a
mirrored harness root has no target configuration of its own.
"""
return harness_config.load_workflow(root, name, repository_config())


# --------------------------------------------------------------------------
# The one honest baseline resolution the per-story validation files share.
Expand Down Expand Up @@ -451,6 +518,7 @@ def _committed(repo: Path, relative: str) -> bool:
architecture_docs:
- .harness/docs/ARCHITECTURE.md
test_command: echo tests-ok
tests_dir: tests/
"""


Expand Down
3 changes: 2 additions & 1 deletion tests/test_artifact_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import context_assembler
import harness_config
import conftest
import schema_validator
import story_coordinator
import story_parser
Expand Down Expand Up @@ -44,7 +45,7 @@
#: into a harness copy holding only orchestration/, schemas/ and tests/,
#: where workflows/ does not exist.
def workflow_definition() -> dict:
return harness_config.load_workflow(
return conftest.shipped_workflow(
Path(context_assembler.__file__).resolve().parents[1],
"story-workflow")

Expand Down
5 changes: 3 additions & 2 deletions tests/test_attempt_archiving.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import pytest

from conftest import commit_setup, first_retry_route, story_diff
import conftest

import context_assembler
import harness_config
Expand All @@ -28,7 +29,7 @@
#: escalates rather than routing it, so every failing verdict below
#: carries one.
RETRY_CATEGORY, RETRY_STAGE = first_retry_route(
harness_config.load_workflow(REPO_ROOT, "story-workflow"))
conftest.shipped_workflow(REPO_ROOT, "story-workflow"))

PASS = {"status": "passed", "blocking_issues": [], "unverified": [],
"retry_recommended": False}
Expand Down Expand Up @@ -261,7 +262,7 @@ def stage_attempt_directory(name: str) -> bool:
"""
stem, sep, number = name.rpartition("-attempt-")
stages = {stage["name"] for stage
in harness_config.load_workflow(REPO_ROOT, "story-workflow")["stages"]}
in conftest.shipped_workflow(REPO_ROOT, "story-workflow")["stages"]}
return bool(sep) and stem in stages and number.isdigit()


Expand Down
4 changes: 2 additions & 2 deletions tests/test_clean_clone_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import story_coordinator
from agent_runner import AgentResult
from conftest import first_retry_route, load_mutant, story_diff
import conftest

#: The two stories this module validates, as `conftest.STORY_ORIGINS`
#: declares them. Every story-range call below names one of these, because a
Expand All @@ -57,8 +58,7 @@
REPO_ROOT = Path(story_coordinator.__file__).resolve().parents[1]
COORDINATOR_PATH = Path(story_coordinator.__file__)
COORDINATOR_SOURCE = COORDINATOR_PATH.read_text(encoding="utf-8")
WORKFLOW = json.loads(
(REPO_ROOT / "workflows" / "story-workflow.json").read_text(encoding="utf-8"))
WORKFLOW = conftest.shipped_workflow()
VERIFIER_STAGE = next(s for s in WORKFLOW["stages"] if s["name"] == "verifier")
#: Since story-028 the clean-clone declaration names both artifacts of the
#: check — the result it writes and the stage a failure routes to — so the
Expand Down
Loading
Loading