Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions docs/agy_cli_agent_testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -255,26 +255,27 @@ supported:
setup:
skills:
# String form: an install target passed straight to
# `agy plugin install`. May be a local plugin directory, a
# `plugin@marketplace` spec, or a git URL (cloned first).
- "cloud-sql-postgresql@gemini-cli-extensions"
# `agy plugin install`. May be a local plugin directory or a git URL
# (cloned first). `agy plugin install` requires the target to resolve
# to a directory, so a bare git URL is cloned before install.
- "/path/to/a/local/plugin"

# Dict form: same, via an explicit target. Git URLs (scheme:// or
# trailing .git) are cloned first, then the clone dir is installed;
# local paths and marketplace specs are installed in place. `url:`
# is conventional; `path:` is accepted as a synonym. Append
# `#<branch-or-tag>` to a git URL to pin a version -- the clone uses
# `git clone --branch`, which resolves branch and tag names only, not
# raw commit SHAs.
# local paths are installed in place. `url:` is conventional; `path:`
# is accepted as a synonym. Append `#<branch-or-tag>` to a git URL to
# pin a version -- the clone uses `git clone --branch`, which resolves
# branch and tag names only, not raw commit SHAs.
- action: install_from_repo
url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git#v1.2.3"
```

> [!NOTE]
> Legacy dict actions (`link`, `install`, `enable`, `disable`,
> `uninstall`) that the gemini-cli generator supports are **not**
> supported here. Use a string target or `install_from_repo`.
> Unsupported entries are logged and skipped.
> A `plugin@marketplace` spec is not a reliable target (unlike `claude_code`/
> `gemini_cli`); use a git URL or local directory. Legacy dict actions
> (`link`, `install`, `enable`, `disable`, `uninstall`) that the gemini-cli
> generator supports are **not** supported here either -- use a string target
> or `install_from_repo`. Unsupported entries are logged and skipped.

### Fake MCP Servers (Testing)

Expand Down
52 changes: 21 additions & 31 deletions evalbench/generators/models/agy_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -540,29 +540,25 @@ def _is_tool_schema_file(path: str) -> bool:
return False
return isinstance(data, dict) and bool(data.get("name"))

# A target is a git URL (to be cloned) rather than a local path or a
# ``plugin@marketplace`` spec when it carries a remote scheme or ends
# in ``.git``.
# A target is a git URL (to be cloned) rather than a local path when it
# carries a remote scheme or ends in ``.git``.
_GIT_URL_PATTERN = re.compile(r"^(https?|git|ssh)://|^git@|\.git(#.*)?$")

def _setup_skills(self, skills: list):
"""Installs skill-bearing plugins via ``agy plugin install``.

Verified against agy v1.0.5: ``agy plugin install <target>`` reads
a plugin manifest (Claude/Gemini/Codex formats), processes any
bundled skills, materializes them under
``<HOME>/.gemini/config/plugins/<name>/`` and records the install
in ``<HOME>/.gemini/config/import_manifest.json``. There is no
``skill`` subcommand and dropping SKILL.md folders on disk
registers nothing.
Skills are delivered as plugins -- there is no ``agy skills``
subcommand -- so the harness shells out to ``agy plugin install``
for each entry. Two input shapes are supported, matching codex_cli
and claude_code:

Two input shapes are supported, matching the cross-CLI convention
used by codex_cli and claude_code:

* ``"<target>"`` -- a local plugin directory, a ``plugin@marketplace``
spec, or a git URL. Git URLs are cloned first, then installed.
* ``"<target>"`` -- a local plugin directory or a git URL (cloned
first; ``agy plugin install`` requires a directory target).
* ``{"action": "install_from_repo", "url"|"path": "..."}`` -- same,
via an explicit dict.

A ``plugin@marketplace`` spec is not a reliable target; use a git
URL or local directory. See docs/agy_cli_agent_testing.md.
"""
if not skills:
return
Expand Down Expand Up @@ -592,8 +588,8 @@ def _setup_skills(self, skills: list):
def _resolve_skill_target(self, skill_config) -> str:
"""Maps a skills-config entry to an ``agy plugin install`` target.

Returns an install target (local dir, ``plugin@marketplace`` spec,
or git URL) or an empty string when the entry is unusable.
Returns an install target (local dir or git URL) or an empty
string when the entry is unusable.
"""
if isinstance(skill_config, str):
return skill_config
Expand All @@ -617,16 +613,8 @@ def _resolve_skill_target(self, skill_config) -> str:
return ""

def _install_agy_plugin(self, target: str, env: dict) -> bool:
"""Runs ``agy plugin install <target>``; returns True on success.

The ``--`` end-of-options delimiter precedes ``target`` so a
config-supplied value beginning with ``--`` is treated as the
positional install target rather than parsed as a flag. (There is no
shell-injection risk -- this is an argv list, not a shell string --
but the delimiter keeps a stray ``--`` value from changing the
command's meaning.)
"""
cmd = [self.agy_bin, "plugin", "install", "--", target]
"""Runs ``agy plugin install <target>``; returns True on success."""
cmd = [self.agy_bin, "plugin", "install", target]
result = self._execute_cli_command(cmd, env=env, cwd=self.fake_home)
if result.returncode != 0:
logging.error(
Expand All @@ -649,14 +637,16 @@ def _log_installed_plugins(self):
self.plugin_manifest_path, e,
)
return
plugins = manifest.get("plugins", manifest)
if isinstance(plugins, dict):
names = sorted(plugins.keys())
elif isinstance(plugins, list):
# The manifest is ``{"imports": [{"name": ...}, ...]}``; older/other
# shapes may use a ``plugins`` list or a name-keyed dict.
plugins = manifest.get("imports", manifest.get("plugins", manifest))
if isinstance(plugins, list):
names = sorted(
p.get("name", str(p)) if isinstance(p, dict) else str(p)
for p in plugins
)
elif isinstance(plugins, dict):
names = sorted(plugins.keys())
else:
names = []
logging.info("agy registered plugins: %s", names)
Expand Down
8 changes: 4 additions & 4 deletions evalbench/test/agy_cli_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ def _install_calls(mock_run):

def test_setup_single_skill_string_runs_plugin_install(mock_run, sandbox):
"""A string entry is passed straight to ``agy plugin install``."""
target = "cloud-sql-postgresql@gemini-cli-extensions"
target = "/path/to/local-plugin"
generator = AgyCliGenerator({"setup": {"skills": [target]}})

calls = _install_calls(mock_run)
assert len(calls) == 1
assert list(calls[0].args[0]) == [
generator.agy_bin, "plugin", "install", "--", target,
generator.agy_bin, "plugin", "install", target,
]


Expand Down Expand Up @@ -99,7 +99,7 @@ def test_install_from_repo_local_path_installs_directly(
calls = _install_calls(mock_run)
assert len(calls) == 1
assert list(calls[0].args[0]) == [
generator.agy_bin, "plugin", "install", "--", local_dir,
generator.agy_bin, "plugin", "install", local_dir,
]


Expand All @@ -125,7 +125,7 @@ def test_install_from_repo_git_url_clones_then_installs(mock_run, sandbox):
calls = _install_calls(mock_run)
assert len(calls) == 1
assert list(calls[0].args[0]) == [
generator.agy_bin, "plugin", "install", "--", expected_clone,
generator.agy_bin, "plugin", "install", expected_clone,
]


Expand Down
Loading