From 87f094510830b063e2e9fea05360f9f4f8431042 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 23 Jun 2026 09:07:43 +0000 Subject: [PATCH 1/2] fix(agy_cli): drop unsupported -- delimiter from plugin install agy's plugin subcommand parser does not honor `--`, so it took the delimiter as the install target and failed with `install target must be a directory: --`. Remove it. Also fix `_log_installed_plugins` to read plugin names from the manifest `imports` array, and correct docs/comments that advertised the unreliable `plugin@marketplace` target form. --- docs/agy_cli_agent_testing.md | 23 ++++++++---- evalbench/generators/models/agy_cli.py | 52 +++++++++++--------------- evalbench/test/agy_cli_test.py | 8 ++-- 3 files changed, 40 insertions(+), 43 deletions(-) diff --git a/docs/agy_cli_agent_testing.md b/docs/agy_cli_agent_testing.md index e20ff2d5..a11e3120 100644 --- a/docs/agy_cli_agent_testing.md +++ b/docs/agy_cli_agent_testing.md @@ -255,21 +255,28 @@ supported: setup: skills: # String form: an install target passed straight to - # `agy plugin install`. May be a local plugin directory, a - # `plugin@marketplace` spec, or a git URL (cloned first). - - "cloud-sql-postgresql@gemini-cli-extensions" + # `agy plugin install`. May be a local plugin directory or a git URL + # (cloned first). `agy plugin install` requires the target to resolve + # to a directory, so a bare git URL is cloned before install. + - "/path/to/a/local/plugin" # Dict form: same, via an explicit target. Git URLs (scheme:// or # trailing .git) are cloned first, then the clone dir is installed; - # local paths and marketplace specs are installed in place. `url:` - # is conventional; `path:` is accepted as a synonym. Append - # `#` to a git URL to pin a version -- the clone uses - # `git clone --branch`, which resolves branch and tag names only, not - # raw commit SHAs. + # local paths are installed in place. `url:` is conventional; `path:` + # is accepted as a synonym. Append `#` to a git URL to + # pin a version -- the clone uses `git clone --branch`, which resolves + # branch and tag names only, not raw commit SHAs. - action: install_from_repo url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git#v1.2.3" ``` +> [!WARNING] +> Unlike the `claude_code` and `gemini_cli` harnesses, a `plugin@marketplace` +> spec is **not** a reliable target here: agy resolves marketplace names +> server-side, and common names such as `gemini-cli-extensions` are not +> recognized (`agy plugin install` fails with `unknown marketplace`). Use a +> git URL or a local plugin directory instead. + > [!NOTE] > Legacy dict actions (`link`, `install`, `enable`, `disable`, > `uninstall`) that the gemini-cli generator supports are **not** diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 85604443..024b9687 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -540,29 +540,25 @@ def _is_tool_schema_file(path: str) -> bool: return False return isinstance(data, dict) and bool(data.get("name")) - # A target is a git URL (to be cloned) rather than a local path or a - # ``plugin@marketplace`` spec when it carries a remote scheme or ends - # in ``.git``. + # A target is a git URL (to be cloned) rather than a local path when it + # carries a remote scheme or ends in ``.git``. _GIT_URL_PATTERN = re.compile(r"^(https?|git|ssh)://|^git@|\.git(#.*)?$") def _setup_skills(self, skills: list): """Installs skill-bearing plugins via ``agy plugin install``. - Verified against agy v1.0.5: ``agy plugin install `` reads - a plugin manifest (Claude/Gemini/Codex formats), processes any - bundled skills, materializes them under - ``/.gemini/config/plugins//`` and records the install - in ``/.gemini/config/import_manifest.json``. There is no - ``skill`` subcommand and dropping SKILL.md folders on disk - registers nothing. + Skills are delivered as plugins -- there is no ``agy skills`` + subcommand -- so the harness shells out to ``agy plugin install`` + for each entry. Two input shapes are supported, matching codex_cli + and claude_code: - Two input shapes are supported, matching the cross-CLI convention - used by codex_cli and claude_code: - - * ``""`` -- a local plugin directory, a ``plugin@marketplace`` - spec, or a git URL. Git URLs are cloned first, then installed. + * ``""`` -- a local plugin directory or a git URL (cloned + first; ``agy plugin install`` requires a directory target). * ``{"action": "install_from_repo", "url"|"path": "..."}`` -- same, via an explicit dict. + + A ``plugin@marketplace`` spec is not a reliable target; use a git + URL or local directory. See docs/agy_cli_agent_testing.md. """ if not skills: return @@ -592,8 +588,8 @@ def _setup_skills(self, skills: list): def _resolve_skill_target(self, skill_config) -> str: """Maps a skills-config entry to an ``agy plugin install`` target. - Returns an install target (local dir, ``plugin@marketplace`` spec, - or git URL) or an empty string when the entry is unusable. + Returns an install target (local dir or git URL) or an empty + string when the entry is unusable. """ if isinstance(skill_config, str): return skill_config @@ -617,16 +613,8 @@ def _resolve_skill_target(self, skill_config) -> str: return "" def _install_agy_plugin(self, target: str, env: dict) -> bool: - """Runs ``agy plugin install ``; returns True on success. - - The ``--`` end-of-options delimiter precedes ``target`` so a - config-supplied value beginning with ``--`` is treated as the - positional install target rather than parsed as a flag. (There is no - shell-injection risk -- this is an argv list, not a shell string -- - but the delimiter keeps a stray ``--`` value from changing the - command's meaning.) - """ - cmd = [self.agy_bin, "plugin", "install", "--", target] + """Runs ``agy plugin install ``; returns True on success.""" + cmd = [self.agy_bin, "plugin", "install", target] result = self._execute_cli_command(cmd, env=env, cwd=self.fake_home) if result.returncode != 0: logging.error( @@ -649,14 +637,16 @@ def _log_installed_plugins(self): self.plugin_manifest_path, e, ) return - plugins = manifest.get("plugins", manifest) - if isinstance(plugins, dict): - names = sorted(plugins.keys()) - elif isinstance(plugins, list): + # The manifest is ``{"imports": [{"name": ...}, ...]}``; older/other + # shapes may use a ``plugins`` list or a name-keyed dict. + plugins = manifest.get("imports", manifest.get("plugins", manifest)) + if isinstance(plugins, list): names = sorted( p.get("name", str(p)) if isinstance(p, dict) else str(p) for p in plugins ) + elif isinstance(plugins, dict): + names = sorted(plugins.keys()) else: names = [] logging.info("agy registered plugins: %s", names) diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index bcfe4743..9d6e43a7 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -64,13 +64,13 @@ def _install_calls(mock_run): def test_setup_single_skill_string_runs_plugin_install(mock_run, sandbox): """A string entry is passed straight to ``agy plugin install``.""" - target = "cloud-sql-postgresql@gemini-cli-extensions" + target = "/path/to/local-plugin" generator = AgyCliGenerator({"setup": {"skills": [target]}}) calls = _install_calls(mock_run) assert len(calls) == 1 assert list(calls[0].args[0]) == [ - generator.agy_bin, "plugin", "install", "--", target, + generator.agy_bin, "plugin", "install", target, ] @@ -99,7 +99,7 @@ def test_install_from_repo_local_path_installs_directly( calls = _install_calls(mock_run) assert len(calls) == 1 assert list(calls[0].args[0]) == [ - generator.agy_bin, "plugin", "install", "--", local_dir, + generator.agy_bin, "plugin", "install", local_dir, ] @@ -125,7 +125,7 @@ def test_install_from_repo_git_url_clones_then_installs(mock_run, sandbox): calls = _install_calls(mock_run) assert len(calls) == 1 assert list(calls[0].args[0]) == [ - generator.agy_bin, "plugin", "install", "--", expected_clone, + generator.agy_bin, "plugin", "install", expected_clone, ] From 6fc850aa82a9005dcc7ad4205164bc159ee018ba Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 23 Jun 2026 09:25:27 +0000 Subject: [PATCH 2/2] docs: consolidate plugin limitations and legacy action warnings in agy CLI testing docs --- docs/agy_cli_agent_testing.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/agy_cli_agent_testing.md b/docs/agy_cli_agent_testing.md index a11e3120..63eb6bc4 100644 --- a/docs/agy_cli_agent_testing.md +++ b/docs/agy_cli_agent_testing.md @@ -270,18 +270,12 @@ setup: url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git#v1.2.3" ``` -> [!WARNING] -> Unlike the `claude_code` and `gemini_cli` harnesses, a `plugin@marketplace` -> spec is **not** a reliable target here: agy resolves marketplace names -> server-side, and common names such as `gemini-cli-extensions` are not -> recognized (`agy plugin install` fails with `unknown marketplace`). Use a -> git URL or a local plugin directory instead. - > [!NOTE] -> Legacy dict actions (`link`, `install`, `enable`, `disable`, -> `uninstall`) that the gemini-cli generator supports are **not** -> supported here. Use a string target or `install_from_repo`. -> Unsupported entries are logged and skipped. +> A `plugin@marketplace` spec is not a reliable target (unlike `claude_code`/ +> `gemini_cli`); use a git URL or local directory. Legacy dict actions +> (`link`, `install`, `enable`, `disable`, `uninstall`) that the gemini-cli +> generator supports are **not** supported here either -- use a string target +> or `install_from_repo`. Unsupported entries are logged and skipped. ### Fake MCP Servers (Testing)