From bce9de0415ed1bc9f19ad306c8d1e6500cd81f8b Mon Sep 17 00:00:00 2001 From: Anionex <1005128408@qq.com> Date: Fri, 7 Aug 2026 13:07:28 +0800 Subject: [PATCH] feat: add Reasonix skill and CLI integration --- .github/workflows/ci.yml | 3 +- CHANGELOG.md | 1 + README.md | 13 +- README_CN.md | 13 +- integrations/reasonix/reasonix-plugin.json | 11 ++ .../reasonix/skills/vision-tools/SKILL.md | 90 ++++++++++++ .../skills/vision-tools/scripts/vision_cli.py | 107 ++++++++++++++ tests/test_reasonix_integration.py | 132 ++++++++++++++++++ 8 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 integrations/reasonix/reasonix-plugin.json create mode 100644 integrations/reasonix/skills/vision-tools/SKILL.md create mode 100755 integrations/reasonix/skills/vision-tools/scripts/vision_cli.py create mode 100644 tests/test_reasonix_integration.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1732963..6503f78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: with: python-version: "3.11" - name: Compile Python entry points - run: python -m py_compile vision_proxy.py vision_client.py ground.py detect.py bin/glance bin/trace bin/crop + run: python -m py_compile vision_proxy.py vision_client.py ground.py detect.py bin/glance bin/trace bin/crop integrations/reasonix/skills/vision-tools/scripts/vision_cli.py - name: Run core tests run: | python tests/test_image_rewrite_shapes.py @@ -30,6 +30,7 @@ jobs: python tests/test_anthropic_rewrite.py python tests/smoke_test_proxy.py python tests/test_vision_client.py + python tests/test_reasonix_integration.py extensions: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 17b00d8..8f6017e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable user-facing changes to agent-vision-toolkit are documented in this f - Five vision CLIs — `glance`, `ground`, `detect`, `trace`, and `crop` — plus the `vision-tools` agent skill. - Optional seamless integration: a local proxy for Codex and Claude Code, and single-file native extensions for Pi, Oh My Pi, and OpenCode. +- A compact native Reasonix 1.21.0 Skill + CLI adapter that reuses an existing toolkit checkout from `PATH`, with no MCP server or runtime sidecar. - Pasted-image and tool-fetched image support with task-aware focus hints, parallel multi-image descriptions, per-request caching, and honest failure notes. - Vision playbooks for long-screenshot OCR, UI restoration, graphic restoration, structure recovery, and GUI operation. - Community contribution, conduct, support, and security policies. diff --git a/README.md b/README.md index a39f2e5..b8be228 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ [![Extensions](https://img.shields.io/badge/-Extensions-3178C6?style=flat-square)](https://github.com/Anionex/agent-vision-toolkit/tree/main/extensions) [![Shell](https://img.shields.io/badge/-Shell-4EAA25?style=flat-square&logo=gnubash&logoColor=white)](https://github.com/Anionex/agent-vision-toolkit/tree/main/bin) -**What it thinks is what it sees — give any text-only coding agent eyes: image Q&A, long-screenshot OCR, frontend UI restoration, and GUI automation, as a vision toolkit plus a skill, with optional drop-in integration for Codex, Claude Code, Pi, Oh My Pi, and OpenCode.** +**What it thinks is what it sees — give any text-only coding agent eyes: image Q&A, long-screenshot OCR, frontend UI restoration, and GUI automation, as a vision toolkit plus a skill, with optional drop-in integration for Codex, Claude Code, Pi, Oh My Pi, and OpenCode, plus a native Skill + CLI adapter for Reasonix.** 🎯 An agent's vision capability doesn't have to live in the model — it can live in the harness. @@ -28,7 +28,7 @@ This repository provides two kinds of components: 1. **Vision tool CLIs** — multiple CLIs, plus a skill that teaches the agent when to use each one. Any agent that can invoke a shell can use them. 2. **Seamless integration** *(optional upgrade)* — a transparent local proxy and single-file native plugins, so **images we paste and the agent's built-in image tools both work seamlessly**, with no extra tool installation or additional prompting. -All code has been verified in real Codex + DeepSeek sessions, and the same pipeline has been live-verified end-to-end in Claude Code, Pi, Oh My Pi, and OpenCode. +All code has been verified in real Codex + DeepSeek sessions, and the same pipeline has been live-verified end-to-end in Claude Code, Pi, Oh My Pi, and OpenCode. The native Reasonix Skill + CLI adapter has also been verified against Reasonix 1.21.0. > If this project helps you or gives you some inspiration, feel free to star🌟 & fork. @@ -154,6 +154,12 @@ npx skills add Anionex/agent-vision-toolkit --skill vision-tools -a codex -g --c Or copy `skills/vision-tools/` into your agent's skills directory (e.g. `~/.codex/skills/`) and restart the agent. +For **Reasonix**, keep the CLI checkout from step 2 on `PATH` and install the compact native adapter instead of the generic skill copy: + +```bash +reasonix plugin install https://github.com/Anionex/agent-vision-toolkit/tree/main/integrations/reasonix --yes +``` + ## The Tools @@ -266,10 +272,13 @@ This layer makes screenshots pasted into an agent work directly, while also prev | **Claude Code** | the same proxy — point `ANTHROPIC_BASE_URL` at it | ✅ verified | | **Pi / Oh My Pi** | one-file native extension ([`extensions/pi/`](extensions/pi/)) | ✅ verified | | **OpenCode** | one-file native plugin ([`extensions/opencode/`](extensions/opencode/)) | ✅ verified | +| **Reasonix** | native Skill + existing CLIs ([`integrations/reasonix/`](integrations/reasonix/)); no MCP/runtime | ✅ verified (1.21.0) | | Any agent with a shell | the toolkit above — no integration needed | ✅ | All entry points share one configuration. Configure it once and use it everywhere. +Reasonix intentionally uses Skill + CLI rather than MCP or a runtime sidecar: its text-only models receive a readable local path for pasted images, and the Skill routes that path through the existing toolkit checkout. + ## How It Works ### Descriptions that keep the task in view diff --git a/README_CN.md b/README_CN.md index de5eebf..4f4cf72 100644 --- a/README_CN.md +++ b/README_CN.md @@ -14,7 +14,7 @@ [![Extensions](https://img.shields.io/badge/-Extensions-3178C6?style=flat-square)](https://github.com/Anionex/agent-vision-toolkit/tree/main/extensions) [![Shell](https://img.shields.io/badge/-Shell-4EAA25?style=flat-square&logo=gnubash&logoColor=white)](https://github.com/Anionex/agent-vision-toolkit/tree/main/bin) -**所想即所见——给任意纯文本 coding agent 装上眼睛：图片问答、长图OCR、前端UI还原、GUI自动化，一套视觉工具箱加一个 skill，并可选无缝接入 Codex、Claude Code、Pi、Oh My Pi、OpenCode。** +**所想即所见——给任意纯文本 coding agent 装上眼睛：图片问答、长图OCR、前端UI还原、GUI自动化，一套视觉工具箱加一个 skill，并可选无缝接入 Codex、Claude Code、Pi、Oh My Pi、OpenCode，另提供 Reasonix 原生 Skill + CLI 适配。** 🎯 agent视觉能力不一定长在模型上，也可以长在harness上。 @@ -28,7 +28,7 @@ 1. **视觉工具CLIs**：多个 CLI，外加一个 skill 告诉 agent 什么时候该用哪个。任何能调用 shell 的 agent 都能使用。 2. **无缝接入**（可选升级）：透明本地代理与单文件原生插件，让**我们粘贴的图片和 agent 内置的看图工具也能完美工作**，不需要额外的工具安装，也不需要额外提示词。 -所有代码均已在真实 Codex + DeepSeek 会话中验证过，同一套管线也在 Claude Code、Pi、Oh My Pi、OpenCode 中完成了真机端到端验证。 +所有代码均已在真实 Codex + DeepSeek 会话中验证过，同一套管线也在 Claude Code、Pi、Oh My Pi、OpenCode 中完成了真机端到端验证；Reasonix 原生 Skill + CLI 适配也已在 Reasonix 1.21.0 上验证通过。 > 如果项目对你有用或为你带来了一些启发，欢迎 star🌟 & fork。 @@ -154,6 +154,12 @@ npx skills add Anionex/agent-vision-toolkit --skill vision-tools -a codex -g --c 也可以把 `skills/vision-tools/` 复制到你的 agent 的 skills 目录（如 `~/.codex/skills/`），重启生效。 +如果使用 **Reasonix**，保留第 2 步的 CLI checkout 并确保其 `bin/` 在 `PATH` 上，然后安装精简的原生适配包，无需再复制通用 skill： + +```bash +reasonix plugin install https://github.com/Anionex/agent-vision-toolkit/tree/main/integrations/reasonix --yes +``` + ## 工具 @@ -264,10 +270,13 @@ crop screenshot.png --region 1563,514,1668,621 -o send-button.png | **Claude Code** | 同一个代理——把 `ANTHROPIC_BASE_URL` 指向它 | ✅ 已验证 | | **Pi / Oh My Pi** | 单文件原生 extension（[`extensions/pi/`](extensions/pi/)） | ✅ 已验证 | | **OpenCode** | 单文件原生 plugin（[`extensions/opencode/`](extensions/opencode/)） | ✅ 已验证 | +| **Reasonix** | 原生 Skill + 现有 CLI（[`integrations/reasonix/`](integrations/reasonix/)），不使用 MCP/runtime | ✅ 已验证（1.21.0） | | 任何有 shell 的 agent | 上面的工具箱——无需接入 | ✅ | 所有入口共享配置。一次配置好后即可多处使用。 +Reasonix 刻意采用 Skill + CLI，而不是 MCP 或 runtime sidecar：纯文本模型会拿到粘贴图片的可读本地路径，再由 Skill 把该路径交给现有工具箱 checkout 处理。 + ## 工作原理 ### 让描述始终对着当前任务 diff --git a/integrations/reasonix/reasonix-plugin.json b/integrations/reasonix/reasonix-plugin.json new file mode 100644 index 0000000..ffbc794 --- /dev/null +++ b/integrations/reasonix/reasonix-plugin.json @@ -0,0 +1,11 @@ +{ + "apiVersion": "reasonix.io/plugin/v1", + "name": "agent-vision-toolkit", + "version": "0.1.0", + "description": "Reasonix skill adapter for the agent-vision-toolkit CLIs", + "contributes": { + "skills": [ + "skills" + ] + } +} diff --git a/integrations/reasonix/skills/vision-tools/SKILL.md b/integrations/reasonix/skills/vision-tools/SKILL.md new file mode 100644 index 0000000..bfeb8a4 --- /dev/null +++ b/integrations/reasonix/skills/vision-tools/SKILL.md @@ -0,0 +1,90 @@ +--- +name: vision-tools +description: >- + Use the installed agent-vision-toolkit CLIs when a Reasonix task involves an + image: describe or OCR it, locate elements, inventory regions, recover exact + geometry, crop a box, compare renders, or inspect a long screenshot. +--- + +# vision-tools for Reasonix + +Reasonix keeps a pasted image's local path available to text-only models. Use +the Skill's CLI dispatcher to inspect that path instead of guessing from the +attachment marker. + +## Preflight the CLI checkout + +Reasonix appends a `## Scripts` section to this skill when it is loaded. Find +the exact absolute path ending in `vision_cli.py` there, then verify that the +toolkit checkout is available: + +```bash +python3 "" --check +``` + +Use the machine's Python 3 launcher (`python` or `py -3`) if it does not expose +the interpreter as `python3`. + +The Reasonix plugin installs this Skill, not a second copy of the toolkit. The +dispatcher finds the existing toolkit checkout through `glance` on `PATH`, or +from the current working tree. If `--check` fails, clone agent-vision-toolkit +and add its `bin/` directory to `PATH` using the repository's normal CLI setup, +then retry. Do not recreate or vendor the vision commands inside a task. + +```bash +git clone https://github.com/Anionex/agent-vision-toolkit.git +export PATH="$PWD/agent-vision-toolkit/bin:$PATH" +``` + +Run every tool through the dispatcher: + +```bash +python3 "" [tool arguments] +``` + +For a Reasonix context block such as ``, pass the `path` +value exactly as shown. If the user wrote an `@path` reference, pass the +filesystem path without the leading `@`. Quote every filesystem path passed to +the shell, including image, output, HTML, and dispatcher paths. + +## Choose the tool by the question + +| Question | Tool invocation | +|---|---| +| What does this image show or say? | `glance ""` | +| What matters for the current request? | `glance "" -q ""` | +| Transcribe visible text exactly | `glance "" --ocr` | +| Where is one named target? | `ground "" ""` | +| What elements or instances are present, and where? | `detect "" [category]` | +| What is the exact local shape or geometry? | `trace "" [-o "output.svg"]` | +| Cut a known pixel box into a reusable file | `crop "" --region X1,Y1,X2,Y2 [-o "output"]` | +| OCR a scrolling screenshot or chat history | `long-screenshot-ocr "" [-o "output.md"]` | +| Compare two renders at pixel level | `pixel-diff "" ""` | +| Extract a foreground graphic | `extract-fg "" ...` | +| Inspect dominant colors | `dominant-colors "" ...` | +| Render HTML to an image | `html-shot "" ...` | + +Run the dispatcher with `--list` to print every supported tool name. All +arguments after the tool name are passed through unchanged, so each tool's +normal `--help` remains available. + +## Operating rules + +1. When the user asks a specific visual question, pass that intent to + `glance -q`; do not settle for a generic description first. +2. For multiple images that must be compared semantically, pass them to one + `glance` call so the vision model sees them together. +3. Use `ground` for one named target and `detect` for an inventory. Their boxes + are suitable for clicking and cropping but can be a few pixels approximate. +4. Use `trace` when shape, offset, or size must come from the actual pixels. +5. Use `crop` and the bundled scripts instead of rewriting their pixel logic in + an ad hoc Python snippet. +6. After changing a UI or graphic, render the result and inspect it again. Do + not claim visual completion from source code alone. +7. Report missing files, optional dependencies, and vision API failures + honestly. Do not invent image contents when a command fails. + +The dispatcher inherits the shared `VISION_API_KEY`, `VISION_BASE_URL`, +`VISION_MODEL`, and optional `LANG` configuration used by the rest of the +toolkit. `glance` itself has no third-party Python dependency; other tools keep +their existing optional dependency requirements. diff --git a/integrations/reasonix/skills/vision-tools/scripts/vision_cli.py b/integrations/reasonix/skills/vision-tools/scripts/vision_cli.py new file mode 100755 index 0000000..c942e5d --- /dev/null +++ b/integrations/reasonix/skills/vision-tools/scripts/vision_cli.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Dispatch Reasonix skill calls to the repository's existing vision CLIs.""" + +from pathlib import Path +import os +import shutil +import sys + + +ENTRY_POINTS = { + "crop": "bin/crop", + "detect": "bin/detect", + "dominant-colors": "skills/vision-tools/scripts/dominant_colors.py", + "extract-fg": "skills/vision-tools/scripts/extract_fg.py", + "glance": "bin/glance", + "ground": "bin/ground", + "html-shot": "skills/vision-tools/scripts/html_shot.py", + "long-screenshot-ocr": "skills/vision-tools/scripts/long_screenshot_ocr.py", + "pixel-diff": "skills/vision-tools/scripts/pixel_diff.py", + "trace": "bin/trace", +} + + +def toolkit_root(glance_path=None, cwd=None): + """Find the toolkit checkout selected by PATH or the current workspace.""" + roots = [] + selected_glance = glance_path or shutil.which("glance") + if selected_glance: + roots.extend(Path(selected_glance).expanduser().resolve().parents) + + current = Path.cwd() if cwd is None else Path(cwd) + current = current.expanduser().resolve() + roots.extend((current, *current.parents)) + + seen = set() + for candidate in roots: + key = str(candidate) + if key in seen: + continue + seen.add(key) + if (candidate / "vision_client.py").is_file() and (candidate / "bin" / "glance").is_file(): + return candidate + raise RuntimeError( + "agent-vision-toolkit CLI checkout not found; add its bin directory to PATH" + ) + + +def normalize_tool(name): + return name.strip().lower().replace("_", "-") + + +def resolve_target(name, glance_path=None, cwd=None): + tool = normalize_tool(name) + relative = ENTRY_POINTS.get(tool) + if relative is None: + raise KeyError(tool) + target = toolkit_root(glance_path=glance_path, cwd=cwd) / relative + if not target.is_file(): + raise RuntimeError(f"toolkit entry point is missing: {relative}") + return target + + +def print_usage(stream): + tools = ", ".join(sorted(ENTRY_POINTS)) + print("usage: vision_cli.py [tool arguments] | --list | --check", file=stream) + print(f"tools: {tools}", file=stream) + + +def main(argv=None): + args = list(sys.argv[1:] if argv is None else argv) + if args == ["--list"]: + print("\n".join(sorted(ENTRY_POINTS))) + return 0 + if args == ["--check"]: + try: + root = toolkit_root() + for tool in ENTRY_POINTS: + resolve_target(tool) + except RuntimeError as exc: + print(f"vision toolkit error: {exc}", file=sys.stderr) + return 1 + print(root) + return 0 + if not args or args[0] in {"-h", "--help"}: + print_usage(sys.stdout if args else sys.stderr) + return 0 if args else 2 + + tool = normalize_tool(args.pop(0)) + try: + target = resolve_target(tool) + except KeyError: + print(f"unknown vision tool: {tool}", file=sys.stderr) + print_usage(sys.stderr) + return 2 + except RuntimeError as exc: + print(f"vision toolkit error: {exc}", file=sys.stderr) + return 1 + + try: + os.execv(sys.executable, [sys.executable, str(target), *args]) + except OSError as exc: + print(f"vision toolkit error: cannot run {target}: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_reasonix_integration.py b/tests/test_reasonix_integration.py new file mode 100644 index 0000000..14f6482 --- /dev/null +++ b/tests/test_reasonix_integration.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Unit and smoke tests for the Reasonix Skill + CLI integration.""" + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +PLUGIN_ROOT = ROOT / "integrations" / "reasonix" +MANIFEST = PLUGIN_ROOT / "reasonix-plugin.json" +SKILL = PLUGIN_ROOT / "skills" / "vision-tools" / "SKILL.md" +DISPATCHER = SKILL.parent / "scripts" / "vision_cli.py" + + +def load_dispatcher(): + spec = importlib.util.spec_from_file_location("reasonix_vision_cli", DISPATCHER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_manifest(): + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + assert manifest["apiVersion"] == "reasonix.io/plugin/v1" + assert manifest["name"] == "agent-vision-toolkit" + assert manifest["version"] == "0.1.0" + assert set(manifest["contributes"]) == {"skills"} + assert manifest["contributes"]["skills"] == ["skills"] + assert "runtime" not in manifest + assert "mcpServers" not in manifest + assert "mcpServers" not in manifest["contributes"] + assert not (ROOT / "reasonix-plugin.json").exists() + + +def test_compact_plugin_package(): + package_bytes = sum(path.stat().st_size for path in PLUGIN_ROOT.rglob("*") if path.is_file()) + assert package_bytes < 1_000_000 + assert not (PLUGIN_ROOT / "vision_client.py").exists() + assert not (PLUGIN_ROOT / "bin").exists() + + +def test_thin_skill_shape(): + text = SKILL.read_text(encoding="utf-8") + assert text.startswith("---\nname: vision-tools\n") + assert "vision_cli.py" in text + assert "not a second copy of the toolkit" in text + assert not (SKILL.parent / "references").exists() + assert len(text.encode("utf-8")) < 10_000 + + +def test_dispatcher_resolution(): + module = load_dispatcher() + glance = ROOT / "bin" / "glance" + assert module.toolkit_root(glance_path=glance, cwd=PLUGIN_ROOT) == ROOT + assert module.resolve_target("glance", glance_path=glance) == ROOT / "bin" / "glance" + assert module.resolve_target("long_screenshot_ocr", glance_path=glance) == ( + ROOT / "skills" / "vision-tools" / "scripts" / "long_screenshot_ocr.py" + ) + for tool in module.ENTRY_POINTS: + assert module.resolve_target(tool, glance_path=glance).is_file() + + +def test_dispatcher_cli(): + env = dict(os.environ) + env["PATH"] = str(ROOT / "bin") + os.pathsep + env.get("PATH", "") + + listed = subprocess.run( + [sys.executable, str(DISPATCHER), "--list"], + text=True, + capture_output=True, + check=True, + ) + names = set(listed.stdout.splitlines()) + assert {"glance", "ground", "detect", "trace", "crop"}.issubset(names) + + checked = subprocess.run( + [sys.executable, str(DISPATCHER), "--check"], + text=True, + capture_output=True, + check=True, + env=env, + ) + assert Path(checked.stdout.strip()).resolve() == ROOT + + for tool in ("glance", "ground", "detect", "trace", "crop"): + help_result = subprocess.run( + [sys.executable, str(DISPATCHER), tool, "--help"], + text=True, + capture_output=True, + check=True, + env=env, + ) + assert "usage:" in help_result.stdout.lower() + + unknown = subprocess.run( + [sys.executable, str(DISPATCHER), "not-a-tool"], + text=True, + capture_output=True, + ) + assert unknown.returncode == 2 + assert "unknown vision tool" in unknown.stderr + + with tempfile.TemporaryDirectory() as empty_cwd: + missing_env = dict(os.environ) + missing_env["PATH"] = "" + missing = subprocess.run( + [sys.executable, str(DISPATCHER), "glance", "--help"], + text=True, + capture_output=True, + cwd=empty_cwd, + env=missing_env, + ) + assert missing.returncode == 1 + assert "add its bin directory to PATH" in missing.stderr + + +def main(): + test_manifest() + test_compact_plugin_package() + test_thin_skill_shape() + test_dispatcher_resolution() + test_dispatcher_cli() + print("REASONIX INTEGRATION TEST PASS") + + +if __name__ == "__main__": + main()