From 25f2f4734c37f09d6b947e7d26bbf25b68aae321 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:45:51 +0800 Subject: [PATCH 01/11] chore: add safe repository ignores --- .gitignore | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75fcd3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.env +/ROADMAP.md + +# Local editor / MCP configuration contains machine-specific absolute paths. +.mcp.json +.cursor/ +.vscode/ +.windsurf/ + +# Local preview and browser artifacts. +.playwright-mcp/ +preview/ +*-preview.png From acdc8981913cc54003fa8c53e40bad1f2011ab9e Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:46:30 +0800 Subject: [PATCH 02/11] chore: add MCP configuration example --- example.config.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 example.config.toml diff --git a/example.config.toml b/example.config.toml new file mode 100644 index 0000000..aa120f7 --- /dev/null +++ b/example.config.toml @@ -0,0 +1,14 @@ +# Example MCP config for Codex / Claude Desktop. +# Copy the [mcp_servers.vision] block into your MCP config. +# Backends: ollama (default), openai_compatible, anthropic, gemini. +# See README.md for backend-specific environment variables. + +[mcp_servers.vision] +type = "stdio" +command = "uv" +args = ["run", "vision-mcp"] +cwd = "/absolute/path/to/shadow-vision" +env = { VISION_BACKEND = "ollama", VISION_MODEL = "qwen3-vl:2b", OLLAMA_URL = "http://127.0.0.1:11434/api/chat" } + +# OpenAI-compatible example: +# env = { VISION_BACKEND = "openai_compatible", VISION_MODEL = "your-vision-model", OPENAI_API_BASE = "https://api.example.com/v1", OPENAI_API_KEY = "sk-...", OPENAI_MAX_TOKENS = "1024", OPENAI_MAX_TOKENS_FIELD = "max_tokens" } From f9e884c97b9ece478dd00c31592337d24d96127a Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:46:52 +0800 Subject: [PATCH 03/11] build: add Python project metadata --- pyproject.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ed04d95 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "vision-mcp" +version = "0.1.0" +description = "Local vision MCP server with pluggable backends (Ollama, OpenAI-compatible, Anthropic, Gemini)" +requires-python = ">=3.11" +dependencies = ["mcp>=1.0.0", "httpx[socks]>=0.27.0"] + +[project.scripts] +vision-mcp = "vision_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["vision_mcp"] + +[tool.uv] +package = true From 23f706e70a8b9a951d7b0febc156043288b4d5a9 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:47:16 +0800 Subject: [PATCH 04/11] feat: add vision MCP package metadata --- vision_mcp/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 vision_mcp/__init__.py diff --git a/vision_mcp/__init__.py b/vision_mcp/__init__.py new file mode 100644 index 0000000..b167937 --- /dev/null +++ b/vision_mcp/__init__.py @@ -0,0 +1,3 @@ +"""Vision MCP server package.""" + +__version__ = "0.1.0" From f159ae1a9a17b85a96ce13ff7a5f19d115b4958b Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:47:41 +0800 Subject: [PATCH 05/11] feat: add backend configuration --- vision_mcp/config.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 vision_mcp/config.py diff --git a/vision_mcp/config.py b/vision_mcp/config.py new file mode 100644 index 0000000..70582ed --- /dev/null +++ b/vision_mcp/config.py @@ -0,0 +1,54 @@ +"""Runtime configuration for the vision MCP server. + +All values are read from environment variables with sensible defaults, +so the server can be configured without touching code. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + + +def _env(primary: str, legacy: str, default: str | None = None) -> str | None: + """Read the new variable name first, then fall back to the legacy name.""" + return os.getenv(primary) or os.getenv(legacy) or default + + +def _optional_int(primary: str, legacy: str) -> int | None: + """Read an optional integer environment variable with a legacy fallback.""" + value = _env(primary, legacy) + return int(value) if value else None + + +@dataclass(frozen=True) +class Config: + backend: str = field(default_factory=lambda: os.getenv("VISION_BACKEND", "ollama")) + model: str = field(default_factory=lambda: os.getenv("VISION_MODEL", "qwen3-vl:2b")) + # Ollama + ollama_url: str = field(default_factory=lambda: os.getenv("OLLAMA_URL", "http://127.0.0.1:11434/api/chat")) + # OpenAI-compatible endpoint. OPENAI_* is preferred; VISION_* remains supported. + api_base: str = field( + default_factory=lambda: _env( + "OPENAI_API_BASE", "VISION_API_BASE", "http://127.0.0.1:11434/v1" + ) or "http://127.0.0.1:11434/v1" + ) + api_key: str = field(default_factory=lambda: _env("OPENAI_API_KEY", "VISION_API_KEY", "") or "") + openai_max_tokens: int | None = field( + default_factory=lambda: _optional_int("OPENAI_MAX_TOKENS", "VISION_MAX_TOKENS") + ) + openai_max_tokens_field: str = field( + default_factory=lambda: _env( + "OPENAI_MAX_TOKENS_FIELD", "VISION_MAX_TOKENS_FIELD", "max_tokens" + ) or "max_tokens" + ) + # Anthropic + anthropic_api_key: str = field(default_factory=lambda: os.getenv("ANTHROPIC_API_KEY", "")) + anthropic_base_url: str = field(default_factory=lambda: os.getenv("ANTHROPIC_BASE_URL", "https://api.anthropic.com")) + anthropic_version: str = field(default_factory=lambda: os.getenv("ANTHROPIC_VERSION", "2023-06-01")) + anthropic_max_tokens: int = field(default_factory=lambda: int(os.getenv("ANTHROPIC_MAX_TOKENS", "1024"))) + # Gemini + gemini_api_key: str = field(default_factory=lambda: os.getenv("GEMINI_API_KEY", "")) + gemini_base_url: str = field(default_factory=lambda: os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com")) + gemini_max_tokens: int = field(default_factory=lambda: int(os.getenv("GEMINI_MAX_TOKENS", "1024"))) + timeout: float = field(default_factory=lambda: float(os.getenv("VISION_TIMEOUT", "180"))) From 0b634ba33cec56a78ffe19d112a1bbb3d79f2837 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:48:30 +0800 Subject: [PATCH 06/11] feat: implement pluggable vision backends --- vision_mcp/backends.py | 220 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 vision_mcp/backends.py diff --git a/vision_mcp/backends.py b/vision_mcp/backends.py new file mode 100644 index 0000000..ae2cd1a --- /dev/null +++ b/vision_mcp/backends.py @@ -0,0 +1,220 @@ +"""Vision backend abstraction. + +The MCP layer only talks to `VisionBackend`, so the backend can be swapped +without touching the server code. Four built-in backends are provided: + +- `ollama`: native Ollama chat API (default) +- `openai_compatible`: any OpenAI-compatible vision endpoint (e.g. LM Studio, + vLLM, or cloud providers) +- `anthropic`: Anthropic Messages API (Claude) +- `gemini`: Google Gemini API +""" + +from __future__ import annotations + +import base64 +from abc import ABC, abstractmethod +from pathlib import Path + +import httpx + +from .config import Config + +# MIME types for common image extensions. +MIME_TYPES = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", +} + + +class VisionBackend(ABC): + """Minimal interface for a vision backend.""" + + @abstractmethod + def analyze(self, prompt: str, image_data: str, mime: str, model: str) -> str: + """Send a prompt plus an image to the backend and return the text answer.""" + + +class OllamaBackend(VisionBackend): + """Native Ollama chat API backend.""" + + def __init__(self, config: Config) -> None: + self.url = config.ollama_url + self.timeout = config.timeout + + def analyze(self, prompt: str, image_data: str, mime: str, model: str) -> str: + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt, "images": [image_data]}], + "stream": False, + } + resp = httpx.post(self.url, json=payload, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + message = data.get("message", {}) + if message.get("content"): + return message["content"] + if message.get("thinking"): + return message["thinking"] + return "no response" + + +class OpenAICompatibleBackend(VisionBackend): + """OpenAI-compatible chat completions backend with image_url support.""" + + def __init__(self, config: Config) -> None: + self.base = config.api_base.rstrip("/") + self.api_key = config.api_key + self.max_tokens = config.openai_max_tokens + self.max_tokens_field = config.openai_max_tokens_field + self.timeout = config.timeout + + def analyze(self, prompt: str, image_data: str, mime: str, model: str) -> str: + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + payload = { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{image_data}"}}, + ], + } + ], + "stream": False, + } + if self.max_tokens is not None: + if self.max_tokens_field not in {"max_tokens", "max_completion_tokens"}: + raise ValueError( + "OPENAI_MAX_TOKENS_FIELD must be max_tokens or max_completion_tokens" + ) + payload[self.max_tokens_field] = self.max_tokens + resp = httpx.post(f"{self.base}/chat/completions", headers=headers, json=payload, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + choices = data.get("choices", []) + if choices and choices[0].get("message", {}).get("content"): + return choices[0]["message"]["content"] + return "no response" + + +def create_backend(config: Config) -> VisionBackend: + """Factory that returns the configured backend.""" + if config.backend == "ollama": + return OllamaBackend(config) + if config.backend in ("openai", "openai_compatible"): + return OpenAICompatibleBackend(config) + raise ValueError(f"unknown vision backend: {config.backend}") + + +def _read_image(image_path: str, image_base64: str | None, mime_type: str | None) -> tuple[str, str]: + """Read an image from a local path or base64 data, returning (base64, mime).""" + if image_base64: + return image_base64, mime_type or "image/png" + path = Path(image_path).expanduser() + if not path.exists(): + raise ValueError(f"image not found: {path}") + data = base64.b64encode(path.read_bytes()).decode() + mime = MIME_TYPES.get(path.suffix.lower(), "image/png") + return data, mime + + +class AnthropicBackend(VisionBackend): + """Anthropic Messages API backend (Claude).""" + + def __init__(self, config: Config) -> None: + self.api_key = config.anthropic_api_key + self.base = config.anthropic_base_url.rstrip("/") + self.version = config.anthropic_version + self.max_tokens = config.anthropic_max_tokens + self.timeout = config.timeout + + def analyze(self, prompt: str, image_data: str, mime: str, model: str) -> str: + headers = { + "x-api-key": self.api_key, + "anthropic-version": self.version, + "Content-Type": "application/json", + } + payload = { + "model": model, + "max_tokens": self.max_tokens, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime, + "data": image_data, + }, + }, + {"type": "text", "text": prompt}, + ], + } + ], + } + resp = httpx.post(f"{self.base}/v1/messages", headers=headers, json=payload, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + texts = [block.get("text", "") for block in data.get("content", []) if block.get("type") == "text"] + return "".join(texts) or "no response" + + +class GeminiBackend(VisionBackend): + """Google Gemini API backend (interactions / generateContent).""" + + def __init__(self, config: Config) -> None: + self.api_key = config.gemini_api_key + self.base = config.gemini_base_url.rstrip("/") + self.max_tokens = config.gemini_max_tokens + self.timeout = config.timeout + + def analyze(self, prompt: str, image_data: str, mime: str, model: str) -> str: + headers = { + "x-goog-api-key": self.api_key, + "Content-Type": "application/json", + } + payload = { + "model": model, + "input": [ + {"type": "text", "text": prompt}, + {"type": "image", "data": image_data, "mime_type": mime}, + ], + "config": {"maxOutputTokens": self.max_tokens}, + } + resp = httpx.post(f"{self.base}/v1beta/interactions", headers=headers, json=payload, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + # The interactions endpoint nests output under candidates[].content. + candidates = data.get("candidates", []) + for candidate in candidates: + content = candidate.get("content", {}) + for part in content.get("parts", []): + if part.get("text"): + return part["text"] + # Fallback: some responses expose outputText directly. + if data.get("outputText"): + return data["outputText"] + return "no response" + + +def create_backend(config: Config) -> VisionBackend: + """Factory that returns the configured backend.""" + if config.backend == "ollama": + return OllamaBackend(config) + if config.backend in ("openai", "openai_compatible"): + return OpenAICompatibleBackend(config) + if config.backend == "anthropic": + return AnthropicBackend(config) + if config.backend == "gemini": + return GeminiBackend(config) + raise ValueError(f"unknown vision backend: {config.backend}") From ccf0ba6998e24ad219b5d6e47d9b90973ad89f8c Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:49:12 +0800 Subject: [PATCH 07/11] feat: add MCP server entrypoint --- vision_mcp/server.py | 105 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 vision_mcp/server.py diff --git a/vision_mcp/server.py b/vision_mcp/server.py new file mode 100644 index 0000000..79f3bab --- /dev/null +++ b/vision_mcp/server.py @@ -0,0 +1,105 @@ +"""MCP server layer. + +Handles tool definitions and dispatch only. All vision logic lives in +`vision_mcp.backends`, so the backend can be swapped independently. +""" + +from __future__ import annotations + +from mcp.server.lowlevel import Server +from mcp.server.stdio import stdio_server +from mcp.types import CallToolRequestParams, CallToolResult, ListToolsResult, PaginatedRequestParams, Tool + +from .backends import _read_image, create_backend +from .config import Config + +OCR_PROMPT = "Extract all text visible in this image. Return only the text, no commentary." +DEFAULT_QUESTION = "Describe this image in detail." + + +def _build_tools(config: Config) -> list[Tool]: + return [ + Tool( + name="vision_ocr", + description="Extract all text from an image. Pass image_path (local file) or image_base64 + mime_type.", + inputSchema={ + "type": "object", + "properties": { + "image_path": {"type": "string", "description": "Local path to the image file"}, + "image_base64": {"type": "string", "description": "Base64-encoded image data"}, + "mime_type": {"type": "string", "description": "MIME type of image_base64 (e.g. image/png)"}, + "model": {"type": "string", "description": "Vision model name", "default": config.model}, + }, + }, + ), + Tool( + name="vision_inspect", + description="Describe or answer questions about an image. Pass image_path (local file) or image_base64 + mime_type.", + inputSchema={ + "type": "object", + "properties": { + "image_path": {"type": "string", "description": "Local path to the image file"}, + "image_base64": {"type": "string", "description": "Base64-encoded image data"}, + "mime_type": {"type": "string", "description": "MIME type of image_base64 (e.g. image/png)"}, + "question": {"type": "string", "description": "Question to ask about the image", "default": DEFAULT_QUESTION}, + "model": {"type": "string", "description": "Vision model name", "default": config.model}, + }, + }, + ), + ] + + +def _make_server() -> Server: + config = Config() + backend = create_backend(config) + tools = _build_tools(config) + + async def _list_tools(_ctx, _params) -> ListToolsResult: + return ListToolsResult(tools=tools) + + async def _call_tool(_ctx, params: CallToolRequestParams) -> CallToolResult: + args = params.arguments or {} + try: + image_path = str(args.get("image_path", "")) + image_base64 = args.get("image_base64") + mime_type = args.get("mime_type") + model = args.get("model", config.model) + data, mime = _read_image(image_path, image_base64, mime_type) + + if params.name == "vision_ocr": + result = backend.analyze(OCR_PROMPT, data, mime, model) + elif params.name == "vision_inspect": + question = str(args.get("question", DEFAULT_QUESTION)) + result = backend.analyze(question, data, mime, model) + else: + return CallToolResult( + content=[{"type": "text", "text": f"unknown tool: {params.name}"}], + isError=True, + ) + return CallToolResult(content=[{"type": "text", "text": result}]) + except Exception as exc: # noqa: BLE001 + return CallToolResult( + content=[{"type": "text", "text": f"error: {exc}"}], + isError=True, + ) + + server = Server("vision-mcp", version="0.1.0") + server.add_request_handler("tools/list", PaginatedRequestParams, _list_tools) + server.add_request_handler("tools/call", CallToolRequestParams, _call_tool) + return server + + +async def main_async() -> None: + server = _make_server() + async with stdio_server() as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) + + +def main() -> None: + import anyio + + anyio.run(main_async) + + +if __name__ == "__main__": + main() From e29e9d177847b291f7ce72e6ed0e523a866598c4 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:49:51 +0800 Subject: [PATCH 08/11] test: cover OpenAI-compatible configuration and payloads --- tests/test_openai_compatible.py | 77 +++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_openai_compatible.py diff --git a/tests/test_openai_compatible.py b/tests/test_openai_compatible.py new file mode 100644 index 0000000..ce70de5 --- /dev/null +++ b/tests/test_openai_compatible.py @@ -0,0 +1,77 @@ +import os +import unittest +from unittest.mock import Mock, patch + +from vision_mcp.backends import OpenAICompatibleBackend +from vision_mcp.config import Config + + +class OpenAICompatibleConfigTests(unittest.TestCase): + def test_new_openai_names_take_precedence_over_legacy_names(self) -> None: + env = { + "OPENAI_API_BASE": "https://new.example/v1", + "VISION_API_BASE": "https://legacy.example/v1", + "OPENAI_API_KEY": "new-key", + "VISION_API_KEY": "legacy-key", + "OPENAI_MAX_TOKENS": "2048", + "VISION_MAX_TOKENS": "1024", + "OPENAI_MAX_TOKENS_FIELD": "max_completion_tokens", + } + with patch.dict(os.environ, env, clear=True): + config = Config() + + self.assertEqual(config.api_base, "https://new.example/v1") + self.assertEqual(config.api_key, "new-key") + self.assertEqual(config.openai_max_tokens, 2048) + self.assertEqual(config.openai_max_tokens_field, "max_completion_tokens") + + def test_legacy_names_remain_supported(self) -> None: + env = { + "VISION_API_BASE": "https://legacy.example/v1", + "VISION_API_KEY": "legacy-key", + "VISION_MAX_TOKENS": "1024", + } + with patch.dict(os.environ, env, clear=True): + config = Config() + + self.assertEqual(config.api_base, "https://legacy.example/v1") + self.assertEqual(config.api_key, "legacy-key") + self.assertEqual(config.openai_max_tokens, 1024) + self.assertEqual(config.openai_max_tokens_field, "max_tokens") + + def test_openai_max_tokens_is_optional(self) -> None: + with patch.dict(os.environ, {}, clear=True): + config = Config() + + self.assertIsNone(config.openai_max_tokens) + + +class OpenAICompatiblePayloadTests(unittest.TestCase): + def test_max_tokens_field_is_selected_explicitly(self) -> None: + config = Config(openai_max_tokens=512, openai_max_tokens_field="max_completion_tokens") + response = Mock() + response.json.return_value = {"choices": [{"message": {"content": "ok"}}]} + + with patch("vision_mcp.backends.httpx.post", return_value=response) as post: + result = OpenAICompatibleBackend(config).analyze("prompt", "image", "image/png", "vision-model") + + self.assertEqual(result, "ok") + payload = post.call_args.kwargs["json"] + self.assertEqual(payload["max_completion_tokens"], 512) + self.assertNotIn("max_tokens", payload) + + def test_token_limit_is_omitted_when_unconfigured(self) -> None: + config = Config(openai_max_tokens=None) + response = Mock() + response.json.return_value = {"choices": [{"message": {"content": "ok"}}]} + + with patch("vision_mcp.backends.httpx.post", return_value=response) as post: + OpenAICompatibleBackend(config).analyze("prompt", "image", "image/png", "vision-model") + + payload = post.call_args.kwargs["json"] + self.assertNotIn("max_tokens", payload) + self.assertNotIn("max_completion_tokens", payload) + + +if __name__ == "__main__": + unittest.main() From d29b4a0a06adeab04f2639b24802720a4b4b0159 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:50:41 +0800 Subject: [PATCH 09/11] docs: add README hero asset --- assets/readme/hero.svg | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 assets/readme/hero.svg diff --git a/assets/readme/hero.svg b/assets/readme/hero.svg new file mode 100644 index 0000000..54215c9 --- /dev/null +++ b/assets/readme/hero.svg @@ -0,0 +1,39 @@ + + 影瞳 Shadow Vision — 给文本模型一双眼睛 + 开源 MCP 视觉服务，让纯文本 LLM 通过 vision_ocr 与 vision_inspect 获得图像理解、OCR 和视觉分析能力。 + + + + + + + + + + + + + OPEN-SOURCE MCP · VISION SERVER + + Shadow Vision + 影瞳 + 给纯文本 LLM 提供图像理解、OCR 与视觉分析， + 让 AI Agent 看见并理解真实世界的信息。 + MIT LICENSE · PYTHON 3.11+ · MCP NATIVE + + + + + + + + ~ uv run vision-mcp + + > vision_inspect("/tmp/ui.png") + "Hero uses a 3-column grid + with a low-contrast CTA." + > vision_ocr("/tmp/receipt.png") + "Total: $12.50 · Tax: $1.05" + "PAID · THANK YOU" + + From 8a60fb86b604ae4e3ebdf574be99fb1e8c20229c Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:52:08 +0800 Subject: [PATCH 10/11] docs: add README workflow asset --- assets/readme/workflow.svg | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 assets/readme/workflow.svg diff --git a/assets/readme/workflow.svg b/assets/readme/workflow.svg new file mode 100644 index 0000000..feac412 --- /dev/null +++ b/assets/readme/workflow.svg @@ -0,0 +1,70 @@ + + 影瞳 Shadow Vision 工作方式 + 纯文本 LLM 通过 MCP 调用 vision_ocr 与 vision_inspect 两个工具，解码到 Ollama、OpenAI-compatible、Anthropic 或 Gemini 四种视觉后端。 + + + + + + + + + + + + + + + Text LLM + no built-in vision + Claude · Codex · GPT + + + + + Shadow Vision MCP + tools the model calls + + vision_ocr + extract text from an image + vision_inspect + describe & answer questions + + + + + + + Ollama + local · qwen3-vl + default backend + + + + + OpenAI + compatible + LM Studio · vLLM + + + + + Anthropic + Claude + + + + + Gemini + Google + + + + + + + + + + + From a9db8eb87b8e610a724875576656a136ac7c08c7 Mon Sep 17 00:00:00 2001 From: Ward Lu Date: Wed, 5 Aug 2026 03:53:09 +0800 Subject: [PATCH 11/11] docs: add public README --- README.md | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..62bace6 --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +

+ 影瞳 Shadow Vision — 开源 MCP 视觉服务，让纯文本 LLM 获得图像理解、OCR 与视觉分析能力 +

+ +# 影瞳 · Shadow Vision + +给纯文本 LLM 添加一双眼睛。影瞳是一个开源 MCP 视觉服务，让 AI Agent 通过 `vision_ocr` 与 `vision_inspect` 看见、理解并分析真实世界的信息，无需切换宿主文本模型。 + +## 为什么不同 + +- **MCP 原生**：适配 Codex、Claude Desktop、Cursor 及其他 MCP 客户端 +- **可插拔后端**：Ollama、OpenAI-compatible、Anthropic、Gemini +- **本地优先**：使用 Ollama 时图片和推理都可以留在本机 +- **输入简单**：支持本地文件路径或 base64 图片数据 + +## 工作原理 + +

+ 纯文本 LLM 通过 MCP 调用 vision_ocr 与 vision_inspect，再连接到 Ollama、OpenAI-compatible、Anthropic 或 Gemini +

+ +文本模型通过 MCP 调用影瞳的两个工具，影瞳把图片和提示词转发到配置的视觉后端，再把文字结果返回给模型。 + +## 快速开始 + +### 1. 安装 + +需要 Python 3.11+ 与 [uv](https://docs.astral.sh/uv/)： + +```bash +git clone https://github.com/WardLu/shadow-vision.git +cd shadow-vision +uv sync +``` + +### 2. 使用本地 Ollama（推荐新手） + +先安装 [Ollama](https://ollama.com/download)。如果没有使用 Ollama 桌面应用，可手动启动服务： + +```bash +ollama serve +ollama pull qwen3-vl:2b +ollama list +``` + +`qwen3-vl:2b` 是默认视觉模型。也可以把 `VISION_MODEL` 换成 `ollama list` 中其他已经下载的视觉模型；文本模型不能直接完成看图。 + +### 3. 注册为 MCP 服务 + +Codex 可以直接执行： + +```bash +codex mcp add vision -- uv run vision-mcp +``` + +或者写入 `~/.codex/config.toml`： + +```toml +[mcp_servers.vision] +type = "stdio" +command = "uv" +args = ["run", "vision-mcp"] +cwd = "/path/to/shadow-vision" +env = { VISION_BACKEND = "ollama", VISION_MODEL = "qwen3-vl:2b", OLLAMA_URL = "http://127.0.0.1:11434/api/chat" } +``` + +重启 MCP 客户端后，直接让模型“看一下这张图片”即可。 + +## 切换模型和后端 + +`VISION_BACKEND` 决定调用方式，`VISION_MODEL` 决定具体视觉模型。修改 MCP 配置中的环境变量后，重启客户端即可。 + +切换本地模型： + +```toml +env = { VISION_BACKEND = "ollama", VISION_MODEL = "你已下载的视觉模型", OLLAMA_URL = "http://127.0.0.1:11434/api/chat" } +``` + +切换到 OpenAI-compatible 服务： + +```toml +env = { VISION_BACKEND = "openai_compatible", VISION_MODEL = "服务端提供的视觉模型名", OPENAI_API_BASE = "https://api.example.com/v1", OPENAI_API_KEY = "sk-...", OPENAI_MAX_TOKENS = "1024", OPENAI_MAX_TOKENS_FIELD = "max_tokens" } +``` + +`OPENAI_*` 表示 OpenAI Chat Completions 兼容协议，也适用于 LM Studio、vLLM 和其他提供 `/v1/chat/completions` 的服务。 + +## 配置后端 + +### 通用变量 + +| 变量 | 默认值 | 说明 | +|---|---|---| +| `VISION_BACKEND` | `ollama` | `ollama` / `openai_compatible` / `anthropic` / `gemini` | +| `VISION_MODEL` | `qwen3-vl:2b` | 视觉模型名称 | +| `VISION_TIMEOUT` | `180` | 请求超时（秒） | + +### Ollama + +| 变量 | 默认值 | 说明 | +|---|---|---| +| `OLLAMA_URL` | `http://127.0.0.1:11434/api/chat` | Ollama 对话端点 | + +使用前执行 `ollama pull <视觉模型名>` 下载模型。 + +### OpenAI-compatible + +| 变量 | 默认值 | 说明 | +|---|---|---| +| `OPENAI_API_BASE` | `http://127.0.0.1:11434/v1` | 兼容服务基础地址 | +| `OPENAI_API_KEY` | 空 | 本地服务通常可留空 | +| `OPENAI_MAX_TOKENS` | 未设置 | 可选输出 token 上限；未设置时不发送 token 限制字段 | +| `OPENAI_MAX_TOKENS_FIELD` | `max_tokens` | 可选：`max_tokens` 或 `max_completion_tokens` | + +不同服务支持的 token 字段不完全一致：支持旧字段就使用 `max_tokens`，只支持新版字段就改成 `max_completion_tokens`，两个字段都不接受时不要设置 `OPENAI_MAX_TOKENS`。旧变量名 `VISION_API_BASE`、`VISION_API_KEY`、`VISION_MAX_TOKENS` 和 `VISION_MAX_TOKENS_FIELD` 仍兼容。 + +### Anthropic / Gemini + +```bash +VISION_BACKEND=anthropic ANTHROPIC_API_KEY=sk-ant-... VISION_MODEL=your-claude-vision-model uv run vision-mcp +VISION_BACKEND=gemini GEMINI_API_KEY=AIza... VISION_MODEL=your-gemini-vision-model uv run vision-mcp +``` + +Anthropic 还支持 `ANTHROPIC_BASE_URL`、`ANTHROPIC_VERSION` 和 `ANTHROPIC_MAX_TOKENS`；Gemini 还支持 `GEMINI_BASE_URL` 和 `GEMINI_MAX_TOKENS`。 + +## 工具 + +### `vision_ocr` + +从截图、票据、文档或表格中提取文字： + +```python +vision_ocr(image_path="/tmp/receipt.png") +``` + +### `vision_inspect` + +描述图片，或回答关于图片的问题： + +```python +vision_inspect(image_path="/tmp/design.png", question="List any UI bugs you see.") +``` + +两个工具都支持： + +- `image_path`：服务器可读的本地图片路径 +- `image_base64` + `mime_type`：base64 编码的图片数据 + +## 本地模型选择与测评 + +Ollama 模型页可以查看模型包大小、上下文窗口和图像能力，但模型包大小不是最低内存要求。建议从 `qwen3-vl:2b` 开始；如果 OCR 或复杂图表理解不足，再比较 `qwen3-vl:4b`、`qwen3-vl:8b` 或文档 OCR 取向的 `minicpm-v4.5:q4_0`。 + +准备 3–5 张真实图片，覆盖 OCR、截图、图表和困难样本，并使用相同提示词比较模型： + +```bash +MODEL=qwen3-vl:2b +IMAGE=/absolute/path/to/test.png + +time ollama run "$MODEL" "$IMAGE" "请准确抄录图片中的全部文字，只输出文字。" +time ollama run "$MODEL" "$IMAGE" "请描述图片内容，并列出你不确定的地方。" +``` + +记录 OCR 错误数量、关键对象和关系是否正确、幻觉、完整响应延迟，以及 `ollama ps` 中的 processor 状态。先直接测试 Ollama，再通过 `vision_ocr` / `vision_inspect` 测试 MCP 链路，可以区分模型问题和 MCP 配置问题。 + +推荐资料： + +- [Ollama Vision 文档](https://docs.ollama.com/capabilities/vision) +- [Ollama Qwen3-VL 模型页](https://ollama.com/library/qwen3-vl) +- [Qwen3-VL 官方仓库](https://github.com/QwenLM/Qwen3-VL) +- [MiniCPM-V 4.5 官方评测](https://github.com/OpenBMB/MiniCPM-V/blob/main/docs/minicpm_v4dot5_en.md) +- [Ollama Context Length 文档](https://docs.ollama.com/context-length) +- [Ollama Modelfile 参数文档](https://docs.ollama.com/modelfile) + +## 支持的 Agent + +所有 Agent 都启动同一命令：`uv run vision-mcp`。 + +| Agent | 配置文件 | +|---|---| +| Codex | `~/.codex/config.toml` | +| Claude Code | `.mcp.json` | +| Cursor | `.cursor/mcp.json` | +| VS Code Copilot | `.vscode/mcp.json` | +| Windsurf | `.windsurf/mcp_config.json` | +| Claude Desktop | `claude_desktop_config.json` | +| OpenCode | `opencode.json` | + +## 开发 + +```bash +uv sync +uv run python -c "import vision_mcp.server; print('ok')" +python -m unittest discover -s tests -v +``` + +## 联系我 + +如果你对 B 端产品、AI 产品开发、供应链数字化或 Shadow 系列产品感兴趣，可以联系我： + +- **X（Twitter）**：[@Gollumgulu](https://x.com/Gollumgulu) +- **小红书 / 微博 / 抖音**：全网同名「Ward 的 AI 产品实战」—— [小红书](https://xhslink.cn/m/4W1NWyRrxv5) · [微博](https://weibo.com/u/8344390431) · [抖音](https://v.douyin.com/1y06PMohfoE/) +- **产品主页**：[Shadow Nexus](https://www.shadow.wang/) +- **Email**：[wardlu@126.com](mailto:wardlu@126.com) + +> 可接 1v1 咨询和项目陪跑：产品诊断 · AI 实施 · 工作流 / Skill · 系统定制 + +## License + +MIT