From 0fae1fc2bff88739a2f2c0eaa835379931740fd6 Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Mon, 10 Aug 2026 09:05:25 +0530 Subject: [PATCH 1/4] feat(serve): run many brains from one process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `serve --brains ` mounts every brain under a directory at its own path, so //mcp is that brain's endpoint. One process, one port, one container. Running a container per brain was the obvious approach and does not scale: each carries its own interpreter and model cache, so a host runs out of memory long before it runs out of useful indexes. Mounted apps do not get their lifespan run by Starlette, so each child's is entered explicitly via AsyncExitStack — without it the MCP session manager never starts and every request hangs. Also here: - build_transport_security() allows the proxied public host. The MCP SDK's DNS-rebinding guard rejects a forwarded Host with 421 Misdirected Request, which behind any reverse proxy means nothing connects. - per-brain tokens via OPEN_INDEX_TOKEN_, so one endpoint being open does not open the rest. - the container entrypoint no longer assumes exactly one brain at /brain; it detects the mode and reconciles each brain before serving. Co-Authored-By: Claude Opus 5 (1M context) --- docker/entrypoint.sh | 79 ++++++++++-- open_index/cli.py | 50 +++++++- open_index/config.py | 16 +++ open_index/mcp_server.py | 192 +++++++++++++++++++++++++++- tests/test_cli_serving.py | 86 +++++++++++++ tests/test_mcp_transport.py | 103 +++++++++++++++ tests/test_multi_brain_serve.py | 218 ++++++++++++++++++++++++++++++++ 7 files changed, 726 insertions(+), 18 deletions(-) create mode 100644 tests/test_multi_brain_serve.py diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e5098cf..5c61445 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,26 +1,58 @@ #!/usr/bin/env sh # Container entrypoint for open-index. # -# Two jobs beyond exec'ing the CLI: +# It runs in one of two modes: # -# 1. Default --brain to $OPEN_INDEX_BRAIN (/brain) so every command works -# without repeating the flag in compose files and `docker run` lines. -# 2. Reconcile file-backed entities into the search index before serving. -# Entities under entities/**/*.json are the git source of truth; a fresh -# container (or a fresh OpenSearch cluster) starts with an empty index, so -# without this the brain answers every query with "no results" and looks -# broken. Index-backed entities are untouched by this — see Brain.index(). +# single-brain one brain mounted at $OPEN_INDEX_BRAIN (/brain). The --brain +# flag is filled in so compose files and `docker run` lines do +# not repeat it. +# many-brains a directory of brains at $OPEN_INDEX_BRAINS_ROOT (/brains), +# selected by `serve --brains` or by the explorer. Nothing is +# prepended; the command already knows where to look. # -# Skip step 2 with OPEN_INDEX_SKIP_INDEX=1 (e.g. a read replica, or when the -# index is large and managed out of band). +# In both modes, file-backed entities are reconciled into the search index +# before serving. Entities under entities/**/*.json are the git source of truth, +# so a fresh container starts with an empty index and would otherwise answer +# every query with "no results" and look broken. +# +# Skip the reconcile with OPEN_INDEX_SKIP_INDEX=1 (a read replica, or an index +# large enough to be managed out of band). set -eu BRAIN="${OPEN_INDEX_BRAIN:-/brain}" +BRAINS_ROOT="${OPEN_INDEX_BRAINS_ROOT:-}" + +# `serve --brains ` selects many-brains mode even without the env var. +for arg in "$@"; do + if [ "$arg" = "--brains" ]; then MULTI_FLAG=1; fi +done + +if [ -n "$BRAINS_ROOT" ] || [ "${MULTI_FLAG:-0}" = "1" ]; then + MULTI=1 + ROOT="${BRAINS_ROOT:-/brains}" +else + MULTI=0 +fi -if [ ! -f "$BRAIN/brain.yaml" ]; then +if [ "$MULTI" = "1" ]; then + if [ ! -d "$ROOT" ]; then + echo "open-index: no directory at $ROOT" >&2 + echo " mount a directory of brains there, e.g. -v \"\$PWD/brains:/brains\"" >&2 + exit 1 + fi + # A brain is a subdirectory holding brain.yaml. An empty root is almost + # always a mis-mounted volume, so say so rather than serving nothing. + if [ -z "$(find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml -print -quit)" ]; then + echo "open-index: no brains under $ROOT" >&2 + echo " each brain is a subdirectory containing brain.yaml" >&2 + echo " (create one with: open-index init )" >&2 + exit 1 + fi +elif [ ! -f "$BRAIN/brain.yaml" ]; then echo "open-index: no brain.yaml at $BRAIN" >&2 echo " mount your brain directory there, e.g. -v \"\$PWD/my-brain:/brain\"" >&2 echo " (create one first with: open-index init my-brain)" >&2 + echo " (or mount a directory of brains and set OPEN_INDEX_BRAINS_ROOT)" >&2 exit 1 fi @@ -50,17 +82,36 @@ except Exception: echo "open-index: OpenSearch is up." fi +reconcile_all() { + # Reconcile every brain under the root. Done here rather than inside `serve` + # so it happens once at container start instead of on every request. + find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml | while read -r f; do + d=$(dirname "$f") + echo "open-index: indexing $(basename "$d") ..." + open-index index --brain "$d" >/dev/null || \ + echo "open-index: WARNING $(basename "$d") failed to index" >&2 + done +} + case "${1:-serve}" in serve|ui|index|search|validate|run|ingest|mcp|mcp-config|list-connectors|add-entity|add-doc-type) COMMAND="$1" shift if [ "$COMMAND" = "serve" ] && [ "${OPEN_INDEX_SKIP_INDEX:-0}" != "1" ]; then - echo "open-index: reconciling file-backed entities into the index ..." - open-index index --brain "$BRAIN" + if [ "$MULTI" = "1" ]; then + reconcile_all + else + echo "open-index: reconciling file-backed entities into the index ..." + open-index index --brain "$BRAIN" + fi fi - set -- "$COMMAND" --brain "$BRAIN" "$@" + if [ "$MULTI" = "1" ]; then + set -- "$COMMAND" "$@" # the command carries its own --brains + else + set -- "$COMMAND" --brain "$BRAIN" "$@" + fi ;; *) # Anything else (sh, python, a raw open-index invocation) runs verbatim. diff --git a/open_index/cli.py b/open_index/cli.py index 6eefe74..60fba11 100644 --- a/open_index/cli.py +++ b/open_index/cli.py @@ -441,6 +441,12 @@ def mcp_config( @app.command() def serve( brain: str = BrainOpt, + brains: Optional[str] = typer.Option( + None, "--brains", envvar="OPEN_INDEX_BRAINS_ROOT", + help="Serve EVERY brain under this directory from one process, each at " + "//mcp. Cheaper than one process per brain: the embedding " + "model is loaded once for all of them.", + ), host: str = typer.Option("0.0.0.0", help="Bind host."), port: int = typer.Option(8080, help="Bind port."), token: Optional[str] = typer.Option( @@ -452,6 +458,12 @@ def serve( help="Externally reachable URL, when behind a proxy/tunnel/load balancer. " "Only used to print correct connection details.", ), + allowed_host: Optional[list[str]] = typer.Option( + None, "--allowed-host", envvar="OPEN_INDEX_ALLOWED_HOSTS", + help="Host header(s) to accept, when behind a proxy or load balancer. " + "Repeatable. The host from --public-url is added automatically. " + "Use '*' to disable the check entirely (trusted proxy only).", + ), read_only: bool = typer.Option( False, "--read-only", help="Opt out of default read+write mode and expose only read tools.", @@ -463,11 +475,42 @@ def serve( OpenSearch backend for a shared, multi-writer brain. """ from open_index.mcp_config import advertised_urls, mask_token - from open_index.mcp_server import serve_http + from open_index.mcp_server import ( + discover_brains, serve_http, serve_http_multi, token_for, + ) + + mode = "read-only" if read_only else "read+write" + + if brains: + found = discover_brains(brains) + if not found: + typer.secho(f"no brains under {brains} — a brain is a directory " + "containing brain.yaml", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + typer.secho(f"open-index · {len(found)} brains · {mode}", + fg=typer.colors.GREEN, bold=True) + typer.echo(f" listening on {host}:{port}") + urls = advertised_urls(host, port, public_url=public_url) + base = urls[-1][1].rsplit("/mcp", 1)[0] + typer.echo("") + for name in found: + gate = "token" if token_for(name, token) else typer.style( + "OPEN", fg=typer.colors.YELLOW) + typer.secho(f" {name:<28} {base}/{name}/mcp ({gate})", + fg=typer.colors.CYAN) + typer.echo("") + typer.echo(" per-brain tokens come from OPEN_INDEX_TOKEN_; " + "--token covers any without one.") + typer.echo(f" directory: {base}/ · health: {base}/healthz") + typer.echo("") + serve_http_multi(brains, host=host, port=port, token=token, + read_only=read_only, public_base_url=base, + allowed_hosts=list(allowed_host or [])) + return root = Path(brain).resolve() b = _open_brain(str(root)) - mode = "read-only" if read_only else "read+write" typer.secho( f"open-index · brain '{b.config.name}' · {mode} · " @@ -502,7 +545,8 @@ def serve( typer.echo("") serve_http(str(root), host=host, port=port, token=token, read_only=read_only, - warn_unauthenticated=False) + warn_unauthenticated=False, public_url=public_url, + allowed_hosts=list(allowed_host or [])) if __name__ == "__main__": # pragma: no cover - module entry point diff --git a/open_index/config.py b/open_index/config.py index 1ad1e5e..9ad116e 100644 --- a/open_index/config.py +++ b/open_index/config.py @@ -220,6 +220,22 @@ def write_doc_type(brain_root: Path, dt: DocType) -> Path: return path +def discover_brains(root: str | Path) -> dict[str, Path]: + """Every brain directory directly under `root`, keyed by directory name. + + A "brain" is any subdirectory holding a brain.yaml, so a root can sit + alongside unrelated folders without confusing anything. + """ + base = Path(root).expanduser().resolve() + if not base.is_dir(): + raise FileNotFoundError(f"no such directory: {base}") + return { + child.name: child + for child in sorted(base.iterdir()) + if (child / "brain.yaml").exists() + } + + def iter_entity_files(brain_dir: str | Path): """Yield every entities/**/*.json path under the brain directory.""" root = Path(brain_dir).expanduser().resolve() diff --git a/open_index/mcp_server.py b/open_index/mcp_server.py index b93619f..196d04e 100644 --- a/open_index/mcp_server.py +++ b/open_index/mcp_server.py @@ -23,6 +23,8 @@ from time import perf_counter from typing import Any, Optional +from pathlib import Path + from open_index.brain import Brain from open_index.schema import DocType, DocTypeDisplay, FieldSpec, RelationshipSpec from open_index.models import Entity, Provenance, Relationship @@ -432,10 +434,168 @@ async def wrapped(scope, receive, send): return wrapped +def _host_of(url: str) -> Optional[str]: + """The host[:port] part of a URL, or None if it isn't parseable.""" + from urllib.parse import urlparse + + parsed = urlparse(url if "://" in url else f"http://{url}") + return parsed.netloc or None + + +def build_transport_security( + public_url: Optional[str] = None, allowed_hosts: Optional[list[str]] = None +): + """Host allow-list for the MCP SDK's DNS-rebinding protection. + + The SDK enables that protection whenever the app is built for a localhost + bind, with a hardcoded localhost-only allow-list. Anything reaching the + server through a reverse proxy or load balancer therefore arrives with a + foreign `Host` header and is rejected with `421 Invalid Host header` — the + server looks up, and every proxied request fails. + + Rather than switch the protection off (which is what passing a non-localhost + bind address to the SDK quietly does), state which hosts are legitimate: + loopback, plus whatever `--public-url` and `--allowed-host` name. + + A literal `*` disables the check — for a brain behind a trusted proxy that + already validates Host. Returns settings, or None when the SDK's own default + is appropriate. + """ + from mcp.server.transport_security import TransportSecuritySettings + + entries = [e.strip() for e in (allowed_hosts or []) if e.strip()] + if "*" in entries: + return TransportSecuritySettings(enable_dns_rebinding_protection=False) + + if public_url: + derived = _host_of(public_url) + if derived: + entries.append(derived) + + hosts = {"127.0.0.1:*", "localhost:*", "[::1]:*", "127.0.0.1", "localhost", "[::1]"} + origins = {"http://127.0.0.1:*", "http://localhost:*", "http://[::1]:*"} + for entry in entries: + bare = entry.split(":", 1)[0] if not entry.startswith("[") else entry + # Both forms: proxies drop the port when it is the scheme default, so + # "example.com" and "example.com:8080" must both pass. + hosts.update({entry, bare, f"{bare}:*"}) + for scheme in ("http", "https"): + origins.update({f"{scheme}://{entry}", f"{scheme}://{bare}", + f"{scheme}://{bare}:*"}) + + return TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=sorted(hosts), + allowed_origins=sorted(origins), + ) + + +def discover_brains(root: str) -> dict[str, "Path"]: + """Brains under `root`. Re-exported from config for the serve entry points.""" + from open_index.config import discover_brains as _discover + + try: + return _discover(root) + except FileNotFoundError as exc: + raise SystemExit(str(exc)) from exc + + +def token_for(name: str, default: Optional[str] = None) -> Optional[str]: + """Per-brain bearer token from the environment, falling back to a shared one. + + `OPEN_INDEX_TOKEN_SALES_EU` gates the brain in `sales-eu/`. Without one, the + shared `--token` applies — which is fine for a private host and wrong for a + shared one, hence the per-brain override. + """ + import os as _os + + key = "OPEN_INDEX_TOKEN_" + name.upper().replace("-", "_").replace(".", "_") + return _os.environ.get(key) or default + + +def build_multi_app( + brains: dict[str, "Path"], + *, + token: Optional[str] = None, + read_only: bool = False, + public_base_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, + host: str = "0.0.0.0", +): + """One ASGI app serving many brains, each mounted at `//mcp`. + + The point is what is *not* duplicated. A process per brain re-loads the + Python runtime and a ~250MB resident embedding model every time, which caps + a modest host at a handful of brains. Here the model is loaded once — the + provider cache is keyed on model configuration, not on brain — so the + marginal cost of a brain is its config and doc_types, and hundreds fit where + a handful did. + + Each brain keeps its own token, its own read/write policy and its own + storage, so this is a packaging change rather than a shared-tenancy one. + """ + from contextlib import AsyncExitStack, asynccontextmanager + + from starlette.applications import Starlette + from starlette.responses import JSONResponse, PlainTextResponse + from starlette.routing import Mount, Route + + base = (public_base_url or "").rstrip("/") + mounted: list = [] # the Starlette children, for their lifespans + routes: list = [] + listing: dict[str, dict] = {} + + for name, path in brains.items(): + brain = Brain.open(path) + server = build_server(brain, read_only=read_only) + public_url = f"{base}/{name}/mcp" if base else None + child = server.streamable_http_app( + transport_security=build_transport_security(public_url, allowed_hosts), + host=host, + ) + mounted.append(child) + + brain_token = token_for(name, token) + app = _bearer_auth_middleware(child, brain_token) if brain_token else child + routes.append(Mount(f"/{name}", app=app)) + + listing[name] = { + "mcp": public_url or f"/{name}/mcp", + "description": brain.config.description, + "entities": sum(brain.counts().values()), + "doc_types": sorted(brain.config.doc_types), + "read_only": read_only, + "authenticated": bool(brain_token), + } + + async def directory(_request): + return JSONResponse(listing) + + async def healthz(_request): + return PlainTextResponse("ok") + + routes += [Route("/", directory), Route("/healthz", healthz)] + + @asynccontextmanager + async def lifespan(_app): + # Starlette does not run a mounted app's lifespan, and the MCP transport + # needs its session manager started — without this every request to a + # mounted brain fails with "Task group is not initialized". + async with AsyncExitStack() as stack: + for child in mounted: + await stack.enter_async_context( + child.router.lifespan_context(child)) + yield + + return Starlette(routes=routes, lifespan=lifespan) + + def serve_http( brain_dir: str, host: str = "0.0.0.0", port: int = 8080, token: Optional[str] = None, read_only: bool = False, warn_unauthenticated: bool = True, + public_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, ) -> None: """Run the MCP server over streamable HTTP so remote/cloud agents can connect by URL. Register `http://:/mcp` as a remote MCP server in the @@ -452,7 +612,10 @@ def serve_http( brain = Brain.open(brain_dir) server = build_server(brain, read_only=read_only) - app = server.streamable_http_app() + app = server.streamable_http_app( + transport_security=build_transport_security(public_url, allowed_hosts), + host=host, + ) if token: app = _bearer_auth_middleware(app, token) elif warn_unauthenticated: @@ -465,3 +628,30 @@ def serve_http( file=sys.stderr, ) uvicorn.run(app, host=host, port=port) + + +def serve_http_multi( + brains_root: str, host: str = "0.0.0.0", port: int = 8080, + token: Optional[str] = None, read_only: bool = False, + public_base_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, +) -> None: + """Serve every brain under `brains_root` from one process.""" + try: + import uvicorn + except ImportError as exc: # pragma: no cover - optional dependency + raise SystemExit( + "the HTTP endpoint needs uvicorn: pip install 'open-index[serve]'" + ) from exc + + brains = discover_brains(brains_root) + if not brains: + raise SystemExit( + f"no brains found under {brains_root} — a brain is a directory " + "containing brain.yaml (create one with `open-index init`)" + ) + app = build_multi_app( + brains, token=token, read_only=read_only, + public_base_url=public_base_url, allowed_hosts=allowed_hosts, host=host, + ) + uvicorn.run(app, host=host, port=port) diff --git a/tests/test_cli_serving.py b/tests/test_cli_serving.py index a5f32ef..ff976aa 100644 --- a/tests/test_cli_serving.py +++ b/tests/test_cli_serving.py @@ -171,3 +171,89 @@ def test_serve_fails_cleanly_on_a_missing_brain(tmp_path, served): result = run("serve", "--brain", tmp_path / "nope") assert result.exit_code == 1 assert "no brain.yaml" in result.output + + +# -- serving many brains from one process -------------------------------------- + + +@pytest.fixture +def brains_root(tmp_path): + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + return root + + +@pytest.fixture +def served_multi(monkeypatch): + captured = {} + monkeypatch.setattr("open_index.mcp_server.serve_http_multi", + lambda root, **kw: captured.update(root=root, **kw)) + return captured + + +def test_brains_flag_serves_every_brain(brains_root, served_multi): + result = run("serve", "--brains", brains_root) + assert result.exit_code == 0 + assert served_multi["root"] == str(brains_root) + + +def test_multi_banner_lists_each_brain_url(brains_root, served_multi): + result = run("serve", "--brains", brains_root, "--public-url", "https://x.example.com") + assert "2 brains" in result.stdout + assert "https://x.example.com/alpha/mcp" in result.stdout + assert "https://x.example.com/beta/mcp" in result.stdout + + +def test_multi_banner_flags_ungated_brains(brains_root, served_multi, monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + result = run("serve", "--brains", brains_root) + assert "OPEN" in result.stdout, "an unauthenticated brain must be called out" + + +def test_multi_banner_shows_a_gated_brain_as_tokened(brains_root, served_multi, monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "t") + result = run("serve", "--brains", brains_root) + assert "token" in result.stdout + + +def test_multi_banner_points_at_the_directory_and_health(brains_root, served_multi): + result = run("serve", "--brains", brains_root) + assert "directory:" in result.stdout and "healthz" in result.stdout + + +def test_brains_root_with_no_brains_is_a_clean_error(tmp_path, served_multi): + empty = tmp_path / "empty" + empty.mkdir() + result = run("serve", "--brains", empty) + assert result.exit_code == 1 + assert "no brains under" in result.output + + +def test_serve_http_multi_starts_uvicorn(brains_root, monkeypatch): + import uvicorn + + from open_index import mcp_server + + captured = {} + monkeypatch.setattr(uvicorn, "run", + lambda app, host, port: captured.update(app=app, port=port)) + mcp_server.serve_http_multi(str(brains_root), port=9001) + assert captured["port"] == 9001 + assert captured["app"] is not None + + +def test_serve_http_multi_refuses_an_empty_root(tmp_path, monkeypatch): + import uvicorn + + from open_index import mcp_server + + monkeypatch.setattr(uvicorn, "run", lambda *a, **k: None) + empty = tmp_path / "none" + empty.mkdir() + with pytest.raises(SystemExit, match="no brains found"): + mcp_server.serve_http_multi(str(empty)) diff --git a/tests/test_mcp_transport.py b/tests/test_mcp_transport.py index 0ffafc5..290d42e 100644 --- a/tests/test_mcp_transport.py +++ b/tests/test_mcp_transport.py @@ -198,3 +198,106 @@ def no_uvicorn(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", no_uvicorn) with pytest.raises(SystemExit, match="open-index\\[serve\\]"): serve_http(str(brain.config.root)) + + +# -- proxy / load-balancer host validation ------------------------------------ +# +# The SDK turns on DNS-rebinding protection with a localhost-only allow-list, so +# every request arriving through a reverse proxy is rejected with +# "421 Invalid Host header". These pin the allow-list we build instead. + + +def test_public_url_host_is_allowed(): + from open_index.mcp_server import build_transport_security + + settings = build_transport_security(public_url="http://203.0.113.10/support/mcp") + assert settings.enable_dns_rebinding_protection is True + assert "203.0.113.10" in settings.allowed_hosts + + +def test_both_with_and_without_port_are_allowed(): + """A proxy drops the port when it is the scheme default; a direct client + sends it. Both must pass or one of the two paths breaks.""" + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_hosts + assert "brain.acme.com" in hosts + assert "brain.acme.com:*" in hosts + + +def test_explicit_host_and_port_survives(): + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security( + public_url="https://brain.acme.com:8443/mcp").allowed_hosts + assert "brain.acme.com:8443" in hosts + assert "brain.acme.com" in hosts + + +def test_loopback_is_always_allowed(): + """Health checks and on-box debugging must keep working.""" + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_hosts + assert {"127.0.0.1", "127.0.0.1:*", "localhost", "localhost:*"} <= set(hosts) + + +def test_extra_allowed_hosts_are_honoured(): + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(allowed_hosts=["lb.internal", "other:9000"]).allowed_hosts + assert "lb.internal" in hosts and "other:9000" in hosts + + +def test_wildcard_disables_the_check(): + """Explicit opt-out for a trusted proxy that already validates Host.""" + from open_index.mcp_server import build_transport_security + + assert build_transport_security( + allowed_hosts=["*"]).enable_dns_rebinding_protection is False + + +def test_origins_cover_http_and_https(): + from open_index.mcp_server import build_transport_security + + origins = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_origins + assert "https://brain.acme.com" in origins + assert "http://brain.acme.com" in origins + + +def test_blank_entries_are_ignored(): + from open_index.mcp_server import build_transport_security + + settings = build_transport_security(allowed_hosts=["", " "]) + assert settings.enable_dns_rebinding_protection is True + + +def test_serve_http_passes_the_allow_list_through(brain, monkeypatch): + """The settings must actually reach streamable_http_app.""" + import uvicorn + + captured = {} + monkeypatch.setattr(uvicorn, "run", lambda app, host, port: None) + + from open_index import mcp_server + + real = mcp_server.build_server + + def spy(b, read_only=False): + server = real(b, read_only=read_only) + original = server.streamable_http_app + + def wrapper(**kwargs): + captured.update(kwargs) + return original(**kwargs) + + server.streamable_http_app = wrapper + return server + + monkeypatch.setattr(mcp_server, "build_server", spy) + mcp_server.serve_http(str(brain.config.root), host="0.0.0.0", + public_url="https://brain.acme.com/mcp", + warn_unauthenticated=False) + + assert captured["host"] == "0.0.0.0" + assert "brain.acme.com" in captured["transport_security"].allowed_hosts diff --git a/tests/test_multi_brain_serve.py b/tests/test_multi_brain_serve.py new file mode 100644 index 0000000..6abb4c8 --- /dev/null +++ b/tests/test_multi_brain_serve.py @@ -0,0 +1,218 @@ +"""Serving many brains from one process. + +A process per brain re-loads the Python runtime and a ~250MB resident embedding +model each time, which caps a modest host at a handful. These cover the mounted +app: routing, per-brain isolation, per-brain auth, and the lifespan wiring that +the MCP transport needs. +""" + +import json +import shutil +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") +pytest.importorskip("starlette") + +from starlette.testclient import TestClient # noqa: E402 + +from open_index.mcp_server import ( # noqa: E402 + build_multi_app, + discover_brains, + token_for, +) + +EXAMPLE = Path(__file__).resolve().parent.parent / "examples" / "support-brain" + +HANDSHAKE = { + "jsonrpc": "2.0", "id": "1", "method": "initialize", + "params": {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}, +} +HEADERS = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream"} + + +def client_for(app): + """A test client whose Host header is one the server accepts. + + Starlette's default is `testserver`, which the SDK's DNS-rebinding + protection rightly rejects with 421 — the same check that makes serving + behind a proxy require an explicit allow-list. + """ + return TestClient(app, base_url="http://localhost") + + +@pytest.fixture +def brains_root(tmp_path): + """Two independent brains side by side, plus a decoy directory.""" + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + (root / "not-a-brain").mkdir() # no brain.yaml + (root / "notes.txt").write_text("x") # not a directory + return root + + +# -- discovery ---------------------------------------------------------------- + + +def test_discovers_only_directories_holding_a_brain_yaml(brains_root): + assert sorted(discover_brains(str(brains_root))) == ["alpha", "beta"] + + +def test_discovery_rejects_a_missing_root(tmp_path): + with pytest.raises(SystemExit, match="no such directory"): + discover_brains(str(tmp_path / "nope")) + + +def test_empty_root_discovers_nothing(tmp_path): + assert discover_brains(str(tmp_path)) == {} + + +# -- per-brain tokens --------------------------------------------------------- + + +def test_token_comes_from_a_per_brain_variable(monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_SALES_EU", "specific") + assert token_for("sales-eu", "shared") == "specific" + + +def test_token_falls_back_to_the_shared_one(monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN_SALES", raising=False) + assert token_for("sales", "shared") == "shared" + + +def test_no_token_at_all_is_none(monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN_SALES", raising=False) + assert token_for("sales") is None + + +# -- routing ------------------------------------------------------------------ + + +def test_each_brain_answers_on_its_own_path(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + for name in ("alpha", "beta"): + response = client.post(f"/{name}/mcp", json=HANDSHAKE, headers=HEADERS) + assert response.status_code == 200, response.text + assert "serverInfo" in response.text + + +def test_an_unknown_brain_path_is_a_404(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + assert client.post("/ghost/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 404 + + +def test_the_directory_lists_every_brain(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + listing = client.get("/").json() + assert set(listing) == {"alpha", "beta"} + assert listing["alpha"]["entities"] > 0 + assert "issue" in listing["alpha"]["doc_types"] + + +def test_health_is_unauthenticated_and_leaks_nothing(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), token="secret") + with client_for(app) as client: + response = client.get("/healthz") + assert response.status_code == 200 + assert response.text.strip() == "ok" + + +def test_public_base_url_is_advertised_per_brain(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), + public_base_url="https://x.example.com") + with client_for(app) as client: + listing = client.get("/").json() + assert listing["alpha"]["mcp"] == "https://x.example.com/alpha/mcp" + + +# -- isolation ---------------------------------------------------------------- + + +def test_a_brains_token_does_not_open_another(brains_root, monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "alpha-token") + monkeypatch.setenv("OPEN_INDEX_TOKEN_BETA", "beta-token") + app = build_multi_app(discover_brains(str(brains_root))) + + with client_for(app) as client: + ok = client.post("/alpha/mcp", json=HANDSHAKE, + headers={**HEADERS, "Authorization": "Bearer alpha-token"}) + crossed = client.post("/beta/mcp", json=HANDSHAKE, + headers={**HEADERS, "Authorization": "Bearer alpha-token"}) + assert ok.status_code == 200 + assert crossed.status_code == 401 + + +def test_an_untokened_brain_stays_open_when_others_are_gated(brains_root, monkeypatch): + """A missing per-brain token must not silently inherit a neighbour's.""" + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "alpha-token") + monkeypatch.delenv("OPEN_INDEX_TOKEN_BETA", raising=False) + app = build_multi_app(discover_brains(str(brains_root))) + + with client_for(app) as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 401 + assert client.post("/beta/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 + + +def test_writing_to_one_brain_does_not_touch_the_other(brains_root): + """Separate storage, not a shared index with a tenant column.""" + from open_index.brain import Brain + from open_index.models import Entity + + Brain.open(brains_root / "alpha").put_entity( + Entity(id="issue:only-alpha", doc_type="issue", name="Only Alpha")) + + assert Brain.open(brains_root / "alpha").get_entity("issue:only-alpha") is not None + assert Brain.open(brains_root / "beta").get_entity("issue:only-alpha") is None + + +def test_read_only_applies_to_every_mounted_brain(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), read_only=True) + with client_for(app) as client: + assert client.get("/").json()["alpha"]["read_only"] is True + + +# -- the shared embedding model, which is the whole point --------------------- + + +def test_all_brains_share_one_embedding_provider(brains_root): + """One process per brain re-loads the model each time; that duplication is + what caps a host at a handful of brains.""" + import open_index.embeddings as emb + from open_index.config import load_brain_config + + emb._provider_cache.clear() + for name in ("alpha", "beta"): + emb.get_embedding_provider(load_brain_config(brains_root / name)) + + assert len(emb._provider_cache) == 1, ( + "each brain built its own provider — the model would be loaded per brain") + + +def test_lifespan_starts_every_mounted_session_manager(brains_root): + """Starlette does not run a mounted app's lifespan. Without wiring it up, + requests fail with 'Task group is not initialized' — so this asserts a real + request works for the *last* mount, not just the first.""" + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + assert client.post("/beta/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 + + +def test_a_foreign_host_header_is_still_rejected(brains_root): + """The proxy allow-list applies per mount, not only to a single-brain serve.""" + app = build_multi_app(discover_brains(str(brains_root))) + with TestClient(app, base_url="http://evil.example.com") as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 421 + + +def test_the_public_host_is_accepted(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), + public_base_url="https://brain.acme.com") + with TestClient(app, base_url="https://brain.acme.com") as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 From 52063cdd15f1385bd8ec9a38fbd4dc9755145343 Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Mon, 10 Aug 2026 09:05:45 +0530 Subject: [PATCH 2/4] =?UTF-8?q?feat(ui):=20a=20usable=20explorer=20?= =?UTF-8?q?=E2=80=94=20help=20first,=20schema,=20readable=20map?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old UI was only useful for search: it showed no structure and no way to see the graph. Rebuilt around what a first-time reader needs, in order: How to use? the default tab — what a doc_type and an entity are, that relationships are optional, the MCP endpoint, and the tool list Schema every doc_type and its fields, so the shape is visible at all Explore search, unchanged in spirit Map the whole index, not just one entity's neighbourhood The map previously drew a label on every node, which at any real size is an unreadable wall of text. Nodes are now bare and identify themselves on hover. Presentation logic lives in view.py with no Streamlit import, so it is testable directly rather than through the app harness. Two fixes that only appear once one process serves many brains: - the index comes from the URL path, and nothing else selects it. Streamlit's own router resolves programmatic pages by an internal identifier, so every path silently rendered whichever brain sorted first; a curl status check cannot catch this because Streamlit returns its shell for any path. Read st.context.url instead. - the help tab derives its MCP endpoint from the page URL. A shared explorer cannot be handed one correct URL as configuration — the answer depends on which index you are looking at. Deriving it also keeps it right behind any proxy or hostname. OPEN_INDEX_PUBLIC_URL remains the fallback for single-brain deployments, where the path carries no index name. Co-Authored-By: Claude Opus 5 (1M context) --- open_index/graph.py | 56 ++++++++ open_index/ui/app.py | 295 +++++++++++++++++++++++++++++++++--------- open_index/ui/view.py | 278 ++++++++++++++++++++++++++++++++++++++- tests/test_graph.py | 64 +++++++++ tests/test_ui_app.py | 155 +++++++++++++++++++--- tests/test_ui_view.py | 245 +++++++++++++++++++++++++++++++++++ 6 files changed, 1019 insertions(+), 74 deletions(-) diff --git a/open_index/graph.py b/open_index/graph.py index 2555bb2..fd3e57b 100644 --- a/open_index/graph.py +++ b/open_index/graph.py @@ -69,6 +69,62 @@ def _label_for(brain: Brain, entity) -> str: return entity.label_for(label_field) +def build_overview_graph( + brain: Brain, + doc_types: Optional[list[str]] = None, + limit: int = 250, +) -> ContextGraph: + """Every entity of the given doc_types, with the edges that run between them. + + The anchor-and-expand view answers "what is around this one thing?". This + answers the other question — "what does this index look like?" — which is + what you want when you have never seen the index before and have no entity + in mind to anchor on. + + Only edges whose *both* ends are in scope are drawn: a dangling half-edge to + a filtered-out doc_type would render as an unexplained line to nowhere. + + `limit` caps the node count, keeping the most-connected entities. A force + layout with thousands of nodes is neither readable nor quick, and the + best-connected ones are the ones worth seeing. Callers should report when + the cap bit — silently showing a subset reads as "this is everything". + """ + entities = brain.backend.all_entities(doc_types) + + degree: dict[str, int] = {e.id: 0 for e in entities} + for entity in entities: + for rel in entity.related_to: + degree[entity.id] = degree.get(entity.id, 0) + 1 + if rel.target in degree: + degree[rel.target] = degree.get(rel.target, 0) + 1 + + if len(entities) > limit: + entities = sorted(entities, key=lambda e: (-degree.get(e.id, 0), e.id))[:limit] + + in_scope = {e.id for e in entities} + graph = ContextGraph(anchors=[]) + for entity in entities: + graph.nodes.append(GraphNode( + id=entity.id, + label=_label_for(brain, entity), + doc_type=entity.doc_type, + color=_color_for(brain, entity.doc_type), + depth=0, + data=entity.to_json(), + )) + + seen: set[tuple[str, str, str]] = set() + for entity in entities: + for source, target, meaning in brain.backend.relationships_from(entity.id): + if target not in in_scope: + continue + key = (source, target, meaning) + if key not in seen: + seen.add(key) + graph.edges.append(GraphEdge(source, target, meaning)) + return graph + + def build_graph(brain: Brain, anchors: str | list[str], depth: int = 1) -> ContextGraph: """Build the map around one or more anchor entities. diff --git a/open_index/ui/app.py b/open_index/ui/app.py index e181efb..8fef0c7 100644 --- a/open_index/ui/app.py +++ b/open_index/ui/app.py @@ -17,12 +17,13 @@ from __future__ import annotations import os +from typing import Optional from time import perf_counter import streamlit as st from open_index.brain import Brain -from open_index.graph import ContextGraph, build_graph +from open_index.graph import ContextGraph, build_graph, build_overview_graph from open_index.ui import view st.set_page_config(page_title="Open Index", page_icon="🧠", layout="wide") @@ -33,6 +34,50 @@ def _open_brain(brain_dir: str) -> Brain: return Brain.open(brain_dir) +@st.cache_resource +def _available_brains(root: str) -> dict: + """Brains under a root, cached so every rerun does not re-stat the disk.""" + from open_index.config import discover_brains + + return {name: str(path) for name, path in discover_brains(root).items()} + + +def _current_url(): + """The URL the browser is on, as reported over the websocket. + + Absent on older Streamlit versions and in bare script runs, so callers must + tolerate None. + """ + try: + return st.context.url + except Exception: + return None + + +def _select_brain(): + """The brain this page shows, chosen solely by the URL path. + + With OPEN_INDEX_BRAINS_ROOT set, one process serves every brain under it — + the alternative is a process, and a ~250MB embedding model, per brain. The + index is the first path segment, so / serves . There is no + in-page switcher on purpose: the URL is the selector, which keeps the + address bar and the screen in agreement and makes every view shareable. + """ + root = os.environ.get("OPEN_INDEX_BRAINS_ROOT") + if not root: + return _open_brain(os.environ.get("OPEN_INDEX_DIR", ".")), None + + brains = _available_brains(root) + if not brains: + st.error(f"No brains found under `{_esc(root)}`.") + st.caption("A brain is a directory containing `brain.yaml`. Create one " + "with `open-index init `.") + st.stop() + + chosen = view.brain_from_url(_current_url(), list(brains)) + # The directory name is also the URL segment, so it is what builds this + # index's endpoint — the brain's configured name may differ from it. + return _open_brain(brains[chosen]), chosen def _theme_type() -> str: @@ -237,48 +282,59 @@ def render_entity(brain: Brain, entity_id: str) -> None: # --------------------------------------------------------------------------- # def render_map(brain: Brain) -> None: + """The whole index at a glance: every entity, coloured by doc_type. + + Anchoring on one entity is the wrong default for someone who has never seen + the index — they have no entity in mind. This shows the shape of the whole + thing and lets them subtract from it. + """ summary = view.summarize(brain) populated = [r.name for r in summary.doc_types if r.count] if not populated: st.info("No entities to map yet. Add some and run `open-index index`.") return - chosen_types = st.multiselect( - "Doc types to include", populated, default=populated, - help="Narrow the map to the concepts you care about.", - ) - scope = chosen_types or populated - - # Auto-anchor rather than waiting for a selection. The old UI drew nothing - # until you picked entities, which looked like a broken canvas. - auto = view.default_anchors(brain, scope) - catalog = {f"{e.name} ({e.id})": e.id - for e in brain.backend.all_entities(scope)} - auto_labels = [label for label, eid in catalog.items() if eid in auto] - - picked = st.multiselect( - "Anchors", list(catalog), default=auto_labels, - help="Starting points. Click any node in the map to expand it.", + chosen = st.multiselect( + "Doc types shown", populated, default=populated, + help="Uncheck a type to drop it and its edges from the map.", ) - anchors = [catalog[label] for label in picked] - - expanded = st.session_state.get("expanded", set()) - if expanded: - cols = st.columns([4, 1]) - cols[0].caption(f"expanded: {', '.join(sorted(expanded))}") - if cols[1].button("↺ reset"): - st.session_state["expanded"] = set() + scope = chosen or populated + + focus = st.session_state.get("map_focus") + if focus: + cols = st.columns([5, 1]) + cols[0].caption(f"Focused on `{focus}` and its immediate neighbours.") + if cols[1].button("↩ show all"): + st.session_state["map_focus"] = None st.rerun() - - anchors = list(dict.fromkeys(anchors + list(expanded))) - if not anchors: - st.info("Select at least one anchor above.") - return - - graph = build_graph(brain, anchors, depth=1) - st.caption(f"{len(graph.nodes)} nodes · {len(graph.edges)} edges — " - "click a node to expand its relationships") - render_graph(brain, graph) + graph = build_graph(brain, [focus], depth=1) + else: + graph = build_overview_graph(brain, scope, limit=view.MAX_GRAPH_NODES) + + total = sum(r.count for r in summary.doc_types if r.name in scope) + if not focus and len(graph.nodes) < total: + st.warning( + f"Showing the {len(graph.nodes)} most-connected of {total} entities. " + "Narrow the doc types above to see the rest." + ) + + canvas, legend = st.columns([4, 1]) + with canvas: + st.caption(f"{len(graph.nodes)} nodes · {len(graph.edges)} edges — " + "hover for detail, click a node to focus on it") + render_graph(brain, graph) + with legend: + st.markdown("**Legend**") + for row in view.legend_rows(brain, graph): + st.markdown( + f"{_dot(row['color'])} `{_esc(row['doc_type'])}`" + f" {row['count']}", + unsafe_allow_html=True, + ) + if graph.edges: + st.markdown("---") + st.caption("Edges are `related_to` links. Hover one to see which " + "relationship it is.") # streamlit-agraph doesn't paint on its first render inside a tab (the canvas # mounts with zero size). One forced rerun remounts it with the tab active. @@ -298,24 +354,8 @@ def render_graph(brain: Brain, graph: ContextGraph) -> None: palette = view.graph_theme(_theme_type()) - nodes = [ - Node(id=n.id, label=n.label, color=n.color, - size=22 if n.is_anchor else 16, shape="dot", - title=f"{n.doc_type} · {n.id}", - # strokeWidth 0 removes vis's white halo, which on a dark canvas - # turns every label into heavy outlined text. - font={"color": palette["node_label"], "size": 15, - "strokeWidth": palette["stroke_width"], "face": "sans-serif"}) - for n in graph.nodes - ] - edges = [ - Edge(source=e.source, target=e.target, label=e.meaning, - color=palette["edge"], - font={"color": palette["edge_label"], "size": 12, - "strokeWidth": palette["stroke_width"], "align": "middle", - "face": "sans-serif"}) - for e in graph.edges - ] + nodes = [Node(**spec) for spec in view.graph_node_specs(graph)] + edges = [Edge(**spec) for spec in view.graph_edge_specs(graph, palette["edge"])] busy = len(graph.nodes) > view.BUSY_GRAPH_NODES config = Config( @@ -335,8 +375,8 @@ def render_graph(brain: Brain, graph: ContextGraph) -> None: _left, middle, _right = st.columns([1, 20, 1]) with middle: clicked = agraph(nodes=nodes, edges=edges, config=config) - if clicked and clicked not in st.session_state.get("expanded", set()): - st.session_state.setdefault("expanded", set()).add(clicked) + if clicked and clicked != st.session_state.get("map_focus"): + st.session_state["map_focus"] = clicked st.rerun() @@ -407,6 +447,139 @@ def render_analytics(brain: Brain) -> None: } for e in events], hide_index=True, use_container_width=True) +def render_schema(brain: Brain) -> None: + """Every doc_type and the shape of it — the reference before you write.""" + summary = view.summarize(brain) + if not summary.has_schema: + st.info("No doc_types defined yet.") + st.caption("A doc_type is a concept this index tracks, plus the fields it " + "stores. Create one with `open-index add-doc-type`, or ask an " + "agent to call `create_doc_type`.") + return + + st.caption( + f"{len(summary.doc_types)} doc_types · {summary.total_entities:,} entities. " + "Every entity id is `:`, and any entity can link to any " + "other through `related_to`." + ) + + for row in summary.doc_types: + doc_type = brain.config.doc_type(row.name) + noun = "entity" if row.count == 1 else "entities" + with st.expander(f"{row.name} · {row.count:,} {noun}", + expanded=len(summary.doc_types) <= 3): + if row.description: + st.markdown(row.description) + st.caption( + f"{_dot(row.color)} source of truth: " + + ("**files** — JSON under `entities/`, git-trackable" + if row.storage == "file" + else "**search index** — DB-owned, not written to files"), + unsafe_allow_html=True, + ) + + fields = view.schema_field_rows(doc_type) + if fields: + st.markdown("**Fields**") + st.table(fields) + else: + st.caption("No fields declared.") + + relationships = view.schema_relationship_rows(brain, row.name) + st.markdown("**Relationships (optional)**") + if relationships: + st.table(relationships) + st.caption("Declared edges are validated against their target " + "doc_type. Undeclared ones still work — they just " + "aren't checked.") + else: + st.caption("None declared or in use. Entities of this type are " + "valid without any; edges are what make the index " + "traversable rather than just searchable.") + + +def render_how_to_use(brain: Brain, url_name: Optional[str] = None) -> None: + """Connecting an agent, the tools it gets, and what the other tabs are for. + + Deliberately the rightmost tab: it is the page people come back to, not the + one they start on. + """ + summary = view.summarize(brain) + # Derived from the URL this page is on, so it is right for *this* index and + # needs no configuration. Falls back to an explicitly configured URL for a + # single-brain deployment, where the page path carries no index name. + mcp_url = (view.mcp_url_for(_current_url(), url_name) + or os.environ.get("OPEN_INDEX_PUBLIC_URL", "")) + read_only = os.environ.get("OPEN_INDEX_READ_ONLY", "").lower() in ("1", "true", "yes") + + st.markdown("### What an index holds") + st.caption( + "Three ideas, and the whole thing follows from them." + ) + for term, what in view.MODEL_GUIDE: + st.markdown(f"- **{term}** — {what}") + + st.divider() + st.markdown(f"### Connect an agent to `{_esc(summary.name)}`") + st.caption( + "This index speaks MCP, so any MCP-capable agent can query it — and, " + "unless the endpoint is read-only, keep it current." + ) + + if mcp_url: + st.markdown("**1. Point your agent at this URL**") + st.code(view.mcp_client_config(mcp_url, server_name=summary.name), language="json") + st.caption("Paste into `.mcp.json` (Claude Code), `.cursor/mcp.json` (Cursor), " + "or any MCP client's server config. Or generate it:") + st.code(f"open-index mcp-config --url {mcp_url} --name {summary.name} > .mcp.json", + language="bash") + else: + st.info("This explorer isn't configured with a public MCP URL " + "(`OPEN_INDEX_PUBLIC_URL`), so the connection block can't be shown.") + st.code("open-index mcp-config --brain > .mcp.json", language="bash") + + st.markdown("**2. Ask it something**") + st.caption("No briefing needed — the navigation guide below is injected into the " + "MCP handshake, so the agent knows this index's doc_types and " + "relationship vocabulary before its first turn.") + + st.divider() + st.markdown("### Tools the agent gets") + + st.markdown("**Reading**") + for name, what in view.READ_TOOLS: + st.markdown(f"- `{name}` — {what}") + + st.markdown("**Writing**") + if read_only: + st.info("This endpoint is **read-only** — the write tools below are not " + "registered on it. Serve without `--read-only` to enable them.") + for name, what in view.WRITE_TOOLS: + st.markdown(f"- `{name}` — {what}") + st.caption("Entity ids are always `:`. Writes are validated " + "against the doc_type schema, and land in the search index (and on " + "disk, for `storage: file` types).") + + st.divider() + st.markdown("### What each tab does") + for name, what in view.TAB_GUIDE: + st.markdown(f"- **{name}** — {what}") + + st.divider() + st.markdown("### What's in this index right now") + st.caption(f"{summary.total_entities:,} entities across " + f"{len(summary.doc_types)} doc_types.") + if summary.doc_types: + st.table([{"doc_type": r.name, "entities": r.count, + "source of truth": "files (git)" if r.storage == "file" else "search index", + "what it holds": r.description or "—"} + for r in summary.doc_types]) + + with st.expander("The full navigation guide the agent receives"): + st.code(brain.navigation_guidelines(include_writes=not read_only), + language="markdown") + + def render_jobs(brain: Brain) -> None: from open_index.connectors.runner import discover_connectors from open_index.scheduling import RunState @@ -443,13 +616,19 @@ def render_jobs(brain: Brain) -> None: def main() -> None: - brain = _open_brain(os.environ.get("OPEN_INDEX_DIR", ".")) + brain, url_name = _select_brain() st.markdown(view.ROW_CSS, unsafe_allow_html=True) options = render_sidebar(brain) - tab_explore, tab_map, tab_analytics, tab_jobs = st.tabs( - ["Explore", "Map", "Analytics", "Jobs"] + # Streamlit opens the first tab, so the help page leftmost means a visitor + # lands on the explanation rather than having to find it. + tab_help, tab_schema, tab_explore, tab_map, tab_analytics, tab_jobs = st.tabs( + [name for name, _ in view.TAB_GUIDE] ) + with tab_help: + render_how_to_use(brain, url_name) + with tab_schema: + render_schema(brain) with tab_explore: render_explore(brain, options) with tab_map: diff --git a/open_index/ui/view.py b/open_index/ui/view.py index e3ba929..bc79714 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -67,6 +67,30 @@ def has_schema(self) -> bool: return bool(self.doc_types) +def brain_from_url(url: Optional[str], available: list[str]) -> Optional[str]: + """Which brain a URL asks for: the first path segment. + + `/support-index` and `/support-index/anything` both select `support-index`, so + the index name lives in the path exactly as it does for the MCP endpoint at + `//mcp`. + + Read from the browser's URL rather than left to Streamlit's own page router: + that router resolves programmatic pages by an internal identifier, so every + path rendered whichever brain sorted first. An unknown or missing segment + falls back to the first brain — a stale link should land somewhere useful + rather than on an error. + """ + if not available: + return None + if not url: + return available[0] + + from urllib.parse import urlparse + + first = urlparse(url).path.strip("/").split("/")[0] + return first if first in available else available[0] + + def color_for(brain: Brain, doc_type: str) -> str: dt = brain.config.doc_type(doc_type) return dt.display.color if dt else DEFAULT_COLOR @@ -207,7 +231,9 @@ def semantic_weight_for(mode: str) -> Optional[float]: # does `f"{width}px"`, so a CSS string like "100%" becomes the invalid value # "100%px", the canvas fails to size, and the graph is stranded in a corner # instead of centred. -GRAPH_WIDTH = 1200 +# Sized to sit beside the legend column rather than to fill the page: at the +# old full-page width the canvas overflowed its column once the legend arrived. +GRAPH_WIDTH = 950 GRAPH_HEIGHT = 650 # Past this many nodes a force layout keeps drifting, so we slow it down rather @@ -262,6 +288,256 @@ def semantic_weight_for(mode: str) -> Optional[float]: """ +def mcp_url_for(current_url: Optional[str], name: Optional[str]) -> Optional[str]: + """This index's MCP endpoint, derived from the URL the browser is on. + + An explorer serving many brains cannot be handed one correct endpoint as + configuration — the answer depends on which index you are looking at. But + the page already knows: it is at `/`, so the endpoint is + `//mcp` on the same origin. Deriving it also means it stays right + behind any proxy or hostname without anything being configured. + + Returns None when the URL or the index name is unknown, so callers can fall + back to explicit configuration. + """ + if not current_url or not name: + return None + + from urllib.parse import urlparse + + parsed = urlparse(current_url) + if not parsed.scheme or not parsed.netloc: + return None + return f"{parsed.scheme}://{parsed.netloc}/{name}/mcp" + + +def mcp_client_config(url: str, server_name: str = "open-index") -> str: + """The JSON block to paste into an agent, as displayed in the How to use tab.""" + import json + + from open_index.mcp_config import normalize_mcp_url + + return json.dumps( + {"mcpServers": {server_name: {"type": "http", "url": normalize_mcp_url(url)}}}, + indent=2, + ) + + +# The three ideas someone needs before anything else on the page makes sense. +# Spelled out because "doc_type" means nothing to a first-time visitor, and the +# help tab is what the app opens on. +MODEL_GUIDE = [ + ("doc_type", + "A **concept** this index tracks, and the fields kept for it — `issue`, " + "`carrier`, `aircraft`. It is the schema, not the data: defining one says " + "what an issue *is*, not that any exist. The Schema tab lists them all."), + ("entity (a doc)", + "**One instance** of a doc_type — one issue, one carrier, one aircraft. " + "Its id is always `:`, so `issue:payment-declined` is an " + "issue called `payment-declined`. This is the document you search for and " + "the agent reads."), + ("relationship", + "An **optional** link from one entity to another, with free text saying " + "what the link means — `product:checkout` → *has common issue* → " + "`issue:payment-declined`. Entities work perfectly well without any. Edges " + "are what turn a list of documents into something you can traverse, so an " + "index with none is a searchable table rather than a graph."), +] + + +# What each tool does, in the order an agent would reach for them. Kept here +# rather than in the page so the list can be asserted against the tools the MCP +# server actually registers — a stale tool list in the docs is worse than none. +READ_TOOLS = [ + ("navigation_guidelines()", + "The whole guide to this index: every doc_type, its fields, the " + "relationship vocabulary in use, and worked examples. Injected into the " + "MCP handshake, so an agent starts oriented."), + ("search_brain(query, doc_types, limit)", + "Free-text search, hybrid keyword + semantic. `doc_types` narrows it."), + ("get_entity(entity_id)", + "One entity with its outgoing AND incoming edges — the incoming direction " + "answers \"what else points at this?\"."), +] + +WRITE_TOOLS = [ + ("put_entity(doc_type, id, name, fields, related_to)", + "Add or update one entity. Upsert — the same id replaces it."), + ("put_entities([...])", + "A whole batch in one call, with optional shared provenance. Use this " + "rather than calling put_entity in a loop."), + ("create_doc_type(doc_type, description, fields, relationships, storage)", + "Define a new concept, when no existing doc_type fits."), +] + +# The tab label for the help page. Leftmost, so it is what a first-time visitor +# lands on, and phrased as the question they are actually asking. +HELP_TAB = "How to use?" + +# What each tab is for, in the order they appear. This list *is* the tab order — +# the page builds its tabs from it, so the two cannot drift apart. +TAB_GUIDE = [ + (HELP_TAB, + "This page: how to connect an agent, what tools it gets, and what " + "everything else here does."), + ("Schema", + "Every doc_type in this index and the shape of it — each field's type, how " + "it is searched, its ranking weight, and the relationship vocabulary that " + "connects types to each other. Read this before writing to the index."), + ("Explore", + "Search the index, or browse by doc_type. Open an entity to see its fields, " + "its attribution, and every relationship in both directions — click through " + "to walk the graph."), + ("Map", + "The same relationships drawn. It auto-anchors on the most-connected " + "entities so it shows something immediately; click any node to expand it, " + "and narrow by doc_type to cut the noise."), + ("Analytics", + "What has been asked of this index, by which client (CLI, agent, this UI) " + "and how often. Zero-result searches are the interesting number: they are " + "the questions this index cannot yet answer."), + ("Jobs", + "Connectors that pull entities in on a schedule, with their last run."), +] + + +def schema_field_rows(doc_type) -> list[dict]: + """One row per field, as the Schema tab tabulates them. + + `search` and `boost` are the two that change behaviour rather than just + describing it, so they are spelled out rather than shown as raw enum values. + """ + searchable = { + "syntactic": "keyword match", + "semantic": "meaning (vector)", + "none": "not searched", + } + rows = [] + for f in doc_type.fields: + rows.append({ + "field": f.name, + "type": f.type, + "searched by": searchable.get(f.search, f.search), + "weight": f"{f.boost:g}×" if f.search != "none" else "—", + "required": "yes" if f.required else "", + "notes": f.description or "", + }) + return rows + + +def schema_relationship_rows(brain, doc_type_name: str) -> list[dict]: + """Declared and observed edges for one doc_type, merged. + + Declared is the vocabulary the schema intends; observed is what the entities + actually use. Showing both together is the point — a declared edge with zero + uses and an undeclared edge in heavy use are both worth noticing. + """ + doc_type = brain.config.doc_type(doc_type_name) + observed = brain.observed_relationships(doc_type_name) + rows = [] + seen = set() + + for spec in (doc_type.relationships if doc_type else []): + seen.add(spec.name) + rows.append({ + "relationship": spec.name, + "points at": spec.target_doc_type or "any", + "declared": "yes", + "in use": observed.get(spec.name, 0), + }) + for meaning, count in sorted(observed.items(), key=lambda kv: -kv[1]): + if meaning not in seen: + rows.append({ + "relationship": meaning, + "points at": "—", + "declared": "no", + "in use": count, + }) + return rows + + +# Cap on nodes drawn at once. Past this a force layout stops settling and the +# picture stops being readable regardless. +MAX_GRAPH_NODES = 250 + + +def node_tooltip(entity_id: str, name: str, doc_type: str, + fields: Optional[dict] = None) -> str: + """What hovering a node shows: the full name, its type, and a few fields. + + vis renders this as plain text, so it is newline-separated rather than HTML. + """ + lines = [name, f"{doc_type} · {entity_id}"] + for key, value in list((fields or {}).items()): + if value in (None, "") or key == "name": + continue + text = " ".join(str(value).split()) + lines.append(f"{key}: {text[:90]}{'…' if len(text) > 90 else ''}") + if len(lines) >= 7: # a tooltip taller than the graph helps nobody + break + return "\n".join(lines) + + +def edge_tooltip(source: str, target: str, meaning: str) -> str: + return f"{source}\n —[{meaning or 'related'}]→\n{target}" + + +def legend_rows(brain, graph) -> list[dict]: + """Doc types present in a drawn graph, with their colour and node count. + + Built from the graph rather than the schema so it describes what is on + screen: a doc_type filtered out, or cut by the node cap, should not appear. + """ + counts: dict[str, int] = {} + for node in graph.nodes: + counts[node.doc_type] = counts.get(node.doc_type, 0) + 1 + return [ + {"doc_type": name, "count": counts[name], "color": color_for(brain, name)} + for name in sorted(counts, key=lambda n: (-counts[n], n)) + ] + + +def graph_node_specs(graph, anchor_size: int = 20, size: int = 13) -> list[dict]: + """Node payloads for the map renderer. + + Nothing is labelled on the canvas. Entity names are long and arbitrary — + drawn beside every dot they overlap each other and their own edges, and no + amount of truncation fixes a dense graph. The picture carries shape and + colour; identity comes from the tooltip, and the legend explains the colours. + + `label` is an empty string rather than None: None serialises to null and vis + draws that literally. + """ + specs = [] + for node in graph.nodes: + fields = {k: v for k, v in (node.data or {}).items() + if k not in ("id", "doc_type", "related_to")} + specs.append({ + "id": node.id, + "label": "", + "color": node.color, + "shape": "dot", + "size": anchor_size if node.is_anchor else size, + "title": node_tooltip(node.id, node.label, node.doc_type, fields), + }) + return specs + + +def graph_edge_specs(graph, color: str) -> list[dict]: + """Edge payloads. Unlabelled for the same reason as nodes — the relationship + is on the tooltip.""" + return [ + { + "source": edge.source, + "target": edge.target, + "label": "", + "title": edge_tooltip(edge.source, edge.target, edge.meaning), + "color": color, + } + for edge in graph.edges + ] + + def graph_theme(theme_type: Optional[str]) -> dict: """Label and edge colours for the map, given Streamlit's active theme. diff --git a/tests/test_graph.py b/tests/test_graph.py index 8a3fed3..f8d9671 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -58,3 +58,67 @@ def test_multiple_anchors(brain): def test_anchor_property(brain): g = build_graph(brain, ["product:checkout"], depth=1) assert g.anchor == "product:checkout" + + +# -- whole-index overview ------------------------------------------------------ + + +def test_overview_includes_every_entity_in_scope(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product"]) + assert {n.id for n in graph.nodes} == { + e.id for e in brain.backend.all_entities(["product"])} + + +def test_overview_only_draws_edges_with_both_ends_in_scope(brain): + """A half-edge to a filtered-out type would render as a line to nowhere.""" + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product"]) + ids = {n.id for n in graph.nodes} + assert all(e.source in ids and e.target in ids for e in graph.edges) + + +def test_overview_keeps_edges_between_included_types(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product", "issue"]) + assert graph.edges, "product→issue edges should survive when both are in scope" + + +def test_overview_colours_nodes_by_doc_type(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + by_type = {n.doc_type: n.color for n in graph.nodes} + assert len(set(by_type.values())) > 1, "doc_types should be visually distinct" + + +def test_overview_cap_keeps_the_most_connected(brain): + """When the cap bites, it must keep the hubs — a random subset of a graph + is far less informative than its best-connected part.""" + from open_index.graph import build_overview_graph + from open_index.ui.view import edge_counts + + kept = {n.id for n in build_overview_graph(brain, limit=3).nodes} + assert len(kept) == 3 + + degree = edge_counts(brain) + dropped = {e.id for e in brain.backend.all_entities()} - kept + assert min(degree.get(i, 0) for i in kept) >= max(degree.get(i, 0) for i in dropped) + + +def test_overview_of_an_empty_scope(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["nonexistent-type"]) + assert graph.nodes == [] and graph.edges == [] + + +def test_overview_deduplicates_edges(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + keys = [(e.source, e.target, e.meaning) for e in graph.edges] + assert len(keys) == len(set(keys)) diff --git a/tests/test_ui_app.py b/tests/test_ui_app.py index d5e8f6b..a20bb07 100644 --- a/tests/test_ui_app.py +++ b/tests/test_ui_app.py @@ -5,6 +5,7 @@ structure you had to hunt for, and a map that opened blank. """ +import os import shutil from pathlib import Path @@ -22,7 +23,9 @@ def _run(brain_dir, monkeypatch): monkeypatch.setenv("OPEN_INDEX_DIR", str(brain_dir)) - return AppTest.from_file(str(APP), default_timeout=60).run() + # Generous timeout: each of these boots a real Streamlit script run, and on + # a loaded machine (the full suite, or a CI runner) 60s flaked once. + return AppTest.from_file(str(APP), default_timeout=180).run() @pytest.fixture @@ -55,10 +58,6 @@ def test_empty_brain_runs_without_exceptions(empty): assert not empty.exception, [e.value for e in empty.exception] -def test_tabs(populated): - assert [t.label for t in populated.tabs] == ["Explore", "Map", "Analytics", "Jobs"] - - # -- structure is visible without hunting for it ------------------------------ @@ -86,20 +85,27 @@ def test_empty_brain_sidebar_explains_what_to_do(empty): # -- the map no longer opens blank -------------------------------------------- -def test_map_preselects_anchors(populated): - """The original complaint: nothing rendered until you made a selection.""" - anchors = next(m for m in populated.multiselect if m.label == "Anchors") - assert anchors.value, "map opened with no anchors selected" - - -def test_map_anchor_options_are_entity_labels(populated): - anchors = next(m for m in populated.multiselect if m.label == "Anchors") - assert all("(" in option and ")" in option for option in anchors.options) +def test_map_draws_without_any_selection(populated): + """The original complaint: nothing rendered until you made a selection. The + map now opens on the whole index, so there is always something to see.""" + populated.tabs[3].run() # Map + assert not populated.exception def test_map_doc_type_filter_defaults_to_everything(populated): - types = next(m for m in populated.multiselect if m.label == "Doc types to include") + types = next(m for m in populated.multiselect if m.label == "Doc types shown") assert set(types.value) == set(types.options) + assert types.options, "every populated doc_type should be filterable" + + +def test_map_filter_lists_only_populated_doc_types(populated, tmp_path): + """A doc_type with no entities would be a dead checkbox.""" + from open_index.brain import Brain + from open_index.schema import DocType + + types = next(m for m in populated.multiselect if m.label == "Doc types shown") + counts = Brain.open(os.environ["OPEN_INDEX_DIR"]).counts() + assert all(counts.get(name, 0) > 0 for name in types.options) # -- explore ------------------------------------------------------------------ @@ -198,3 +204,122 @@ def test_row_css_inherits_colour_through_the_label_element(): """Streamlit wraps button labels in

, which otherwise keeps its own colour and ignores the inherit on the button.""" assert "> button p{color:inherit" in view.ROW_CSS + + +# -- How to use ---------------------------------------------------------------- + + +def test_help_tab_explains_the_model_before_the_connection_block(populated): + """The vocabulary has to land before "point your agent at this URL" means + anything to a first-time visitor.""" + markdown = [m.value for m in populated.markdown] + headings = [m for m in markdown if m.startswith("###")] + assert any("What an index holds" in h for h in headings) + assert headings.index(next(h for h in headings if "What an index holds" in h)) \ + < headings.index(next(h for h in headings if "Connect an agent" in h)) + + +def test_schema_marks_relationships_optional(populated): + populated.tabs[1].run() # Schema + assert not populated.exception + assert any("Relationships (optional)" in m.value for m in populated.markdown) + + +def test_help_is_the_first_tab_so_it_opens_on_load(populated): + """Streamlit selects the first tab, so a first-time visitor lands on the + explanation instead of having to find it.""" + assert [t.label for t in populated.tabs][0] == view.HELP_TAB == "How to use?" + + +def test_schema_sits_between_help_and_explore(populated): + labels = [t.label for t in populated.tabs] + assert labels[:3] == ["How to use?", "Schema", "Explore"] + + +def test_tab_guide_matches_the_tabs_actually_rendered(populated): + """A stale list of what-each-tab-does is worse than none.""" + assert [t.label for t in populated.tabs] == [n for n, _ in view.TAB_GUIDE] + + +def test_documented_tools_match_the_registered_mcp_tools(brain): + """The tab must not advertise a tool the server does not expose, nor miss one.""" + pytest.importorskip("mcp") + import asyncio as _asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain) + registered = {t.name for t in + _asyncio.new_event_loop().run_until_complete(server.list_tools())} + documented = {name.split("(")[0] for name, _ in view.READ_TOOLS + view.WRITE_TOOLS} + assert documented == registered, ( + f"docs vs server mismatch: only-in-docs={documented - registered}, " + f"only-on-server={registered - documented}") + + +def test_read_only_tools_are_the_documented_read_set(brain): + pytest.importorskip("mcp") + import asyncio as _asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain, read_only=True) + registered = {t.name for t in + _asyncio.new_event_loop().run_until_complete(server.list_tools())} + assert {n.split("(")[0] for n, _ in view.READ_TOOLS} == registered + + +def test_connection_block_is_valid_json_with_the_url(): + import json + + block = json.loads(view.mcp_client_config("https://x.example.com/demo/mcp", "demo")) + assert block["mcpServers"]["demo"]["url"] == "https://x.example.com/demo/mcp" + assert block["mcpServers"]["demo"]["type"] == "http" + + +def test_connection_block_appends_the_mcp_path(): + import json + + block = json.loads(view.mcp_client_config("https://x.example.com/demo")) + assert block["mcpServers"]["open-index"]["url"].endswith("/mcp") + + +# -- one UI process serving many brains ---------------------------------------- + + +@pytest.fixture +def multi(tmp_path, monkeypatch): + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) + return AppTest.from_file(str(APP), default_timeout=180).run() + + +def test_an_empty_brains_root_explains_itself(tmp_path, monkeypatch): + empty = tmp_path / "none" + empty.mkdir() + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(empty)) + at = AppTest.from_file(str(APP), default_timeout=180).run() + assert any("No brains found" in e.value for e in at.error) + + +def test_single_brain_mode_shows_no_picker(populated): + """OPEN_INDEX_DIR alone must behave exactly as before.""" + assert not any(s.label.startswith("Index") for s in populated.sidebar.selectbox) + + +def test_many_brains_render_without_error(multi): + assert not multi.exception, [e.value for e in multi.exception] + + +def test_no_index_switcher_widget_anywhere(multi): + """The URL is the only selector; a dropdown would let the address bar and + the screen disagree about which index you are looking at.""" + assert not multi.sidebar.selectbox diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index eca21f6..28ff901 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -226,3 +226,248 @@ def test_every_theme_defines_the_full_palette(): keys = {"node_label", "edge_label", "edge", "stroke_width"} for theme in ("dark", "light"): assert keys <= set(view.graph_theme(theme)) + + +# -- Schema tab ---------------------------------------------------------------- + + +def test_schema_field_rows_describe_search_behaviour(brain): + dt = brain.config.doc_type("issue") + rows = {r["field"]: r for r in view.schema_field_rows(dt)} + assert rows["name"]["searched by"] == "keyword match" + assert rows["description"]["searched by"] == "meaning (vector)" + assert rows["name"]["weight"] == "6×" + + +def test_unsearched_fields_show_no_weight(brain): + from open_index.schema import DocType, FieldSpec + + dt = DocType(doc_type="t", fields=[FieldSpec(name="blob", search="none", boost=9)]) + row = view.schema_field_rows(dt)[0] + assert row["searched by"] == "not searched" + assert row["weight"] == "—", "a weight on an unsearched field is misleading" + + +def test_required_fields_are_marked(brain): + from open_index.schema import DocType, FieldSpec + + dt = DocType(doc_type="t", fields=[FieldSpec(name="owner", required=True)]) + assert view.schema_field_rows(dt)[0]["required"] == "yes" + + +def test_relationship_rows_merge_declared_and_observed(brain): + rows = {r["relationship"]: r for r in view.schema_relationship_rows(brain, "product")} + assert "has common issue" in rows + assert rows["has common issue"]["declared"] == "yes" + assert rows["has common issue"]["points at"] == "issue" + assert rows["has common issue"]["in use"] > 0 + + +def test_undeclared_relationships_in_use_are_still_listed(brain): + """An edge nobody declared but everything uses is worth seeing.""" + from open_index.models import Entity, Relationship + + brain.put_entity(Entity( + id="product:improvised", doc_type="product", name="Improvised", + related_to=[Relationship(target="issue:no-results", + relationship_edge_meaning="totally made up")])) + rows = {r["relationship"]: r for r in view.schema_relationship_rows(brain, "product")} + assert rows["totally made up"]["declared"] == "no" + assert rows["totally made up"]["in use"] == 1 + + +def test_declared_but_unused_relationships_show_zero(brain): + from open_index.schema import DocType, RelationshipSpec + + brain.create_doc_type(DocType( + doc_type="unused_rel", + relationships=[RelationshipSpec(name="never used", target_doc_type="issue")])) + row = view.schema_relationship_rows(brain, "unused_rel")[0] + assert row["declared"] == "yes" and row["in use"] == 0 + + +def test_help_tab_is_first_in_the_guide(): + assert view.TAB_GUIDE[0][0] == view.HELP_TAB + assert [n for n, _ in view.TAB_GUIDE][:3] == ["How to use?", "Schema", "Explore"] + + +# -- map readability ----------------------------------------------------------- +# +# Long entity names drawn next to their dot were the main thing making the map +# unreadable; the full text moves to the hover tooltip. + + +def test_nodes_carry_no_label(brain): + """Entity names are long and arbitrary; drawn beside every dot they overlap + each other. Identity comes from the tooltip instead.""" + from open_index.graph import build_overview_graph + + specs = view.graph_node_specs(build_overview_graph(brain)) + assert specs + assert all(s["label"] == "" for s in specs) + # Empty string, not None: None serialises to null and vis draws that. + assert all(s["label"] is not None for s in specs) + + +def test_every_node_carries_its_identity_on_hover(brain): + from open_index.graph import build_overview_graph + + specs = {s["id"]: s for s in view.graph_node_specs(build_overview_graph(brain))} + spec = specs["product:checkout"] + assert "Checkout" in spec["title"] + assert "product:checkout" in spec["title"] + + +def test_edges_carry_no_label_but_name_the_relationship(brain): + from open_index.graph import build_overview_graph + + specs = view.graph_edge_specs(build_overview_graph(brain), "#ccc") + assert specs + assert all(s["label"] == "" for s in specs) + assert any("has common issue" in s["title"] for s in specs) + + +def test_node_colour_comes_from_the_doc_type(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + by_id = {n.id: n.color for n in graph.nodes} + assert all(s["color"] == by_id[s["id"]] for s in view.graph_node_specs(graph)) + + +def test_node_tooltip_carries_the_full_name_and_type(): + tip = view.node_tooltip("issue:x", "A very long name that got truncated", + "issue", {"severity": "high"}) + assert "A very long name that got truncated" in tip + assert "issue:x" in tip + assert "severity: high" in tip + + +def test_node_tooltip_skips_empty_fields_and_caps_length(): + tip = view.node_tooltip("t:x", "N", "t", + {f"f{i}": "v" for i in range(20)} | {"blank": ""}) + assert "blank" not in tip + assert len(tip.splitlines()) <= 7 + + +def test_edge_tooltip_names_the_relationship(): + tip = view.edge_tooltip("a:1", "b:2", "depends on") + assert "depends on" in tip and "a:1" in tip and "b:2" in tip + + +def test_edge_tooltip_handles_an_unlabelled_edge(): + assert "related" in view.edge_tooltip("a:1", "b:2", "") + + +def test_legend_describes_what_is_on_screen(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product", "issue"]) + rows = view.legend_rows(brain, graph) + assert {r["doc_type"] for r in rows} <= {"product", "issue"} + assert all(r["color"].startswith("#") for r in rows) + assert sum(r["count"] for r in rows) == len(graph.nodes) + + +def test_legend_is_ordered_by_count(brain): + from open_index.graph import build_overview_graph + + rows = view.legend_rows(brain, build_overview_graph(brain)) + assert [r["count"] for r in rows] == sorted([r["count"] for r in rows], reverse=True) + + +def test_graph_width_fits_beside_the_legend(): + assert view.GRAPH_WIDTH <= 1000 + + +# -- the model explanation on the help tab ------------------------------------ + + +def test_model_guide_covers_the_three_ideas(): + terms = " ".join(t for t, _ in view.MODEL_GUIDE).lower() + assert "doc_type" in terms + assert "entity" in terms and "doc" in terms # both names for an instance + assert "relationship" in terms + + +def test_model_guide_says_relationships_are_optional(): + text = dict(view.MODEL_GUIDE)["relationship"].lower() + assert "optional" in text + assert "without any" in text, "should say entities are valid with no edges" + + +def test_model_guide_explains_the_id_convention(): + """The single most common write failure, so it belongs in the explanation.""" + text = " ".join(w for _, w in view.MODEL_GUIDE) + assert ":" in text + + +def test_model_guide_distinguishes_schema_from_data(): + text = dict(view.MODEL_GUIDE)["doc_type"].lower() + assert "schema" in text + + +# -- one UI process, many brains: the URL is the selector ---------------------- + + +def test_the_url_path_selects_the_brain(): + assert view.brain_from_url("https://h/sales-index", + ["support-index", "sales-index"]) == "sales-index" + + +def test_a_deeper_path_selects_on_the_first_segment(): + assert view.brain_from_url("https://h/sales-index/x", + ["alpha", "sales-index"]) == "sales-index" + + +def test_the_root_path_falls_back_to_the_first_brain(): + assert view.brain_from_url("https://h/", ["alpha", "beta"]) == "alpha" + + +def test_an_unknown_path_falls_back_rather_than_erroring(): + """A stale link should land somewhere useful.""" + assert view.brain_from_url("https://h/deleted", ["alpha", "beta"]) == "alpha" + + +def test_a_missing_url_falls_back(): + assert view.brain_from_url(None, ["alpha", "beta"]) == "alpha" + + +def test_a_query_string_does_not_affect_selection(): + assert view.brain_from_url("https://h/beta?x=1", ["alpha", "beta"]) == "beta" + + +def test_no_brains_selects_nothing(): + assert view.brain_from_url("https://h/x", []) is None + + +# -- the endpoint shown on the help tab ---------------------------------------- +# +# A shared explorer cannot be handed one correct MCP URL as configuration: the +# answer depends on which index you are looking at. It is derived from the page +# URL instead, which also keeps it right behind any proxy or hostname. + + +def test_endpoint_is_derived_from_the_page_url(): + assert view.mcp_url_for("https://brain.acme.com/sales-index", "sales-index") \ + == "https://brain.acme.com/sales-index/mcp" + + +def test_endpoint_ignores_the_rest_of_the_path_and_query(): + assert view.mcp_url_for("https://h:8443/sales-index/x?q=1", "sales-index") \ + == "https://h:8443/sales-index/mcp" + + +def test_endpoint_uses_the_directory_name_not_the_configured_brain_name(): + """The URL segment is the directory; brain.yaml's name may differ.""" + assert view.mcp_url_for("https://h/dir-name", "dir-name").endswith("/dir-name/mcp") + + +def test_endpoint_keeps_the_scheme(): + assert view.mcp_url_for("http://localhost:8501/a", "a").startswith("http://") + + +def test_no_endpoint_without_a_url_or_a_name(): + assert view.mcp_url_for(None, "a") is None + assert view.mcp_url_for("https://h/a", None) is None + assert view.mcp_url_for("/relative/path", "a") is None From ab1505f63182803ebc316e5e33e7df400920ac2c Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Mon, 10 Aug 2026 09:05:45 +0530 Subject: [PATCH 3/4] fix(opensearch): a numeric or date field no longer breaks every search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuzzy matching is only valid on keyword and text fields, so as soon as a brain had one numeric or date field OpenSearch rejected the whole query — every search, not just ones touching that field. Non-string fields are now excluded from the fuzzy field list, and date_detection is off so a string that happens to look like a date is not silently remapped. Writes also create the index if it is missing, rather than letting OpenSearch auto-create one with a guessed dynamic mapping that then behaves differently from a deliberately created index. Co-Authored-By: Claude Opus 5 (1M context) --- open_index/storage/opensearch_backend.py | 36 +++++++++++++++++ tests/test_opensearch_builders.py | 50 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/open_index/storage/opensearch_backend.py b/open_index/storage/opensearch_backend.py index cbb8349..92fc3bb 100644 --- a/open_index/storage/opensearch_backend.py +++ b/open_index/storage/opensearch_backend.py @@ -100,6 +100,11 @@ def mapping(dimension: int = 384) -> dict: return { "settings": {"index": {"knn": True}}, "mappings": { + # A field declared `string` whose values look like dates would + # otherwise be mapped as `date` by dynamic detection, and then + # rejected by fuzzy search. The schema decides the type here, + # not a guess from the first document indexed. + "date_detection": False, "properties": { "id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, @@ -137,6 +142,11 @@ def _base_mapping(self) -> dict: """Mapping without the embedding field (for keyword-only brains).""" return { "mappings": { + # A field declared `string` whose values look like dates would + # otherwise be mapped as `date` by dynamic detection, and then + # rejected by fuzzy search. The schema decides the type here, + # not a guess from the first document indexed. + "date_detection": False, "properties": { "id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, @@ -261,6 +271,13 @@ def _search_fields(self, doc_types: Optional[list[str]]) -> list[str]: continue if f.name == label: name_boost = max(name_boost, f.boost) + # Only text-ish fields belong in a fuzzy full-text query. + # OpenSearch maps a `number` field to long and a timestamp to + # date, and rejects the whole query with + # "Can only use fuzzy queries on keyword and text fields" if + # either is listed — so one numeric field breaks all search. + if f.type not in ("string", "text"): + continue boosts[f.name] = max(boosts.get(f.name, 1.0), f.boost) fields = [f"name^{name_boost:g}"] fields += [f"fields.{name}^{b:g}" for name, b in boosts.items()] @@ -373,9 +390,27 @@ def ensure_schema(self, doc_types: dict[str, DocType]) -> None: "is the cluster running? (search.backend: opensearch)" ) from exc + def _ensure_index_exists(self) -> None: + """Recreate the index with our mapping if it has gone missing. + + `ensure_schema` runs once when the brain is opened. If the index is + dropped after that — a cluster rebuild, a manual DELETE, a restored + snapshot — OpenSearch would auto-create it on the next write using + *dynamic* mapping, silently typing `doc_type` as text and breaking every + aggregation, and typing date-shaped strings as dates and breaking fuzzy + search. Both fail long after the write that caused them, so it is worth + one existence check on the write path. + """ + if self._client.indices.exists(index=self.index): + return + logger.warning("index %s was missing; recreating it with the schema mapping", + self.index) + self._client.indices.create(index=self.index, body=self._mapping()) + def upsert_entity(self, entity: Entity, doc_type: Optional[DocType] = None) -> None: if doc_type is not None: self._doc_types[doc_type.doc_type] = doc_type + self._ensure_index_exists() self._client.index( index=self.index, id=entity.id, body=self._doc_with_embedding(entity), refresh=True ) @@ -391,6 +426,7 @@ def upsert_many(self, items: list[tuple[Entity, Optional[DocType]]]) -> None: for _entity, doc_type in items: if doc_type is not None: self._doc_types[doc_type.doc_type] = doc_type + self._ensure_index_exists() payload: list[dict] = [] for entity, _doc_type in items: diff --git a/tests/test_opensearch_builders.py b/tests/test_opensearch_builders.py index e8eec6f..9b6dde2 100644 --- a/tests/test_opensearch_builders.py +++ b/tests/test_opensearch_builders.py @@ -153,3 +153,53 @@ def test_reserved_keys_cover_the_metadata_fields(): from open_index.storage.opensearch_backend import _RESERVED_KEYS assert {"provenance", "valid_from", "valid_to"} <= _RESERVED_KEYS + + +# -- fuzzy search must not be handed non-text fields --------------------------- +# +# OpenSearch rejects the entire query with "Can only use fuzzy queries on keyword +# and text fields" if a numeric or date field is listed in multi_match. One +# `number` field in the schema therefore broke *all* search — found with a real +# dataset, not caught by the earlier synthetic ones. + + +def test_numeric_fields_are_kept_out_of_the_fuzzy_query(): + dt = DocType(doc_type="defect", fields=[ + FieldSpec(name="name", type="string", boost=6), + FieldSpec(name="symptom", type="text"), + FieldSpec(name="manhours", type="number"), + ]) + fields = _backend_with_types([dt])._search_fields(None) + joined = " ".join(fields) + assert "fields.symptom" in joined + assert "manhours" not in joined, "a numeric field would break the whole query" + + +def test_boolean_and_timestamp_fields_are_excluded_too(): + dt = DocType(doc_type="t", fields=[ + FieldSpec(name="name", type="string"), + FieldSpec(name="active", type="boolean"), + FieldSpec(name="seen_at", type="timestamp"), + ]) + joined = " ".join(_backend_with_types([dt])._search_fields(None)) + assert "active" not in joined and "seen_at" not in joined + + +def test_text_fields_keep_their_boosts(): + dt = DocType(doc_type="t", fields=[ + FieldSpec(name="title", type="string", boost=6), + FieldSpec(name="body", type="text", boost=2), + FieldSpec(name="count", type="number", boost=9), + ]) + fields = _backend_with_types([dt])._search_fields(None) + assert "fields.title^6" in fields + assert "fields.body^2" in fields + assert not any("count" in f for f in fields) + + +def test_date_detection_is_disabled_in_the_mapping(): + """A `string` field holding e.g. 2026-04-12 would otherwise be mapped as a + date, and then rejected by fuzzy search.""" + assert OpenSearchBackend.mapping()["mappings"]["date_detection"] is False + backend = _backend_with_types([DocType(doc_type="t")]) + assert backend._base_mapping()["mappings"]["date_detection"] is False From 327d2c2e145c08db0ba1130814ab09ac70199773 Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Mon, 10 Aug 2026 09:05:45 +0530 Subject: [PATCH 4/4] docs: deploying many indexes behind one host How to run a directory of brains from one process, what each index's MCP endpoint looks like, and how to point an agent at one. Also ignores the rendered fleet deployment state: those files describe one specific machine and do not belong in the repo. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 8 ++++++++ README.md | 3 ++- docs/deployment.md | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48b835e..d92efbe 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,11 @@ dist/ .coverage coverage.xml htmlcov/ + +# Generated fleet deployment state (host names, index lists, credentials). +# `deploy/fleet/fleet.py` renders these for a specific machine; they are not +# part of the tool. Keep the example file, ignore the rendered output. +deploy/fleet/indexes.yml +deploy/fleet/docker-compose.generated.yml +deploy/fleet/caddy/ +deploy/fleet/secrets/ diff --git a/README.md b/README.md index 932a2ab..bad859d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Prefer containers, or need a brain several agents share? → | `open-index search [-t doc_type]` | Search from the terminal. | | `open-index ui` | Launch the Streamlit explorer (Explore / **Map** / Analytics / Jobs). | | `open-index mcp [--read-only]` | Run the MCP context layer over stdio. **Read+write by default**; `--read-only` opts out of writes. | -| `open-index serve [--port --token --read-only]` | Serve the same read+write MCP context layer over **HTTP** for remote agents (bearer-token auth). | +| `open-index serve [--port --token --read-only]` | Serve the MCP context layer over **HTTP** for remote agents (bearer-token auth). | +| `open-index serve --brains ` | Serve **every** brain under a directory from one process, each at `//mcp`. | | `open-index mcp-config [--url --token]` | Print the MCP connection block to paste into your agent. | ### A context layer for domain-specialized agents diff --git a/docs/deployment.md b/docs/deployment.md index 960ffb1..6991448 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -265,6 +265,47 @@ chmod -R g+rwX /path/to/my-brain You can still read and edit the files; writes from inside the container land as uid 10001 with your group. +### Many brains from one process + +`serve --brain

` runs one brain. For more than a handful, `--brains +` serves every brain under a directory from a single process, each at +`//mcp`: + +```bash +open-index serve --brains /srv/brains --port 8080 +# /srv/brains/support/ → /support/mcp +# /srv/brains/sales/ → /sales/mcp +``` + +This matters because of what is *not* duplicated. A process per brain re-loads +the Python runtime and a ~250MB resident embedding model each time, so a modest +host tops out at a handful. In one process the model is loaded once and a brain +costs only its config and doc_types. + +Measured on the bundled example brain, SQLite-backed: + +| Brains | One process | One process per brain | +|---|---|---| +| 50 | 345 MB | ~13 GB | +| 200 | **373 MB** | ~53 GB | + +That is **1.8MB of marginal cost per brain**, and 200 mount in ~4 seconds. + +Each brain keeps its own storage, its own read/write policy and its own token — +`OPEN_INDEX_TOKEN_` gates one brain (`OPEN_INDEX_TOKEN_SALES_EU` for +`sales-eu/`), and `--token` covers any without one. Nothing is shared between +brains except the process and the model. + +Two extras come with it: `GET /` lists every brain with its URL, entity count +and doc_types, and `GET /healthz` is an unauthenticated probe for a load +balancer. + +> **Prefer SQLite here.** With OpenSearch, every brain is a separate cluster +> index and therefore a shard; the working guidance is ~20 shards per GB of +> heap, so hundreds of brains would hit that ceiling long before RAM. SQLite +> gives each brain its own file and no shard cost at all. Use OpenSearch for the +> few brains that genuinely need concurrent writers or >10k entities. + ### Running one-off commands ```bash