diff --git a/.gitignore b/.gitignore index 48b835e..d92efbe 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,11 @@ dist/ .coverage coverage.xml htmlcov/ + +# Generated fleet deployment state (host names, index lists, credentials). +# `deploy/fleet/fleet.py` renders these for a specific machine; they are not +# part of the tool. Keep the example file, ignore the rendered output. +deploy/fleet/indexes.yml +deploy/fleet/docker-compose.generated.yml +deploy/fleet/caddy/ +deploy/fleet/secrets/ diff --git a/README.md b/README.md index 932a2ab..bad859d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Prefer containers, or need a brain several agents share? → | `open-index search [-t doc_type]` | Search from the terminal. | | `open-index ui` | Launch the Streamlit explorer (Explore / **Map** / Analytics / Jobs). | | `open-index mcp [--read-only]` | Run the MCP context layer over stdio. **Read+write by default**; `--read-only` opts out of writes. | -| `open-index serve [--port --token --read-only]` | Serve the same read+write MCP context layer over **HTTP** for remote agents (bearer-token auth). | +| `open-index serve [--port --token --read-only]` | Serve the MCP context layer over **HTTP** for remote agents (bearer-token auth). | +| `open-index serve --brains ` | Serve **every** brain under a directory from one process, each at `//mcp`. | | `open-index mcp-config [--url --token]` | Print the MCP connection block to paste into your agent. | ### A context layer for domain-specialized agents diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e5098cf..5c61445 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,26 +1,58 @@ #!/usr/bin/env sh # Container entrypoint for open-index. # -# Two jobs beyond exec'ing the CLI: +# It runs in one of two modes: # -# 1. Default --brain to $OPEN_INDEX_BRAIN (/brain) so every command works -# without repeating the flag in compose files and `docker run` lines. -# 2. Reconcile file-backed entities into the search index before serving. -# Entities under entities/**/*.json are the git source of truth; a fresh -# container (or a fresh OpenSearch cluster) starts with an empty index, so -# without this the brain answers every query with "no results" and looks -# broken. Index-backed entities are untouched by this — see Brain.index(). +# single-brain one brain mounted at $OPEN_INDEX_BRAIN (/brain). The --brain +# flag is filled in so compose files and `docker run` lines do +# not repeat it. +# many-brains a directory of brains at $OPEN_INDEX_BRAINS_ROOT (/brains), +# selected by `serve --brains` or by the explorer. Nothing is +# prepended; the command already knows where to look. # -# Skip step 2 with OPEN_INDEX_SKIP_INDEX=1 (e.g. a read replica, or when the -# index is large and managed out of band). +# In both modes, file-backed entities are reconciled into the search index +# before serving. Entities under entities/**/*.json are the git source of truth, +# so a fresh container starts with an empty index and would otherwise answer +# every query with "no results" and look broken. +# +# Skip the reconcile with OPEN_INDEX_SKIP_INDEX=1 (a read replica, or an index +# large enough to be managed out of band). set -eu BRAIN="${OPEN_INDEX_BRAIN:-/brain}" +BRAINS_ROOT="${OPEN_INDEX_BRAINS_ROOT:-}" + +# `serve --brains ` selects many-brains mode even without the env var. +for arg in "$@"; do + if [ "$arg" = "--brains" ]; then MULTI_FLAG=1; fi +done + +if [ -n "$BRAINS_ROOT" ] || [ "${MULTI_FLAG:-0}" = "1" ]; then + MULTI=1 + ROOT="${BRAINS_ROOT:-/brains}" +else + MULTI=0 +fi -if [ ! -f "$BRAIN/brain.yaml" ]; then +if [ "$MULTI" = "1" ]; then + if [ ! -d "$ROOT" ]; then + echo "open-index: no directory at $ROOT" >&2 + echo " mount a directory of brains there, e.g. -v \"\$PWD/brains:/brains\"" >&2 + exit 1 + fi + # A brain is a subdirectory holding brain.yaml. An empty root is almost + # always a mis-mounted volume, so say so rather than serving nothing. + if [ -z "$(find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml -print -quit)" ]; then + echo "open-index: no brains under $ROOT" >&2 + echo " each brain is a subdirectory containing brain.yaml" >&2 + echo " (create one with: open-index init )" >&2 + exit 1 + fi +elif [ ! -f "$BRAIN/brain.yaml" ]; then echo "open-index: no brain.yaml at $BRAIN" >&2 echo " mount your brain directory there, e.g. -v \"\$PWD/my-brain:/brain\"" >&2 echo " (create one first with: open-index init my-brain)" >&2 + echo " (or mount a directory of brains and set OPEN_INDEX_BRAINS_ROOT)" >&2 exit 1 fi @@ -50,17 +82,36 @@ except Exception: echo "open-index: OpenSearch is up." fi +reconcile_all() { + # Reconcile every brain under the root. Done here rather than inside `serve` + # so it happens once at container start instead of on every request. + find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml | while read -r f; do + d=$(dirname "$f") + echo "open-index: indexing $(basename "$d") ..." + open-index index --brain "$d" >/dev/null || \ + echo "open-index: WARNING $(basename "$d") failed to index" >&2 + done +} + case "${1:-serve}" in serve|ui|index|search|validate|run|ingest|mcp|mcp-config|list-connectors|add-entity|add-doc-type) COMMAND="$1" shift if [ "$COMMAND" = "serve" ] && [ "${OPEN_INDEX_SKIP_INDEX:-0}" != "1" ]; then - echo "open-index: reconciling file-backed entities into the index ..." - open-index index --brain "$BRAIN" + if [ "$MULTI" = "1" ]; then + reconcile_all + else + echo "open-index: reconciling file-backed entities into the index ..." + open-index index --brain "$BRAIN" + fi fi - set -- "$COMMAND" --brain "$BRAIN" "$@" + if [ "$MULTI" = "1" ]; then + set -- "$COMMAND" "$@" # the command carries its own --brains + else + set -- "$COMMAND" --brain "$BRAIN" "$@" + fi ;; *) # Anything else (sh, python, a raw open-index invocation) runs verbatim. diff --git a/docs/deployment.md b/docs/deployment.md index 960ffb1..6991448 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -265,6 +265,47 @@ chmod -R g+rwX /path/to/my-brain You can still read and edit the files; writes from inside the container land as uid 10001 with your group. +### Many brains from one process + +`serve --brain ` runs one brain. For more than a handful, `--brains +` serves every brain under a directory from a single process, each at +`//mcp`: + +```bash +open-index serve --brains /srv/brains --port 8080 +# /srv/brains/support/ → /support/mcp +# /srv/brains/sales/ → /sales/mcp +``` + +This matters because of what is *not* duplicated. A process per brain re-loads +the Python runtime and a ~250MB resident embedding model each time, so a modest +host tops out at a handful. In one process the model is loaded once and a brain +costs only its config and doc_types. + +Measured on the bundled example brain, SQLite-backed: + +| Brains | One process | One process per brain | +|---|---|---| +| 50 | 345 MB | ~13 GB | +| 200 | **373 MB** | ~53 GB | + +That is **1.8MB of marginal cost per brain**, and 200 mount in ~4 seconds. + +Each brain keeps its own storage, its own read/write policy and its own token — +`OPEN_INDEX_TOKEN_` gates one brain (`OPEN_INDEX_TOKEN_SALES_EU` for +`sales-eu/`), and `--token` covers any without one. Nothing is shared between +brains except the process and the model. + +Two extras come with it: `GET /` lists every brain with its URL, entity count +and doc_types, and `GET /healthz` is an unauthenticated probe for a load +balancer. + +> **Prefer SQLite here.** With OpenSearch, every brain is a separate cluster +> index and therefore a shard; the working guidance is ~20 shards per GB of +> heap, so hundreds of brains would hit that ceiling long before RAM. SQLite +> gives each brain its own file and no shard cost at all. Use OpenSearch for the +> few brains that genuinely need concurrent writers or >10k entities. + ### Running one-off commands ```bash diff --git a/open_index/cli.py b/open_index/cli.py index 6eefe74..60fba11 100644 --- a/open_index/cli.py +++ b/open_index/cli.py @@ -441,6 +441,12 @@ def mcp_config( @app.command() def serve( brain: str = BrainOpt, + brains: Optional[str] = typer.Option( + None, "--brains", envvar="OPEN_INDEX_BRAINS_ROOT", + help="Serve EVERY brain under this directory from one process, each at " + "//mcp. Cheaper than one process per brain: the embedding " + "model is loaded once for all of them.", + ), host: str = typer.Option("0.0.0.0", help="Bind host."), port: int = typer.Option(8080, help="Bind port."), token: Optional[str] = typer.Option( @@ -452,6 +458,12 @@ def serve( help="Externally reachable URL, when behind a proxy/tunnel/load balancer. " "Only used to print correct connection details.", ), + allowed_host: Optional[list[str]] = typer.Option( + None, "--allowed-host", envvar="OPEN_INDEX_ALLOWED_HOSTS", + help="Host header(s) to accept, when behind a proxy or load balancer. " + "Repeatable. The host from --public-url is added automatically. " + "Use '*' to disable the check entirely (trusted proxy only).", + ), read_only: bool = typer.Option( False, "--read-only", help="Opt out of default read+write mode and expose only read tools.", @@ -463,11 +475,42 @@ def serve( OpenSearch backend for a shared, multi-writer brain. """ from open_index.mcp_config import advertised_urls, mask_token - from open_index.mcp_server import serve_http + from open_index.mcp_server import ( + discover_brains, serve_http, serve_http_multi, token_for, + ) + + mode = "read-only" if read_only else "read+write" + + if brains: + found = discover_brains(brains) + if not found: + typer.secho(f"no brains under {brains} — a brain is a directory " + "containing brain.yaml", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + typer.secho(f"open-index · {len(found)} brains · {mode}", + fg=typer.colors.GREEN, bold=True) + typer.echo(f" listening on {host}:{port}") + urls = advertised_urls(host, port, public_url=public_url) + base = urls[-1][1].rsplit("/mcp", 1)[0] + typer.echo("") + for name in found: + gate = "token" if token_for(name, token) else typer.style( + "OPEN", fg=typer.colors.YELLOW) + typer.secho(f" {name:<28} {base}/{name}/mcp ({gate})", + fg=typer.colors.CYAN) + typer.echo("") + typer.echo(" per-brain tokens come from OPEN_INDEX_TOKEN_; " + "--token covers any without one.") + typer.echo(f" directory: {base}/ · health: {base}/healthz") + typer.echo("") + serve_http_multi(brains, host=host, port=port, token=token, + read_only=read_only, public_base_url=base, + allowed_hosts=list(allowed_host or [])) + return root = Path(brain).resolve() b = _open_brain(str(root)) - mode = "read-only" if read_only else "read+write" typer.secho( f"open-index · brain '{b.config.name}' · {mode} · " @@ -502,7 +545,8 @@ def serve( typer.echo("") serve_http(str(root), host=host, port=port, token=token, read_only=read_only, - warn_unauthenticated=False) + warn_unauthenticated=False, public_url=public_url, + allowed_hosts=list(allowed_host or [])) if __name__ == "__main__": # pragma: no cover - module entry point diff --git a/open_index/config.py b/open_index/config.py index 1ad1e5e..9ad116e 100644 --- a/open_index/config.py +++ b/open_index/config.py @@ -220,6 +220,22 @@ def write_doc_type(brain_root: Path, dt: DocType) -> Path: return path +def discover_brains(root: str | Path) -> dict[str, Path]: + """Every brain directory directly under `root`, keyed by directory name. + + A "brain" is any subdirectory holding a brain.yaml, so a root can sit + alongside unrelated folders without confusing anything. + """ + base = Path(root).expanduser().resolve() + if not base.is_dir(): + raise FileNotFoundError(f"no such directory: {base}") + return { + child.name: child + for child in sorted(base.iterdir()) + if (child / "brain.yaml").exists() + } + + def iter_entity_files(brain_dir: str | Path): """Yield every entities/**/*.json path under the brain directory.""" root = Path(brain_dir).expanduser().resolve() diff --git a/open_index/graph.py b/open_index/graph.py index 2555bb2..fd3e57b 100644 --- a/open_index/graph.py +++ b/open_index/graph.py @@ -69,6 +69,62 @@ def _label_for(brain: Brain, entity) -> str: return entity.label_for(label_field) +def build_overview_graph( + brain: Brain, + doc_types: Optional[list[str]] = None, + limit: int = 250, +) -> ContextGraph: + """Every entity of the given doc_types, with the edges that run between them. + + The anchor-and-expand view answers "what is around this one thing?". This + answers the other question — "what does this index look like?" — which is + what you want when you have never seen the index before and have no entity + in mind to anchor on. + + Only edges whose *both* ends are in scope are drawn: a dangling half-edge to + a filtered-out doc_type would render as an unexplained line to nowhere. + + `limit` caps the node count, keeping the most-connected entities. A force + layout with thousands of nodes is neither readable nor quick, and the + best-connected ones are the ones worth seeing. Callers should report when + the cap bit — silently showing a subset reads as "this is everything". + """ + entities = brain.backend.all_entities(doc_types) + + degree: dict[str, int] = {e.id: 0 for e in entities} + for entity in entities: + for rel in entity.related_to: + degree[entity.id] = degree.get(entity.id, 0) + 1 + if rel.target in degree: + degree[rel.target] = degree.get(rel.target, 0) + 1 + + if len(entities) > limit: + entities = sorted(entities, key=lambda e: (-degree.get(e.id, 0), e.id))[:limit] + + in_scope = {e.id for e in entities} + graph = ContextGraph(anchors=[]) + for entity in entities: + graph.nodes.append(GraphNode( + id=entity.id, + label=_label_for(brain, entity), + doc_type=entity.doc_type, + color=_color_for(brain, entity.doc_type), + depth=0, + data=entity.to_json(), + )) + + seen: set[tuple[str, str, str]] = set() + for entity in entities: + for source, target, meaning in brain.backend.relationships_from(entity.id): + if target not in in_scope: + continue + key = (source, target, meaning) + if key not in seen: + seen.add(key) + graph.edges.append(GraphEdge(source, target, meaning)) + return graph + + def build_graph(brain: Brain, anchors: str | list[str], depth: int = 1) -> ContextGraph: """Build the map around one or more anchor entities. diff --git a/open_index/mcp_server.py b/open_index/mcp_server.py index b93619f..196d04e 100644 --- a/open_index/mcp_server.py +++ b/open_index/mcp_server.py @@ -23,6 +23,8 @@ from time import perf_counter from typing import Any, Optional +from pathlib import Path + from open_index.brain import Brain from open_index.schema import DocType, DocTypeDisplay, FieldSpec, RelationshipSpec from open_index.models import Entity, Provenance, Relationship @@ -432,10 +434,168 @@ async def wrapped(scope, receive, send): return wrapped +def _host_of(url: str) -> Optional[str]: + """The host[:port] part of a URL, or None if it isn't parseable.""" + from urllib.parse import urlparse + + parsed = urlparse(url if "://" in url else f"http://{url}") + return parsed.netloc or None + + +def build_transport_security( + public_url: Optional[str] = None, allowed_hosts: Optional[list[str]] = None +): + """Host allow-list for the MCP SDK's DNS-rebinding protection. + + The SDK enables that protection whenever the app is built for a localhost + bind, with a hardcoded localhost-only allow-list. Anything reaching the + server through a reverse proxy or load balancer therefore arrives with a + foreign `Host` header and is rejected with `421 Invalid Host header` — the + server looks up, and every proxied request fails. + + Rather than switch the protection off (which is what passing a non-localhost + bind address to the SDK quietly does), state which hosts are legitimate: + loopback, plus whatever `--public-url` and `--allowed-host` name. + + A literal `*` disables the check — for a brain behind a trusted proxy that + already validates Host. Returns settings, or None when the SDK's own default + is appropriate. + """ + from mcp.server.transport_security import TransportSecuritySettings + + entries = [e.strip() for e in (allowed_hosts or []) if e.strip()] + if "*" in entries: + return TransportSecuritySettings(enable_dns_rebinding_protection=False) + + if public_url: + derived = _host_of(public_url) + if derived: + entries.append(derived) + + hosts = {"127.0.0.1:*", "localhost:*", "[::1]:*", "127.0.0.1", "localhost", "[::1]"} + origins = {"http://127.0.0.1:*", "http://localhost:*", "http://[::1]:*"} + for entry in entries: + bare = entry.split(":", 1)[0] if not entry.startswith("[") else entry + # Both forms: proxies drop the port when it is the scheme default, so + # "example.com" and "example.com:8080" must both pass. + hosts.update({entry, bare, f"{bare}:*"}) + for scheme in ("http", "https"): + origins.update({f"{scheme}://{entry}", f"{scheme}://{bare}", + f"{scheme}://{bare}:*"}) + + return TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=sorted(hosts), + allowed_origins=sorted(origins), + ) + + +def discover_brains(root: str) -> dict[str, "Path"]: + """Brains under `root`. Re-exported from config for the serve entry points.""" + from open_index.config import discover_brains as _discover + + try: + return _discover(root) + except FileNotFoundError as exc: + raise SystemExit(str(exc)) from exc + + +def token_for(name: str, default: Optional[str] = None) -> Optional[str]: + """Per-brain bearer token from the environment, falling back to a shared one. + + `OPEN_INDEX_TOKEN_SALES_EU` gates the brain in `sales-eu/`. Without one, the + shared `--token` applies — which is fine for a private host and wrong for a + shared one, hence the per-brain override. + """ + import os as _os + + key = "OPEN_INDEX_TOKEN_" + name.upper().replace("-", "_").replace(".", "_") + return _os.environ.get(key) or default + + +def build_multi_app( + brains: dict[str, "Path"], + *, + token: Optional[str] = None, + read_only: bool = False, + public_base_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, + host: str = "0.0.0.0", +): + """One ASGI app serving many brains, each mounted at `//mcp`. + + The point is what is *not* duplicated. A process per brain re-loads the + Python runtime and a ~250MB resident embedding model every time, which caps + a modest host at a handful of brains. Here the model is loaded once — the + provider cache is keyed on model configuration, not on brain — so the + marginal cost of a brain is its config and doc_types, and hundreds fit where + a handful did. + + Each brain keeps its own token, its own read/write policy and its own + storage, so this is a packaging change rather than a shared-tenancy one. + """ + from contextlib import AsyncExitStack, asynccontextmanager + + from starlette.applications import Starlette + from starlette.responses import JSONResponse, PlainTextResponse + from starlette.routing import Mount, Route + + base = (public_base_url or "").rstrip("/") + mounted: list = [] # the Starlette children, for their lifespans + routes: list = [] + listing: dict[str, dict] = {} + + for name, path in brains.items(): + brain = Brain.open(path) + server = build_server(brain, read_only=read_only) + public_url = f"{base}/{name}/mcp" if base else None + child = server.streamable_http_app( + transport_security=build_transport_security(public_url, allowed_hosts), + host=host, + ) + mounted.append(child) + + brain_token = token_for(name, token) + app = _bearer_auth_middleware(child, brain_token) if brain_token else child + routes.append(Mount(f"/{name}", app=app)) + + listing[name] = { + "mcp": public_url or f"/{name}/mcp", + "description": brain.config.description, + "entities": sum(brain.counts().values()), + "doc_types": sorted(brain.config.doc_types), + "read_only": read_only, + "authenticated": bool(brain_token), + } + + async def directory(_request): + return JSONResponse(listing) + + async def healthz(_request): + return PlainTextResponse("ok") + + routes += [Route("/", directory), Route("/healthz", healthz)] + + @asynccontextmanager + async def lifespan(_app): + # Starlette does not run a mounted app's lifespan, and the MCP transport + # needs its session manager started — without this every request to a + # mounted brain fails with "Task group is not initialized". + async with AsyncExitStack() as stack: + for child in mounted: + await stack.enter_async_context( + child.router.lifespan_context(child)) + yield + + return Starlette(routes=routes, lifespan=lifespan) + + def serve_http( brain_dir: str, host: str = "0.0.0.0", port: int = 8080, token: Optional[str] = None, read_only: bool = False, warn_unauthenticated: bool = True, + public_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, ) -> None: """Run the MCP server over streamable HTTP so remote/cloud agents can connect by URL. Register `http://:/mcp` as a remote MCP server in the @@ -452,7 +612,10 @@ def serve_http( brain = Brain.open(brain_dir) server = build_server(brain, read_only=read_only) - app = server.streamable_http_app() + app = server.streamable_http_app( + transport_security=build_transport_security(public_url, allowed_hosts), + host=host, + ) if token: app = _bearer_auth_middleware(app, token) elif warn_unauthenticated: @@ -465,3 +628,30 @@ def serve_http( file=sys.stderr, ) uvicorn.run(app, host=host, port=port) + + +def serve_http_multi( + brains_root: str, host: str = "0.0.0.0", port: int = 8080, + token: Optional[str] = None, read_only: bool = False, + public_base_url: Optional[str] = None, + allowed_hosts: Optional[list[str]] = None, +) -> None: + """Serve every brain under `brains_root` from one process.""" + try: + import uvicorn + except ImportError as exc: # pragma: no cover - optional dependency + raise SystemExit( + "the HTTP endpoint needs uvicorn: pip install 'open-index[serve]'" + ) from exc + + brains = discover_brains(brains_root) + if not brains: + raise SystemExit( + f"no brains found under {brains_root} — a brain is a directory " + "containing brain.yaml (create one with `open-index init`)" + ) + app = build_multi_app( + brains, token=token, read_only=read_only, + public_base_url=public_base_url, allowed_hosts=allowed_hosts, host=host, + ) + uvicorn.run(app, host=host, port=port) diff --git a/open_index/storage/opensearch_backend.py b/open_index/storage/opensearch_backend.py index cbb8349..92fc3bb 100644 --- a/open_index/storage/opensearch_backend.py +++ b/open_index/storage/opensearch_backend.py @@ -100,6 +100,11 @@ def mapping(dimension: int = 384) -> dict: return { "settings": {"index": {"knn": True}}, "mappings": { + # A field declared `string` whose values look like dates would + # otherwise be mapped as `date` by dynamic detection, and then + # rejected by fuzzy search. The schema decides the type here, + # not a guess from the first document indexed. + "date_detection": False, "properties": { "id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, @@ -137,6 +142,11 @@ def _base_mapping(self) -> dict: """Mapping without the embedding field (for keyword-only brains).""" return { "mappings": { + # A field declared `string` whose values look like dates would + # otherwise be mapped as `date` by dynamic detection, and then + # rejected by fuzzy search. The schema decides the type here, + # not a guess from the first document indexed. + "date_detection": False, "properties": { "id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, @@ -261,6 +271,13 @@ def _search_fields(self, doc_types: Optional[list[str]]) -> list[str]: continue if f.name == label: name_boost = max(name_boost, f.boost) + # Only text-ish fields belong in a fuzzy full-text query. + # OpenSearch maps a `number` field to long and a timestamp to + # date, and rejects the whole query with + # "Can only use fuzzy queries on keyword and text fields" if + # either is listed — so one numeric field breaks all search. + if f.type not in ("string", "text"): + continue boosts[f.name] = max(boosts.get(f.name, 1.0), f.boost) fields = [f"name^{name_boost:g}"] fields += [f"fields.{name}^{b:g}" for name, b in boosts.items()] @@ -373,9 +390,27 @@ def ensure_schema(self, doc_types: dict[str, DocType]) -> None: "is the cluster running? (search.backend: opensearch)" ) from exc + def _ensure_index_exists(self) -> None: + """Recreate the index with our mapping if it has gone missing. + + `ensure_schema` runs once when the brain is opened. If the index is + dropped after that — a cluster rebuild, a manual DELETE, a restored + snapshot — OpenSearch would auto-create it on the next write using + *dynamic* mapping, silently typing `doc_type` as text and breaking every + aggregation, and typing date-shaped strings as dates and breaking fuzzy + search. Both fail long after the write that caused them, so it is worth + one existence check on the write path. + """ + if self._client.indices.exists(index=self.index): + return + logger.warning("index %s was missing; recreating it with the schema mapping", + self.index) + self._client.indices.create(index=self.index, body=self._mapping()) + def upsert_entity(self, entity: Entity, doc_type: Optional[DocType] = None) -> None: if doc_type is not None: self._doc_types[doc_type.doc_type] = doc_type + self._ensure_index_exists() self._client.index( index=self.index, id=entity.id, body=self._doc_with_embedding(entity), refresh=True ) @@ -391,6 +426,7 @@ def upsert_many(self, items: list[tuple[Entity, Optional[DocType]]]) -> None: for _entity, doc_type in items: if doc_type is not None: self._doc_types[doc_type.doc_type] = doc_type + self._ensure_index_exists() payload: list[dict] = [] for entity, _doc_type in items: diff --git a/open_index/ui/app.py b/open_index/ui/app.py index e181efb..8fef0c7 100644 --- a/open_index/ui/app.py +++ b/open_index/ui/app.py @@ -17,12 +17,13 @@ from __future__ import annotations import os +from typing import Optional from time import perf_counter import streamlit as st from open_index.brain import Brain -from open_index.graph import ContextGraph, build_graph +from open_index.graph import ContextGraph, build_graph, build_overview_graph from open_index.ui import view st.set_page_config(page_title="Open Index", page_icon="🧠", layout="wide") @@ -33,6 +34,50 @@ def _open_brain(brain_dir: str) -> Brain: return Brain.open(brain_dir) +@st.cache_resource +def _available_brains(root: str) -> dict: + """Brains under a root, cached so every rerun does not re-stat the disk.""" + from open_index.config import discover_brains + + return {name: str(path) for name, path in discover_brains(root).items()} + + +def _current_url(): + """The URL the browser is on, as reported over the websocket. + + Absent on older Streamlit versions and in bare script runs, so callers must + tolerate None. + """ + try: + return st.context.url + except Exception: + return None + + +def _select_brain(): + """The brain this page shows, chosen solely by the URL path. + + With OPEN_INDEX_BRAINS_ROOT set, one process serves every brain under it — + the alternative is a process, and a ~250MB embedding model, per brain. The + index is the first path segment, so / serves . There is no + in-page switcher on purpose: the URL is the selector, which keeps the + address bar and the screen in agreement and makes every view shareable. + """ + root = os.environ.get("OPEN_INDEX_BRAINS_ROOT") + if not root: + return _open_brain(os.environ.get("OPEN_INDEX_DIR", ".")), None + + brains = _available_brains(root) + if not brains: + st.error(f"No brains found under `{_esc(root)}`.") + st.caption("A brain is a directory containing `brain.yaml`. Create one " + "with `open-index init `.") + st.stop() + + chosen = view.brain_from_url(_current_url(), list(brains)) + # The directory name is also the URL segment, so it is what builds this + # index's endpoint — the brain's configured name may differ from it. + return _open_brain(brains[chosen]), chosen def _theme_type() -> str: @@ -237,48 +282,59 @@ def render_entity(brain: Brain, entity_id: str) -> None: # --------------------------------------------------------------------------- # def render_map(brain: Brain) -> None: + """The whole index at a glance: every entity, coloured by doc_type. + + Anchoring on one entity is the wrong default for someone who has never seen + the index — they have no entity in mind. This shows the shape of the whole + thing and lets them subtract from it. + """ summary = view.summarize(brain) populated = [r.name for r in summary.doc_types if r.count] if not populated: st.info("No entities to map yet. Add some and run `open-index index`.") return - chosen_types = st.multiselect( - "Doc types to include", populated, default=populated, - help="Narrow the map to the concepts you care about.", - ) - scope = chosen_types or populated - - # Auto-anchor rather than waiting for a selection. The old UI drew nothing - # until you picked entities, which looked like a broken canvas. - auto = view.default_anchors(brain, scope) - catalog = {f"{e.name} ({e.id})": e.id - for e in brain.backend.all_entities(scope)} - auto_labels = [label for label, eid in catalog.items() if eid in auto] - - picked = st.multiselect( - "Anchors", list(catalog), default=auto_labels, - help="Starting points. Click any node in the map to expand it.", + chosen = st.multiselect( + "Doc types shown", populated, default=populated, + help="Uncheck a type to drop it and its edges from the map.", ) - anchors = [catalog[label] for label in picked] - - expanded = st.session_state.get("expanded", set()) - if expanded: - cols = st.columns([4, 1]) - cols[0].caption(f"expanded: {', '.join(sorted(expanded))}") - if cols[1].button("↺ reset"): - st.session_state["expanded"] = set() + scope = chosen or populated + + focus = st.session_state.get("map_focus") + if focus: + cols = st.columns([5, 1]) + cols[0].caption(f"Focused on `{focus}` and its immediate neighbours.") + if cols[1].button("↩ show all"): + st.session_state["map_focus"] = None st.rerun() - - anchors = list(dict.fromkeys(anchors + list(expanded))) - if not anchors: - st.info("Select at least one anchor above.") - return - - graph = build_graph(brain, anchors, depth=1) - st.caption(f"{len(graph.nodes)} nodes · {len(graph.edges)} edges — " - "click a node to expand its relationships") - render_graph(brain, graph) + graph = build_graph(brain, [focus], depth=1) + else: + graph = build_overview_graph(brain, scope, limit=view.MAX_GRAPH_NODES) + + total = sum(r.count for r in summary.doc_types if r.name in scope) + if not focus and len(graph.nodes) < total: + st.warning( + f"Showing the {len(graph.nodes)} most-connected of {total} entities. " + "Narrow the doc types above to see the rest." + ) + + canvas, legend = st.columns([4, 1]) + with canvas: + st.caption(f"{len(graph.nodes)} nodes · {len(graph.edges)} edges — " + "hover for detail, click a node to focus on it") + render_graph(brain, graph) + with legend: + st.markdown("**Legend**") + for row in view.legend_rows(brain, graph): + st.markdown( + f"{_dot(row['color'])} `{_esc(row['doc_type'])}`" + f" {row['count']}", + unsafe_allow_html=True, + ) + if graph.edges: + st.markdown("---") + st.caption("Edges are `related_to` links. Hover one to see which " + "relationship it is.") # streamlit-agraph doesn't paint on its first render inside a tab (the canvas # mounts with zero size). One forced rerun remounts it with the tab active. @@ -298,24 +354,8 @@ def render_graph(brain: Brain, graph: ContextGraph) -> None: palette = view.graph_theme(_theme_type()) - nodes = [ - Node(id=n.id, label=n.label, color=n.color, - size=22 if n.is_anchor else 16, shape="dot", - title=f"{n.doc_type} · {n.id}", - # strokeWidth 0 removes vis's white halo, which on a dark canvas - # turns every label into heavy outlined text. - font={"color": palette["node_label"], "size": 15, - "strokeWidth": palette["stroke_width"], "face": "sans-serif"}) - for n in graph.nodes - ] - edges = [ - Edge(source=e.source, target=e.target, label=e.meaning, - color=palette["edge"], - font={"color": palette["edge_label"], "size": 12, - "strokeWidth": palette["stroke_width"], "align": "middle", - "face": "sans-serif"}) - for e in graph.edges - ] + nodes = [Node(**spec) for spec in view.graph_node_specs(graph)] + edges = [Edge(**spec) for spec in view.graph_edge_specs(graph, palette["edge"])] busy = len(graph.nodes) > view.BUSY_GRAPH_NODES config = Config( @@ -335,8 +375,8 @@ def render_graph(brain: Brain, graph: ContextGraph) -> None: _left, middle, _right = st.columns([1, 20, 1]) with middle: clicked = agraph(nodes=nodes, edges=edges, config=config) - if clicked and clicked not in st.session_state.get("expanded", set()): - st.session_state.setdefault("expanded", set()).add(clicked) + if clicked and clicked != st.session_state.get("map_focus"): + st.session_state["map_focus"] = clicked st.rerun() @@ -407,6 +447,139 @@ def render_analytics(brain: Brain) -> None: } for e in events], hide_index=True, use_container_width=True) +def render_schema(brain: Brain) -> None: + """Every doc_type and the shape of it — the reference before you write.""" + summary = view.summarize(brain) + if not summary.has_schema: + st.info("No doc_types defined yet.") + st.caption("A doc_type is a concept this index tracks, plus the fields it " + "stores. Create one with `open-index add-doc-type`, or ask an " + "agent to call `create_doc_type`.") + return + + st.caption( + f"{len(summary.doc_types)} doc_types · {summary.total_entities:,} entities. " + "Every entity id is `:`, and any entity can link to any " + "other through `related_to`." + ) + + for row in summary.doc_types: + doc_type = brain.config.doc_type(row.name) + noun = "entity" if row.count == 1 else "entities" + with st.expander(f"{row.name} · {row.count:,} {noun}", + expanded=len(summary.doc_types) <= 3): + if row.description: + st.markdown(row.description) + st.caption( + f"{_dot(row.color)} source of truth: " + + ("**files** — JSON under `entities/`, git-trackable" + if row.storage == "file" + else "**search index** — DB-owned, not written to files"), + unsafe_allow_html=True, + ) + + fields = view.schema_field_rows(doc_type) + if fields: + st.markdown("**Fields**") + st.table(fields) + else: + st.caption("No fields declared.") + + relationships = view.schema_relationship_rows(brain, row.name) + st.markdown("**Relationships (optional)**") + if relationships: + st.table(relationships) + st.caption("Declared edges are validated against their target " + "doc_type. Undeclared ones still work — they just " + "aren't checked.") + else: + st.caption("None declared or in use. Entities of this type are " + "valid without any; edges are what make the index " + "traversable rather than just searchable.") + + +def render_how_to_use(brain: Brain, url_name: Optional[str] = None) -> None: + """Connecting an agent, the tools it gets, and what the other tabs are for. + + Deliberately the rightmost tab: it is the page people come back to, not the + one they start on. + """ + summary = view.summarize(brain) + # Derived from the URL this page is on, so it is right for *this* index and + # needs no configuration. Falls back to an explicitly configured URL for a + # single-brain deployment, where the page path carries no index name. + mcp_url = (view.mcp_url_for(_current_url(), url_name) + or os.environ.get("OPEN_INDEX_PUBLIC_URL", "")) + read_only = os.environ.get("OPEN_INDEX_READ_ONLY", "").lower() in ("1", "true", "yes") + + st.markdown("### What an index holds") + st.caption( + "Three ideas, and the whole thing follows from them." + ) + for term, what in view.MODEL_GUIDE: + st.markdown(f"- **{term}** — {what}") + + st.divider() + st.markdown(f"### Connect an agent to `{_esc(summary.name)}`") + st.caption( + "This index speaks MCP, so any MCP-capable agent can query it — and, " + "unless the endpoint is read-only, keep it current." + ) + + if mcp_url: + st.markdown("**1. Point your agent at this URL**") + st.code(view.mcp_client_config(mcp_url, server_name=summary.name), language="json") + st.caption("Paste into `.mcp.json` (Claude Code), `.cursor/mcp.json` (Cursor), " + "or any MCP client's server config. Or generate it:") + st.code(f"open-index mcp-config --url {mcp_url} --name {summary.name} > .mcp.json", + language="bash") + else: + st.info("This explorer isn't configured with a public MCP URL " + "(`OPEN_INDEX_PUBLIC_URL`), so the connection block can't be shown.") + st.code("open-index mcp-config --brain > .mcp.json", language="bash") + + st.markdown("**2. Ask it something**") + st.caption("No briefing needed — the navigation guide below is injected into the " + "MCP handshake, so the agent knows this index's doc_types and " + "relationship vocabulary before its first turn.") + + st.divider() + st.markdown("### Tools the agent gets") + + st.markdown("**Reading**") + for name, what in view.READ_TOOLS: + st.markdown(f"- `{name}` — {what}") + + st.markdown("**Writing**") + if read_only: + st.info("This endpoint is **read-only** — the write tools below are not " + "registered on it. Serve without `--read-only` to enable them.") + for name, what in view.WRITE_TOOLS: + st.markdown(f"- `{name}` — {what}") + st.caption("Entity ids are always `:`. Writes are validated " + "against the doc_type schema, and land in the search index (and on " + "disk, for `storage: file` types).") + + st.divider() + st.markdown("### What each tab does") + for name, what in view.TAB_GUIDE: + st.markdown(f"- **{name}** — {what}") + + st.divider() + st.markdown("### What's in this index right now") + st.caption(f"{summary.total_entities:,} entities across " + f"{len(summary.doc_types)} doc_types.") + if summary.doc_types: + st.table([{"doc_type": r.name, "entities": r.count, + "source of truth": "files (git)" if r.storage == "file" else "search index", + "what it holds": r.description or "—"} + for r in summary.doc_types]) + + with st.expander("The full navigation guide the agent receives"): + st.code(brain.navigation_guidelines(include_writes=not read_only), + language="markdown") + + def render_jobs(brain: Brain) -> None: from open_index.connectors.runner import discover_connectors from open_index.scheduling import RunState @@ -443,13 +616,19 @@ def render_jobs(brain: Brain) -> None: def main() -> None: - brain = _open_brain(os.environ.get("OPEN_INDEX_DIR", ".")) + brain, url_name = _select_brain() st.markdown(view.ROW_CSS, unsafe_allow_html=True) options = render_sidebar(brain) - tab_explore, tab_map, tab_analytics, tab_jobs = st.tabs( - ["Explore", "Map", "Analytics", "Jobs"] + # Streamlit opens the first tab, so the help page leftmost means a visitor + # lands on the explanation rather than having to find it. + tab_help, tab_schema, tab_explore, tab_map, tab_analytics, tab_jobs = st.tabs( + [name for name, _ in view.TAB_GUIDE] ) + with tab_help: + render_how_to_use(brain, url_name) + with tab_schema: + render_schema(brain) with tab_explore: render_explore(brain, options) with tab_map: diff --git a/open_index/ui/view.py b/open_index/ui/view.py index e3ba929..bc79714 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -67,6 +67,30 @@ def has_schema(self) -> bool: return bool(self.doc_types) +def brain_from_url(url: Optional[str], available: list[str]) -> Optional[str]: + """Which brain a URL asks for: the first path segment. + + `/support-index` and `/support-index/anything` both select `support-index`, so + the index name lives in the path exactly as it does for the MCP endpoint at + `//mcp`. + + Read from the browser's URL rather than left to Streamlit's own page router: + that router resolves programmatic pages by an internal identifier, so every + path rendered whichever brain sorted first. An unknown or missing segment + falls back to the first brain — a stale link should land somewhere useful + rather than on an error. + """ + if not available: + return None + if not url: + return available[0] + + from urllib.parse import urlparse + + first = urlparse(url).path.strip("/").split("/")[0] + return first if first in available else available[0] + + def color_for(brain: Brain, doc_type: str) -> str: dt = brain.config.doc_type(doc_type) return dt.display.color if dt else DEFAULT_COLOR @@ -207,7 +231,9 @@ def semantic_weight_for(mode: str) -> Optional[float]: # does `f"{width}px"`, so a CSS string like "100%" becomes the invalid value # "100%px", the canvas fails to size, and the graph is stranded in a corner # instead of centred. -GRAPH_WIDTH = 1200 +# Sized to sit beside the legend column rather than to fill the page: at the +# old full-page width the canvas overflowed its column once the legend arrived. +GRAPH_WIDTH = 950 GRAPH_HEIGHT = 650 # Past this many nodes a force layout keeps drifting, so we slow it down rather @@ -262,6 +288,256 @@ def semantic_weight_for(mode: str) -> Optional[float]: """ +def mcp_url_for(current_url: Optional[str], name: Optional[str]) -> Optional[str]: + """This index's MCP endpoint, derived from the URL the browser is on. + + An explorer serving many brains cannot be handed one correct endpoint as + configuration — the answer depends on which index you are looking at. But + the page already knows: it is at `/`, so the endpoint is + `//mcp` on the same origin. Deriving it also means it stays right + behind any proxy or hostname without anything being configured. + + Returns None when the URL or the index name is unknown, so callers can fall + back to explicit configuration. + """ + if not current_url or not name: + return None + + from urllib.parse import urlparse + + parsed = urlparse(current_url) + if not parsed.scheme or not parsed.netloc: + return None + return f"{parsed.scheme}://{parsed.netloc}/{name}/mcp" + + +def mcp_client_config(url: str, server_name: str = "open-index") -> str: + """The JSON block to paste into an agent, as displayed in the How to use tab.""" + import json + + from open_index.mcp_config import normalize_mcp_url + + return json.dumps( + {"mcpServers": {server_name: {"type": "http", "url": normalize_mcp_url(url)}}}, + indent=2, + ) + + +# The three ideas someone needs before anything else on the page makes sense. +# Spelled out because "doc_type" means nothing to a first-time visitor, and the +# help tab is what the app opens on. +MODEL_GUIDE = [ + ("doc_type", + "A **concept** this index tracks, and the fields kept for it — `issue`, " + "`carrier`, `aircraft`. It is the schema, not the data: defining one says " + "what an issue *is*, not that any exist. The Schema tab lists them all."), + ("entity (a doc)", + "**One instance** of a doc_type — one issue, one carrier, one aircraft. " + "Its id is always `:`, so `issue:payment-declined` is an " + "issue called `payment-declined`. This is the document you search for and " + "the agent reads."), + ("relationship", + "An **optional** link from one entity to another, with free text saying " + "what the link means — `product:checkout` → *has common issue* → " + "`issue:payment-declined`. Entities work perfectly well without any. Edges " + "are what turn a list of documents into something you can traverse, so an " + "index with none is a searchable table rather than a graph."), +] + + +# What each tool does, in the order an agent would reach for them. Kept here +# rather than in the page so the list can be asserted against the tools the MCP +# server actually registers — a stale tool list in the docs is worse than none. +READ_TOOLS = [ + ("navigation_guidelines()", + "The whole guide to this index: every doc_type, its fields, the " + "relationship vocabulary in use, and worked examples. Injected into the " + "MCP handshake, so an agent starts oriented."), + ("search_brain(query, doc_types, limit)", + "Free-text search, hybrid keyword + semantic. `doc_types` narrows it."), + ("get_entity(entity_id)", + "One entity with its outgoing AND incoming edges — the incoming direction " + "answers \"what else points at this?\"."), +] + +WRITE_TOOLS = [ + ("put_entity(doc_type, id, name, fields, related_to)", + "Add or update one entity. Upsert — the same id replaces it."), + ("put_entities([...])", + "A whole batch in one call, with optional shared provenance. Use this " + "rather than calling put_entity in a loop."), + ("create_doc_type(doc_type, description, fields, relationships, storage)", + "Define a new concept, when no existing doc_type fits."), +] + +# The tab label for the help page. Leftmost, so it is what a first-time visitor +# lands on, and phrased as the question they are actually asking. +HELP_TAB = "How to use?" + +# What each tab is for, in the order they appear. This list *is* the tab order — +# the page builds its tabs from it, so the two cannot drift apart. +TAB_GUIDE = [ + (HELP_TAB, + "This page: how to connect an agent, what tools it gets, and what " + "everything else here does."), + ("Schema", + "Every doc_type in this index and the shape of it — each field's type, how " + "it is searched, its ranking weight, and the relationship vocabulary that " + "connects types to each other. Read this before writing to the index."), + ("Explore", + "Search the index, or browse by doc_type. Open an entity to see its fields, " + "its attribution, and every relationship in both directions — click through " + "to walk the graph."), + ("Map", + "The same relationships drawn. It auto-anchors on the most-connected " + "entities so it shows something immediately; click any node to expand it, " + "and narrow by doc_type to cut the noise."), + ("Analytics", + "What has been asked of this index, by which client (CLI, agent, this UI) " + "and how often. Zero-result searches are the interesting number: they are " + "the questions this index cannot yet answer."), + ("Jobs", + "Connectors that pull entities in on a schedule, with their last run."), +] + + +def schema_field_rows(doc_type) -> list[dict]: + """One row per field, as the Schema tab tabulates them. + + `search` and `boost` are the two that change behaviour rather than just + describing it, so they are spelled out rather than shown as raw enum values. + """ + searchable = { + "syntactic": "keyword match", + "semantic": "meaning (vector)", + "none": "not searched", + } + rows = [] + for f in doc_type.fields: + rows.append({ + "field": f.name, + "type": f.type, + "searched by": searchable.get(f.search, f.search), + "weight": f"{f.boost:g}×" if f.search != "none" else "—", + "required": "yes" if f.required else "", + "notes": f.description or "", + }) + return rows + + +def schema_relationship_rows(brain, doc_type_name: str) -> list[dict]: + """Declared and observed edges for one doc_type, merged. + + Declared is the vocabulary the schema intends; observed is what the entities + actually use. Showing both together is the point — a declared edge with zero + uses and an undeclared edge in heavy use are both worth noticing. + """ + doc_type = brain.config.doc_type(doc_type_name) + observed = brain.observed_relationships(doc_type_name) + rows = [] + seen = set() + + for spec in (doc_type.relationships if doc_type else []): + seen.add(spec.name) + rows.append({ + "relationship": spec.name, + "points at": spec.target_doc_type or "any", + "declared": "yes", + "in use": observed.get(spec.name, 0), + }) + for meaning, count in sorted(observed.items(), key=lambda kv: -kv[1]): + if meaning not in seen: + rows.append({ + "relationship": meaning, + "points at": "—", + "declared": "no", + "in use": count, + }) + return rows + + +# Cap on nodes drawn at once. Past this a force layout stops settling and the +# picture stops being readable regardless. +MAX_GRAPH_NODES = 250 + + +def node_tooltip(entity_id: str, name: str, doc_type: str, + fields: Optional[dict] = None) -> str: + """What hovering a node shows: the full name, its type, and a few fields. + + vis renders this as plain text, so it is newline-separated rather than HTML. + """ + lines = [name, f"{doc_type} · {entity_id}"] + for key, value in list((fields or {}).items()): + if value in (None, "") or key == "name": + continue + text = " ".join(str(value).split()) + lines.append(f"{key}: {text[:90]}{'…' if len(text) > 90 else ''}") + if len(lines) >= 7: # a tooltip taller than the graph helps nobody + break + return "\n".join(lines) + + +def edge_tooltip(source: str, target: str, meaning: str) -> str: + return f"{source}\n —[{meaning or 'related'}]→\n{target}" + + +def legend_rows(brain, graph) -> list[dict]: + """Doc types present in a drawn graph, with their colour and node count. + + Built from the graph rather than the schema so it describes what is on + screen: a doc_type filtered out, or cut by the node cap, should not appear. + """ + counts: dict[str, int] = {} + for node in graph.nodes: + counts[node.doc_type] = counts.get(node.doc_type, 0) + 1 + return [ + {"doc_type": name, "count": counts[name], "color": color_for(brain, name)} + for name in sorted(counts, key=lambda n: (-counts[n], n)) + ] + + +def graph_node_specs(graph, anchor_size: int = 20, size: int = 13) -> list[dict]: + """Node payloads for the map renderer. + + Nothing is labelled on the canvas. Entity names are long and arbitrary — + drawn beside every dot they overlap each other and their own edges, and no + amount of truncation fixes a dense graph. The picture carries shape and + colour; identity comes from the tooltip, and the legend explains the colours. + + `label` is an empty string rather than None: None serialises to null and vis + draws that literally. + """ + specs = [] + for node in graph.nodes: + fields = {k: v for k, v in (node.data or {}).items() + if k not in ("id", "doc_type", "related_to")} + specs.append({ + "id": node.id, + "label": "", + "color": node.color, + "shape": "dot", + "size": anchor_size if node.is_anchor else size, + "title": node_tooltip(node.id, node.label, node.doc_type, fields), + }) + return specs + + +def graph_edge_specs(graph, color: str) -> list[dict]: + """Edge payloads. Unlabelled for the same reason as nodes — the relationship + is on the tooltip.""" + return [ + { + "source": edge.source, + "target": edge.target, + "label": "", + "title": edge_tooltip(edge.source, edge.target, edge.meaning), + "color": color, + } + for edge in graph.edges + ] + + def graph_theme(theme_type: Optional[str]) -> dict: """Label and edge colours for the map, given Streamlit's active theme. diff --git a/tests/test_cli_serving.py b/tests/test_cli_serving.py index a5f32ef..ff976aa 100644 --- a/tests/test_cli_serving.py +++ b/tests/test_cli_serving.py @@ -171,3 +171,89 @@ def test_serve_fails_cleanly_on_a_missing_brain(tmp_path, served): result = run("serve", "--brain", tmp_path / "nope") assert result.exit_code == 1 assert "no brain.yaml" in result.output + + +# -- serving many brains from one process -------------------------------------- + + +@pytest.fixture +def brains_root(tmp_path): + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + return root + + +@pytest.fixture +def served_multi(monkeypatch): + captured = {} + monkeypatch.setattr("open_index.mcp_server.serve_http_multi", + lambda root, **kw: captured.update(root=root, **kw)) + return captured + + +def test_brains_flag_serves_every_brain(brains_root, served_multi): + result = run("serve", "--brains", brains_root) + assert result.exit_code == 0 + assert served_multi["root"] == str(brains_root) + + +def test_multi_banner_lists_each_brain_url(brains_root, served_multi): + result = run("serve", "--brains", brains_root, "--public-url", "https://x.example.com") + assert "2 brains" in result.stdout + assert "https://x.example.com/alpha/mcp" in result.stdout + assert "https://x.example.com/beta/mcp" in result.stdout + + +def test_multi_banner_flags_ungated_brains(brains_root, served_multi, monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + result = run("serve", "--brains", brains_root) + assert "OPEN" in result.stdout, "an unauthenticated brain must be called out" + + +def test_multi_banner_shows_a_gated_brain_as_tokened(brains_root, served_multi, monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "t") + result = run("serve", "--brains", brains_root) + assert "token" in result.stdout + + +def test_multi_banner_points_at_the_directory_and_health(brains_root, served_multi): + result = run("serve", "--brains", brains_root) + assert "directory:" in result.stdout and "healthz" in result.stdout + + +def test_brains_root_with_no_brains_is_a_clean_error(tmp_path, served_multi): + empty = tmp_path / "empty" + empty.mkdir() + result = run("serve", "--brains", empty) + assert result.exit_code == 1 + assert "no brains under" in result.output + + +def test_serve_http_multi_starts_uvicorn(brains_root, monkeypatch): + import uvicorn + + from open_index import mcp_server + + captured = {} + monkeypatch.setattr(uvicorn, "run", + lambda app, host, port: captured.update(app=app, port=port)) + mcp_server.serve_http_multi(str(brains_root), port=9001) + assert captured["port"] == 9001 + assert captured["app"] is not None + + +def test_serve_http_multi_refuses_an_empty_root(tmp_path, monkeypatch): + import uvicorn + + from open_index import mcp_server + + monkeypatch.setattr(uvicorn, "run", lambda *a, **k: None) + empty = tmp_path / "none" + empty.mkdir() + with pytest.raises(SystemExit, match="no brains found"): + mcp_server.serve_http_multi(str(empty)) diff --git a/tests/test_graph.py b/tests/test_graph.py index 8a3fed3..f8d9671 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -58,3 +58,67 @@ def test_multiple_anchors(brain): def test_anchor_property(brain): g = build_graph(brain, ["product:checkout"], depth=1) assert g.anchor == "product:checkout" + + +# -- whole-index overview ------------------------------------------------------ + + +def test_overview_includes_every_entity_in_scope(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product"]) + assert {n.id for n in graph.nodes} == { + e.id for e in brain.backend.all_entities(["product"])} + + +def test_overview_only_draws_edges_with_both_ends_in_scope(brain): + """A half-edge to a filtered-out type would render as a line to nowhere.""" + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product"]) + ids = {n.id for n in graph.nodes} + assert all(e.source in ids and e.target in ids for e in graph.edges) + + +def test_overview_keeps_edges_between_included_types(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product", "issue"]) + assert graph.edges, "product→issue edges should survive when both are in scope" + + +def test_overview_colours_nodes_by_doc_type(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + by_type = {n.doc_type: n.color for n in graph.nodes} + assert len(set(by_type.values())) > 1, "doc_types should be visually distinct" + + +def test_overview_cap_keeps_the_most_connected(brain): + """When the cap bites, it must keep the hubs — a random subset of a graph + is far less informative than its best-connected part.""" + from open_index.graph import build_overview_graph + from open_index.ui.view import edge_counts + + kept = {n.id for n in build_overview_graph(brain, limit=3).nodes} + assert len(kept) == 3 + + degree = edge_counts(brain) + dropped = {e.id for e in brain.backend.all_entities()} - kept + assert min(degree.get(i, 0) for i in kept) >= max(degree.get(i, 0) for i in dropped) + + +def test_overview_of_an_empty_scope(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["nonexistent-type"]) + assert graph.nodes == [] and graph.edges == [] + + +def test_overview_deduplicates_edges(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + keys = [(e.source, e.target, e.meaning) for e in graph.edges] + assert len(keys) == len(set(keys)) diff --git a/tests/test_mcp_transport.py b/tests/test_mcp_transport.py index 0ffafc5..290d42e 100644 --- a/tests/test_mcp_transport.py +++ b/tests/test_mcp_transport.py @@ -198,3 +198,106 @@ def no_uvicorn(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", no_uvicorn) with pytest.raises(SystemExit, match="open-index\\[serve\\]"): serve_http(str(brain.config.root)) + + +# -- proxy / load-balancer host validation ------------------------------------ +# +# The SDK turns on DNS-rebinding protection with a localhost-only allow-list, so +# every request arriving through a reverse proxy is rejected with +# "421 Invalid Host header". These pin the allow-list we build instead. + + +def test_public_url_host_is_allowed(): + from open_index.mcp_server import build_transport_security + + settings = build_transport_security(public_url="http://203.0.113.10/support/mcp") + assert settings.enable_dns_rebinding_protection is True + assert "203.0.113.10" in settings.allowed_hosts + + +def test_both_with_and_without_port_are_allowed(): + """A proxy drops the port when it is the scheme default; a direct client + sends it. Both must pass or one of the two paths breaks.""" + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_hosts + assert "brain.acme.com" in hosts + assert "brain.acme.com:*" in hosts + + +def test_explicit_host_and_port_survives(): + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security( + public_url="https://brain.acme.com:8443/mcp").allowed_hosts + assert "brain.acme.com:8443" in hosts + assert "brain.acme.com" in hosts + + +def test_loopback_is_always_allowed(): + """Health checks and on-box debugging must keep working.""" + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_hosts + assert {"127.0.0.1", "127.0.0.1:*", "localhost", "localhost:*"} <= set(hosts) + + +def test_extra_allowed_hosts_are_honoured(): + from open_index.mcp_server import build_transport_security + + hosts = build_transport_security(allowed_hosts=["lb.internal", "other:9000"]).allowed_hosts + assert "lb.internal" in hosts and "other:9000" in hosts + + +def test_wildcard_disables_the_check(): + """Explicit opt-out for a trusted proxy that already validates Host.""" + from open_index.mcp_server import build_transport_security + + assert build_transport_security( + allowed_hosts=["*"]).enable_dns_rebinding_protection is False + + +def test_origins_cover_http_and_https(): + from open_index.mcp_server import build_transport_security + + origins = build_transport_security(public_url="https://brain.acme.com/mcp").allowed_origins + assert "https://brain.acme.com" in origins + assert "http://brain.acme.com" in origins + + +def test_blank_entries_are_ignored(): + from open_index.mcp_server import build_transport_security + + settings = build_transport_security(allowed_hosts=["", " "]) + assert settings.enable_dns_rebinding_protection is True + + +def test_serve_http_passes_the_allow_list_through(brain, monkeypatch): + """The settings must actually reach streamable_http_app.""" + import uvicorn + + captured = {} + monkeypatch.setattr(uvicorn, "run", lambda app, host, port: None) + + from open_index import mcp_server + + real = mcp_server.build_server + + def spy(b, read_only=False): + server = real(b, read_only=read_only) + original = server.streamable_http_app + + def wrapper(**kwargs): + captured.update(kwargs) + return original(**kwargs) + + server.streamable_http_app = wrapper + return server + + monkeypatch.setattr(mcp_server, "build_server", spy) + mcp_server.serve_http(str(brain.config.root), host="0.0.0.0", + public_url="https://brain.acme.com/mcp", + warn_unauthenticated=False) + + assert captured["host"] == "0.0.0.0" + assert "brain.acme.com" in captured["transport_security"].allowed_hosts diff --git a/tests/test_multi_brain_serve.py b/tests/test_multi_brain_serve.py new file mode 100644 index 0000000..6abb4c8 --- /dev/null +++ b/tests/test_multi_brain_serve.py @@ -0,0 +1,218 @@ +"""Serving many brains from one process. + +A process per brain re-loads the Python runtime and a ~250MB resident embedding +model each time, which caps a modest host at a handful. These cover the mounted +app: routing, per-brain isolation, per-brain auth, and the lifespan wiring that +the MCP transport needs. +""" + +import json +import shutil +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") +pytest.importorskip("starlette") + +from starlette.testclient import TestClient # noqa: E402 + +from open_index.mcp_server import ( # noqa: E402 + build_multi_app, + discover_brains, + token_for, +) + +EXAMPLE = Path(__file__).resolve().parent.parent / "examples" / "support-brain" + +HANDSHAKE = { + "jsonrpc": "2.0", "id": "1", "method": "initialize", + "params": {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}, +} +HEADERS = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream"} + + +def client_for(app): + """A test client whose Host header is one the server accepts. + + Starlette's default is `testserver`, which the SDK's DNS-rebinding + protection rightly rejects with 421 — the same check that makes serving + behind a proxy require an explicit allow-list. + """ + return TestClient(app, base_url="http://localhost") + + +@pytest.fixture +def brains_root(tmp_path): + """Two independent brains side by side, plus a decoy directory.""" + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + (root / "not-a-brain").mkdir() # no brain.yaml + (root / "notes.txt").write_text("x") # not a directory + return root + + +# -- discovery ---------------------------------------------------------------- + + +def test_discovers_only_directories_holding_a_brain_yaml(brains_root): + assert sorted(discover_brains(str(brains_root))) == ["alpha", "beta"] + + +def test_discovery_rejects_a_missing_root(tmp_path): + with pytest.raises(SystemExit, match="no such directory"): + discover_brains(str(tmp_path / "nope")) + + +def test_empty_root_discovers_nothing(tmp_path): + assert discover_brains(str(tmp_path)) == {} + + +# -- per-brain tokens --------------------------------------------------------- + + +def test_token_comes_from_a_per_brain_variable(monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_SALES_EU", "specific") + assert token_for("sales-eu", "shared") == "specific" + + +def test_token_falls_back_to_the_shared_one(monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN_SALES", raising=False) + assert token_for("sales", "shared") == "shared" + + +def test_no_token_at_all_is_none(monkeypatch): + monkeypatch.delenv("OPEN_INDEX_TOKEN_SALES", raising=False) + assert token_for("sales") is None + + +# -- routing ------------------------------------------------------------------ + + +def test_each_brain_answers_on_its_own_path(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + for name in ("alpha", "beta"): + response = client.post(f"/{name}/mcp", json=HANDSHAKE, headers=HEADERS) + assert response.status_code == 200, response.text + assert "serverInfo" in response.text + + +def test_an_unknown_brain_path_is_a_404(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + assert client.post("/ghost/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 404 + + +def test_the_directory_lists_every_brain(brains_root): + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + listing = client.get("/").json() + assert set(listing) == {"alpha", "beta"} + assert listing["alpha"]["entities"] > 0 + assert "issue" in listing["alpha"]["doc_types"] + + +def test_health_is_unauthenticated_and_leaks_nothing(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), token="secret") + with client_for(app) as client: + response = client.get("/healthz") + assert response.status_code == 200 + assert response.text.strip() == "ok" + + +def test_public_base_url_is_advertised_per_brain(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), + public_base_url="https://x.example.com") + with client_for(app) as client: + listing = client.get("/").json() + assert listing["alpha"]["mcp"] == "https://x.example.com/alpha/mcp" + + +# -- isolation ---------------------------------------------------------------- + + +def test_a_brains_token_does_not_open_another(brains_root, monkeypatch): + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "alpha-token") + monkeypatch.setenv("OPEN_INDEX_TOKEN_BETA", "beta-token") + app = build_multi_app(discover_brains(str(brains_root))) + + with client_for(app) as client: + ok = client.post("/alpha/mcp", json=HANDSHAKE, + headers={**HEADERS, "Authorization": "Bearer alpha-token"}) + crossed = client.post("/beta/mcp", json=HANDSHAKE, + headers={**HEADERS, "Authorization": "Bearer alpha-token"}) + assert ok.status_code == 200 + assert crossed.status_code == 401 + + +def test_an_untokened_brain_stays_open_when_others_are_gated(brains_root, monkeypatch): + """A missing per-brain token must not silently inherit a neighbour's.""" + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA", "alpha-token") + monkeypatch.delenv("OPEN_INDEX_TOKEN_BETA", raising=False) + app = build_multi_app(discover_brains(str(brains_root))) + + with client_for(app) as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 401 + assert client.post("/beta/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 + + +def test_writing_to_one_brain_does_not_touch_the_other(brains_root): + """Separate storage, not a shared index with a tenant column.""" + from open_index.brain import Brain + from open_index.models import Entity + + Brain.open(brains_root / "alpha").put_entity( + Entity(id="issue:only-alpha", doc_type="issue", name="Only Alpha")) + + assert Brain.open(brains_root / "alpha").get_entity("issue:only-alpha") is not None + assert Brain.open(brains_root / "beta").get_entity("issue:only-alpha") is None + + +def test_read_only_applies_to_every_mounted_brain(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), read_only=True) + with client_for(app) as client: + assert client.get("/").json()["alpha"]["read_only"] is True + + +# -- the shared embedding model, which is the whole point --------------------- + + +def test_all_brains_share_one_embedding_provider(brains_root): + """One process per brain re-loads the model each time; that duplication is + what caps a host at a handful of brains.""" + import open_index.embeddings as emb + from open_index.config import load_brain_config + + emb._provider_cache.clear() + for name in ("alpha", "beta"): + emb.get_embedding_provider(load_brain_config(brains_root / name)) + + assert len(emb._provider_cache) == 1, ( + "each brain built its own provider — the model would be loaded per brain") + + +def test_lifespan_starts_every_mounted_session_manager(brains_root): + """Starlette does not run a mounted app's lifespan. Without wiring it up, + requests fail with 'Task group is not initialized' — so this asserts a real + request works for the *last* mount, not just the first.""" + with client_for(build_multi_app(discover_brains(str(brains_root)))) as client: + assert client.post("/beta/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 + + +def test_a_foreign_host_header_is_still_rejected(brains_root): + """The proxy allow-list applies per mount, not only to a single-brain serve.""" + app = build_multi_app(discover_brains(str(brains_root))) + with TestClient(app, base_url="http://evil.example.com") as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 421 + + +def test_the_public_host_is_accepted(brains_root): + app = build_multi_app(discover_brains(str(brains_root)), + public_base_url="https://brain.acme.com") + with TestClient(app, base_url="https://brain.acme.com") as client: + assert client.post("/alpha/mcp", json=HANDSHAKE, headers=HEADERS).status_code == 200 diff --git a/tests/test_opensearch_builders.py b/tests/test_opensearch_builders.py index e8eec6f..9b6dde2 100644 --- a/tests/test_opensearch_builders.py +++ b/tests/test_opensearch_builders.py @@ -153,3 +153,53 @@ def test_reserved_keys_cover_the_metadata_fields(): from open_index.storage.opensearch_backend import _RESERVED_KEYS assert {"provenance", "valid_from", "valid_to"} <= _RESERVED_KEYS + + +# -- fuzzy search must not be handed non-text fields --------------------------- +# +# OpenSearch rejects the entire query with "Can only use fuzzy queries on keyword +# and text fields" if a numeric or date field is listed in multi_match. One +# `number` field in the schema therefore broke *all* search — found with a real +# dataset, not caught by the earlier synthetic ones. + + +def test_numeric_fields_are_kept_out_of_the_fuzzy_query(): + dt = DocType(doc_type="defect", fields=[ + FieldSpec(name="name", type="string", boost=6), + FieldSpec(name="symptom", type="text"), + FieldSpec(name="manhours", type="number"), + ]) + fields = _backend_with_types([dt])._search_fields(None) + joined = " ".join(fields) + assert "fields.symptom" in joined + assert "manhours" not in joined, "a numeric field would break the whole query" + + +def test_boolean_and_timestamp_fields_are_excluded_too(): + dt = DocType(doc_type="t", fields=[ + FieldSpec(name="name", type="string"), + FieldSpec(name="active", type="boolean"), + FieldSpec(name="seen_at", type="timestamp"), + ]) + joined = " ".join(_backend_with_types([dt])._search_fields(None)) + assert "active" not in joined and "seen_at" not in joined + + +def test_text_fields_keep_their_boosts(): + dt = DocType(doc_type="t", fields=[ + FieldSpec(name="title", type="string", boost=6), + FieldSpec(name="body", type="text", boost=2), + FieldSpec(name="count", type="number", boost=9), + ]) + fields = _backend_with_types([dt])._search_fields(None) + assert "fields.title^6" in fields + assert "fields.body^2" in fields + assert not any("count" in f for f in fields) + + +def test_date_detection_is_disabled_in_the_mapping(): + """A `string` field holding e.g. 2026-04-12 would otherwise be mapped as a + date, and then rejected by fuzzy search.""" + assert OpenSearchBackend.mapping()["mappings"]["date_detection"] is False + backend = _backend_with_types([DocType(doc_type="t")]) + assert backend._base_mapping()["mappings"]["date_detection"] is False diff --git a/tests/test_ui_app.py b/tests/test_ui_app.py index d5e8f6b..a20bb07 100644 --- a/tests/test_ui_app.py +++ b/tests/test_ui_app.py @@ -5,6 +5,7 @@ structure you had to hunt for, and a map that opened blank. """ +import os import shutil from pathlib import Path @@ -22,7 +23,9 @@ def _run(brain_dir, monkeypatch): monkeypatch.setenv("OPEN_INDEX_DIR", str(brain_dir)) - return AppTest.from_file(str(APP), default_timeout=60).run() + # Generous timeout: each of these boots a real Streamlit script run, and on + # a loaded machine (the full suite, or a CI runner) 60s flaked once. + return AppTest.from_file(str(APP), default_timeout=180).run() @pytest.fixture @@ -55,10 +58,6 @@ def test_empty_brain_runs_without_exceptions(empty): assert not empty.exception, [e.value for e in empty.exception] -def test_tabs(populated): - assert [t.label for t in populated.tabs] == ["Explore", "Map", "Analytics", "Jobs"] - - # -- structure is visible without hunting for it ------------------------------ @@ -86,20 +85,27 @@ def test_empty_brain_sidebar_explains_what_to_do(empty): # -- the map no longer opens blank -------------------------------------------- -def test_map_preselects_anchors(populated): - """The original complaint: nothing rendered until you made a selection.""" - anchors = next(m for m in populated.multiselect if m.label == "Anchors") - assert anchors.value, "map opened with no anchors selected" - - -def test_map_anchor_options_are_entity_labels(populated): - anchors = next(m for m in populated.multiselect if m.label == "Anchors") - assert all("(" in option and ")" in option for option in anchors.options) +def test_map_draws_without_any_selection(populated): + """The original complaint: nothing rendered until you made a selection. The + map now opens on the whole index, so there is always something to see.""" + populated.tabs[3].run() # Map + assert not populated.exception def test_map_doc_type_filter_defaults_to_everything(populated): - types = next(m for m in populated.multiselect if m.label == "Doc types to include") + types = next(m for m in populated.multiselect if m.label == "Doc types shown") assert set(types.value) == set(types.options) + assert types.options, "every populated doc_type should be filterable" + + +def test_map_filter_lists_only_populated_doc_types(populated, tmp_path): + """A doc_type with no entities would be a dead checkbox.""" + from open_index.brain import Brain + from open_index.schema import DocType + + types = next(m for m in populated.multiselect if m.label == "Doc types shown") + counts = Brain.open(os.environ["OPEN_INDEX_DIR"]).counts() + assert all(counts.get(name, 0) > 0 for name in types.options) # -- explore ------------------------------------------------------------------ @@ -198,3 +204,122 @@ def test_row_css_inherits_colour_through_the_label_element(): """Streamlit wraps button labels in

, which otherwise keeps its own colour and ignores the inherit on the button.""" assert "> button p{color:inherit" in view.ROW_CSS + + +# -- How to use ---------------------------------------------------------------- + + +def test_help_tab_explains_the_model_before_the_connection_block(populated): + """The vocabulary has to land before "point your agent at this URL" means + anything to a first-time visitor.""" + markdown = [m.value for m in populated.markdown] + headings = [m for m in markdown if m.startswith("###")] + assert any("What an index holds" in h for h in headings) + assert headings.index(next(h for h in headings if "What an index holds" in h)) \ + < headings.index(next(h for h in headings if "Connect an agent" in h)) + + +def test_schema_marks_relationships_optional(populated): + populated.tabs[1].run() # Schema + assert not populated.exception + assert any("Relationships (optional)" in m.value for m in populated.markdown) + + +def test_help_is_the_first_tab_so_it_opens_on_load(populated): + """Streamlit selects the first tab, so a first-time visitor lands on the + explanation instead of having to find it.""" + assert [t.label for t in populated.tabs][0] == view.HELP_TAB == "How to use?" + + +def test_schema_sits_between_help_and_explore(populated): + labels = [t.label for t in populated.tabs] + assert labels[:3] == ["How to use?", "Schema", "Explore"] + + +def test_tab_guide_matches_the_tabs_actually_rendered(populated): + """A stale list of what-each-tab-does is worse than none.""" + assert [t.label for t in populated.tabs] == [n for n, _ in view.TAB_GUIDE] + + +def test_documented_tools_match_the_registered_mcp_tools(brain): + """The tab must not advertise a tool the server does not expose, nor miss one.""" + pytest.importorskip("mcp") + import asyncio as _asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain) + registered = {t.name for t in + _asyncio.new_event_loop().run_until_complete(server.list_tools())} + documented = {name.split("(")[0] for name, _ in view.READ_TOOLS + view.WRITE_TOOLS} + assert documented == registered, ( + f"docs vs server mismatch: only-in-docs={documented - registered}, " + f"only-on-server={registered - documented}") + + +def test_read_only_tools_are_the_documented_read_set(brain): + pytest.importorskip("mcp") + import asyncio as _asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain, read_only=True) + registered = {t.name for t in + _asyncio.new_event_loop().run_until_complete(server.list_tools())} + assert {n.split("(")[0] for n, _ in view.READ_TOOLS} == registered + + +def test_connection_block_is_valid_json_with_the_url(): + import json + + block = json.loads(view.mcp_client_config("https://x.example.com/demo/mcp", "demo")) + assert block["mcpServers"]["demo"]["url"] == "https://x.example.com/demo/mcp" + assert block["mcpServers"]["demo"]["type"] == "http" + + +def test_connection_block_appends_the_mcp_path(): + import json + + block = json.loads(view.mcp_client_config("https://x.example.com/demo")) + assert block["mcpServers"]["open-index"]["url"].endswith("/mcp") + + +# -- one UI process serving many brains ---------------------------------------- + + +@pytest.fixture +def multi(tmp_path, monkeypatch): + from open_index.brain import Brain + + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) + return AppTest.from_file(str(APP), default_timeout=180).run() + + +def test_an_empty_brains_root_explains_itself(tmp_path, monkeypatch): + empty = tmp_path / "none" + empty.mkdir() + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(empty)) + at = AppTest.from_file(str(APP), default_timeout=180).run() + assert any("No brains found" in e.value for e in at.error) + + +def test_single_brain_mode_shows_no_picker(populated): + """OPEN_INDEX_DIR alone must behave exactly as before.""" + assert not any(s.label.startswith("Index") for s in populated.sidebar.selectbox) + + +def test_many_brains_render_without_error(multi): + assert not multi.exception, [e.value for e in multi.exception] + + +def test_no_index_switcher_widget_anywhere(multi): + """The URL is the only selector; a dropdown would let the address bar and + the screen disagree about which index you are looking at.""" + assert not multi.sidebar.selectbox diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index eca21f6..28ff901 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -226,3 +226,248 @@ def test_every_theme_defines_the_full_palette(): keys = {"node_label", "edge_label", "edge", "stroke_width"} for theme in ("dark", "light"): assert keys <= set(view.graph_theme(theme)) + + +# -- Schema tab ---------------------------------------------------------------- + + +def test_schema_field_rows_describe_search_behaviour(brain): + dt = brain.config.doc_type("issue") + rows = {r["field"]: r for r in view.schema_field_rows(dt)} + assert rows["name"]["searched by"] == "keyword match" + assert rows["description"]["searched by"] == "meaning (vector)" + assert rows["name"]["weight"] == "6×" + + +def test_unsearched_fields_show_no_weight(brain): + from open_index.schema import DocType, FieldSpec + + dt = DocType(doc_type="t", fields=[FieldSpec(name="blob", search="none", boost=9)]) + row = view.schema_field_rows(dt)[0] + assert row["searched by"] == "not searched" + assert row["weight"] == "—", "a weight on an unsearched field is misleading" + + +def test_required_fields_are_marked(brain): + from open_index.schema import DocType, FieldSpec + + dt = DocType(doc_type="t", fields=[FieldSpec(name="owner", required=True)]) + assert view.schema_field_rows(dt)[0]["required"] == "yes" + + +def test_relationship_rows_merge_declared_and_observed(brain): + rows = {r["relationship"]: r for r in view.schema_relationship_rows(brain, "product")} + assert "has common issue" in rows + assert rows["has common issue"]["declared"] == "yes" + assert rows["has common issue"]["points at"] == "issue" + assert rows["has common issue"]["in use"] > 0 + + +def test_undeclared_relationships_in_use_are_still_listed(brain): + """An edge nobody declared but everything uses is worth seeing.""" + from open_index.models import Entity, Relationship + + brain.put_entity(Entity( + id="product:improvised", doc_type="product", name="Improvised", + related_to=[Relationship(target="issue:no-results", + relationship_edge_meaning="totally made up")])) + rows = {r["relationship"]: r for r in view.schema_relationship_rows(brain, "product")} + assert rows["totally made up"]["declared"] == "no" + assert rows["totally made up"]["in use"] == 1 + + +def test_declared_but_unused_relationships_show_zero(brain): + from open_index.schema import DocType, RelationshipSpec + + brain.create_doc_type(DocType( + doc_type="unused_rel", + relationships=[RelationshipSpec(name="never used", target_doc_type="issue")])) + row = view.schema_relationship_rows(brain, "unused_rel")[0] + assert row["declared"] == "yes" and row["in use"] == 0 + + +def test_help_tab_is_first_in_the_guide(): + assert view.TAB_GUIDE[0][0] == view.HELP_TAB + assert [n for n, _ in view.TAB_GUIDE][:3] == ["How to use?", "Schema", "Explore"] + + +# -- map readability ----------------------------------------------------------- +# +# Long entity names drawn next to their dot were the main thing making the map +# unreadable; the full text moves to the hover tooltip. + + +def test_nodes_carry_no_label(brain): + """Entity names are long and arbitrary; drawn beside every dot they overlap + each other. Identity comes from the tooltip instead.""" + from open_index.graph import build_overview_graph + + specs = view.graph_node_specs(build_overview_graph(brain)) + assert specs + assert all(s["label"] == "" for s in specs) + # Empty string, not None: None serialises to null and vis draws that. + assert all(s["label"] is not None for s in specs) + + +def test_every_node_carries_its_identity_on_hover(brain): + from open_index.graph import build_overview_graph + + specs = {s["id"]: s for s in view.graph_node_specs(build_overview_graph(brain))} + spec = specs["product:checkout"] + assert "Checkout" in spec["title"] + assert "product:checkout" in spec["title"] + + +def test_edges_carry_no_label_but_name_the_relationship(brain): + from open_index.graph import build_overview_graph + + specs = view.graph_edge_specs(build_overview_graph(brain), "#ccc") + assert specs + assert all(s["label"] == "" for s in specs) + assert any("has common issue" in s["title"] for s in specs) + + +def test_node_colour_comes_from_the_doc_type(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain) + by_id = {n.id: n.color for n in graph.nodes} + assert all(s["color"] == by_id[s["id"]] for s in view.graph_node_specs(graph)) + + +def test_node_tooltip_carries_the_full_name_and_type(): + tip = view.node_tooltip("issue:x", "A very long name that got truncated", + "issue", {"severity": "high"}) + assert "A very long name that got truncated" in tip + assert "issue:x" in tip + assert "severity: high" in tip + + +def test_node_tooltip_skips_empty_fields_and_caps_length(): + tip = view.node_tooltip("t:x", "N", "t", + {f"f{i}": "v" for i in range(20)} | {"blank": ""}) + assert "blank" not in tip + assert len(tip.splitlines()) <= 7 + + +def test_edge_tooltip_names_the_relationship(): + tip = view.edge_tooltip("a:1", "b:2", "depends on") + assert "depends on" in tip and "a:1" in tip and "b:2" in tip + + +def test_edge_tooltip_handles_an_unlabelled_edge(): + assert "related" in view.edge_tooltip("a:1", "b:2", "") + + +def test_legend_describes_what_is_on_screen(brain): + from open_index.graph import build_overview_graph + + graph = build_overview_graph(brain, ["product", "issue"]) + rows = view.legend_rows(brain, graph) + assert {r["doc_type"] for r in rows} <= {"product", "issue"} + assert all(r["color"].startswith("#") for r in rows) + assert sum(r["count"] for r in rows) == len(graph.nodes) + + +def test_legend_is_ordered_by_count(brain): + from open_index.graph import build_overview_graph + + rows = view.legend_rows(brain, build_overview_graph(brain)) + assert [r["count"] for r in rows] == sorted([r["count"] for r in rows], reverse=True) + + +def test_graph_width_fits_beside_the_legend(): + assert view.GRAPH_WIDTH <= 1000 + + +# -- the model explanation on the help tab ------------------------------------ + + +def test_model_guide_covers_the_three_ideas(): + terms = " ".join(t for t, _ in view.MODEL_GUIDE).lower() + assert "doc_type" in terms + assert "entity" in terms and "doc" in terms # both names for an instance + assert "relationship" in terms + + +def test_model_guide_says_relationships_are_optional(): + text = dict(view.MODEL_GUIDE)["relationship"].lower() + assert "optional" in text + assert "without any" in text, "should say entities are valid with no edges" + + +def test_model_guide_explains_the_id_convention(): + """The single most common write failure, so it belongs in the explanation.""" + text = " ".join(w for _, w in view.MODEL_GUIDE) + assert ":" in text + + +def test_model_guide_distinguishes_schema_from_data(): + text = dict(view.MODEL_GUIDE)["doc_type"].lower() + assert "schema" in text + + +# -- one UI process, many brains: the URL is the selector ---------------------- + + +def test_the_url_path_selects_the_brain(): + assert view.brain_from_url("https://h/sales-index", + ["support-index", "sales-index"]) == "sales-index" + + +def test_a_deeper_path_selects_on_the_first_segment(): + assert view.brain_from_url("https://h/sales-index/x", + ["alpha", "sales-index"]) == "sales-index" + + +def test_the_root_path_falls_back_to_the_first_brain(): + assert view.brain_from_url("https://h/", ["alpha", "beta"]) == "alpha" + + +def test_an_unknown_path_falls_back_rather_than_erroring(): + """A stale link should land somewhere useful.""" + assert view.brain_from_url("https://h/deleted", ["alpha", "beta"]) == "alpha" + + +def test_a_missing_url_falls_back(): + assert view.brain_from_url(None, ["alpha", "beta"]) == "alpha" + + +def test_a_query_string_does_not_affect_selection(): + assert view.brain_from_url("https://h/beta?x=1", ["alpha", "beta"]) == "beta" + + +def test_no_brains_selects_nothing(): + assert view.brain_from_url("https://h/x", []) is None + + +# -- the endpoint shown on the help tab ---------------------------------------- +# +# A shared explorer cannot be handed one correct MCP URL as configuration: the +# answer depends on which index you are looking at. It is derived from the page +# URL instead, which also keeps it right behind any proxy or hostname. + + +def test_endpoint_is_derived_from_the_page_url(): + assert view.mcp_url_for("https://brain.acme.com/sales-index", "sales-index") \ + == "https://brain.acme.com/sales-index/mcp" + + +def test_endpoint_ignores_the_rest_of_the_path_and_query(): + assert view.mcp_url_for("https://h:8443/sales-index/x?q=1", "sales-index") \ + == "https://h:8443/sales-index/mcp" + + +def test_endpoint_uses_the_directory_name_not_the_configured_brain_name(): + """The URL segment is the directory; brain.yaml's name may differ.""" + assert view.mcp_url_for("https://h/dir-name", "dir-name").endswith("/dir-name/mcp") + + +def test_endpoint_keeps_the_scheme(): + assert view.mcp_url_for("http://localhost:8501/a", "a").startswith("http://") + + +def test_no_endpoint_without_a_url_or_a_name(): + assert view.mcp_url_for(None, "a") is None + assert view.mcp_url_for("https://h/a", None) is None + assert view.mcp_url_for("/relative/path", "a") is None