feat: many indexes on one host — usable explorer, URL-path routing, proxy-safe MCP

.gitignore

@@ -26,3 +26,11 @@ dist/
 .coverage
 coverage.xml
 htmlcov/
+
+# Generated fleet deployment state (host names, index lists, credentials).
+# `deploy/fleet/fleet.py` renders these for a specific machine; they are not
+# part of the tool. Keep the example file, ignore the rendered output.
+deploy/fleet/indexes.yml
+deploy/fleet/docker-compose.generated.yml
+deploy/fleet/caddy/
+deploy/fleet/secrets/

README.md

@@ -49,7 +49,8 @@ Prefer containers, or need a brain several agents share? →
 | `open-index search <query> [-t doc_type]` | Search from the terminal. |
 | `open-index ui` | Launch the Streamlit explorer (Explore / **Map** / Analytics / Jobs). |
 | `open-index mcp [--read-only]` | Run the MCP context layer over stdio. **Read+write by default**; `--read-only` opts out of writes. |
-| `open-index serve [--port --token --read-only]` | Serve the same read+write MCP context layer over **HTTP** for remote agents (bearer-token auth). |
+| `open-index serve [--port --token --read-only]` | Serve the MCP context layer over **HTTP** for remote agents (bearer-token auth). |
+| `open-index serve --brains <root>` | Serve **every** brain under a directory from one process, each at `/<name>/mcp`. |
 | `open-index mcp-config [--url --token]` | Print the MCP connection block to paste into your agent. |
 
 ### A context layer for domain-specialized agents

docker/entrypoint.sh

@@ -1,26 +1,58 @@
 #!/usr/bin/env sh
 # Container entrypoint for open-index.
 #
-# Two jobs beyond exec'ing the CLI:
+# It runs in one of two modes:
 #
-#   1. Default --brain to $OPEN_INDEX_BRAIN (/brain) so every command works
-#      without repeating the flag in compose files and `docker run` lines.
-#   2. Reconcile file-backed entities into the search index before serving.
-#      Entities under entities/**/*.json are the git source of truth; a fresh
-#      container (or a fresh OpenSearch cluster) starts with an empty index, so
-#      without this the brain answers every query with "no results" and looks
-#      broken. Index-backed entities are untouched by this — see Brain.index().
+#   single-brain  one brain mounted at $OPEN_INDEX_BRAIN (/brain). The --brain
+#                 flag is filled in so compose files and `docker run` lines do
+#                 not repeat it.
+#   many-brains   a directory of brains at $OPEN_INDEX_BRAINS_ROOT (/brains),
+#                 selected by `serve --brains` or by the explorer. Nothing is
+#                 prepended; the command already knows where to look.
 #
-# Skip step 2 with OPEN_INDEX_SKIP_INDEX=1 (e.g. a read replica, or when the
-# index is large and managed out of band).
+# In both modes, file-backed entities are reconciled into the search index
+# before serving. Entities under entities/**/*.json are the git source of truth,
+# so a fresh container starts with an empty index and would otherwise answer
+# every query with "no results" and look broken.
+#
+# Skip the reconcile with OPEN_INDEX_SKIP_INDEX=1 (a read replica, or an index
+# large enough to be managed out of band).
 set -eu
 
 BRAIN="${OPEN_INDEX_BRAIN:-/brain}"
+BRAINS_ROOT="${OPEN_INDEX_BRAINS_ROOT:-}"
+
+# `serve --brains <root>` selects many-brains mode even without the env var.
+for arg in "$@"; do
+  if [ "$arg" = "--brains" ]; then MULTI_FLAG=1; fi
+done
+
+if [ -n "$BRAINS_ROOT" ] || [ "${MULTI_FLAG:-0}" = "1" ]; then
+  MULTI=1
+  ROOT="${BRAINS_ROOT:-/brains}"
+else
+  MULTI=0
+fi
 
-if [ ! -f "$BRAIN/brain.yaml" ]; then
+if [ "$MULTI" = "1" ]; then
+  if [ ! -d "$ROOT" ]; then
+    echo "open-index: no directory at $ROOT" >&2
+    echo "  mount a directory of brains there, e.g. -v \"\$PWD/brains:/brains\"" >&2
+    exit 1
+  fi
+  # A brain is a subdirectory holding brain.yaml. An empty root is almost
+  # always a mis-mounted volume, so say so rather than serving nothing.
+  if [ -z "$(find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml -print -quit)" ]; then
+    echo "open-index: no brains under $ROOT" >&2
+    echo "  each brain is a subdirectory containing brain.yaml" >&2
+    echo "  (create one with: open-index init <name>)" >&2
+    exit 1
+  fi
+elif [ ! -f "$BRAIN/brain.yaml" ]; then
   echo "open-index: no brain.yaml at $BRAIN" >&2
   echo "  mount your brain directory there, e.g. -v \"\$PWD/my-brain:/brain\"" >&2
   echo "  (create one first with: open-index init my-brain)" >&2
+  echo "  (or mount a directory of brains and set OPEN_INDEX_BRAINS_ROOT)" >&2
   exit 1
 fi
 
@@ -50,17 +82,36 @@ except Exception:
   echo "open-index: OpenSearch is up."
 fi
 
+reconcile_all() {
+  # Reconcile every brain under the root. Done here rather than inside `serve`
+  # so it happens once at container start instead of on every request.
+  find "$ROOT" -mindepth 2 -maxdepth 2 -name brain.yaml | while read -r f; do
+    d=$(dirname "$f")
+    echo "open-index: indexing $(basename "$d") ..."
+    open-index index --brain "$d" >/dev/null || \
+      echo "open-index: WARNING $(basename "$d") failed to index" >&2
+  done
+}
+
 case "${1:-serve}" in
   serve|ui|index|search|validate|run|ingest|mcp|mcp-config|list-connectors|add-entity|add-doc-type)
     COMMAND="$1"
     shift
 
     if [ "$COMMAND" = "serve" ] && [ "${OPEN_INDEX_SKIP_INDEX:-0}" != "1" ]; then
-      echo "open-index: reconciling file-backed entities into the index ..."
-      open-index index --brain "$BRAIN"
+      if [ "$MULTI" = "1" ]; then
+        reconcile_all
+      else
+        echo "open-index: reconciling file-backed entities into the index ..."
+        open-index index --brain "$BRAIN"
+      fi
     fi
 
-    set -- "$COMMAND" --brain "$BRAIN" "$@"
+    if [ "$MULTI" = "1" ]; then
+      set -- "$COMMAND" "$@"          # the command carries its own --brains
+    else
+      set -- "$COMMAND" --brain "$BRAIN" "$@"
+    fi
     ;;
   *)
     # Anything else (sh, python, a raw open-index invocation) runs verbatim.

docs/deployment.md

@@ -265,6 +265,47 @@ chmod -R g+rwX /path/to/my-brain
 You can still read and edit the files; writes from inside the container land as
 uid 10001 with your group.
 
+### Many brains from one process
+
+`serve --brain <dir>` runs one brain. For more than a handful, `--brains
+<root>` serves every brain under a directory from a single process, each at
+`/<name>/mcp`:
+
+```bash
+open-index serve --brains /srv/brains --port 8080
+#   /srv/brains/support/  →  /support/mcp
+#   /srv/brains/sales/    →  /sales/mcp
+```
+
+This matters because of what is *not* duplicated. A process per brain re-loads
+the Python runtime and a ~250MB resident embedding model each time, so a modest
+host tops out at a handful. In one process the model is loaded once and a brain
+costs only its config and doc_types.
+
+Measured on the bundled example brain, SQLite-backed:
+
+| Brains | One process | One process per brain |
+|---|---|---|
+| 50 | 345 MB | ~13 GB |
+| 200 | **373 MB** | ~53 GB |
+
+That is **1.8MB of marginal cost per brain**, and 200 mount in ~4 seconds.
+
+Each brain keeps its own storage, its own read/write policy and its own token —
+`OPEN_INDEX_TOKEN_<NAME>` gates one brain (`OPEN_INDEX_TOKEN_SALES_EU` for
+`sales-eu/`), and `--token` covers any without one. Nothing is shared between
+brains except the process and the model.
+
+Two extras come with it: `GET /` lists every brain with its URL, entity count
+and doc_types, and `GET /healthz` is an unauthenticated probe for a load
+balancer.
+
+> **Prefer SQLite here.** With OpenSearch, every brain is a separate cluster
+> index and therefore a shard; the working guidance is ~20 shards per GB of
+> heap, so hundreds of brains would hit that ceiling long before RAM. SQLite
+> gives each brain its own file and no shard cost at all. Use OpenSearch for the
+> few brains that genuinely need concurrent writers or >10k entities.
+
 ### Running one-off commands
 
 ```bash

open_index/cli.py

@@ -441,6 +441,12 @@ def mcp_config(
 @app.command()
 def serve(
     brain: str = BrainOpt,
+    brains: Optional[str] = typer.Option(
+        None, "--brains", envvar="OPEN_INDEX_BRAINS_ROOT",
+        help="Serve EVERY brain under this directory from one process, each at "
+             "/<name>/mcp. Cheaper than one process per brain: the embedding "
+             "model is loaded once for all of them.",
+    ),
     host: str = typer.Option("0.0.0.0", help="Bind host."),
     port: int = typer.Option(8080, help="Bind port."),
     token: Optional[str] = typer.Option(
@@ -452,6 +458,12 @@ def serve(
         help="Externally reachable URL, when behind a proxy/tunnel/load balancer. "
              "Only used to print correct connection details.",
     ),
+    allowed_host: Optional[list[str]] = typer.Option(
+        None, "--allowed-host", envvar="OPEN_INDEX_ALLOWED_HOSTS",
+        help="Host header(s) to accept, when behind a proxy or load balancer. "
+             "Repeatable. The host from --public-url is added automatically. "
+             "Use '*' to disable the check entirely (trusted proxy only).",
+    ),
     read_only: bool = typer.Option(
         False, "--read-only",
         help="Opt out of default read+write mode and expose only read tools.",
@@ -463,11 +475,42 @@ def serve(
     OpenSearch backend for a shared, multi-writer brain.
     """
     from open_index.mcp_config import advertised_urls, mask_token
-    from open_index.mcp_server import serve_http
+    from open_index.mcp_server import (
+        discover_brains, serve_http, serve_http_multi, token_for,
+    )
+
+    mode = "read-only" if read_only else "read+write"
+
+    if brains:
+        found = discover_brains(brains)
+        if not found:
+            typer.secho(f"no brains under {brains} — a brain is a directory "
+                        "containing brain.yaml", fg=typer.colors.RED, err=True)
+            raise typer.Exit(1)
+
+        typer.secho(f"open-index · {len(found)} brains · {mode}",
+                    fg=typer.colors.GREEN, bold=True)
+        typer.echo(f"  listening on {host}:{port}")
+        urls = advertised_urls(host, port, public_url=public_url)
+        base = urls[-1][1].rsplit("/mcp", 1)[0]
+        typer.echo("")
+        for name in found:
+            gate = "token" if token_for(name, token) else typer.style(
+                "OPEN", fg=typer.colors.YELLOW)
+            typer.secho(f"  {name:<28} {base}/{name}/mcp  ({gate})",
+                        fg=typer.colors.CYAN)
+        typer.echo("")
+        typer.echo("  per-brain tokens come from OPEN_INDEX_TOKEN_<NAME>; "
+                   "--token covers any without one.")
+        typer.echo(f"  directory: {base}/   ·   health: {base}/healthz")
+        typer.echo("")
+        serve_http_multi(brains, host=host, port=port, token=token,
+                         read_only=read_only, public_base_url=base,
+                         allowed_hosts=list(allowed_host or []))
+        return
 
     root = Path(brain).resolve()
     b = _open_brain(str(root))
-    mode = "read-only" if read_only else "read+write"
 
     typer.secho(
         f"open-index · brain '{b.config.name}' · {mode} · "
@@ -502,7 +545,8 @@ def serve(
     typer.echo("")
 
     serve_http(str(root), host=host, port=port, token=token, read_only=read_only,
-               warn_unauthenticated=False)
+               warn_unauthenticated=False, public_url=public_url,
+               allowed_hosts=list(allowed_host or []))
 
 
 if __name__ == "__main__":  # pragma: no cover - module entry point

open_index/config.py

@@ -220,6 +220,22 @@ def write_doc_type(brain_root: Path, dt: DocType) -> Path:
     return path
 
 
+def discover_brains(root: str | Path) -> dict[str, Path]:
+    """Every brain directory directly under `root`, keyed by directory name.
+
+    A "brain" is any subdirectory holding a brain.yaml, so a root can sit
+    alongside unrelated folders without confusing anything.
+    """
+    base = Path(root).expanduser().resolve()
+    if not base.is_dir():
+        raise FileNotFoundError(f"no such directory: {base}")
+    return {
+        child.name: child
+        for child in sorted(base.iterdir())
+        if (child / "brain.yaml").exists()
+    }
+
+
 def iter_entity_files(brain_dir: str | Path):
     """Yield every entities/**/*.json path under the brain directory."""
     root = Path(brain_dir).expanduser().resolve()

open_index/graph.py

@@ -69,6 +69,62 @@ def _label_for(brain: Brain, entity) -> str:
     return entity.label_for(label_field)
 
 
+def build_overview_graph(
+    brain: Brain,
+    doc_types: Optional[list[str]] = None,
+    limit: int = 250,
+) -> ContextGraph:
+    """Every entity of the given doc_types, with the edges that run between them.
+
+    The anchor-and-expand view answers "what is around this one thing?". This
+    answers the other question — "what does this index look like?" — which is
+    what you want when you have never seen the index before and have no entity
+    in mind to anchor on.
+
+    Only edges whose *both* ends are in scope are drawn: a dangling half-edge to
+    a filtered-out doc_type would render as an unexplained line to nowhere.
+
+    `limit` caps the node count, keeping the most-connected entities. A force
+    layout with thousands of nodes is neither readable nor quick, and the
+    best-connected ones are the ones worth seeing. Callers should report when
+    the cap bit — silently showing a subset reads as "this is everything".
+    """
+    entities = brain.backend.all_entities(doc_types)
+
+    degree: dict[str, int] = {e.id: 0 for e in entities}
+    for entity in entities:
+        for rel in entity.related_to:
+            degree[entity.id] = degree.get(entity.id, 0) + 1
+            if rel.target in degree:
+                degree[rel.target] = degree.get(rel.target, 0) + 1
+
+    if len(entities) > limit:
+        entities = sorted(entities, key=lambda e: (-degree.get(e.id, 0), e.id))[:limit]
+
+    in_scope = {e.id for e in entities}
+    graph = ContextGraph(anchors=[])
+    for entity in entities:
+        graph.nodes.append(GraphNode(
+            id=entity.id,
+            label=_label_for(brain, entity),
+            doc_type=entity.doc_type,
+            color=_color_for(brain, entity.doc_type),
+            depth=0,
+            data=entity.to_json(),
+        ))
+
+    seen: set[tuple[str, str, str]] = set()
+    for entity in entities:
+        for source, target, meaning in brain.backend.relationships_from(entity.id):
+            if target not in in_scope:
+                continue
+            key = (source, target, meaning)
+            if key not in seen:
+                seen.add(key)
+                graph.edges.append(GraphEdge(source, target, meaning))
+    return graph
+
+
 def build_graph(brain: Brain, anchors: str | list[str], depth: int = 1) -> ContextGraph:
     """Build the map around one or more anchor entities.