diff --git a/docs/docs.json b/docs/docs.json index d44be4d..b0cddfa 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -48,7 +48,8 @@ { "group": "Reference", "pages": [ - "reference/cli" + "reference/cli", + "reference/http-api" ] } ] diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index 1c924bb..da486b0 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -36,6 +36,9 @@ changing dimensions. | Command | What it does | |---|---| | `open-index search [-t doc_type]` | Search from the terminal. | +| `open-index delete ` | Delete one entity, its edges and its file (`--yes` to skip the prompt). | +| `open-index lookup ` | Find an entity by the id its source system knows it by. | +| `open-index trace ` | What one turn retrieved: each query, each document, rank and score. | | `open-index ui` | Launch the explorer (How to use / Schema / Explore / **Map** / Analytics / Jobs). | ## Serving over MCP diff --git a/docs/reference/http-api.mdx b/docs/reference/http-api.mdx new file mode 100644 index 0000000..bcf309c --- /dev/null +++ b/docs/reference/http-api.mdx @@ -0,0 +1,131 @@ +--- +title: HTTP API +description: The JSON API over a brain — the same operations MCP exposes, for everything that is not an agent. +--- + +MCP is how agents talk to an index. This is how everything else does: a script, +a cron job, a service, `curl`. + +It mirrors the MCP tools rather than inventing a second vocabulary, and calls the +same `Brain` methods underneath — so the two cannot disagree about what a search +means or what a write validates. + +The API is served by `open-index ui`, alongside the explorer, so one process and +one port serve the UI, the API and the MCP endpoint for every index on the host. + +``` +https:////api/v1/... one of several indexes +http://localhost:8501/api/v1/... a single brain +``` + +## Reading + +### Search + +```bash +curl "$HOST/api/v1/search?q=payment+declined&mode=hybrid&limit=10" +``` + +| Parameter | | +|---|---| +| `q` | free text; omit it to list | +| `mode` | `hybrid` (default), `keyword`, `semantic` | +| `doc_type` | repeatable — `&doc_type=issue&doc_type=product` | +| `filter.` | exact match, e.g. `filter.tenant_id=acme` | +| `limit` | default 20 | + +Every result carries `match`, saying **why** it came back: + +```json +{ + "id": "issue:payment-declined", + "score": 0.937, + "match": { "type": "both", "keyword_score": 1.0, "semantic_score": 0.79 } +} +``` + +`type` is `keyword`, `semantic`, `both`, `filter` or `none`. Read it before +trusting a result — a semantic-only hit at a low score is a guess, not a fact. + +**Filters are hard predicates, not ranking hints.** A document that does not +match cannot be returned at any score, in any mode. Only fields declared +`filterable: true` can be filtered, and filtering on any other field returns +`400` rather than being ignored — so a filter never silently fails open. This +matters when the filter is carrying a tenant or user boundary. + +### Entities + +```bash +curl "$HOST/api/v1/entities/issue:payment-declined" +curl "$HOST/api/v1/entities?id=issue:a&id=issue:b" # batch +curl "$HOST/api/v1/entities/by-external-id/CRM-4471" # your own id +curl "$HOST/api/v1/schema" +``` + +A single entity comes back with its relationships in **both** directions — +incoming answers "what else points at this?", which the entity's own document +cannot. + +The batch endpoint reports `missing` rather than padding the list with nulls. + +### Traces + +```bash +curl "$HOST/api/v1/traces/turn-8f21c3" +``` + +Every read records the `X-Trace-Id` header it arrived with. Send one and you can +later recover exactly which documents that turn retrieved, at what rank, with +what score, and on what kind of match — which is how you work out afterwards what +the index actually fed an agent. + +## Writing + +```bash +curl -X PUT "$HOST/api/v1/entities/issue:payment-declined" \ + -H 'Content-Type: application/json' \ + -d '{"doc_type": "issue", "name": "Payment declined", "severity": "high", + "external_id": "CRM-4471"}' + +curl -X DELETE "$HOST/api/v1/entities/issue:payment-declined" +``` + +`PUT` is an upsert. The id in the URL wins: a body id that disagrees is a `400`, +so `PUT /entities/a` can never write entity `b`. + +`DELETE` removes the entity, its file (for a `storage: file` doc_type), and every +edge naming it in either direction. Deleting only the index row would leave the +file to resurrect it on the next reindex — a pause, not a delete. + +Prefer correcting an entity over deleting it: an id that once resolved and now +404s breaks anything still holding a reference. + +## Authentication + +Off unless a token is configured. Set `OPEN_INDEX_TOKEN` (or a per-index +`OPEN_INDEX_TOKEN_`) and **writes** require it: + +```bash +curl -X PUT "$HOST/api/v1/entities/issue:x" \ + -H "Authorization: Bearer $OPEN_INDEX_TOKEN" ... +``` + +Reads stay open either way, matching how the MCP endpoint already behaves. A +deployment holding real data sets a token; one serving public demo data does not, +and nothing changes for it. + + +With no token set, anyone who can reach the endpoint can write to it and delete +from it. That is the right default for disposable demo data and the wrong one for +anything you would miss. + + +## Status codes + +| | | +|---|---| +| `400` | your request — bad mode, unfilterable field, malformed body, id mismatch | +| `401` | a write without a valid token, when one is configured | +| `404` | unknown index, entity, or external id | +| `422` | a well-formed entity that failed schema validation | +| `500` | the write could not complete; the message says what state things are in | diff --git a/open_index/api.py b/open_index/api.py new file mode 100644 index 0000000..13e1bfc --- /dev/null +++ b/open_index/api.py @@ -0,0 +1,220 @@ +"""A JSON HTTP API over a brain — the same operations MCP exposes, over HTTP. + +MCP is for agents; this is for everything else: a script, a job, a service, a +`curl`. It deliberately mirrors the MCP tools rather than inventing a second +vocabulary, and calls the same `Brain` methods, so the two cannot disagree about +what a search means or what a write validates. + +Mounted at `//api/v1` alongside the explorer, so one process and one port +serve the UI, the API and the MCP endpoint for every brain on the host. + +Auth is off unless a token is configured. `OPEN_INDEX_TOKEN` (or the per-brain +`OPEN_INDEX_TOKEN_`) gates *writes* only — reads stay open, matching how +the MCP endpoint already behaves. A deployment holding real data sets the token; +one serving public demo data does not, and nothing changes for it. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from open_index.brain import Brain + + +def token_for(name: Optional[str]) -> Optional[str]: + """The write token for this brain, if one is configured.""" + if name: + specific = os.environ.get( + "OPEN_INDEX_TOKEN_" + name.upper().replace("-", "_").replace(".", "_")) + if specific: + return specific + return os.environ.get("OPEN_INDEX_TOKEN") or None + + +def _entity_payload(brain: Brain, entity) -> dict[str, Any]: + """One entity as the API returns it: the document plus both edge directions.""" + payload = entity.to_json() + payload["relationships"] = { + "outgoing": [{"target": t, "meaning": m} + for (_s, t, m) in brain.backend.relationships_from(entity.id)], + "incoming": [{"source": s, "meaning": m} + for (s, _t, m) in brain.backend.relationships_to(entity.id)], + } + return payload + + +def build_routes(resolve, prefix: str = ""): + """Routes for the JSON API. + + `resolve(request)` returns `(name, Brain)` — supplied by the caller so the + API and the explorer agree on which index a path refers to instead of each + working it out separately. + """ + from starlette.responses import JSONResponse + from starlette.routing import Route + + def error(message: str, status: int, **extra): + return JSONResponse({"error": message, **extra}, status_code=status) + + def authorized(request, name: Optional[str]) -> bool: + expected = token_for(name) + if not expected: + return True # no token configured: writes are open + header = request.headers.get("authorization", "") + scheme, _, value = header.partition(" ") + return scheme.lower() == "bearer" and value == expected + + def with_brain(handler, *, write: bool = False): + async def endpoint(request): + name, brain = resolve(request) + if brain is None: + return error("unknown index", 404) + if write and not authorized(request, name): + # 401 with a challenge, not 403: the caller can fix this by + # presenting a token, and should be told how. + return JSONResponse( + {"error": "a bearer token is required to write to this index"}, + status_code=401, + headers={"WWW-Authenticate": 'Bearer realm="open-index"'}, + ) + return await handler(request, brain) + return endpoint + + # -- reads ---------------------------------------------------------------- + + async def search(request, brain: Brain): + params = request.query_params + try: + limit = int(params.get("limit", 20)) + except ValueError: + return error("limit must be an integer", 400) + + filters: dict[str, Any] = {} + # filter.=, so a filter needs no JSON body on a GET. + for key, value in params.multi_items(): + if key.startswith("filter."): + filters[key[len("filter."):]] = value + + try: + results = brain.search( + query=params.get("q"), + doc_types=[t for t in params.getlist("doc_type") if t] or None, + limit=limit, + mode=params.get("mode", "hybrid"), + filters=filters or None, + source="api", + ) + except ValueError as exc: + # An unknown mode or a filter on an undeclared field. The caller can + # fix both, and the message says how — so it is a 400, not a 500. + return error(str(exc), 400) + + return JSONResponse({ + "query": params.get("q"), + "mode": params.get("mode", "hybrid"), + "filters": filters, + "total": results.total, + "doc_type_counts": results.doc_type_counts, + "limited": results.limited, + "results": results.results, + }) + + async def get_one(request, brain: Brain): + entity_id = request.path_params["entity_id"] + entity = brain.get_entity(entity_id, source="api") + if entity is None: + return error(f"no entity '{entity_id}'", 404) + return JSONResponse(_entity_payload(brain, entity)) + + async def get_many(request, brain: Brain): + ids = [i for i in request.query_params.getlist("id") if i] + found = brain.get_entities(ids, source="api") + return JSONResponse({ + "requested": len(ids), + "found": len(found), + "missing": sorted(set(ids) - {e.id for e in found}), + "entities": [_entity_payload(brain, e) for e in found], + }) + + async def by_external(request, brain: Brain): + external_id = request.path_params["external_id"] + entity = brain.get_by_external_id(external_id, source="api") + if entity is None: + return error(f"no entity with external_id '{external_id}'", 404) + return JSONResponse(_entity_payload(brain, entity)) + + async def schema(request, brain: Brain): + from open_index.config import doc_type_to_yaml_dict + + counts = brain.counts() + return JSONResponse({ + "name": brain.config.name, + "description": brain.config.description, + "doc_types": [ + {**doc_type_to_yaml_dict(dt), "count": counts.get(name, 0)} + for name, dt in brain.config.doc_types.items() + ], + }) + + async def trace_lookup(request, brain: Brain): + trace_id = request.path_params["trace_id"] + return JSONResponse({"trace_id": trace_id, + "reads": brain.analytics_by_trace(trace_id)}) + + # -- writes --------------------------------------------------------------- + + async def put_one(request, brain: Brain): + from open_index.models import Entity + + entity_id = request.path_params["entity_id"] + try: + body = await request.json() + except Exception: + return error("body must be JSON", 400) + if not isinstance(body, dict): + return error("body must be a JSON object", 400) + + body = dict(body) + # The URL is the authority on which entity this is. Accepting a body id + # that disagrees would let PUT /entities/a write entity b. + if body.get("id") not in (None, entity_id): + return error("id in the body does not match the URL", 400, + url_id=entity_id, body_id=body.get("id")) + body["id"] = entity_id + body.setdefault("doc_type", entity_id.split(":", 1)[0]) + + try: + entity = Entity.from_dict(body) + except Exception as exc: + return error(f"invalid entity: {exc}", 400) + try: + path = brain.put_entity(entity) + except ValueError as exc: + return error(str(exc), 422) + return JSONResponse({"written": entity.id, + "file": str(path) if path else None}) + + async def delete_one(request, brain: Brain): + entity_id = request.path_params["entity_id"] + try: + deleted = brain.delete_entity(entity_id, source="api") + except RuntimeError as exc: + return error(str(exc), 500) + if not deleted: + return error(f"no entity '{entity_id}'", 404) + return JSONResponse({"deleted": entity_id}) + + p = prefix + return [ + Route(f"{p}/search", with_brain(search)), + Route(f"{p}/schema", with_brain(schema)), + Route(f"{p}/entities", with_brain(get_many)), + Route(f"{p}/entities/by-external-id/{{external_id:path}}", with_brain(by_external)), + Route(f"{p}/traces/{{trace_id}}", with_brain(trace_lookup)), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(get_one), methods=["GET"]), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(put_one, write=True), + methods=["PUT"]), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(delete_one, write=True), + methods=["DELETE"]), + ] diff --git a/open_index/brain.py b/open_index/brain.py index 0e66e61..f68997c 100644 --- a/open_index/brain.py +++ b/open_index/brain.py @@ -212,6 +212,88 @@ def _write_entity_file(self, entity: Entity) -> Path: path.write_text(json.dumps(entity.to_json(), indent=2, ensure_ascii=False) + "\n") return path + def get_by_external_id( + self, external_id: str, source: Optional[str] = None + ) -> Optional[Entity]: + """Find an entity by the id its source system knows it by.""" + started = perf_counter() + entity = self.backend.get_by_external_id(external_id) + if source: + self._record_fetch( + source=source, operation="lookup_by_external_id", started=started, + entity_id=external_id, result_count=int(entity is not None), + result_doc_types=({entity.doc_type: 1} if entity else {}), + results=([{"id": entity.id, "doc_type": entity.doc_type, + "score": None, + "match": {"type": "lookup", "keyword_score": None, + "semantic_score": None}}] + if entity else []), + ) + return entity + + def get_entities( + self, entity_ids: list[str], source: Optional[str] = None + ) -> list[Entity]: + """Several entities in one call. Missing ids are skipped, not None-padded. + + Exists because agents were emulating it with N round trips, which on a + remote endpoint is N times the latency for the same answer. + """ + started = perf_counter() + found = [e for e in (self.backend.get_entity(i) for i in entity_ids) + if e is not None] + if source: + counts: dict[str, int] = {} + for e in found: + counts[e.doc_type] = counts.get(e.doc_type, 0) + 1 + self._record_fetch( + source=source, operation="get_entities", started=started, + entity_id=",".join(entity_ids[:10]), result_count=len(found), + result_doc_types=counts, + results=[{"id": e.id, "doc_type": e.doc_type, "score": None, + "match": {"type": "lookup", "keyword_score": None, + "semantic_score": None}} + for e in found], + ) + return found + + def delete_entity(self, entity_id: str, source: Optional[str] = None) -> bool: + """Remove one entity everywhere it exists. True if it was there. + + For a `storage: file` doc_type the JSON file is the source of truth, so + deleting only the index row would resurrect the entity on the next + `open-index index` — the delete has to reach the file or it is not a + delete, it is a pause. + + The file goes first. If the index delete fails after the file is gone, + a reconcile removes the row; the other order leaves a file that silently + restores a deleted entity. + """ + started = perf_counter() + entity = self.backend.get_entity(entity_id) + if entity is None: + return False + + path = None + if entity.doc_type in self._file_backed_types() and self.config.root is not None: + path = self.entity_path(entity) + try: + path.unlink(missing_ok=True) + except OSError as exc: + raise RuntimeError( + f"could not remove {path}: {exc}. The entity is still indexed; " + "nothing was deleted." + ) from exc + + deleted = self.backend.delete_entity(entity_id) + if source: + self._record_fetch( + source=source, operation="delete_entity", started=started, + entity_id=entity_id, result_count=int(deleted), + result_doc_types={entity.doc_type: 1} if deleted else {}, + ) + return deleted + def _file_backed_types(self) -> set[str]: return {n for n, dt in self.config.doc_types.items() if dt.storage == "file"} @@ -545,6 +627,14 @@ def _guide_read_section() -> list[str]: "Only fields declared `filterable` can be filtered, and filtering on any other\n" "field is an error rather than being ignored — so a filter never silently\n" "returns everything. The error names the fields you *can* filter.", + "\n## Identifiers", + "An entity id is always `:` — that is how you can tell a\n" + "document's type by looking at its id.\n\n" + "If the thing already has an id in your system — a ticket key, a CRM record\n" + "id, a UUID — pass it as `external_id` when writing, and\n" + "`lookup_by_external_id(...)` finds the entity without you having to know or\n" + "construct the open-index id. Use `get_entities([...])` to fetch several at\n" + "once rather than calling `get_entity` in a loop.", "\n## Making a retrieval debuggable", "Pass `trace_id=\"\"` to `search_brain` and\n" "`get_entity`. Every document returned is then recoverable by that id later,\n" @@ -580,6 +670,10 @@ def _guide_write_section() -> list[str]: "\n## How to write", "Read and write access is the default. Write back durable, validated domain\n" "knowledge whenever work reveals something worth retaining.", + "Deleting is possible (`delete_entity`) but rarely right: prefer correcting\n" + "an entity, because an id that once resolved and now 404s breaks anything\n" + "still holding a reference to it. Delete removes the entity, its file, and\n" + "every edge naming it in either direction.", "\n### Add or update an entity — `put_entity`", "```python\n" "put_entity(\n" diff --git a/open_index/cli.py b/open_index/cli.py index 348dc2c..5a1770a 100644 --- a/open_index/cli.py +++ b/open_index/cli.py @@ -380,6 +380,57 @@ def ui( serve_ui(host=host, port=port) +@app.command() +def delete( + entity_id: str = typer.Argument(..., help='Entity id, e.g. "issue:payment-declined".'), + brain: str = BrainOpt, + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation."), +): + """Delete one entity, its edges in both directions, and its file. + + Irreversible. For a `storage: file` doc_type the JSON file goes too — + removing only the index row would resurrect it on the next `index`. + """ + b = _open_brain(brain) + entity = b.get_entity(entity_id) + if entity is None: + typer.secho(f"no entity '{entity_id}'", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + inbound = b.backend.relationships_to(entity_id) + if not yes: + typer.echo(f"{entity.id} ({entity.doc_type}) {entity.name}") + if inbound: + # Worth seeing before deleting: these edges disappear with it, and + # the entities holding them are not otherwise mentioned. + typer.secho(f" {len(inbound)} entity(ies) point at this and will " + "lose that edge:", fg=typer.colors.YELLOW) + for src, _t, meaning in inbound[:10]: + typer.echo(f" {src} —[{meaning or 'related'}]→") + typer.confirm("delete it?", abort=True) + + if b.delete_entity(entity_id, source="cli"): + typer.secho(f"✓ deleted {entity_id}", fg=typer.colors.GREEN) + else: + typer.secho(f"no entity '{entity_id}'", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + +@app.command() +def lookup( + external_id: str = typer.Argument(..., help="The source system's id."), + brain: str = BrainOpt, +): + """Find an entity by the id its source system knows it by.""" + b = _open_brain(brain) + entity = b.get_by_external_id(external_id, source="cli") + if entity is None: + typer.secho(f"no entity with external_id '{external_id}'", + fg=typer.colors.YELLOW) + raise typer.Exit(1) + typer.echo(json.dumps(entity.to_json(), indent=2)) + + @app.command() def trace( trace_id: str = typer.Argument(..., help="The trace id to look up."), diff --git a/open_index/mcp_server.py b/open_index/mcp_server.py index 34189f0..afa423a 100644 --- a/open_index/mcp_server.py +++ b/open_index/mcp_server.py @@ -186,6 +186,35 @@ def get_entity(entity_id: str, trace_id: Optional[str] = None) -> str: } return json.dumps(payload, indent=2) + @server.tool() + def get_entities(entity_ids: list[str], trace_id: Optional[str] = None) -> str: + """Fetch several entities by id in one call. + + Prefer this over calling get_entity in a loop: on a remote endpoint the + loop costs one round trip per id for the same answer. Ids that do not + exist are omitted rather than returned as nulls — check the count.""" + with _trace_scope(trace_id): + found = brain.get_entities(entity_ids, source="mcp") + return json.dumps( + {"requested": len(entity_ids), "found": len(found), + "missing": sorted(set(entity_ids) - {e.id for e in found}), + "entities": [e.to_json() for e in found]}, + indent=2, + ) + + @server.tool() + def lookup_by_external_id(external_id: str, trace_id: Optional[str] = None) -> str: + """Find an entity by the id its source system knows it by. + + Use when you hold your own identifier — a ticket key, a CRM record id, a + UUID — and do not know this index's `:` id. Set + `external_id` when writing to make an entity findable this way.""" + with _trace_scope(trace_id): + entity = brain.get_by_external_id(external_id, source="mcp") + if entity is None: + return json.dumps({"error": f"no entity with external_id '{external_id}'"}) + return json.dumps(entity.to_json(), indent=2) + # ---- write ------------------------------------------------------------ # if read_only: @@ -196,6 +225,7 @@ def put_entity( doc_type: str, id: str, name: str = "", + external_id: Optional[str] = None, fields: Optional[dict[str, Any]] = None, # dict[str, Any], not dict[str, str]: an edge may carry a nested # "provenance" object. The MCP SDK builds this tool's input schema from @@ -214,6 +244,10 @@ def put_entity( id: Must be ":", e.g. "issue:payment-declined". The prefix has to match `doc_type`. Chars: a-z A-Z 0-9 . _ - name: Human-readable label. + external_id: The id the source system knows this by — a ticket key, + a CRM record id, a UUID. Free-form, and optional. Set it and + `lookup_by_external_id` finds this entity without the caller + needing to know or construct the `:` id. fields: The doc_type's schema fields, e.g. {"severity": "high", "status": "open"}. Do NOT put id/doc_type/ name/related_to in here — they are separate arguments. @@ -281,6 +315,7 @@ def put_entity( id=id, doc_type=doc_type, name=name, + external_id=external_id, fields=fields or {}, related_to=[ Relationship( @@ -376,6 +411,29 @@ def put_entities( "paths": [str(p) for p in result.paths], }, indent=2) + @server.tool() + def delete_entity(entity_id: str) -> str: + """Delete one entity, its edges in both directions, and its file. + + Irreversible, and the file goes too for a `storage: file` doc_type — + removing only the index row would resurrect the entity on the next + reindex, which is a pause, not a delete. + + Edges pointing *at* it are removed as well: an edge left dangling still + renders on the map and in neighbour lists, and reads as corruption + rather than as a deletion. + + Prefer correcting an entity with put_entity over deleting it — an id + that once resolved and now 404s breaks anything holding a reference.""" + try: + deleted = brain.delete_entity(entity_id, source="mcp") + except RuntimeError as exc: + return json.dumps({"error": str(exc)}) + if not deleted: + return json.dumps({"deleted": False, + "error": f"no entity '{entity_id}'"}) + return json.dumps({"deleted": True, "entity_id": entity_id}) + @server.tool() def create_doc_type( doc_type: str, diff --git a/open_index/models.py b/open_index/models.py index cb8cc0d..f37da5d 100644 --- a/open_index/models.py +++ b/open_index/models.py @@ -89,6 +89,12 @@ class Entity(BaseModel): id: str doc_type: str name: str = "" + # Your identifier for this thing, if it already has one — a CRM record id, a + # ticket key, a UUID. Free-form, unlike `id`, which stays `:` + # so an agent can tell an entity's type by looking at it. Set it at write + # time and `lookup_by_external_id` finds the entity without you having to + # know or construct the open-index id. + external_id: Optional[str] = None related_to: list[Relationship] = Field(default_factory=list) # Arbitrary schema-defined fields (description, owner, status, ...). fields: dict[str, Any] = Field(default_factory=dict) @@ -130,8 +136,8 @@ def from_dict(cls, data: dict) -> "Entity": """Build an Entity from a loose JSON/dict, folding unknown keys into `fields` and coercing `related_to` shorthand into Relationships.""" data = dict(data) - reserved = {"id", "doc_type", "name", "related_to", "fields", - "provenance", "valid_from", "valid_to"} + reserved = {"id", "doc_type", "name", "external_id", "related_to", + "fields", "provenance", "valid_from", "valid_to"} explicit_fields = dict(data.pop("fields", {}) or {}) related = data.pop("related_to", []) or [] @@ -151,6 +157,7 @@ def from_dict(cls, data: dict) -> "Entity": "id": core.get("id"), "doc_type": core.get("doc_type"), "name": core.get("name", ""), + "external_id": core.get("external_id"), "related_to": norm_related, "fields": explicit_fields, "provenance": core.get("provenance"), @@ -177,6 +184,10 @@ def label_for(self, label_field: str) -> str: def to_json(self) -> dict: """Flat, file-friendly JSON (fields hoisted to the top level).""" out: dict[str, Any] = {"id": self.id, "doc_type": self.doc_type, "name": self.name} + # Only when set: writing `"external_id": null` onto every entity would + # add noise to every file diff for a feature most brains never use. + if self.external_id: + out["external_id"] = self.external_id out.update(self.fields) if self.related_to: # Drop empty provenance rather than writing `"provenance": null` onto diff --git a/open_index/storage/base.py b/open_index/storage/base.py index 37f0c6e..3dc55b7 100644 --- a/open_index/storage/base.py +++ b/open_index/storage/base.py @@ -239,6 +239,29 @@ def counts(self) -> dict[str, int]: """Total entity count per doc_type.""" ... + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + """The entity carrying this caller-supplied id, or None. + + `external_id` is free-form and its uniqueness is the caller's business, + not something the store enforces. Implementations return the single + match, or the first by id when a caller has reused one — deterministic + rather than arbitrary, so the same lookup gives the same answer twice. + """ + ... + + def delete_entity(self, entity_id: str) -> bool: + """Remove one entity and everything derived from it. True if it existed. + + Implementations must also drop its search-index row, its embedding, and + every edge naming it in *either* direction. An edge left pointing at a + deleted entity is a dangling reference that still renders on the map and + in neighbour lists, which reads as data corruption rather than as a + deletion. + + Idempotent: deleting an absent id returns False, it does not raise. + """ + ... + def delete_by_doc_type(self, doc_types: list[str]) -> None: """Remove all entities (and their edges) of the given doc_types. diff --git a/open_index/storage/opensearch_backend.py b/open_index/storage/opensearch_backend.py index 3103e74..870c82a 100644 --- a/open_index/storage/opensearch_backend.py +++ b/open_index/storage/opensearch_backend.py @@ -46,8 +46,8 @@ # Reserved top-level keys; everything else on an entity is a schema field. # `embedding` is reserved so user schema fields can never collide with the vector # payload stored at the top level of each OpenSearch document. -_RESERVED_KEYS = {"id", "doc_type", "name", "related_to", "embedding", - "provenance", "valid_from", "valid_to"} +_RESERVED_KEYS = {"id", "doc_type", "name", "external_id", "related_to", + "embedding", "provenance", "valid_from", "valid_to"} _MAX_ALL = 10_000 # cap for all_entities / relationship scans @@ -110,6 +110,9 @@ def mapping(dimension: int = 384) -> dict: "date_detection": False, "properties": { "id": {"type": "keyword"}, + # Exact-match only: it is an identifier a caller looks up + # by, never something to tokenise and rank. + "external_id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, "name": {"type": "text", "fields": {"kw": {"type": "keyword"}}}, "related_to": { @@ -152,6 +155,9 @@ def _base_mapping(self) -> dict: "date_detection": False, "properties": { "id": {"type": "keyword"}, + # Exact-match only: it is an identifier a caller looks up + # by, never something to tokenise and rank. + "external_id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, "name": {"type": "text", "fields": {"kw": {"type": "keyword"}}}, "related_to": { @@ -207,6 +213,11 @@ def entity_to_doc(entity: Entity) -> dict: ], "fields": dict(entity.fields), } + # This backend maps fields explicitly, so anything not named here is + # dropped on write — which for an id a caller looks entities up by would + # be a lookup that silently never matches. + if entity.external_id: + doc["external_id"] = entity.external_id if entity.provenance is not None: doc["provenance"] = entity.provenance.model_dump(exclude_none=True) # Validity bounds decide whether a claim holds at a given instant; drop @@ -247,7 +258,7 @@ def doc_to_entity(source: dict) -> Entity: for r in source.get("related_to", []) ], } - for key in ("provenance", "valid_from", "valid_to"): + for key in ("external_id", "provenance", "valid_from", "valid_to"): if source.get(key): flat[key] = source[key] flat.update(source.get("fields", {})) @@ -717,6 +728,50 @@ def search( total=total, results=results, doc_type_counts=doc_type_counts, limited=total > limit, ) + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + if not external_id: + return None + res = self._client.search(index=self.index, body={ + "size": 1, + "query": {"term": {"external_id": external_id}}, + # Deterministic when a caller has reused an external id: the same + # lookup must give the same answer twice. + "sort": [{"id": "asc"}], + }) + hits = res["hits"]["hits"] + return self.doc_to_entity(hits[0]["_source"]) if hits else None + + def delete_entity(self, entity_id: str) -> bool: + from opensearchpy.exceptions import NotFoundError + + try: + self._client.delete(index=self.index, id=entity_id, refresh=True) + except NotFoundError: + return False + # Edges live on the source document, so removing an entity does not + # remove the edges pointing *at* it. Strip those too, or the deleted id + # keeps appearing as an incoming neighbour on documents that survive. + self._client.update_by_query( + index=self.index, + body={ + "query": {"nested": { + "path": "related_to", + "query": {"term": {"related_to.target": entity_id}}, + "ignore_unmapped": True, + }}, + "script": { + "source": ( + "if (ctx._source.related_to != null) {" + " ctx._source.related_to.removeIf(r -> r.target == params.t) }" + ), + "params": {"t": entity_id}, + }, + }, + refresh=True, + conflicts="proceed", + ) + return True + def delete_by_doc_type(self, doc_types: list[str]) -> None: if not doc_types: return diff --git a/open_index/storage/sqlite_backend.py b/open_index/storage/sqlite_backend.py index a353f77..35372c2 100644 --- a/open_index/storage/sqlite_backend.py +++ b/open_index/storage/sqlite_backend.py @@ -49,7 +49,7 @@ # Reserved top-level keys in an entity's flat JSON; everything else is a field. # `embedding` is reserved so user schema fields can never collide with the vector # payload stored in the OpenSearch doc / SQLite entity_embeddings table. -_RESERVED_KEYS = {"id", "doc_type", "name", "related_to", "embedding"} +_RESERVED_KEYS = {"id", "doc_type", "name", "external_id", "related_to", "embedding"} def _fields_of(data: dict) -> dict: @@ -115,6 +115,11 @@ def _create_tables(self) -> None: tokenize = 'unicode61' ); + -- Expression index: external_id lives inside the JSON blob, and + -- without this every lookup by it scans the table. + CREATE INDEX IF NOT EXISTS idx_entities_external + ON entities(json_extract(data, '$.external_id')); + CREATE TABLE IF NOT EXISTS entity_embeddings ( entity_id TEXT PRIMARY KEY, model TEXT NOT NULL, @@ -365,6 +370,38 @@ def _store_embeddings_many(self, entities: list[Entity]) -> None: ) self._conn.commit() + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + if not external_id: + return None + row = self._conn.execute( + "SELECT data FROM entities WHERE json_extract(data, '$.external_id') = ? " + "ORDER BY id LIMIT 1", + (external_id,), + ).fetchone() + return Entity.from_dict(json.loads(row["data"])) if row else None + + def delete_entity(self, entity_id: str) -> bool: + with self._lock: + cur = self._conn.cursor() + row = cur.execute( + "SELECT rowid FROM entities WHERE id = ?", (entity_id,) + ).fetchone() + if row is None: + return False + # FTS is an external-content-free table keyed by rowid, so it does + # not follow the delete on its own. + cur.execute("DELETE FROM entities_fts WHERE rowid = ?", (row["rowid"],)) + cur.execute("DELETE FROM entities WHERE id = ?", (entity_id,)) + cur.execute("DELETE FROM entity_embeddings WHERE entity_id = ?", (entity_id,)) + # Both directions: an edge that still points at a deleted entity + # renders as a dangling node on the map and a "(missing)" neighbour. + cur.execute( + "DELETE FROM relationships WHERE source_id = ? OR target_id = ?", + (entity_id, entity_id), + ) + self._conn.commit() + return True + def delete_by_doc_type(self, doc_types: list[str]) -> None: if not doc_types: return diff --git a/open_index/ui/view.py b/open_index/ui/view.py index 01fa309..e689505 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -283,6 +283,12 @@ def mcp_client_config(url: str, server_name: str = "open-index") -> str: ("get_entity(entity_id)", "One entity with its outgoing AND incoming edges — the incoming direction " "answers \"what else points at this?\"."), + ("get_entities([ids])", + "Several entities in one call. One round trip instead of N, which matters " + "on a remote endpoint."), + ("lookup_by_external_id(external_id)", + "Find an entity by the id its source system knows it by — a ticket key, a " + "CRM record id — when you don't know this index's `:` id."), ] WRITE_TOOLS = [ @@ -293,6 +299,10 @@ def mcp_client_config(url: str, server_name: str = "open-index") -> str: "rather than calling put_entity in a loop."), ("create_doc_type(doc_type, description, fields, relationships, storage)", "Define a new concept, when no existing doc_type fits."), + ("delete_entity(entity_id)", + "Remove one entity, its edges in both directions, and its file. " + "Irreversible — prefer correcting an entity over deleting it, since an id " + "that once resolved and now 404s breaks anything holding a reference."), ] # The tab label for the help page. Leftmost, so it is what a first-time visitor diff --git a/open_index/ui/web.py b/open_index/ui/web.py index 3e607bc..ddc0b86 100644 --- a/open_index/ui/web.py +++ b/open_index/ui/web.py @@ -470,6 +470,14 @@ def healthz(request): routes.append(Route("/entity/{entity_id:path}", entity)) routes.append(Route("/api/graph", graph_json)) + # The JSON API, mounted beside the explorer so one process and one port + # serve the UI, the API and MCP for every brain. Registered before the + # /{name} page routes: /api/... must not be read as an index named "api". + from open_index.api import build_routes as api_routes + + routes += api_routes(resolve, prefix="/api/v1") + routes += api_routes(resolve, prefix="/{name}/api/v1") + routes.append(Route("/{name}", make(page_help, "help.html"))) for slug, fn, tpl in pages: routes.append(Route(f"/{{name}}/{slug}", make(fn, tpl))) diff --git a/tests/test_crud_and_api.py b/tests/test_crud_and_api.py new file mode 100644 index 0000000..9490362 --- /dev/null +++ b/tests/test_crud_and_api.py @@ -0,0 +1,409 @@ +"""Delete, external ids, and the JSON HTTP API. + +The properties worth defending: + + a delete reaches the file. For a `storage: file` doc_type the JSON on disk is + the source of truth, so deleting only the index row is a pause, not a delete — + the entity returns on the next reindex. + + a delete takes its edges with it, in both directions. An edge left pointing at + a removed entity still renders on the map and in neighbour lists, and reads as + corruption rather than as a deletion. + + writes are gated only when a token is configured, and reads never are. +""" + +import os +import shutil + +import pytest +from starlette.testclient import TestClient + +from open_index.brain import Brain +from open_index.models import Entity, Relationship + +EXAMPLE = "examples/support-brain" + + +@pytest.fixture +def brain(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + d = tmp_path / "b" + shutil.copytree(EXAMPLE, d) + b = Brain.open(d) + b.index() + return b + + +# -- delete -------------------------------------------------------------------- + + +def test_delete_removes_the_entity(brain): + target = brain.backend.all_entities()[0].id + assert brain.delete_entity(target) is True + assert brain.get_entity(target) is None + + +def test_delete_removes_the_file_so_a_reindex_does_not_resurrect_it(brain): + """The failure this guards: delete the row, reindex, and it is back.""" + entity = next(e for e in brain.backend.all_entities() + if e.doc_type in brain._file_backed_types()) + path = brain.entity_path(entity) + assert path.exists() + + brain.delete_entity(entity.id) + assert not path.exists() + + brain.index() + assert brain.get_entity(entity.id) is None + + +def test_delete_removes_edges_pointing_at_it(brain): + """An edge surviving its target renders as a dangling node on the map.""" + target = brain.backend.all_entities()[0] + brain.put_entity(Entity( + id="issue:linker", doc_type="issue", name="linker", + related_to=[Relationship(target=target.id, + relationship_edge_meaning="refers to")])) + assert brain.backend.relationships_to(target.id) + + brain.delete_entity(target.id) + assert brain.backend.relationships_to(target.id) == [] + assert brain.backend.relationships_from("issue:linker") == [] + + +def test_deleting_something_absent_is_false_not_an_error(brain): + assert brain.delete_entity("issue:never-existed") is False + + +def test_delete_is_idempotent(brain): + target = brain.backend.all_entities()[0].id + assert brain.delete_entity(target) is True + assert brain.delete_entity(target) is False + + +def test_a_delete_that_cannot_remove_the_file_deletes_nothing(brain, monkeypatch): + """Better to fail wholly than to unindex an entity whose file remains: that + file silently restores it on the next reindex.""" + entity = next(e for e in brain.backend.all_entities() + if e.doc_type in brain._file_backed_types()) + + def refuse(self, missing_ok=False): + raise OSError("read-only file system") + + monkeypatch.setattr("pathlib.Path.unlink", refuse) + with pytest.raises(RuntimeError, match="still indexed"): + brain.delete_entity(entity.id) + assert brain.get_entity(entity.id) is not None + + +def test_delete_is_recorded_in_the_audit_trail(brain): + target = brain.backend.all_entities()[0].id + brain.delete_entity(target, source="cli") + assert brain.analytics_events(limit=1)[0]["operation"] == "delete_entity" + + +# -- external ids -------------------------------------------------------------- + + +def test_an_entity_can_carry_the_id_its_source_system_knows_it_by(brain): + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-4471")) + found = brain.get_by_external_id("CRM-4471") + assert found is not None and found.id == "issue:crm" + + +def test_an_unknown_external_id_is_none(brain): + assert brain.get_by_external_id("nope") is None + assert brain.get_by_external_id("") is None + + +def test_the_external_id_survives_a_file_round_trip(brain): + """It is written to disk, so a reindex must not drop it.""" + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-1")) + brain.index() + reopened = Brain.open(brain.config.root) + assert reopened.get_by_external_id("CRM-1") is not None + + +def test_the_canonical_id_keeps_its_shape(brain): + """external_id is free-form precisely so `id` does not have to be: an agent + reads the doc_type off the id.""" + with pytest.raises(ValueError, match="must look like"): + Entity(id="not-a-valid-id", doc_type="issue") + + +def test_external_id_is_not_treated_as_a_schema_field(brain): + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-2")) + assert "external_id" not in brain.get_entity("issue:crm").fields + + +def test_a_batch_lookup_skips_what_is_missing(brain): + ids = [e.id for e in brain.backend.all_entities()[:2]] + ["issue:ghost"] + found = brain.get_entities(ids) + assert len(found) == 2 + assert all(e is not None for e in found) + + +# -- the HTTP API -------------------------------------------------------------- + + +@pytest.fixture +def api(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + d = tmp_path / "support-brain" + shutil.copytree(EXAMPLE, d) + b = Brain.open(d) + b.index() + b.put_entity(Entity(id="issue:crm", doc_type="issue", name="from crm", + external_id="CRM-9", fields={"description": "x"})) + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_search_over_http(api): + body = api.get("/api/v1/search", params={"q": "payment"}).json() + assert body["total"] and body["results"][0]["match"] + + +def test_search_exposes_the_modes(api): + body = api.get("/api/v1/search", params={"q": "payment", "mode": "keyword"}).json() + assert all(r["match"]["type"] == "keyword" for r in body["results"]) + + +def test_an_unknown_mode_is_a_400_not_a_500(api): + """The caller can fix it, and the message says how.""" + r = api.get("/api/v1/search", params={"q": "x", "mode": "telepathy"}) + assert r.status_code == 400 + assert "unknown search mode" in r.json()["error"] + + +def test_filtering_on_an_undeclared_field_is_a_400(api): + r = api.get("/api/v1/search", params={"filter.nope": "1"}) + assert r.status_code == 400 + assert "cannot filter on" in r.json()["error"] + + +def test_get_one_entity(api): + body = api.get("/api/v1/entities/issue:crm").json() + assert body["id"] == "issue:crm" + assert "relationships" in body + + +def test_a_missing_entity_is_a_404(api): + assert api.get("/api/v1/entities/issue:ghost").status_code == 404 + + +def test_lookup_by_external_id_over_http(api): + assert api.get("/api/v1/entities/by-external-id/CRM-9").json()["id"] == "issue:crm" + + +def test_batch_reports_what_was_missing(api): + body = api.get("/api/v1/entities", + params={"id": ["issue:crm", "issue:ghost"]}).json() + assert body["found"] == 1 + assert body["missing"] == ["issue:ghost"] + + +def test_the_schema_endpoint_describes_the_index(api): + body = api.get("/api/v1/schema").json() + assert body["doc_types"] and "count" in body["doc_types"][0] + + +def test_put_creates_an_entity(api): + r = api.put("/api/v1/entities/issue:via-api", + json={"doc_type": "issue", "name": "made over http"}) + assert r.status_code == 200 + assert api.get("/api/v1/entities/issue:via-api").status_code == 200 + + +def test_put_rejects_a_body_id_that_contradicts_the_url(api): + """Otherwise PUT /entities/a could write entity b.""" + r = api.put("/api/v1/entities/issue:a", json={"id": "issue:b", + "doc_type": "issue"}) + assert r.status_code == 400 + + +def test_put_rejects_an_invalid_entity_with_422(api): + r = api.put("/api/v1/entities/nope:x", json={"doc_type": "nope"}) + assert r.status_code in (400, 422) + + +def test_put_rejects_a_non_json_body(api): + r = api.put("/api/v1/entities/issue:x", content=b"not json") + assert r.status_code == 400 + + +def test_delete_over_http(api): + api.put("/api/v1/entities/issue:temp", json={"doc_type": "issue"}) + assert api.delete("/api/v1/entities/issue:temp").status_code == 200 + assert api.delete("/api/v1/entities/issue:temp").status_code == 404 + + +def test_a_trace_is_readable_over_http(api): + api.get("/api/v1/search", params={"q": "payment"}, + headers={"X-Trace-Id": "api-turn-1"}) + body = api.get("/api/v1/traces/api-turn-1").json() + assert body["reads"] and body["reads"][0]["results"] + + +# -- auth ---------------------------------------------------------------------- + + +@pytest.fixture +def gated(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home2")) + (tmp_path / "home2").mkdir() + d = tmp_path / "gated-brain" + shutil.copytree(EXAMPLE, d) + Brain.open(d).index() + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.setenv("OPEN_INDEX_TOKEN", "s3cret") + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_reads_stay_open_when_a_token_is_configured(gated): + """Matching how the MCP endpoint already behaves: the token gates writes.""" + assert gated.get("/api/v1/search", params={"q": "payment"}).status_code == 200 + + +def test_writes_need_the_token(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}) + assert r.status_code == 401 + assert "Bearer" in r.headers.get("www-authenticate", "") + + +def test_a_wrong_token_is_refused(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}, + headers={"Authorization": "Bearer wrong"}) + assert r.status_code == 401 + + +def test_the_right_token_is_accepted(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}, + headers={"Authorization": "Bearer s3cret"}) + assert r.status_code == 200 + + +def test_delete_is_gated_too(gated): + assert gated.delete("/api/v1/entities/issue:anything").status_code == 401 + + +def test_writes_are_open_when_no_token_is_set(api): + """The demo-host case: nothing configured, nothing gated.""" + assert api.put("/api/v1/entities/issue:open", + json={"doc_type": "issue"}).status_code == 200 + + +def test_a_per_brain_token_overrides_the_shared_one(monkeypatch): + from open_index.api import token_for + + monkeypatch.setenv("OPEN_INDEX_TOKEN", "shared") + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA_INDEX", "specific") + assert token_for("alpha-index") == "specific" + assert token_for("beta") == "shared" + monkeypatch.delenv("OPEN_INDEX_TOKEN") + assert token_for("beta") is None + + +# -- multi-brain routing ------------------------------------------------------- + + +@pytest.fixture +def many(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home3")) + (tmp_path / "home3").mkdir() + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_the_api_is_per_index(many): + assert many.get("/alpha/api/v1/search", params={"q": "payment"}).status_code == 200 + assert many.get("/beta/api/v1/schema").status_code == 200 + + +def test_the_api_on_an_unknown_index_is_a_404(many): + assert many.get("/zzz/api/v1/search").status_code == 404 + + +def test_the_api_route_is_not_mistaken_for_an_index(many): + """/api/... must not resolve as an index literally named "api".""" + assert many.get("/alpha").status_code == 200 + assert many.get("/alpha/api/v1/schema").json()["doc_types"] + + +def test_a_write_lands_in_the_right_index(many): + many.put("/alpha/api/v1/entities/issue:only-alpha", json={"doc_type": "issue"}) + assert many.get("/alpha/api/v1/entities/issue:only-alpha").status_code == 200 + assert many.get("/beta/api/v1/entities/issue:only-alpha").status_code == 404 + + +# -- the error paths a caller can actually hit --------------------------------- + + +def test_a_non_numeric_limit_is_a_400(api): + r = api.get("/api/v1/search", params={"q": "x", "limit": "many"}) + assert r.status_code == 400 + assert "integer" in r.json()["error"] + + +def test_an_unknown_external_id_over_http_is_a_404(api): + assert api.get("/api/v1/entities/by-external-id/NOPE-1").status_code == 404 + + +def test_a_json_body_that_is_not_an_object_is_a_400(api): + r = api.put("/api/v1/entities/issue:x", json=["not", "an", "object"]) + assert r.status_code == 400 + assert "object" in r.json()["error"] + + +def test_an_unconstructable_entity_is_a_400(api): + """A field of the wrong shape fails at model construction, before validation + against the doc_type — still the caller's mistake, still a 4xx.""" + r = api.put("/api/v1/entities/issue:bad", + json={"doc_type": "issue", "related_to": [{"no_target": 1}]}) + assert r.status_code == 400 + assert "invalid entity" in r.json()["error"] + + +def test_a_delete_that_cannot_remove_the_file_is_a_500_not_a_silent_success( + api, tmp_path, monkeypatch): + """The entity is still indexed, so reporting success would be a lie.""" + api.put("/api/v1/entities/issue:stuck", json={"doc_type": "issue"}) + + def refuse(self, missing_ok=False): + raise OSError("read-only file system") + + monkeypatch.setattr("pathlib.Path.unlink", refuse) + r = api.delete("/api/v1/entities/issue:stuck") + assert r.status_code == 500 + assert "still indexed" in r.json()["error"] diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index b483929..7395f12 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -36,7 +36,9 @@ def srv(brain): def test_default_server_exposes_read_and_write_tools(srv): assert {"navigation_guidelines", "search_brain", "get_entity", - "put_entity", "put_entities", "create_doc_type"} == srv.tools() + "get_entities", "lookup_by_external_id", + "put_entity", "put_entities", "create_doc_type", + "delete_entity"} == srv.tools() def test_default_prompt_positions_brain_as_read_write_domain_context(srv): diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index f5df166..b28cdf4 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -336,3 +336,40 @@ def test_model_guide_distinguishes_schema_from_data(): assert "schema" in text + + +# -- the help tab's tool list must match the server's actual tools ------------- + + +def _documented_tool_names(): + """Tool names as the help tab lists them, e.g. 'search_brain(...)'.""" + return {name.split("(")[0] + for name, _ in list(view.READ_TOOLS) + list(view.WRITE_TOOLS)} + + +def test_the_help_tab_lists_exactly_the_tools_the_server_registers(brain): + """A stale tool list on the page is worse than none: it tells a reader an + agent can do something it cannot, or hides something it can. This assertion + is the only thing keeping the two in step. + """ + pytest.importorskip("mcp") + import asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain) + registered = {t.name for t in asyncio.run(server.list_tools())} + # navigation_guidelines is described in prose on the page, not as a row. + assert _documented_tool_names() | {"navigation_guidelines"} == registered + + +def test_the_read_only_tools_are_exactly_the_documented_read_ones(brain): + pytest.importorskip("mcp") + import asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain, read_only=True) + registered = {t.name for t in asyncio.run(server.list_tools())} + documented = {name.split("(")[0] for name, _ in view.READ_TOOLS} + assert documented | {"navigation_guidelines"} == registered