From a2fbb3989cfe61f9860d462d800da598ea7e21d9 Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Tue, 11 Aug 2026 23:11:50 +0530 Subject: [PATCH] feat(api): delete, external ids, and a JSON HTTP API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes CRUD and puts it on a second surface. The explorer stays read-only — writes keep going through a validated path. **delete_entity**, end to end through both backends, Brain, CLI, MCP and the API. Three things it has to get right: the file for a `storage: file` doc_type the JSON on disk is the source of truth, so removing only the index row is a pause, not a delete — the entity returns on the next reindex. the edges in both directions. An edge outliving its target still renders on the map and in neighbour lists, and reads as corruption rather than as a deletion. the order the file goes first. If the index delete then fails, a reconcile cleans up; the other order leaves a file that silently restores a deleted entity. A delete that cannot remove the file deletes nothing and says so, rather than unindexing an entity whose file will bring it back. **external_id** — the id the source system already knows a thing by: a ticket key, a CRM record, a UUID. `id` keeps its `:` shape, because that is what lets an agent read a document's type off its id; the free-form identifier goes beside it, with `lookup_by_external_id` and a CLI `lookup`. Also `get_entities([ids])`, which agents were emulating with N round trips. OpenSearch maps fields explicitly, so external_id had to be added to the mapping and both conversion directions — omitted, it would have been dropped on write and the lookup would have silently never matched. **A JSON API** at `//api/v1`, mounted beside the explorer so one process and one port serve the UI, the API and MCP for every index. It mirrors the MCP tools rather than inventing a second vocabulary, and calls the same Brain methods, so the two cannot disagree about what a search means. - search with mode and `filter.`, returning the same `match` block - entities: get, batch, by-external-id, PUT, DELETE - schema, and trace lookup - the URL is authoritative on PUT: a body id that disagrees is a 400, so PUT /entities/a can never write entity b - a caller's mistake is a 4xx with a message that says how to fix it — an unknown mode and an unfilterable field are 400, not 500 Auth is off unless a token is configured, and gates writes only, matching the MCP endpoint. A deployment with real data sets OPEN_INDEX_TOKEN; one serving public demo data does not, and nothing changes for it. Found while doing this: the assertion tying the help tab's tool list to the tools the server actually registers was lost when the Streamlit test suite was replaced. It is restored, and it immediately did its job — the page had gone stale against three new tools. A tool list that lies about what an agent can do is worse than no list. 629 tests pass; api.py at 100%. Co-Authored-By: Claude Opus 5 (1M context) --- docs/docs.json | 3 +- docs/reference/cli.mdx | 3 + docs/reference/http-api.mdx | 131 ++++++++ open_index/api.py | 220 ++++++++++++ open_index/brain.py | 94 ++++++ open_index/cli.py | 51 +++ open_index/mcp_server.py | 58 ++++ open_index/models.py | 15 +- open_index/storage/base.py | 23 ++ open_index/storage/opensearch_backend.py | 61 +++- open_index/storage/sqlite_backend.py | 39 ++- open_index/ui/view.py | 10 + open_index/ui/web.py | 8 + tests/test_crud_and_api.py | 409 +++++++++++++++++++++++ tests/test_mcp_server.py | 4 +- tests/test_ui_view.py | 37 ++ 16 files changed, 1158 insertions(+), 8 deletions(-) create mode 100644 docs/reference/http-api.mdx create mode 100644 open_index/api.py create mode 100644 tests/test_crud_and_api.py diff --git a/docs/docs.json b/docs/docs.json index d44be4d..b0cddfa 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -48,7 +48,8 @@ { "group": "Reference", "pages": [ - "reference/cli" + "reference/cli", + "reference/http-api" ] } ] diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index 1c924bb..da486b0 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -36,6 +36,9 @@ changing dimensions. | Command | What it does | |---|---| | `open-index search [-t doc_type]` | Search from the terminal. | +| `open-index delete ` | Delete one entity, its edges and its file (`--yes` to skip the prompt). | +| `open-index lookup ` | Find an entity by the id its source system knows it by. | +| `open-index trace ` | What one turn retrieved: each query, each document, rank and score. | | `open-index ui` | Launch the explorer (How to use / Schema / Explore / **Map** / Analytics / Jobs). | ## Serving over MCP diff --git a/docs/reference/http-api.mdx b/docs/reference/http-api.mdx new file mode 100644 index 0000000..bcf309c --- /dev/null +++ b/docs/reference/http-api.mdx @@ -0,0 +1,131 @@ +--- +title: HTTP API +description: The JSON API over a brain — the same operations MCP exposes, for everything that is not an agent. +--- + +MCP is how agents talk to an index. This is how everything else does: a script, +a cron job, a service, `curl`. + +It mirrors the MCP tools rather than inventing a second vocabulary, and calls the +same `Brain` methods underneath — so the two cannot disagree about what a search +means or what a write validates. + +The API is served by `open-index ui`, alongside the explorer, so one process and +one port serve the UI, the API and the MCP endpoint for every index on the host. + +``` +https:////api/v1/... one of several indexes +http://localhost:8501/api/v1/... a single brain +``` + +## Reading + +### Search + +```bash +curl "$HOST/api/v1/search?q=payment+declined&mode=hybrid&limit=10" +``` + +| Parameter | | +|---|---| +| `q` | free text; omit it to list | +| `mode` | `hybrid` (default), `keyword`, `semantic` | +| `doc_type` | repeatable — `&doc_type=issue&doc_type=product` | +| `filter.` | exact match, e.g. `filter.tenant_id=acme` | +| `limit` | default 20 | + +Every result carries `match`, saying **why** it came back: + +```json +{ + "id": "issue:payment-declined", + "score": 0.937, + "match": { "type": "both", "keyword_score": 1.0, "semantic_score": 0.79 } +} +``` + +`type` is `keyword`, `semantic`, `both`, `filter` or `none`. Read it before +trusting a result — a semantic-only hit at a low score is a guess, not a fact. + +**Filters are hard predicates, not ranking hints.** A document that does not +match cannot be returned at any score, in any mode. Only fields declared +`filterable: true` can be filtered, and filtering on any other field returns +`400` rather than being ignored — so a filter never silently fails open. This +matters when the filter is carrying a tenant or user boundary. + +### Entities + +```bash +curl "$HOST/api/v1/entities/issue:payment-declined" +curl "$HOST/api/v1/entities?id=issue:a&id=issue:b" # batch +curl "$HOST/api/v1/entities/by-external-id/CRM-4471" # your own id +curl "$HOST/api/v1/schema" +``` + +A single entity comes back with its relationships in **both** directions — +incoming answers "what else points at this?", which the entity's own document +cannot. + +The batch endpoint reports `missing` rather than padding the list with nulls. + +### Traces + +```bash +curl "$HOST/api/v1/traces/turn-8f21c3" +``` + +Every read records the `X-Trace-Id` header it arrived with. Send one and you can +later recover exactly which documents that turn retrieved, at what rank, with +what score, and on what kind of match — which is how you work out afterwards what +the index actually fed an agent. + +## Writing + +```bash +curl -X PUT "$HOST/api/v1/entities/issue:payment-declined" \ + -H 'Content-Type: application/json' \ + -d '{"doc_type": "issue", "name": "Payment declined", "severity": "high", + "external_id": "CRM-4471"}' + +curl -X DELETE "$HOST/api/v1/entities/issue:payment-declined" +``` + +`PUT` is an upsert. The id in the URL wins: a body id that disagrees is a `400`, +so `PUT /entities/a` can never write entity `b`. + +`DELETE` removes the entity, its file (for a `storage: file` doc_type), and every +edge naming it in either direction. Deleting only the index row would leave the +file to resurrect it on the next reindex — a pause, not a delete. + +Prefer correcting an entity over deleting it: an id that once resolved and now +404s breaks anything still holding a reference. + +## Authentication + +Off unless a token is configured. Set `OPEN_INDEX_TOKEN` (or a per-index +`OPEN_INDEX_TOKEN_`) and **writes** require it: + +```bash +curl -X PUT "$HOST/api/v1/entities/issue:x" \ + -H "Authorization: Bearer $OPEN_INDEX_TOKEN" ... +``` + +Reads stay open either way, matching how the MCP endpoint already behaves. A +deployment holding real data sets a token; one serving public demo data does not, +and nothing changes for it. + + +With no token set, anyone who can reach the endpoint can write to it and delete +from it. That is the right default for disposable demo data and the wrong one for +anything you would miss. + + +## Status codes + +| | | +|---|---| +| `400` | your request — bad mode, unfilterable field, malformed body, id mismatch | +| `401` | a write without a valid token, when one is configured | +| `404` | unknown index, entity, or external id | +| `422` | a well-formed entity that failed schema validation | +| `500` | the write could not complete; the message says what state things are in | diff --git a/open_index/api.py b/open_index/api.py new file mode 100644 index 0000000..13e1bfc --- /dev/null +++ b/open_index/api.py @@ -0,0 +1,220 @@ +"""A JSON HTTP API over a brain — the same operations MCP exposes, over HTTP. + +MCP is for agents; this is for everything else: a script, a job, a service, a +`curl`. It deliberately mirrors the MCP tools rather than inventing a second +vocabulary, and calls the same `Brain` methods, so the two cannot disagree about +what a search means or what a write validates. + +Mounted at `//api/v1` alongside the explorer, so one process and one port +serve the UI, the API and the MCP endpoint for every brain on the host. + +Auth is off unless a token is configured. `OPEN_INDEX_TOKEN` (or the per-brain +`OPEN_INDEX_TOKEN_`) gates *writes* only — reads stay open, matching how +the MCP endpoint already behaves. A deployment holding real data sets the token; +one serving public demo data does not, and nothing changes for it. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from open_index.brain import Brain + + +def token_for(name: Optional[str]) -> Optional[str]: + """The write token for this brain, if one is configured.""" + if name: + specific = os.environ.get( + "OPEN_INDEX_TOKEN_" + name.upper().replace("-", "_").replace(".", "_")) + if specific: + return specific + return os.environ.get("OPEN_INDEX_TOKEN") or None + + +def _entity_payload(brain: Brain, entity) -> dict[str, Any]: + """One entity as the API returns it: the document plus both edge directions.""" + payload = entity.to_json() + payload["relationships"] = { + "outgoing": [{"target": t, "meaning": m} + for (_s, t, m) in brain.backend.relationships_from(entity.id)], + "incoming": [{"source": s, "meaning": m} + for (s, _t, m) in brain.backend.relationships_to(entity.id)], + } + return payload + + +def build_routes(resolve, prefix: str = ""): + """Routes for the JSON API. + + `resolve(request)` returns `(name, Brain)` — supplied by the caller so the + API and the explorer agree on which index a path refers to instead of each + working it out separately. + """ + from starlette.responses import JSONResponse + from starlette.routing import Route + + def error(message: str, status: int, **extra): + return JSONResponse({"error": message, **extra}, status_code=status) + + def authorized(request, name: Optional[str]) -> bool: + expected = token_for(name) + if not expected: + return True # no token configured: writes are open + header = request.headers.get("authorization", "") + scheme, _, value = header.partition(" ") + return scheme.lower() == "bearer" and value == expected + + def with_brain(handler, *, write: bool = False): + async def endpoint(request): + name, brain = resolve(request) + if brain is None: + return error("unknown index", 404) + if write and not authorized(request, name): + # 401 with a challenge, not 403: the caller can fix this by + # presenting a token, and should be told how. + return JSONResponse( + {"error": "a bearer token is required to write to this index"}, + status_code=401, + headers={"WWW-Authenticate": 'Bearer realm="open-index"'}, + ) + return await handler(request, brain) + return endpoint + + # -- reads ---------------------------------------------------------------- + + async def search(request, brain: Brain): + params = request.query_params + try: + limit = int(params.get("limit", 20)) + except ValueError: + return error("limit must be an integer", 400) + + filters: dict[str, Any] = {} + # filter.=, so a filter needs no JSON body on a GET. + for key, value in params.multi_items(): + if key.startswith("filter."): + filters[key[len("filter."):]] = value + + try: + results = brain.search( + query=params.get("q"), + doc_types=[t for t in params.getlist("doc_type") if t] or None, + limit=limit, + mode=params.get("mode", "hybrid"), + filters=filters or None, + source="api", + ) + except ValueError as exc: + # An unknown mode or a filter on an undeclared field. The caller can + # fix both, and the message says how — so it is a 400, not a 500. + return error(str(exc), 400) + + return JSONResponse({ + "query": params.get("q"), + "mode": params.get("mode", "hybrid"), + "filters": filters, + "total": results.total, + "doc_type_counts": results.doc_type_counts, + "limited": results.limited, + "results": results.results, + }) + + async def get_one(request, brain: Brain): + entity_id = request.path_params["entity_id"] + entity = brain.get_entity(entity_id, source="api") + if entity is None: + return error(f"no entity '{entity_id}'", 404) + return JSONResponse(_entity_payload(brain, entity)) + + async def get_many(request, brain: Brain): + ids = [i for i in request.query_params.getlist("id") if i] + found = brain.get_entities(ids, source="api") + return JSONResponse({ + "requested": len(ids), + "found": len(found), + "missing": sorted(set(ids) - {e.id for e in found}), + "entities": [_entity_payload(brain, e) for e in found], + }) + + async def by_external(request, brain: Brain): + external_id = request.path_params["external_id"] + entity = brain.get_by_external_id(external_id, source="api") + if entity is None: + return error(f"no entity with external_id '{external_id}'", 404) + return JSONResponse(_entity_payload(brain, entity)) + + async def schema(request, brain: Brain): + from open_index.config import doc_type_to_yaml_dict + + counts = brain.counts() + return JSONResponse({ + "name": brain.config.name, + "description": brain.config.description, + "doc_types": [ + {**doc_type_to_yaml_dict(dt), "count": counts.get(name, 0)} + for name, dt in brain.config.doc_types.items() + ], + }) + + async def trace_lookup(request, brain: Brain): + trace_id = request.path_params["trace_id"] + return JSONResponse({"trace_id": trace_id, + "reads": brain.analytics_by_trace(trace_id)}) + + # -- writes --------------------------------------------------------------- + + async def put_one(request, brain: Brain): + from open_index.models import Entity + + entity_id = request.path_params["entity_id"] + try: + body = await request.json() + except Exception: + return error("body must be JSON", 400) + if not isinstance(body, dict): + return error("body must be a JSON object", 400) + + body = dict(body) + # The URL is the authority on which entity this is. Accepting a body id + # that disagrees would let PUT /entities/a write entity b. + if body.get("id") not in (None, entity_id): + return error("id in the body does not match the URL", 400, + url_id=entity_id, body_id=body.get("id")) + body["id"] = entity_id + body.setdefault("doc_type", entity_id.split(":", 1)[0]) + + try: + entity = Entity.from_dict(body) + except Exception as exc: + return error(f"invalid entity: {exc}", 400) + try: + path = brain.put_entity(entity) + except ValueError as exc: + return error(str(exc), 422) + return JSONResponse({"written": entity.id, + "file": str(path) if path else None}) + + async def delete_one(request, brain: Brain): + entity_id = request.path_params["entity_id"] + try: + deleted = brain.delete_entity(entity_id, source="api") + except RuntimeError as exc: + return error(str(exc), 500) + if not deleted: + return error(f"no entity '{entity_id}'", 404) + return JSONResponse({"deleted": entity_id}) + + p = prefix + return [ + Route(f"{p}/search", with_brain(search)), + Route(f"{p}/schema", with_brain(schema)), + Route(f"{p}/entities", with_brain(get_many)), + Route(f"{p}/entities/by-external-id/{{external_id:path}}", with_brain(by_external)), + Route(f"{p}/traces/{{trace_id}}", with_brain(trace_lookup)), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(get_one), methods=["GET"]), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(put_one, write=True), + methods=["PUT"]), + Route(f"{p}/entities/{{entity_id:path}}", with_brain(delete_one, write=True), + methods=["DELETE"]), + ] diff --git a/open_index/brain.py b/open_index/brain.py index 0e66e61..f68997c 100644 --- a/open_index/brain.py +++ b/open_index/brain.py @@ -212,6 +212,88 @@ def _write_entity_file(self, entity: Entity) -> Path: path.write_text(json.dumps(entity.to_json(), indent=2, ensure_ascii=False) + "\n") return path + def get_by_external_id( + self, external_id: str, source: Optional[str] = None + ) -> Optional[Entity]: + """Find an entity by the id its source system knows it by.""" + started = perf_counter() + entity = self.backend.get_by_external_id(external_id) + if source: + self._record_fetch( + source=source, operation="lookup_by_external_id", started=started, + entity_id=external_id, result_count=int(entity is not None), + result_doc_types=({entity.doc_type: 1} if entity else {}), + results=([{"id": entity.id, "doc_type": entity.doc_type, + "score": None, + "match": {"type": "lookup", "keyword_score": None, + "semantic_score": None}}] + if entity else []), + ) + return entity + + def get_entities( + self, entity_ids: list[str], source: Optional[str] = None + ) -> list[Entity]: + """Several entities in one call. Missing ids are skipped, not None-padded. + + Exists because agents were emulating it with N round trips, which on a + remote endpoint is N times the latency for the same answer. + """ + started = perf_counter() + found = [e for e in (self.backend.get_entity(i) for i in entity_ids) + if e is not None] + if source: + counts: dict[str, int] = {} + for e in found: + counts[e.doc_type] = counts.get(e.doc_type, 0) + 1 + self._record_fetch( + source=source, operation="get_entities", started=started, + entity_id=",".join(entity_ids[:10]), result_count=len(found), + result_doc_types=counts, + results=[{"id": e.id, "doc_type": e.doc_type, "score": None, + "match": {"type": "lookup", "keyword_score": None, + "semantic_score": None}} + for e in found], + ) + return found + + def delete_entity(self, entity_id: str, source: Optional[str] = None) -> bool: + """Remove one entity everywhere it exists. True if it was there. + + For a `storage: file` doc_type the JSON file is the source of truth, so + deleting only the index row would resurrect the entity on the next + `open-index index` — the delete has to reach the file or it is not a + delete, it is a pause. + + The file goes first. If the index delete fails after the file is gone, + a reconcile removes the row; the other order leaves a file that silently + restores a deleted entity. + """ + started = perf_counter() + entity = self.backend.get_entity(entity_id) + if entity is None: + return False + + path = None + if entity.doc_type in self._file_backed_types() and self.config.root is not None: + path = self.entity_path(entity) + try: + path.unlink(missing_ok=True) + except OSError as exc: + raise RuntimeError( + f"could not remove {path}: {exc}. The entity is still indexed; " + "nothing was deleted." + ) from exc + + deleted = self.backend.delete_entity(entity_id) + if source: + self._record_fetch( + source=source, operation="delete_entity", started=started, + entity_id=entity_id, result_count=int(deleted), + result_doc_types={entity.doc_type: 1} if deleted else {}, + ) + return deleted + def _file_backed_types(self) -> set[str]: return {n for n, dt in self.config.doc_types.items() if dt.storage == "file"} @@ -545,6 +627,14 @@ def _guide_read_section() -> list[str]: "Only fields declared `filterable` can be filtered, and filtering on any other\n" "field is an error rather than being ignored — so a filter never silently\n" "returns everything. The error names the fields you *can* filter.", + "\n## Identifiers", + "An entity id is always `:` — that is how you can tell a\n" + "document's type by looking at its id.\n\n" + "If the thing already has an id in your system — a ticket key, a CRM record\n" + "id, a UUID — pass it as `external_id` when writing, and\n" + "`lookup_by_external_id(...)` finds the entity without you having to know or\n" + "construct the open-index id. Use `get_entities([...])` to fetch several at\n" + "once rather than calling `get_entity` in a loop.", "\n## Making a retrieval debuggable", "Pass `trace_id=\"\"` to `search_brain` and\n" "`get_entity`. Every document returned is then recoverable by that id later,\n" @@ -580,6 +670,10 @@ def _guide_write_section() -> list[str]: "\n## How to write", "Read and write access is the default. Write back durable, validated domain\n" "knowledge whenever work reveals something worth retaining.", + "Deleting is possible (`delete_entity`) but rarely right: prefer correcting\n" + "an entity, because an id that once resolved and now 404s breaks anything\n" + "still holding a reference to it. Delete removes the entity, its file, and\n" + "every edge naming it in either direction.", "\n### Add or update an entity — `put_entity`", "```python\n" "put_entity(\n" diff --git a/open_index/cli.py b/open_index/cli.py index 348dc2c..5a1770a 100644 --- a/open_index/cli.py +++ b/open_index/cli.py @@ -380,6 +380,57 @@ def ui( serve_ui(host=host, port=port) +@app.command() +def delete( + entity_id: str = typer.Argument(..., help='Entity id, e.g. "issue:payment-declined".'), + brain: str = BrainOpt, + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation."), +): + """Delete one entity, its edges in both directions, and its file. + + Irreversible. For a `storage: file` doc_type the JSON file goes too — + removing only the index row would resurrect it on the next `index`. + """ + b = _open_brain(brain) + entity = b.get_entity(entity_id) + if entity is None: + typer.secho(f"no entity '{entity_id}'", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + inbound = b.backend.relationships_to(entity_id) + if not yes: + typer.echo(f"{entity.id} ({entity.doc_type}) {entity.name}") + if inbound: + # Worth seeing before deleting: these edges disappear with it, and + # the entities holding them are not otherwise mentioned. + typer.secho(f" {len(inbound)} entity(ies) point at this and will " + "lose that edge:", fg=typer.colors.YELLOW) + for src, _t, meaning in inbound[:10]: + typer.echo(f" {src} —[{meaning or 'related'}]→") + typer.confirm("delete it?", abort=True) + + if b.delete_entity(entity_id, source="cli"): + typer.secho(f"✓ deleted {entity_id}", fg=typer.colors.GREEN) + else: + typer.secho(f"no entity '{entity_id}'", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + +@app.command() +def lookup( + external_id: str = typer.Argument(..., help="The source system's id."), + brain: str = BrainOpt, +): + """Find an entity by the id its source system knows it by.""" + b = _open_brain(brain) + entity = b.get_by_external_id(external_id, source="cli") + if entity is None: + typer.secho(f"no entity with external_id '{external_id}'", + fg=typer.colors.YELLOW) + raise typer.Exit(1) + typer.echo(json.dumps(entity.to_json(), indent=2)) + + @app.command() def trace( trace_id: str = typer.Argument(..., help="The trace id to look up."), diff --git a/open_index/mcp_server.py b/open_index/mcp_server.py index 34189f0..afa423a 100644 --- a/open_index/mcp_server.py +++ b/open_index/mcp_server.py @@ -186,6 +186,35 @@ def get_entity(entity_id: str, trace_id: Optional[str] = None) -> str: } return json.dumps(payload, indent=2) + @server.tool() + def get_entities(entity_ids: list[str], trace_id: Optional[str] = None) -> str: + """Fetch several entities by id in one call. + + Prefer this over calling get_entity in a loop: on a remote endpoint the + loop costs one round trip per id for the same answer. Ids that do not + exist are omitted rather than returned as nulls — check the count.""" + with _trace_scope(trace_id): + found = brain.get_entities(entity_ids, source="mcp") + return json.dumps( + {"requested": len(entity_ids), "found": len(found), + "missing": sorted(set(entity_ids) - {e.id for e in found}), + "entities": [e.to_json() for e in found]}, + indent=2, + ) + + @server.tool() + def lookup_by_external_id(external_id: str, trace_id: Optional[str] = None) -> str: + """Find an entity by the id its source system knows it by. + + Use when you hold your own identifier — a ticket key, a CRM record id, a + UUID — and do not know this index's `:` id. Set + `external_id` when writing to make an entity findable this way.""" + with _trace_scope(trace_id): + entity = brain.get_by_external_id(external_id, source="mcp") + if entity is None: + return json.dumps({"error": f"no entity with external_id '{external_id}'"}) + return json.dumps(entity.to_json(), indent=2) + # ---- write ------------------------------------------------------------ # if read_only: @@ -196,6 +225,7 @@ def put_entity( doc_type: str, id: str, name: str = "", + external_id: Optional[str] = None, fields: Optional[dict[str, Any]] = None, # dict[str, Any], not dict[str, str]: an edge may carry a nested # "provenance" object. The MCP SDK builds this tool's input schema from @@ -214,6 +244,10 @@ def put_entity( id: Must be ":", e.g. "issue:payment-declined". The prefix has to match `doc_type`. Chars: a-z A-Z 0-9 . _ - name: Human-readable label. + external_id: The id the source system knows this by — a ticket key, + a CRM record id, a UUID. Free-form, and optional. Set it and + `lookup_by_external_id` finds this entity without the caller + needing to know or construct the `:` id. fields: The doc_type's schema fields, e.g. {"severity": "high", "status": "open"}. Do NOT put id/doc_type/ name/related_to in here — they are separate arguments. @@ -281,6 +315,7 @@ def put_entity( id=id, doc_type=doc_type, name=name, + external_id=external_id, fields=fields or {}, related_to=[ Relationship( @@ -376,6 +411,29 @@ def put_entities( "paths": [str(p) for p in result.paths], }, indent=2) + @server.tool() + def delete_entity(entity_id: str) -> str: + """Delete one entity, its edges in both directions, and its file. + + Irreversible, and the file goes too for a `storage: file` doc_type — + removing only the index row would resurrect the entity on the next + reindex, which is a pause, not a delete. + + Edges pointing *at* it are removed as well: an edge left dangling still + renders on the map and in neighbour lists, and reads as corruption + rather than as a deletion. + + Prefer correcting an entity with put_entity over deleting it — an id + that once resolved and now 404s breaks anything holding a reference.""" + try: + deleted = brain.delete_entity(entity_id, source="mcp") + except RuntimeError as exc: + return json.dumps({"error": str(exc)}) + if not deleted: + return json.dumps({"deleted": False, + "error": f"no entity '{entity_id}'"}) + return json.dumps({"deleted": True, "entity_id": entity_id}) + @server.tool() def create_doc_type( doc_type: str, diff --git a/open_index/models.py b/open_index/models.py index cb8cc0d..f37da5d 100644 --- a/open_index/models.py +++ b/open_index/models.py @@ -89,6 +89,12 @@ class Entity(BaseModel): id: str doc_type: str name: str = "" + # Your identifier for this thing, if it already has one — a CRM record id, a + # ticket key, a UUID. Free-form, unlike `id`, which stays `:` + # so an agent can tell an entity's type by looking at it. Set it at write + # time and `lookup_by_external_id` finds the entity without you having to + # know or construct the open-index id. + external_id: Optional[str] = None related_to: list[Relationship] = Field(default_factory=list) # Arbitrary schema-defined fields (description, owner, status, ...). fields: dict[str, Any] = Field(default_factory=dict) @@ -130,8 +136,8 @@ def from_dict(cls, data: dict) -> "Entity": """Build an Entity from a loose JSON/dict, folding unknown keys into `fields` and coercing `related_to` shorthand into Relationships.""" data = dict(data) - reserved = {"id", "doc_type", "name", "related_to", "fields", - "provenance", "valid_from", "valid_to"} + reserved = {"id", "doc_type", "name", "external_id", "related_to", + "fields", "provenance", "valid_from", "valid_to"} explicit_fields = dict(data.pop("fields", {}) or {}) related = data.pop("related_to", []) or [] @@ -151,6 +157,7 @@ def from_dict(cls, data: dict) -> "Entity": "id": core.get("id"), "doc_type": core.get("doc_type"), "name": core.get("name", ""), + "external_id": core.get("external_id"), "related_to": norm_related, "fields": explicit_fields, "provenance": core.get("provenance"), @@ -177,6 +184,10 @@ def label_for(self, label_field: str) -> str: def to_json(self) -> dict: """Flat, file-friendly JSON (fields hoisted to the top level).""" out: dict[str, Any] = {"id": self.id, "doc_type": self.doc_type, "name": self.name} + # Only when set: writing `"external_id": null` onto every entity would + # add noise to every file diff for a feature most brains never use. + if self.external_id: + out["external_id"] = self.external_id out.update(self.fields) if self.related_to: # Drop empty provenance rather than writing `"provenance": null` onto diff --git a/open_index/storage/base.py b/open_index/storage/base.py index 37f0c6e..3dc55b7 100644 --- a/open_index/storage/base.py +++ b/open_index/storage/base.py @@ -239,6 +239,29 @@ def counts(self) -> dict[str, int]: """Total entity count per doc_type.""" ... + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + """The entity carrying this caller-supplied id, or None. + + `external_id` is free-form and its uniqueness is the caller's business, + not something the store enforces. Implementations return the single + match, or the first by id when a caller has reused one — deterministic + rather than arbitrary, so the same lookup gives the same answer twice. + """ + ... + + def delete_entity(self, entity_id: str) -> bool: + """Remove one entity and everything derived from it. True if it existed. + + Implementations must also drop its search-index row, its embedding, and + every edge naming it in *either* direction. An edge left pointing at a + deleted entity is a dangling reference that still renders on the map and + in neighbour lists, which reads as data corruption rather than as a + deletion. + + Idempotent: deleting an absent id returns False, it does not raise. + """ + ... + def delete_by_doc_type(self, doc_types: list[str]) -> None: """Remove all entities (and their edges) of the given doc_types. diff --git a/open_index/storage/opensearch_backend.py b/open_index/storage/opensearch_backend.py index 3103e74..870c82a 100644 --- a/open_index/storage/opensearch_backend.py +++ b/open_index/storage/opensearch_backend.py @@ -46,8 +46,8 @@ # Reserved top-level keys; everything else on an entity is a schema field. # `embedding` is reserved so user schema fields can never collide with the vector # payload stored at the top level of each OpenSearch document. -_RESERVED_KEYS = {"id", "doc_type", "name", "related_to", "embedding", - "provenance", "valid_from", "valid_to"} +_RESERVED_KEYS = {"id", "doc_type", "name", "external_id", "related_to", + "embedding", "provenance", "valid_from", "valid_to"} _MAX_ALL = 10_000 # cap for all_entities / relationship scans @@ -110,6 +110,9 @@ def mapping(dimension: int = 384) -> dict: "date_detection": False, "properties": { "id": {"type": "keyword"}, + # Exact-match only: it is an identifier a caller looks up + # by, never something to tokenise and rank. + "external_id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, "name": {"type": "text", "fields": {"kw": {"type": "keyword"}}}, "related_to": { @@ -152,6 +155,9 @@ def _base_mapping(self) -> dict: "date_detection": False, "properties": { "id": {"type": "keyword"}, + # Exact-match only: it is an identifier a caller looks up + # by, never something to tokenise and rank. + "external_id": {"type": "keyword"}, "doc_type": {"type": "keyword"}, "name": {"type": "text", "fields": {"kw": {"type": "keyword"}}}, "related_to": { @@ -207,6 +213,11 @@ def entity_to_doc(entity: Entity) -> dict: ], "fields": dict(entity.fields), } + # This backend maps fields explicitly, so anything not named here is + # dropped on write — which for an id a caller looks entities up by would + # be a lookup that silently never matches. + if entity.external_id: + doc["external_id"] = entity.external_id if entity.provenance is not None: doc["provenance"] = entity.provenance.model_dump(exclude_none=True) # Validity bounds decide whether a claim holds at a given instant; drop @@ -247,7 +258,7 @@ def doc_to_entity(source: dict) -> Entity: for r in source.get("related_to", []) ], } - for key in ("provenance", "valid_from", "valid_to"): + for key in ("external_id", "provenance", "valid_from", "valid_to"): if source.get(key): flat[key] = source[key] flat.update(source.get("fields", {})) @@ -717,6 +728,50 @@ def search( total=total, results=results, doc_type_counts=doc_type_counts, limited=total > limit, ) + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + if not external_id: + return None + res = self._client.search(index=self.index, body={ + "size": 1, + "query": {"term": {"external_id": external_id}}, + # Deterministic when a caller has reused an external id: the same + # lookup must give the same answer twice. + "sort": [{"id": "asc"}], + }) + hits = res["hits"]["hits"] + return self.doc_to_entity(hits[0]["_source"]) if hits else None + + def delete_entity(self, entity_id: str) -> bool: + from opensearchpy.exceptions import NotFoundError + + try: + self._client.delete(index=self.index, id=entity_id, refresh=True) + except NotFoundError: + return False + # Edges live on the source document, so removing an entity does not + # remove the edges pointing *at* it. Strip those too, or the deleted id + # keeps appearing as an incoming neighbour on documents that survive. + self._client.update_by_query( + index=self.index, + body={ + "query": {"nested": { + "path": "related_to", + "query": {"term": {"related_to.target": entity_id}}, + "ignore_unmapped": True, + }}, + "script": { + "source": ( + "if (ctx._source.related_to != null) {" + " ctx._source.related_to.removeIf(r -> r.target == params.t) }" + ), + "params": {"t": entity_id}, + }, + }, + refresh=True, + conflicts="proceed", + ) + return True + def delete_by_doc_type(self, doc_types: list[str]) -> None: if not doc_types: return diff --git a/open_index/storage/sqlite_backend.py b/open_index/storage/sqlite_backend.py index a353f77..35372c2 100644 --- a/open_index/storage/sqlite_backend.py +++ b/open_index/storage/sqlite_backend.py @@ -49,7 +49,7 @@ # Reserved top-level keys in an entity's flat JSON; everything else is a field. # `embedding` is reserved so user schema fields can never collide with the vector # payload stored in the OpenSearch doc / SQLite entity_embeddings table. -_RESERVED_KEYS = {"id", "doc_type", "name", "related_to", "embedding"} +_RESERVED_KEYS = {"id", "doc_type", "name", "external_id", "related_to", "embedding"} def _fields_of(data: dict) -> dict: @@ -115,6 +115,11 @@ def _create_tables(self) -> None: tokenize = 'unicode61' ); + -- Expression index: external_id lives inside the JSON blob, and + -- without this every lookup by it scans the table. + CREATE INDEX IF NOT EXISTS idx_entities_external + ON entities(json_extract(data, '$.external_id')); + CREATE TABLE IF NOT EXISTS entity_embeddings ( entity_id TEXT PRIMARY KEY, model TEXT NOT NULL, @@ -365,6 +370,38 @@ def _store_embeddings_many(self, entities: list[Entity]) -> None: ) self._conn.commit() + def get_by_external_id(self, external_id: str) -> Optional[Entity]: + if not external_id: + return None + row = self._conn.execute( + "SELECT data FROM entities WHERE json_extract(data, '$.external_id') = ? " + "ORDER BY id LIMIT 1", + (external_id,), + ).fetchone() + return Entity.from_dict(json.loads(row["data"])) if row else None + + def delete_entity(self, entity_id: str) -> bool: + with self._lock: + cur = self._conn.cursor() + row = cur.execute( + "SELECT rowid FROM entities WHERE id = ?", (entity_id,) + ).fetchone() + if row is None: + return False + # FTS is an external-content-free table keyed by rowid, so it does + # not follow the delete on its own. + cur.execute("DELETE FROM entities_fts WHERE rowid = ?", (row["rowid"],)) + cur.execute("DELETE FROM entities WHERE id = ?", (entity_id,)) + cur.execute("DELETE FROM entity_embeddings WHERE entity_id = ?", (entity_id,)) + # Both directions: an edge that still points at a deleted entity + # renders as a dangling node on the map and a "(missing)" neighbour. + cur.execute( + "DELETE FROM relationships WHERE source_id = ? OR target_id = ?", + (entity_id, entity_id), + ) + self._conn.commit() + return True + def delete_by_doc_type(self, doc_types: list[str]) -> None: if not doc_types: return diff --git a/open_index/ui/view.py b/open_index/ui/view.py index 01fa309..e689505 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -283,6 +283,12 @@ def mcp_client_config(url: str, server_name: str = "open-index") -> str: ("get_entity(entity_id)", "One entity with its outgoing AND incoming edges — the incoming direction " "answers \"what else points at this?\"."), + ("get_entities([ids])", + "Several entities in one call. One round trip instead of N, which matters " + "on a remote endpoint."), + ("lookup_by_external_id(external_id)", + "Find an entity by the id its source system knows it by — a ticket key, a " + "CRM record id — when you don't know this index's `:` id."), ] WRITE_TOOLS = [ @@ -293,6 +299,10 @@ def mcp_client_config(url: str, server_name: str = "open-index") -> str: "rather than calling put_entity in a loop."), ("create_doc_type(doc_type, description, fields, relationships, storage)", "Define a new concept, when no existing doc_type fits."), + ("delete_entity(entity_id)", + "Remove one entity, its edges in both directions, and its file. " + "Irreversible — prefer correcting an entity over deleting it, since an id " + "that once resolved and now 404s breaks anything holding a reference."), ] # The tab label for the help page. Leftmost, so it is what a first-time visitor diff --git a/open_index/ui/web.py b/open_index/ui/web.py index 3e607bc..ddc0b86 100644 --- a/open_index/ui/web.py +++ b/open_index/ui/web.py @@ -470,6 +470,14 @@ def healthz(request): routes.append(Route("/entity/{entity_id:path}", entity)) routes.append(Route("/api/graph", graph_json)) + # The JSON API, mounted beside the explorer so one process and one port + # serve the UI, the API and MCP for every brain. Registered before the + # /{name} page routes: /api/... must not be read as an index named "api". + from open_index.api import build_routes as api_routes + + routes += api_routes(resolve, prefix="/api/v1") + routes += api_routes(resolve, prefix="/{name}/api/v1") + routes.append(Route("/{name}", make(page_help, "help.html"))) for slug, fn, tpl in pages: routes.append(Route(f"/{{name}}/{slug}", make(fn, tpl))) diff --git a/tests/test_crud_and_api.py b/tests/test_crud_and_api.py new file mode 100644 index 0000000..9490362 --- /dev/null +++ b/tests/test_crud_and_api.py @@ -0,0 +1,409 @@ +"""Delete, external ids, and the JSON HTTP API. + +The properties worth defending: + + a delete reaches the file. For a `storage: file` doc_type the JSON on disk is + the source of truth, so deleting only the index row is a pause, not a delete — + the entity returns on the next reindex. + + a delete takes its edges with it, in both directions. An edge left pointing at + a removed entity still renders on the map and in neighbour lists, and reads as + corruption rather than as a deletion. + + writes are gated only when a token is configured, and reads never are. +""" + +import os +import shutil + +import pytest +from starlette.testclient import TestClient + +from open_index.brain import Brain +from open_index.models import Entity, Relationship + +EXAMPLE = "examples/support-brain" + + +@pytest.fixture +def brain(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + d = tmp_path / "b" + shutil.copytree(EXAMPLE, d) + b = Brain.open(d) + b.index() + return b + + +# -- delete -------------------------------------------------------------------- + + +def test_delete_removes_the_entity(brain): + target = brain.backend.all_entities()[0].id + assert brain.delete_entity(target) is True + assert brain.get_entity(target) is None + + +def test_delete_removes_the_file_so_a_reindex_does_not_resurrect_it(brain): + """The failure this guards: delete the row, reindex, and it is back.""" + entity = next(e for e in brain.backend.all_entities() + if e.doc_type in brain._file_backed_types()) + path = brain.entity_path(entity) + assert path.exists() + + brain.delete_entity(entity.id) + assert not path.exists() + + brain.index() + assert brain.get_entity(entity.id) is None + + +def test_delete_removes_edges_pointing_at_it(brain): + """An edge surviving its target renders as a dangling node on the map.""" + target = brain.backend.all_entities()[0] + brain.put_entity(Entity( + id="issue:linker", doc_type="issue", name="linker", + related_to=[Relationship(target=target.id, + relationship_edge_meaning="refers to")])) + assert brain.backend.relationships_to(target.id) + + brain.delete_entity(target.id) + assert brain.backend.relationships_to(target.id) == [] + assert brain.backend.relationships_from("issue:linker") == [] + + +def test_deleting_something_absent_is_false_not_an_error(brain): + assert brain.delete_entity("issue:never-existed") is False + + +def test_delete_is_idempotent(brain): + target = brain.backend.all_entities()[0].id + assert brain.delete_entity(target) is True + assert brain.delete_entity(target) is False + + +def test_a_delete_that_cannot_remove_the_file_deletes_nothing(brain, monkeypatch): + """Better to fail wholly than to unindex an entity whose file remains: that + file silently restores it on the next reindex.""" + entity = next(e for e in brain.backend.all_entities() + if e.doc_type in brain._file_backed_types()) + + def refuse(self, missing_ok=False): + raise OSError("read-only file system") + + monkeypatch.setattr("pathlib.Path.unlink", refuse) + with pytest.raises(RuntimeError, match="still indexed"): + brain.delete_entity(entity.id) + assert brain.get_entity(entity.id) is not None + + +def test_delete_is_recorded_in_the_audit_trail(brain): + target = brain.backend.all_entities()[0].id + brain.delete_entity(target, source="cli") + assert brain.analytics_events(limit=1)[0]["operation"] == "delete_entity" + + +# -- external ids -------------------------------------------------------------- + + +def test_an_entity_can_carry_the_id_its_source_system_knows_it_by(brain): + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-4471")) + found = brain.get_by_external_id("CRM-4471") + assert found is not None and found.id == "issue:crm" + + +def test_an_unknown_external_id_is_none(brain): + assert brain.get_by_external_id("nope") is None + assert brain.get_by_external_id("") is None + + +def test_the_external_id_survives_a_file_round_trip(brain): + """It is written to disk, so a reindex must not drop it.""" + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-1")) + brain.index() + reopened = Brain.open(brain.config.root) + assert reopened.get_by_external_id("CRM-1") is not None + + +def test_the_canonical_id_keeps_its_shape(brain): + """external_id is free-form precisely so `id` does not have to be: an agent + reads the doc_type off the id.""" + with pytest.raises(ValueError, match="must look like"): + Entity(id="not-a-valid-id", doc_type="issue") + + +def test_external_id_is_not_treated_as_a_schema_field(brain): + brain.put_entity(Entity(id="issue:crm", doc_type="issue", name="n", + external_id="CRM-2")) + assert "external_id" not in brain.get_entity("issue:crm").fields + + +def test_a_batch_lookup_skips_what_is_missing(brain): + ids = [e.id for e in brain.backend.all_entities()[:2]] + ["issue:ghost"] + found = brain.get_entities(ids) + assert len(found) == 2 + assert all(e is not None for e in found) + + +# -- the HTTP API -------------------------------------------------------------- + + +@pytest.fixture +def api(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + d = tmp_path / "support-brain" + shutil.copytree(EXAMPLE, d) + b = Brain.open(d) + b.index() + b.put_entity(Entity(id="issue:crm", doc_type="issue", name="from crm", + external_id="CRM-9", fields={"description": "x"})) + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_search_over_http(api): + body = api.get("/api/v1/search", params={"q": "payment"}).json() + assert body["total"] and body["results"][0]["match"] + + +def test_search_exposes_the_modes(api): + body = api.get("/api/v1/search", params={"q": "payment", "mode": "keyword"}).json() + assert all(r["match"]["type"] == "keyword" for r in body["results"]) + + +def test_an_unknown_mode_is_a_400_not_a_500(api): + """The caller can fix it, and the message says how.""" + r = api.get("/api/v1/search", params={"q": "x", "mode": "telepathy"}) + assert r.status_code == 400 + assert "unknown search mode" in r.json()["error"] + + +def test_filtering_on_an_undeclared_field_is_a_400(api): + r = api.get("/api/v1/search", params={"filter.nope": "1"}) + assert r.status_code == 400 + assert "cannot filter on" in r.json()["error"] + + +def test_get_one_entity(api): + body = api.get("/api/v1/entities/issue:crm").json() + assert body["id"] == "issue:crm" + assert "relationships" in body + + +def test_a_missing_entity_is_a_404(api): + assert api.get("/api/v1/entities/issue:ghost").status_code == 404 + + +def test_lookup_by_external_id_over_http(api): + assert api.get("/api/v1/entities/by-external-id/CRM-9").json()["id"] == "issue:crm" + + +def test_batch_reports_what_was_missing(api): + body = api.get("/api/v1/entities", + params={"id": ["issue:crm", "issue:ghost"]}).json() + assert body["found"] == 1 + assert body["missing"] == ["issue:ghost"] + + +def test_the_schema_endpoint_describes_the_index(api): + body = api.get("/api/v1/schema").json() + assert body["doc_types"] and "count" in body["doc_types"][0] + + +def test_put_creates_an_entity(api): + r = api.put("/api/v1/entities/issue:via-api", + json={"doc_type": "issue", "name": "made over http"}) + assert r.status_code == 200 + assert api.get("/api/v1/entities/issue:via-api").status_code == 200 + + +def test_put_rejects_a_body_id_that_contradicts_the_url(api): + """Otherwise PUT /entities/a could write entity b.""" + r = api.put("/api/v1/entities/issue:a", json={"id": "issue:b", + "doc_type": "issue"}) + assert r.status_code == 400 + + +def test_put_rejects_an_invalid_entity_with_422(api): + r = api.put("/api/v1/entities/nope:x", json={"doc_type": "nope"}) + assert r.status_code in (400, 422) + + +def test_put_rejects_a_non_json_body(api): + r = api.put("/api/v1/entities/issue:x", content=b"not json") + assert r.status_code == 400 + + +def test_delete_over_http(api): + api.put("/api/v1/entities/issue:temp", json={"doc_type": "issue"}) + assert api.delete("/api/v1/entities/issue:temp").status_code == 200 + assert api.delete("/api/v1/entities/issue:temp").status_code == 404 + + +def test_a_trace_is_readable_over_http(api): + api.get("/api/v1/search", params={"q": "payment"}, + headers={"X-Trace-Id": "api-turn-1"}) + body = api.get("/api/v1/traces/api-turn-1").json() + assert body["reads"] and body["reads"][0]["results"] + + +# -- auth ---------------------------------------------------------------------- + + +@pytest.fixture +def gated(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home2")) + (tmp_path / "home2").mkdir() + d = tmp_path / "gated-brain" + shutil.copytree(EXAMPLE, d) + Brain.open(d).index() + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.setenv("OPEN_INDEX_TOKEN", "s3cret") + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_reads_stay_open_when_a_token_is_configured(gated): + """Matching how the MCP endpoint already behaves: the token gates writes.""" + assert gated.get("/api/v1/search", params={"q": "payment"}).status_code == 200 + + +def test_writes_need_the_token(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}) + assert r.status_code == 401 + assert "Bearer" in r.headers.get("www-authenticate", "") + + +def test_a_wrong_token_is_refused(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}, + headers={"Authorization": "Bearer wrong"}) + assert r.status_code == 401 + + +def test_the_right_token_is_accepted(gated): + r = gated.put("/api/v1/entities/issue:x", json={"doc_type": "issue"}, + headers={"Authorization": "Bearer s3cret"}) + assert r.status_code == 200 + + +def test_delete_is_gated_too(gated): + assert gated.delete("/api/v1/entities/issue:anything").status_code == 401 + + +def test_writes_are_open_when_no_token_is_set(api): + """The demo-host case: nothing configured, nothing gated.""" + assert api.put("/api/v1/entities/issue:open", + json={"doc_type": "issue"}).status_code == 200 + + +def test_a_per_brain_token_overrides_the_shared_one(monkeypatch): + from open_index.api import token_for + + monkeypatch.setenv("OPEN_INDEX_TOKEN", "shared") + monkeypatch.setenv("OPEN_INDEX_TOKEN_ALPHA_INDEX", "specific") + assert token_for("alpha-index") == "specific" + assert token_for("beta") == "shared" + monkeypatch.delenv("OPEN_INDEX_TOKEN") + assert token_for("beta") is None + + +# -- multi-brain routing ------------------------------------------------------- + + +@pytest.fixture +def many(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home3")) + (tmp_path / "home3").mkdir() + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.delenv("OPEN_INDEX_TOKEN", raising=False) + + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return TestClient(web.build_app()) + + +def test_the_api_is_per_index(many): + assert many.get("/alpha/api/v1/search", params={"q": "payment"}).status_code == 200 + assert many.get("/beta/api/v1/schema").status_code == 200 + + +def test_the_api_on_an_unknown_index_is_a_404(many): + assert many.get("/zzz/api/v1/search").status_code == 404 + + +def test_the_api_route_is_not_mistaken_for_an_index(many): + """/api/... must not resolve as an index literally named "api".""" + assert many.get("/alpha").status_code == 200 + assert many.get("/alpha/api/v1/schema").json()["doc_types"] + + +def test_a_write_lands_in_the_right_index(many): + many.put("/alpha/api/v1/entities/issue:only-alpha", json={"doc_type": "issue"}) + assert many.get("/alpha/api/v1/entities/issue:only-alpha").status_code == 200 + assert many.get("/beta/api/v1/entities/issue:only-alpha").status_code == 404 + + +# -- the error paths a caller can actually hit --------------------------------- + + +def test_a_non_numeric_limit_is_a_400(api): + r = api.get("/api/v1/search", params={"q": "x", "limit": "many"}) + assert r.status_code == 400 + assert "integer" in r.json()["error"] + + +def test_an_unknown_external_id_over_http_is_a_404(api): + assert api.get("/api/v1/entities/by-external-id/NOPE-1").status_code == 404 + + +def test_a_json_body_that_is_not_an_object_is_a_400(api): + r = api.put("/api/v1/entities/issue:x", json=["not", "an", "object"]) + assert r.status_code == 400 + assert "object" in r.json()["error"] + + +def test_an_unconstructable_entity_is_a_400(api): + """A field of the wrong shape fails at model construction, before validation + against the doc_type — still the caller's mistake, still a 4xx.""" + r = api.put("/api/v1/entities/issue:bad", + json={"doc_type": "issue", "related_to": [{"no_target": 1}]}) + assert r.status_code == 400 + assert "invalid entity" in r.json()["error"] + + +def test_a_delete_that_cannot_remove_the_file_is_a_500_not_a_silent_success( + api, tmp_path, monkeypatch): + """The entity is still indexed, so reporting success would be a lie.""" + api.put("/api/v1/entities/issue:stuck", json={"doc_type": "issue"}) + + def refuse(self, missing_ok=False): + raise OSError("read-only file system") + + monkeypatch.setattr("pathlib.Path.unlink", refuse) + r = api.delete("/api/v1/entities/issue:stuck") + assert r.status_code == 500 + assert "still indexed" in r.json()["error"] diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index b483929..7395f12 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -36,7 +36,9 @@ def srv(brain): def test_default_server_exposes_read_and_write_tools(srv): assert {"navigation_guidelines", "search_brain", "get_entity", - "put_entity", "put_entities", "create_doc_type"} == srv.tools() + "get_entities", "lookup_by_external_id", + "put_entity", "put_entities", "create_doc_type", + "delete_entity"} == srv.tools() def test_default_prompt_positions_brain_as_read_write_domain_context(srv): diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index f5df166..b28cdf4 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -336,3 +336,40 @@ def test_model_guide_distinguishes_schema_from_data(): assert "schema" in text + + +# -- the help tab's tool list must match the server's actual tools ------------- + + +def _documented_tool_names(): + """Tool names as the help tab lists them, e.g. 'search_brain(...)'.""" + return {name.split("(")[0] + for name, _ in list(view.READ_TOOLS) + list(view.WRITE_TOOLS)} + + +def test_the_help_tab_lists_exactly_the_tools_the_server_registers(brain): + """A stale tool list on the page is worse than none: it tells a reader an + agent can do something it cannot, or hides something it can. This assertion + is the only thing keeping the two in step. + """ + pytest.importorskip("mcp") + import asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain) + registered = {t.name for t in asyncio.run(server.list_tools())} + # navigation_guidelines is described in prose on the page, not as a row. + assert _documented_tool_names() | {"navigation_guidelines"} == registered + + +def test_the_read_only_tools_are_exactly_the_documented_read_ones(brain): + pytest.importorskip("mcp") + import asyncio + + from open_index.mcp_server import build_server + + server = build_server(brain, read_only=True) + registered = {t.name for t in asyncio.run(server.list_tools())} + documented = {name.split("(")[0] for name, _ in view.READ_TOOLS} + assert documented | {"navigation_guidelines"} == registered