From 9d216dfe510b1a60ea620e35233ab427c3749f13 Mon Sep 17 00:00:00 2001 From: atimics Date: Sat, 11 Jul 2026 18:27:24 -0700 Subject: [PATCH 01/16] Add x402 product API --- .dockerignore | 26 ++ API_QUICKREF.md | 42 ++ AWS_X402_DEPLOY.md | 157 +++++++ Dockerfile.x402 | 15 + PRODUCT.md | 50 +++ README.md | 11 +- X402_API.md | 95 ++++ apiquickref.py | 3 +- docs/PACKAGING.md | 1 + .../holographic_catalog_p04.py | 20 + holographic_product.py | 356 +++++++++++++++ holographic_x402_api.py | 414 ++++++++++++++++++ lecore.py | 19 +- requirements-x402.txt | 2 + setup.py | 3 +- tests/test_holographic_product.py | 66 +++ tests/test_holographic_x402_api.py | 100 +++++ tests/test_lecore.py | 5 +- 18 files changed, 1374 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 AWS_X402_DEPLOY.md create mode 100644 Dockerfile.x402 create mode 100644 PRODUCT.md create mode 100644 X402_API.md create mode 100644 holographic_product.py create mode 100644 holographic_x402_api.py create mode 100644 requirements-x402.txt create mode 100644 tests/test_holographic_product.py create mode 100644 tests/test_holographic_x402_api.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4b883791 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +.git +.gitignore +.dockerignore +.pytest_cache +.mypy_cache +.ruff_cache +__pycache__ +node_modules +dist +.next +.vinext +.wrangler +lecore-site +*.py[cod] +*.pyo +.venv +venv +env +.env +.env.* +metrics +holostuff.zip +current backlogs +*_backlog.md +PANEL_*_backlog.md +RENDER_PIPELINE_BACKLOG.md diff --git a/API_QUICKREF.md b/API_QUICKREF.md index 7a32471d..fa6b3f49 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -2,6 +2,48 @@ *A scannable, one-line-per-symbol map of the app-building surface -- auto-generated by `apiquickref.py`. For the full engine (every module), see REFERENCE.md.* +## Product wedge + +### `holographic_product` +*holographic_product.py -- the small product-facing leCore facade.* + +- **class `MemoryEntry`** -- One stored memory item. + - `to_dict(self)` -- Return a JSON-safe representation of this memory entry. + - `from_dict(cls, data)` -- Build a memory entry from `to_dict` data. +- **class `LocalAgentCore`** -- Product facade for local agent memory, skill routing, and evidence. + - `entries(self)` -- A copy of the stored entries, in insertion order. + - `remember(self, text, label=None, metadata=None, id=None)` -- Store one local memory. + - `remember_many(self, items)` -- Store several memories. + - `recall(self, query, k=3, abstain=None)` -- Return the nearest stored memories for `query`, best first. + - `suggest(self, task, k=5)` -- Suggest capabilities for a plain-English task. + - `route(self, task)` -- Route a task to one capability when confident, otherwise return options. + - `evidence(self)` -- Return a machine-readable product readiness snapshot. + - `dashboard(self, html=False)` -- Return the evidence dashboard as a dict, or static HTML with `html=True`. + - `dashboard_html(data)` -- Render an evidence snapshot as a dependency-free static HTML dashboard. + - `to_state(self)` -- Serialize configuration and entries. + - `from_state(cls, state)` -- Rebuild a core from `to_state` data. + - `save(self, path)` -- Write the product state to JSON and return the path. + - `load(cls, path)` -- Load a product state saved by `save`. +- `demo()` -- Build a tiny ready-to-query product demo. + +### `holographic_x402_api` +*holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API.* + +- **class `PaidRoute`** -- One x402-protected route. + - `key(self)` -- The route key shape expected by x402 middleware, e.g. +- **class `X402Config`** -- Seller configuration for the x402-paid API. + - `from_env(cls, require_pay_to=True)` -- Build config from LECORE_X402_* environment variables. + - `to_public_dict(self)` -- Public, JSON-safe view of the payment configuration. +- `optional_dependency_help()` -- Install hint for the optional paid API dependencies. +- `leos_token_offer()` -- Public metadata for the leOS CA-only offer. +- `landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. +- `payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. +- `x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. +- `x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. +- `create_app(core=None, config=None, paid=True, admin_token=None)` -- Create the FastAPI app. +- `load_core(path)` -- Load a persisted core if present, otherwise return the demo core. +- `main(argv=None)` -- CLI entry point for local x402 API serving. + ## Scene authoring ### `holographic_scene_doc` diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md new file mode 100644 index 00000000..1c83108a --- /dev/null +++ b/AWS_X402_DEPLOY.md @@ -0,0 +1,157 @@ +# AWS x402 Deployment + +This is the production shape for serving `LocalAgentCore` as an x402-paid API +on AWS. + +## Short Answer + +Yes, we can launch this on AWS. For the **seller** side of x402, the service +does **not** need a wallet private key in the container. It only needs: + +- the public receiving wallet address (`LECORE_X402_PAY_TO`) +- x402/facilitator configuration +- an admin token for seller-only memory writes + +The receiving wallet should be a cold wallet, hardware wallet, Safe/multisig, +or a custody wallet. The API simply tells x402 where funds should go. + +Only build an AWS-hosted signing wallet if the app itself must **spend** funds +or pay upstream APIs as a buyer. + +## Recommended AWS Architecture + +- **ECS Fargate** runs the `Dockerfile.x402` container. +- **Application Load Balancer** terminates HTTPS and forwards to port `4021`. +- **ECR** stores the container image. +- **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN` and production + facilitator credentials. +- **SSM Parameter Store or plain task env** stores non-secret config like + `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, and `LECORE_X402_NETWORK`. +- **CloudWatch Logs** captures service logs. +- **AWS WAF** can rate-limit and block bad traffic at the ALB. + +Protected paid routes: + +- `POST /v1/recall` +- `POST /v1/route` +- `GET /v1/dashboard` + +Free routes: + +- `GET /health` +- `GET /pricing` + +Seller-only route: + +- `POST /admin/remember`, guarded by `X-Admin-Token` + +## Build And Push + +```bash +aws ecr create-repository --repository-name lecore-x402 + +ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +REGION="${AWS_REGION:-us-west-2}" +IMAGE="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/lecore-x402:latest" + +aws ecr get-login-password --region "$REGION" \ + | docker login --username AWS --password-stdin "$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" + +docker build -f Dockerfile.x402 -t "$IMAGE" . +docker push "$IMAGE" +``` + +## Runtime Environment + +Non-secret environment variables: + +```text +LECORE_X402_PAY_TO=0xYourReceivingWallet +LECORE_X402_PRICE=$0.001 +LECORE_X402_NETWORK=eip155:8453 +LECORE_X402_FACILITATOR_URL=https://api.cdp.coinbase.com/platform/v2/x402 +``` + +Secrets Manager values: + +```text +LECORE_X402_ADMIN_TOKEN= +CDP_API_KEY_ID= +CDP_API_KEY_SECRET= +``` + +Use ECS task definition `secrets` entries for secrets, not literal environment +variables in the task definition. + +## Wallet Storage Decision + +### Seller API, Recommended + +Do **not** store a private key in AWS. + +The API receives payments; it does not spend. x402 payment verification and +settlement happen through the facilitator. The service only advertises +`payTo`. + +Best receiving wallet options: + +- Safe/multisig +- hardware wallet +- cold wallet +- custodial account dedicated to receipts + +### Buyer/Spender API, If Needed Later + +If the leCore agent itself needs to pay other x402 APIs, use a separate signer +service: + +1. Create an AWS KMS asymmetric signing key with `ECC_SECG_P256K1`. +2. Derive the public Ethereum address from `kms:GetPublicKey`. +3. Allow only a narrow IAM role to call `kms:Sign`. +4. Sign EIP-712/EIP-3009 payload digests through KMS. +5. Enforce spend limits in application logic before every signing request. +6. Log every signing request with CloudTrail and app-level audit records. + +This keeps the private key non-exportable: it never appears in the container. + +### High-Assurance Signer + +For larger balances or stronger isolation, put the signing service in **AWS +Nitro Enclaves** and allow KMS decrypt/sign only when enclave attestation +matches the expected image measurement. + +### Last Resort + +Storing a raw private key in Secrets Manager is acceptable only for testnet or +very small hot-wallet balances. If used, wrap it with strict IAM, rotation +plans, spend limits, CloudTrail alarms, and a tiny blast radius. + +## First Production Checklist + +- Use mainnet network id and production facilitator URL. +- Put the ALB behind HTTPS only. +- Keep `/admin/remember` private or blocked from the public ALB path. +- Keep paid route configs explicit; avoid wildcard paid routes at first. +- Add WAF rate limits. +- Add CloudWatch alarms on 5xx, 402 spikes, and admin write attempts. +- Keep customer memory isolated before offering paid writes. +- Do not put secrets or PII in x402 route descriptions or payment metadata. + +## Local Smoke Before AWS + +```bash +pip install ".[x402]" +export LECORE_X402_PAY_TO="0xYourReceivingWallet" +export LECORE_X402_ADMIN_TOKEN="local-admin-secret" +python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 +``` + +Then: + +```bash +curl http://127.0.0.1:4021/health +curl http://127.0.0.1:4021/pricing +curl -X POST http://127.0.0.1:4021/v1/route \ + -H "Content-Type: application/json" \ + -d '{"task":"search local agent memory"}' +``` diff --git a/Dockerfile.x402 b/Dockerfile.x402 new file mode 100644 index 00000000..e5332891 --- /dev/null +++ b/Dockerfile.x402 @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +COPY . /app + +RUN python -m pip install --no-cache-dir --upgrade pip \ + && python -m pip install --no-cache-dir ".[x402]" + +EXPOSE 4021 + +CMD ["python", "holographic_x402_api.py", "--host", "0.0.0.0", "--port", "4021"] diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 00000000..a53d96e0 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,50 @@ +# leCore Product Wedge + +The first product surface is **LocalAgentCore**: a small facade for local agent +memory, capability routing, and readiness evidence. + +It deliberately narrows the promise. The full repo is a broad research engine; +this surface is the five-minute path for a builder who wants a deterministic, +inspectable local substrate. + +```python +from holographic_product import LocalAgentCore + +core = LocalAgentCore(dim=512, seed=0) +core.remember("local agents need deterministic durable memory", label="memory") +core.remember("capability routing should act when confident", label="routing") + +print(core.recall("deterministic local memory")[0]) +print(core.route("render a scene with global illumination")) +print(core.dashboard()) +``` + +The same object is available from the friendly import surface: + +```python +import lecore + +core = lecore.product.LocalAgentCore() +``` + +## What It Productizes + +- **Memory:** local text memories encoded through `UniversalEncoder` and recalled + through the shared `Index` home. +- **Routing:** plain-English tasks routed through the existing skill catalog. +- **Evidence:** a JSON/static-HTML dashboard with memory counts, capability + counts, determinism checks, and optional C-kernel availability. +- **Persistence:** `save(path)` and `LocalAgentCore.load(path)` round-trip the + stable state as JSON. Vectors are rebuilt from seed, text context, and entries. +- **Paid API publishing:** `holographic_x402_api.py` serves the product wedge as + an optional x402-paid FastAPI service. See [`X402_API.md`](X402_API.md). + +## Honest Scope + +This is not a neural database, a hosted service, or a general semantic model. +Out of the box it matches by the deterministic holographic text geometry it is +given. Better domain recall comes from adding domain memories, teaching text +context, or layering a specialized encoder on the same facade. + +The product rule is simple: the public wedge stays small, auditable, local, and +measured. The research garden remains available behind it. diff --git a/README.md b/README.md index 9cb25925..fa40bfa1 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ pip install "leos-core[symbolic]" # pip install .[symbolic] design-ti pip install "leos-core[zig]" # pip install .[zig] native batch kernels, 2-5x (ziglang -- whole # toolchain in one wheel, bit-identical in safe mode) pip install "leos-core[images]" # pip install .[images] jpg/webp/... image I/O (Pillow, no Flask) +pip install "leos-core[x402]" # pip install .[x402] paid API publishing (x402, FastAPI) pip install "leos-core[dev]" # pip install .[dev] run the tests and make plots (pytest, matplotlib) pip install "leos-core[all]" # pip install .[all] everything portable, one shot pip install "leos-core[ui,jit]" # pip install .[ui,jit] ...or combine whichever you want @@ -187,6 +188,13 @@ Like leOS, leCore is **free and open source**, and the work that keeps it free i `find_capability` first, wire every capability to a mind faculty (so it is `/invoke`-able), register it in the catalog so it is discoverable, and run the reachability/gap audits — the discipline that keeps the codebase from growing gaps or isolating code in tests. Read this before making code changes. +- **[`PRODUCT.md`](PRODUCT.md)** — the **narrow product wedge**: `LocalAgentCore`, a small stable facade for local + agent memory, capability routing, persistence, and the readiness dashboard. Start here if you want the five-minute + "use it in an agent" path rather than the whole research surface. +- **[`X402_API.md`](X402_API.md)** — the **paid API publishing guide**: serve the product wedge over FastAPI with + optional x402 middleware, per-route pricing, and admin-gated memory writes. +- **[`AWS_X402_DEPLOY.md`](AWS_X402_DEPLOY.md)** — the **AWS launch guide**: ECS/Fargate deployment, Secrets Manager + config, and when to use KMS or Nitro Enclaves for wallet signing. - **[`CAPABILITIES.md`](CAPABILITIES.md)** — the **front-door menu**: a plain-language, grouped list of what leCore can do and the one call that starts each job. The friendliest place to begin if you're deciding whether the engine already does the thing you need. Generated from the live capability catalog by `capdoc.py` and kept in sync by CI. @@ -200,7 +208,8 @@ Like leOS, leCore is **free and open source**, and the work that keeps it free i way around. It's generated from the code by `docgen.py` and kept in sync automatically by CI, so it never drifts from what's actually there. - **[`API_QUICKREF.md`](API_QUICKREF.md)** — the **app-builder's quick reference**: one scannable line per public - class/function for the modules you actually touch when building on leCore (scene, mesh, camera, render, ship). + class/function for the modules you actually touch when building on leCore (product, scene, mesh, camera, render, + ship). - **[`SERVICE.md`](SERVICE.md)** — the **standalone HTTP service**: every endpoint (data store, jobs, and the agent-facing skills API) with `curl` examples, for driving leCore as an app rather than a library. - **[`GALLERY.md`](GALLERY.md)** — a **visual showcase**: renders, procedural patterns, memory/reconstruction demos, and performance charts, straight from the engine's tests (the visual companion to the code reference). diff --git a/X402_API.md b/X402_API.md new file mode 100644 index 00000000..2226af2a --- /dev/null +++ b/X402_API.md @@ -0,0 +1,95 @@ +# x402 API Publishing + +Yes: leCore can be published as a paid API with x402. + +The implementation lives in `holographic_x402_api.py`. It wraps +`LocalAgentCore` with a small FastAPI app and applies x402 middleware only to +the public read/compute routes: + +- `POST /v1/recall` +- `POST /v1/route` +- `GET /v1/dashboard` + +Free routes: + +- `GET /health` +- `GET /pricing` + +Admin route: + +- `POST /admin/remember`, guarded by `X-Admin-Token` + +This split is deliberate. Paid customers can use the memory/router/dashboard, +but they cannot mutate the shared memory store unless they also hold the admin +token. + +## Install + +```bash +pip install ".[x402]" +``` + +The core package still needs only NumPy. The `x402` extra pulls in the optional +FastAPI/x402/uvicorn stack. + +## Testnet Run + +The default network is Base Sepolia (`eip155:84532`) and the default facilitator +is the signup-free x402.org testnet facilitator. + +```bash +export LECORE_X402_PAY_TO="0xYourReceivingWallet" +export LECORE_X402_PRICE="$0.001" +export LECORE_X402_ADMIN_TOKEN="local-admin-secret" + +python holographic_x402_api.py --host 127.0.0.1 --port 4021 +``` + +Inspect pricing: + +```bash +curl http://127.0.0.1:4021/pricing +``` + +Add memories locally as the seller: + +```bash +curl -X POST http://127.0.0.1:4021/admin/remember \ + -H "Content-Type: application/json" \ + -H "X-Admin-Token: local-admin-secret" \ + -d '{"text":"local agents need deterministic durable memory","label":"memory"}' +``` + +Requests to paid routes return `402 Payment Required` unless the client retries +with a valid x402 payment payload: + +```bash +curl -X POST http://127.0.0.1:4021/v1/recall \ + -H "Content-Type: application/json" \ + -d '{"query":"deterministic local memory"}' +``` + +## Local Unpaid Smoke Test + +Use this only for development: + +```bash +python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 +``` + +## Production Notes + +- Use a real receiving wallet and a production facilitator. +- Put the API behind HTTPS. +- Keep route prices explicit; avoid wildcard paid route configs for this first + product surface. +- Keep writes admin-only, or move customer writes into isolated per-customer + stores before charging for them. +- Treat x402 payment metadata as public enough to avoid putting secrets or PII + in route descriptions. + +The implementation follows the current x402 seller shape: FastAPI middleware, +`RouteConfig`, `PaymentOption`, an `exact` EVM scheme, and a facilitator-backed +resource server. + +For AWS hosting, see [`AWS_X402_DEPLOY.md`](AWS_X402_DEPLOY.md). diff --git a/apiquickref.py b/apiquickref.py index dec17c42..d66a14b2 100644 --- a/apiquickref.py +++ b/apiquickref.py @@ -21,11 +21,12 @@ # ---------------------------------------------------------------------------------------------------------- # THE CURATED SURFACE. Edit this list to change what the quick reference covers. Grouped by the job a builder -# is doing, in the order they meet it: author a scene -> model geometry -> aim a camera -> render -> ship. +# is doing, in the order they meet it: product wedge -> author a scene -> model geometry -> aim a camera -> render -> ship. # Kept deliberately SHORT -- the point is a page you can scan, not a full index (that is REFERENCE.md). # ---------------------------------------------------------------------------------------------------------- CURATED = [ + ("Product wedge", ["holographic_product", "holographic_x402_api"]), ("Scene authoring", ["holographic_scene_doc", "holographic_modifier"]), ("Geometry / SDF", ["holographic_sdf", "holographic_sdfscene", "holographic_mesh"]), ("Transforms", ["holographic_transform"]), diff --git a/docs/PACKAGING.md b/docs/PACKAGING.md index 13ab0cf3..3c464b06 100644 --- a/docs/PACKAGING.md +++ b/docs/PACKAGING.md @@ -132,6 +132,7 @@ The core requires **only NumPy**. Everything else is declared as a named "extra" | `zig` | `ziglang` | native batch kernels + raymarcher (`holographic_zigrun`, `zigmarch`); ships the whole Zig toolchain, no system compiler needed | | `wgsl` | `wgpu` | **the vendor-neutral GPU path** (`holographic_wgpurun`): compute on Vulkan / Metal / DX12 / WebGPU, so it works on Apple silicon, AMD and Intel Arc as well as NVIDIA. Prebuilt wheels, no system toolchain | | `gpu` | `cupy` | the CuPy backend (`holographic_backend`) — **NVIDIA/CUDA only**; see the CuPy note | +| `x402` | `x402[fastapi,evm]`, `uvicorn` | paid API publishing (`holographic_x402_api`) | | `ui` | `flask`, `pillow` | the browser UI (`app.py`) and image load/save | | `images` | `pillow` | image I/O beyond stdlib PNG (jpg/webp/…) without pulling in Flask — a headless subset of `ui` | | `dev` | `pytest`, `matplotlib`, `nltk` | running the test suite, generating plots, and loading the text corpora the benchmarks/ablations use | diff --git a/holographic/caching_and_storage/holographic_catalog_p04.py b/holographic/caching_and_storage/holographic_catalog_p04.py index e0bb2f95..2eab132b 100644 --- a/holographic/caching_and_storage/holographic_catalog_p04.py +++ b/holographic/caching_and_storage/holographic_catalog_p04.py @@ -1202,6 +1202,26 @@ def register_p04(c): "llm bridge", "notify the agent", "push notification", "on render done", "connect an agent", "send message to agent", "mailbox", "inbox", "trigger the llm", "watch for events", "task done event")) + # --- product and paid API publishing --------------------------------------------------------------- + c.register_capability( + "Local agent core (memory + routing)", + "the PRODUCT-FACING wedge: LocalAgentCore gives a local agent deterministic text memory " + "(remember/recall), skill routing over the live capability catalog, JSON persistence, and " + "a readiness dashboard with C-kernel status. This is the small stable door for embedding " + "leCore without learning the whole research surface first.", + example="from holographic_product import LocalAgentCore; core = LocalAgentCore(); core.remember('local agent memory'); core.recall('agent memory')", + native=True, + aliases=("product", "productization", "agent memory", "local memory", "durable memory", "recall", + "skill routing", "dashboard", "first user", "facade", "local agent core")) + c.register_capability( + "x402 paid API publisher", + "publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, " + "task routing, and the evidence dashboard protected by x402 middleware, with free " + "health/pricing routes and admin-token-gated memory writes.", + example="from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...'))", + native=False, + aliases=("x402", "paid api", "payment required", "402", "monetize api", "micropayment", + "agent payments", "pay per request", "fastapi", "api publishing", "sell api", "paid route")) # --- agent-friendly discovery: describe / suggest / route / autocomplete over the whole engine --- c.register_capability("Agent skills (discover & route)", "the AGENT-FRIENDLY layer: mind.skills() lists every " "capability + method with how to CALL it (skill descriptions, real signatures); " diff --git a/holographic_product.py b/holographic_product.py new file mode 100644 index 00000000..ae85e16c --- /dev/null +++ b/holographic_product.py @@ -0,0 +1,356 @@ +"""holographic_product.py -- the small product-facing leCore facade. + +WHY THIS EXISTS +--------------- +The research engine is intentionally broad: memory, geometry, rendering, +simulation, jobs, skills, and more all share the same holographic substrate. +That is useful for research, but a first-time product user needs one narrow, +reliable door. + +`LocalAgentCore` is that door. It packages the current production wedge: + + * local deterministic text memory (`remember` / `recall`) + * agent skill routing through the existing capability catalog (`route`) + * an evidence snapshot and static HTML dashboard (`dashboard`) + +It does not replace `UnifiedMind` or hide the research surface. It is a small, +boring facade over the stable pieces, meant to be easy to install, test, demo, +and embed. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import html +import json +from pathlib import Path +import re +from typing import Any, Dict, Iterable, List, Optional + +import numpy as np + +from holographic.caching_and_storage.holographic_index import Index +from holographic.agents_and_reasoning.holographic_mind import UniversalEncoder + + +_WORD_RE = re.compile(r"[a-z0-9_]+") + + +def _tokens(text: Any) -> List[str]: + """Deterministic product tokenization: lower-case content tokens, no hidden NLP dependency.""" + if isinstance(text, (list, tuple)): + return [str(t).lower() for t in text if str(t).strip()] + return _WORD_RE.findall(str(text).lower()) + + +@dataclass +class MemoryEntry: + """One stored memory item.""" + + id: str + text: str + label: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Return a JSON-safe representation of this memory entry.""" + return { + "id": self.id, + "text": self.text, + "label": self.label, + "metadata": dict(self.metadata), + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "MemoryEntry": + """Build a memory entry from `to_dict` data.""" + return cls( + id=str(data["id"]), + text=str(data.get("text", "")), + label=data.get("label"), + metadata=dict(data.get("metadata") or {}), + ) + + +class LocalAgentCore: + """Product facade for local agent memory, skill routing, and evidence. + + The API is deliberately small: + + core = LocalAgentCore() + core.remember("local agents need deterministic memory", label="memory") + core.recall("deterministic local memory") + core.route("render a scene") + core.dashboard() + + Text memory uses the existing `UniversalEncoder` and `Index` homes. It is + deterministic, local-only, and query-safe: `recall()` does not mutate the + stored corpus or teach the encoder new query words. + """ + + def __init__(self, dim: int = 512, seed: int = 0, route_threshold: float = 0.6): + self.dim = int(dim) + self.seed = int(seed) + self.route_threshold = float(route_threshold) + self._entries: List[MemoryEntry] = [] + self._encoder = UniversalEncoder(self.dim, seed=self.seed) + self._vectors: Optional[np.ndarray] = None + self._index: Optional[Index] = None + self._next_id = 1 + + # ---- memory --------------------------------------------------------------------------------------- + @property + def entries(self) -> List[MemoryEntry]: + """A copy of the stored entries, in insertion order.""" + return list(self._entries) + + def remember( + self, + text: Any, + label: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + id: Optional[str] = None, + ) -> Dict[str, Any]: + """Store one local memory. Returns the stored entry as a plain dict.""" + entry_id = str(id) if id is not None else self._allocate_id() + if any(e.id == entry_id for e in self._entries): + raise ValueError("memory id already exists: %s" % entry_id) + entry = MemoryEntry(entry_id, str(text), label, dict(metadata or {})) + self._entries.append(entry) + self._rebuild_index() + return entry.to_dict() + + def remember_many(self, items: Iterable[Any]) -> List[Dict[str, Any]]: + """Store several memories. Each item may be text or a dict with text/label/metadata/id.""" + stored = [] + for item in items: + if isinstance(item, dict): + stored.append(self.remember( + item.get("text", ""), + label=item.get("label"), + metadata=item.get("metadata"), + id=item.get("id"), + )) + else: + stored.append(self.remember(item)) + return stored + + def recall(self, query: Any, k: int = 3, abstain: Optional[float] = None) -> List[Dict[str, Any]]: + """Return the nearest stored memories for `query`, best first. + + `abstain` is passed to `Index.nearest`; when set, noisy matches can + return an empty list instead of a guess. + """ + if not self._entries or self._index is None: + return [] + q = self._encode_text(query) + hits = self._index.nearest(q, k=min(int(k), len(self._entries)), abstain=abstain) + by_id = {entry.id: entry for entry in self._entries} + out = [] + for entry_id, score in hits: + entry = by_id[str(entry_id)] + row = entry.to_dict() + row["score"] = float(score) + out.append(row) + return out + + # ---- agent routing -------------------------------------------------------------------------------- + def suggest(self, task: str, k: int = 5) -> List[Dict[str, Any]]: + """Suggest capabilities for a plain-English task.""" + from holographic.misc import holographic_skills as skills + + return skills.suggest(task, k=k) + + def route(self, task: str) -> Dict[str, Any]: + """Route a task to one capability when confident, otherwise return options.""" + from holographic.misc import holographic_skills as skills + + routed = skills.route(task, act_threshold=self.route_threshold) + out = {"task": str(task)} + out.update(routed) + return out + + # ---- evidence / dashboard ------------------------------------------------------------------------- + def evidence(self) -> Dict[str, Any]: + """Return a machine-readable product readiness snapshot.""" + from holographic.caching_and_storage.holographic_catalog import default_catalog + + c_kernel = self._c_kernel_status() + route_probe = self.route("search a big pile of vectors") + return { + "name": "leCore LocalAgentCore", + "status": "ready" if self._deterministic_probe() else "check", + "memory": { + "entries": len(self._entries), + "dim": self.dim, + "index_method": self._index.method if self._index is not None else None, + "query_mutates_store": False, + }, + "routing": { + "capabilities": len(default_catalog()), + "probe_decision": route_probe.get("decision"), + "probe_skill": (route_probe.get("skill") or {}).get("name"), + }, + "c_kernel": c_kernel, + "checks": { + "deterministic_encoding": self._deterministic_probe(), + "local_only": True, + "no_model_weights": True, + }, + } + + def dashboard(self, html: bool = False) -> Any: + """Return the evidence dashboard as a dict, or static HTML with `html=True`.""" + data = self.evidence() + return self.dashboard_html(data) if html else data + + @staticmethod + def dashboard_html(data: Dict[str, Any]) -> str: + """Render an evidence snapshot as a dependency-free static HTML dashboard.""" + memory = data.get("memory", {}) + routing = data.get("routing", {}) + c_kernel = data.get("c_kernel", {}) + checks = data.get("checks", {}) + + def esc(value: Any) -> str: + return html.escape("" if value is None else str(value)) + + rows = [ + ("Status", data.get("status")), + ("Memories", memory.get("entries")), + ("Dimension", memory.get("dim")), + ("Index", memory.get("index_method") or "empty"), + ("Capabilities", routing.get("capabilities")), + ("Route Probe", "%s: %s" % (routing.get("probe_decision"), routing.get("probe_skill"))), + ("C Kernel", "available" if c_kernel.get("available") else "not built"), + ("C Path", c_kernel.get("path") or ""), + ("Deterministic", checks.get("deterministic_encoding")), + ("Local Only", checks.get("local_only")), + ("No Model Weights", checks.get("no_model_weights")), + ] + body = "\n".join( + "%s%s" % (esc(k), esc(v)) + for k, v in rows + ) + return """ + + + + leCore LocalAgentCore Dashboard + + + +
+

leCore LocalAgentCore

+

Local deterministic memory, skill routing, and readiness evidence.

+ + %s +
+
+ +""" % body + + # ---- persistence ---------------------------------------------------------------------------------- + def to_state(self) -> Dict[str, Any]: + """Serialize configuration and entries. Vectors are seed/context-derived and rebuilt on load.""" + return { + "dim": self.dim, + "seed": self.seed, + "route_threshold": self.route_threshold, + "next_id": self._next_id, + "entries": [entry.to_dict() for entry in self._entries], + } + + @classmethod + def from_state(cls, state: Dict[str, Any]) -> "LocalAgentCore": + """Rebuild a core from `to_state` data.""" + core = cls( + dim=int(state.get("dim", 512)), + seed=int(state.get("seed", 0)), + route_threshold=float(state.get("route_threshold", 0.6)), + ) + core._entries = [MemoryEntry.from_dict(row) for row in state.get("entries", [])] + core._next_id = int(state.get("next_id", len(core._entries) + 1)) + core._rebuild_index() + return core + + def save(self, path: Any) -> str: + """Write the product state to JSON and return the path.""" + p = Path(path) + p.write_text(json.dumps(self.to_state(), indent=2, sort_keys=True), encoding="utf-8") + return str(p) + + @classmethod + def load(cls, path: Any) -> "LocalAgentCore": + """Load a product state saved by `save`.""" + data = json.loads(Path(path).read_text(encoding="utf-8")) + return cls.from_state(data) + + # ---- internals ------------------------------------------------------------------------------------ + def _allocate_id(self) -> str: + entry_id = "m%d" % self._next_id + self._next_id += 1 + return entry_id + + def _encode_text(self, text: Any) -> np.ndarray: + toks = _tokens(text) + return self._encoder.encode(toks, modality="text") + + def _rebuild_index(self) -> None: + self._encoder = UniversalEncoder(self.dim, seed=self.seed) + for entry in self._entries: + toks = _tokens(entry.text) + if toks: + self._encoder.learn_text([toks]) + if not self._entries: + self._vectors = None + self._index = None + return + self._vectors = np.stack([self._encode_text(entry.text) for entry in self._entries]) + self._index = Index(self._vectors, labels=[entry.id for entry in self._entries], method="exact", seed=self.seed) + + def _deterministic_probe(self) -> bool: + a = self._encode_text("deterministic local memory") + b = self._encode_text("deterministic local memory") + return bool(np.allclose(a, b)) + + @staticmethod + def _c_kernel_status() -> Dict[str, Any]: + try: + import holographic_c + + return { + "available": bool(holographic_c.available()), + "path": holographic_c.backend_path(), + } + except Exception as exc: # pragma: no cover - defensive dashboard reporting + return {"available": False, "path": None, "error": "%s: %s" % (type(exc).__name__, exc)} + + +def demo() -> LocalAgentCore: + """Build a tiny ready-to-query product demo.""" + core = LocalAgentCore(dim=512, seed=0) + core.remember("local agents need deterministic durable memory", label="agent-memory") + core.remember("capability routing should act when confident and choose when ambiguous", label="routing") + core.remember("the C kernel accelerates the audited vector algebra hot path", label="c-kernel") + return core + + +def _selftest() -> None: + core = demo() + assert core.recall("deterministic local memory")[0]["label"] == "agent-memory" + assert core.route("start pause resume cancel a job")["decision"] == "act" + assert core.dashboard()["checks"]["deterministic_encoding"] + print("OK: holographic_product self-test passed") + + +if __name__ == "__main__": + _selftest() diff --git a/holographic_x402_api.py b/holographic_x402_api.py new file mode 100644 index 00000000..e1e1477a --- /dev/null +++ b/holographic_x402_api.py @@ -0,0 +1,414 @@ +"""holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. + +WHY THIS EXISTS +--------------- +`LocalAgentCore` is the narrow product wedge. This module makes it sellable as +an HTTP API without making x402, FastAPI, or uvicorn core dependencies. + +The boundary is intentionally conservative: + + * public read/compute routes are x402-paid + * health/pricing routes are free + * memory writes are admin-token gated, not pay-to-write + +That keeps the paid surface useful while preventing customers from poisoning a +shared memory store just because they paid for one request. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from html import escape +import argparse +import os +from pathlib import Path +from string import Template +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from holographic_product import LocalAgentCore, demo + + +DEFAULT_FACILITATOR_URL = "https://x402.org/facilitator" +DEFAULT_NETWORK = "eip155:84532" # Base Sepolia, safe default for testnet publishing. +DEFAULT_PRICE = "$0.0011" +LEOS_SITE_URL = "https://discoverleos.com/" +LEOS_TOKEN_CA = "5xgsnby6P9zqGK71J7H4yJLxzqPvNbC7rDZxNzjHmj7e" +LEOS_TOKEN_PRICE = "$0.0010" + + +@dataclass(frozen=True) +class PaidRoute: + """One x402-protected route.""" + + method: str + path: str + description: str + price: Optional[str] = None + mime_type: str = "application/json" + + @property + def key(self) -> str: + """The route key shape expected by x402 middleware, e.g. `POST /v1/recall`.""" + return "%s %s" % (self.method.upper(), self.path) + + +DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = ( + PaidRoute("POST", "/v1/recall", "Recall nearest memories from a LocalAgentCore instance"), + PaidRoute("POST", "/v1/route", "Route a plain-English task to a leCore capability"), + PaidRoute("GET", "/v1/dashboard", "Read the LocalAgentCore evidence dashboard"), +) + + +LANDING_PAGE_TEMPLATE = Template(""" + + + + +leCore x402 API + + + + +
+
+ + +

Live on AWS$network_label$price per callleOS CA $leos_token_price

leCore x402 API

Buy the small, useful surface of leCore: local agent memory, capability routing, and a readiness dashboard, sold as paid HTTP primitives instead of another subscription dashboard.

View pricingSee x402 challengeleOS site
+ +
+
Endpointhttps://lecore.rati.foundation
Paymentx402 exact scheme
Buyer shapeinspect, pay, call
+

Why you would buy it

Because most agents do not need a platform. They need a few reliable cognitive calls.

You pay for an answerable primitive, not a monthly seat.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: local agent memory plus capability routing.

The implementation is deployed, health-checked, and already returning x402 payment challenges.

+

leOS token offer

A slightly cheaper CA-only price for the leOS token.

Token price$leos_token_price
CA$leos_token_ca
leOS website
+

What the payment unlocks

Three paid routes, each small enough to understand.

POST

Recall

/v1/recall

Pull nearest memories from a compact local agent core without shipping a whole application stack.

POST

Route

/v1/route

Send a plain-language task and get the leCore capability it should use, with evidence attached.

GET

Dashboard

/v1/dashboard

Read the readiness surface: memory counts, capability map, abstention behavior, and route coverage.

+

Good first buyers

Teams who want the leCore idea without adopting the whole repo.

Agent memory for prototypes that should remember without a database rollout.

Capability routing for tools that need to pick the right leCore subsystem before doing work.

Evidence dashboards for teams deciding whether a local vector system is ready to productize.

A working x402 seller endpoint to copy when you want pay-per-call APIs instead of subscriptions.

+

Proof it is real

It is already deployed, priced, and protected.

The free endpoints show health and pricing. Paid endpoints return a real x402 payment challenge. The receiving address is public, while admin writes stay out of the paid customer path.

Price
$price
Network
$network_name
Receiver
$pay_to_short
Status
Healthy
+

The pitch

Buy it when you want a local-memory agent primitive that can pay for itself one request at a time.

Inspect the offerCheck live health
+
+ +""") + + +@dataclass(frozen=True) +class X402Config: + """Seller configuration for the x402-paid API.""" + + pay_to: str + price: str = DEFAULT_PRICE + network: str = DEFAULT_NETWORK + facilitator_url: str = DEFAULT_FACILITATOR_URL + scheme: str = "exact" + routes: Tuple[PaidRoute, ...] = DEFAULT_PAID_ROUTES + + def __post_init__(self) -> None: + if not self.pay_to: + raise ValueError("pay_to is required") + if not self.price.startswith("$"): + raise ValueError("x402 price must include a dollar prefix, e.g. '$0.001'") + if not self.network: + raise ValueError("network is required") + if not self.facilitator_url: + raise ValueError("facilitator_url is required") + + @classmethod + def from_env(cls, require_pay_to: bool = True) -> "X402Config": + """Build config from LECORE_X402_* environment variables.""" + pay_to = os.environ.get("LECORE_X402_PAY_TO", "") + if require_pay_to and not pay_to: + raise ValueError("set LECORE_X402_PAY_TO to the receiving wallet address") + return cls( + pay_to=pay_to or "0xYourAddress", + price=os.environ.get("LECORE_X402_PRICE", DEFAULT_PRICE), + network=os.environ.get("LECORE_X402_NETWORK", DEFAULT_NETWORK), + facilitator_url=os.environ.get("LECORE_X402_FACILITATOR_URL", DEFAULT_FACILITATOR_URL), + ) + + def to_public_dict(self) -> Dict[str, Any]: + """Public, JSON-safe view of the payment configuration.""" + return { + "pay_to": self.pay_to, + "price": self.price, + "network": self.network, + "facilitator_url": self.facilitator_url, + "scheme": self.scheme, + } + + +def optional_dependency_help() -> str: + """Install hint for the optional paid API dependencies.""" + return 'Install the optional API dependencies with: pip install ".[x402]" (includes FastAPI and EVM x402 support)' + + +def _landing_nodes() -> str: + """CSS-positioned visual nodes for the marketing page hero.""" + nodes = [] + for index in range(34): + size = 9 if index % 5 == 0 else 7 if index % 3 == 0 else 5 + nodes.append( + '' + % ((index * 29) % 100, (index * 47 + 11) % 100, (index % 9) * -0.45, size) + ) + return "".join(nodes) + + +def _short_address(address: str) -> str: + """Compact public wallet display.""" + if len(address) <= 12: + return address + return "%s...%s" % (address[:6], address[-4:]) + + +def _network_name(network: str) -> str: + """Human label for known x402 network ids.""" + return {"eip155:84532": "Base Sepolia", "eip155:8453": "Base"}.get(network, network) + + +def leos_token_offer() -> Dict[str, str]: + """Public metadata for the leOS CA-only offer.""" + return { + "name": "leOS CA offer", + "site": LEOS_SITE_URL, + "ca": LEOS_TOKEN_CA, + "price": LEOS_TOKEN_PRICE, + "note": "Only the CA is needed for this token offer.", + } + + +def landing_page_html(config: X402Config) -> str: + """Render the buyer-facing landing page served from `/`.""" + network_name = _network_name(config.network) + offer = leos_token_offer() + return LANDING_PAGE_TEMPLATE.substitute( + nodes=_landing_nodes(), + price=escape(config.price), + network=escape(config.network), + network_label=escape("%s x402" % network_name), + network_name=escape(network_name), + pay_to_short=escape(_short_address(config.pay_to)), + leos_site_url=escape(offer["site"], quote=True), + leos_token_ca=escape(offer["ca"]), + leos_token_price=escape(offer["price"]), + ) + + +def payment_manifest(config: X402Config) -> List[Dict[str, Any]]: + """Plain JSON route manifest, useful for docs, `/pricing`, and tests.""" + out = [] + for route in config.routes: + price = route.price or config.price + out.append({ + "route": route.key, + "description": route.description, + "mime_type": route.mime_type, + "accepts": [{ + "scheme": config.scheme, + "price": price, + "network": config.network, + "pay_to": config.pay_to, + }], + }) + return out + + +def x402_route_configs(config: X402Config) -> Dict[str, Any]: + """Build x402 SDK RouteConfig objects for the protected routes.""" + try: + from x402.http import PaymentOption + from x402.http.types import RouteConfig + except ImportError as exc: + raise RuntimeError(optional_dependency_help()) from exc + + routes = {} + for route in config.routes: + routes[route.key] = RouteConfig( + accepts=[ + PaymentOption( + scheme=config.scheme, + pay_to=config.pay_to, + price=route.price or config.price, + network=config.network, + ) + ], + mime_type=route.mime_type, + description=route.description, + ) + return routes + + +def x402_resource_server(config: X402Config) -> Any: + """Create an x402 resource server wired to the configured facilitator.""" + try: + from x402.http import FacilitatorConfig, HTTPFacilitatorClient + from x402.mechanisms.evm.exact import ExactEvmServerScheme + from x402.server import x402ResourceServer + except ImportError as exc: + raise RuntimeError(optional_dependency_help()) from exc + + facilitator = HTTPFacilitatorClient(FacilitatorConfig(url=config.facilitator_url)) + server = x402ResourceServer(facilitator) + server.register(config.network, ExactEvmServerScheme()) + return server + + +def create_app( + core: Optional[LocalAgentCore] = None, + config: Optional[X402Config] = None, + paid: bool = True, + admin_token: Optional[str] = None, +) -> Any: + """Create the FastAPI app. + + With `paid=True`, the public `/v1/*` read/compute routes are protected by + x402 middleware. Set `paid=False` for local development smoke tests. + """ + try: + from fastapi import FastAPI, Header, HTTPException + from fastapi.responses import HTMLResponse + except ImportError as exc: + raise RuntimeError(optional_dependency_help()) from exc + + app = FastAPI(title="leCore x402 API", version="0.1.0") + core = core or demo() + config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) + + if paid: + try: + from x402.http.middleware.fastapi import PaymentMiddlewareASGI + except ImportError as exc: + raise RuntimeError(optional_dependency_help()) from exc + app.add_middleware( + PaymentMiddlewareASGI, + routes=x402_route_configs(config), + server=x402_resource_server(config), + ) + + def require_admin(header_value: Optional[str]) -> None: + if not admin_token: + raise HTTPException(status_code=403, detail="admin writes are disabled") + if header_value != admin_token: + raise HTTPException(status_code=401, detail="invalid admin token") + + @app.get("/", response_class=HTMLResponse, include_in_schema=False) + async def landing() -> str: + return landing_page_html(config) + + @app.get("/health") + async def health() -> Dict[str, Any]: + return { + "ok": True, + "name": "leCore x402 API", + "paid": bool(paid), + "memory": core.evidence()["memory"], + } + + @app.get("/pricing") + async def pricing() -> Dict[str, Any]: + return { + "ok": True, + "x402": config.to_public_dict(), + "token_offer": leos_token_offer(), + "routes": payment_manifest(config), + } + + @app.post("/v1/recall") + async def recall(payload: Dict[str, Any]) -> Dict[str, Any]: + query = payload.get("query") + if query is None: + raise HTTPException(status_code=400, detail="POST /v1/recall needs {query}") + return { + "ok": True, + "query": query, + "hits": core.recall(query, k=int(payload.get("k", 3)), abstain=payload.get("abstain")), + } + + @app.post("/v1/route") + async def route(payload: Dict[str, Any]) -> Dict[str, Any]: + task = payload.get("task") + if task is None: + raise HTTPException(status_code=400, detail="POST /v1/route needs {task}") + return {"ok": True, "route": core.route(str(task))} + + @app.get("/v1/dashboard") + async def dashboard() -> Dict[str, Any]: + return {"ok": True, "dashboard": core.dashboard()} + + @app.post("/admin/remember") + async def remember(payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None)) -> Dict[str, Any]: + require_admin(x_admin_token) + text = payload.get("text") + if text is None: + raise HTTPException(status_code=400, detail="POST /admin/remember needs {text}") + return { + "ok": True, + "memory": core.remember(text, label=payload.get("label"), metadata=payload.get("metadata")), + } + + return app + + +def load_core(path: Optional[str]) -> LocalAgentCore: + """Load a persisted core if present, otherwise return the demo core.""" + if path and Path(path).exists(): + return LocalAgentCore.load(path) + return demo() + + +def main(argv: Optional[Iterable[str]] = None) -> None: + """CLI entry point for local x402 API serving.""" + p = argparse.ArgumentParser(description="Serve LocalAgentCore as an x402-paid API") + p.add_argument("--host", default=os.environ.get("LECORE_X402_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.environ.get("LECORE_X402_PORT", "4021"))) + p.add_argument("--state", default=os.environ.get("LECORE_X402_STATE")) + p.add_argument("--pay-to", default=os.environ.get("LECORE_X402_PAY_TO", "")) + p.add_argument("--price", default=os.environ.get("LECORE_X402_PRICE", DEFAULT_PRICE)) + p.add_argument("--network", default=os.environ.get("LECORE_X402_NETWORK", DEFAULT_NETWORK)) + p.add_argument("--facilitator-url", default=os.environ.get("LECORE_X402_FACILITATOR_URL", DEFAULT_FACILITATOR_URL)) + p.add_argument("--admin-token", default=os.environ.get("LECORE_X402_ADMIN_TOKEN")) + p.add_argument("--unpaid-dev", action="store_true", help="Disable x402 middleware for local development only") + args = p.parse_args(list(argv) if argv is not None else None) + + paid = not args.unpaid_dev + config = X402Config( + pay_to=args.pay_to or ("0xYourAddress" if not paid else ""), + price=args.price, + network=args.network, + facilitator_url=args.facilitator_url, + ) + app = create_app(load_core(args.state), config=config, paid=paid, admin_token=args.admin_token) + try: + import uvicorn + except ImportError as exc: + raise RuntimeError(optional_dependency_help()) from exc + uvicorn.run(app, host=args.host, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/lecore.py b/lecore.py index d4f40953..9b9a2169 100644 --- a/lecore.py +++ b/lecore.py @@ -2,9 +2,10 @@ # # The engine is ~436 `holographic_*.py` modules organized into family packages. A newcomer shouldn't need to know which # one holds `Scene` versus `RenderSession` versus `look_at`. This module gathers the handful of things -# most callers actually want into five plain-English areas, so that after `pip install lecore` you can: +# most callers actually want into plain-English areas, so that after `pip install lecore` you can: # # import lecore +# core = lecore.product.LocalAgentCore() # local memory + routing + dashboard # doc = lecore.scene.Scene(dim=1024, seed=0) # build a scene # img = lecore.render.path_trace(sdf, camera) # render it # M = lecore.transform.look_at(eye, target) # aim a camera @@ -29,7 +30,7 @@ # --------------------------------------------------------------------------------------------------- -# The five curated areas. +# The curated areas. # # Each area is imported here and packed into a SimpleNamespace below. We keep the imports grouped by # area (not alphabetised) so it reads as "here is everything the `scene` builder needs", etc. If any @@ -40,6 +41,9 @@ # scene -- author and store a scene document (objects, handles, transforms, undo snapshots). from holographic.scene_and_pipeline.holographic_scene_doc import Scene, SceneObject +# product -- the narrowed first-user surface: local agent memory, skill routing, and readiness evidence. +from holographic_product import LocalAgentCore + # model -- edit geometry: the modifier stack, object description, SDF primitives, key mesh verbs. from holographic.misc.holographic_modifier import ModifierStack, describe_object from holographic.mesh_and_geometry.holographic_sdf import sphere, box # SDF primitives more live in holographic_sdf @@ -80,8 +84,10 @@ def _area(**members): return types.SimpleNamespace(**members) -# The five areas. These are the ONLY place a member is listed; areas() reads its map straight off these +# The areas. These are the ONLY place a member is listed; areas() reads its map straight off these # namespaces (see below) so the docs and the objects can never drift out of sync. +product = _area(LocalAgentCore=LocalAgentCore) + scene = _area(Scene=Scene, SceneObject=SceneObject) model = _area( @@ -111,9 +117,10 @@ def _area(**members): ) -# The names of the five areas, in the order a builder meets them (author -> model -> render -> sim -> -# aim). Kept as a tuple so `areas()` and any future __all__ share one source of truth. -_AREA_NAMES = ("scene", "model", "render", "sim", "transform") +# The names of the areas, in the order a product user tends to meet them (product wedge first, then +# author -> model -> render -> sim -> aim). Kept as a tuple so `areas()` and any future __all__ share +# one source of truth. +_AREA_NAMES = ("product", "scene", "model", "render", "sim", "transform") def areas(): diff --git a/requirements-x402.txt b/requirements-x402.txt new file mode 100644 index 00000000..45aae28b --- /dev/null +++ b/requirements-x402.txt @@ -0,0 +1,2 @@ +x402[fastapi,evm] +uvicorn diff --git a/setup.py b/setup.py index 6c883f41..ca3f27bc 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def read_version(): long_description_content_type="text/markdown", author="AnOversizedMooseWithSocks", url="https://github.com/AnOversizedMooseWithSocks/leCore", - py_modules=["lecore", "holographic_service"], # <- top-level: the import-lecore shim + the standalone HTTP service (from holographic_service import serve) + py_modules=["lecore", "holographic_service", "holographic_product", "holographic_x402_api"], packages=engine_packages + ["lecore_data"], # <- the real holographic/ package tree + the runtime data package # The runtime data (the WordNet dictionary, material property JSON) ships as the small `lecore_data` PACKAGE, so # it is carried into the wheel and resolves the same from a clone or an install (see lecore_data/__init__.py). @@ -84,6 +84,7 @@ def read_version(): # `cupy-cuda12x` instead, so it is best installed by hand (and left # out of `all`, which is why `wgsl` and `gpu` are separate extras # rather than one). + "x402": ["x402[fastapi,evm]", "uvicorn"], # paid API publishing (holographic_x402_api) # -- optional tooling -- "ui": ["flask", "pillow"], # the browser UI (app.py) + image load/save "images": ["pillow"], # image I/O beyond stdlib PNG (jpg/webp/... via mind.save_render) -- diff --git a/tests/test_holographic_product.py b/tests/test_holographic_product.py new file mode 100644 index 00000000..e525ec81 --- /dev/null +++ b/tests/test_holographic_product.py @@ -0,0 +1,66 @@ +"""Tests for the product-facing LocalAgentCore facade.""" + +from holographic_product import LocalAgentCore, demo + + +def test_local_agent_core_remembers_and_recalls(): + core = LocalAgentCore(dim=256, seed=0) + core.remember("render scenes with global illumination and light caches", label="render") + core.remember("local agents need deterministic durable memory", label="memory") + + hits = core.recall("deterministic local memory", k=2) + assert hits[0]["label"] == "memory" + assert hits[0]["score"] >= hits[1]["score"] + + +def test_recall_is_query_safe_and_deterministic(): + core = demo() + before = core.to_state() + a = core.recall("deterministic local memory") + b = core.recall("deterministic local memory") + after = core.to_state() + + assert a == b + assert before == after + + +def test_route_uses_existing_skill_catalog(): + core = LocalAgentCore(dim=128, seed=0) + routed = core.route("start pause resume cancel a job") + + assert routed["task"] == "start pause resume cancel a job" + assert routed["decision"] == "act" + assert "call" in routed["skill"] + + +def test_dashboard_reports_product_evidence(): + core = demo() + data = core.dashboard() + page = core.dashboard(html=True) + + assert data["name"] == "leCore LocalAgentCore" + assert data["memory"]["entries"] == 3 + assert data["checks"]["deterministic_encoding"] is True + assert "c_kernel" in data + assert "leCore LocalAgentCore" in page + assert "No Model Weights" in page + + +def test_save_load_roundtrip(tmp_path): + path = tmp_path / "agent-core.json" + core = demo() + core.save(path) + + loaded = LocalAgentCore.load(path) + + assert loaded.to_state() == core.to_state() + assert loaded.recall("audited c kernel hot path")[0]["label"] == "c-kernel" + + +def test_lecore_exports_product_area(): + import lecore + + assert "product" in lecore.areas() + core = lecore.product.LocalAgentCore(dim=128, seed=1) + core.remember("agent memory product facade", label="product") + assert core.recall("agent memory")[0]["label"] == "product" diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py new file mode 100644 index 00000000..e9315035 --- /dev/null +++ b/tests/test_holographic_x402_api.py @@ -0,0 +1,100 @@ +"""Tests for the optional x402-paid API publisher.""" + +import pytest + +from holographic_x402_api import ( + DEFAULT_NETWORK, + DEFAULT_PRICE, + LEOS_SITE_URL, + LEOS_TOKEN_CA, + LEOS_TOKEN_PRICE, + X402Config, + create_app, + landing_page_html, + leos_token_offer, + optional_dependency_help, + payment_manifest, +) + + +def test_default_x402_config_uses_testnet_price_shape(): + cfg = X402Config(pay_to="0xabc") + + assert cfg.network == DEFAULT_NETWORK + assert cfg.price == DEFAULT_PRICE and cfg.price.startswith("$") + assert cfg.facilitator_url == "https://x402.org/facilitator" + + +def test_payment_manifest_protects_specific_read_routes_only(): + manifest = payment_manifest(X402Config(pay_to="0xabc")) + routes = {row["route"] for row in manifest} + + assert routes == {"POST /v1/recall", "POST /v1/route", "GET /v1/dashboard"} + assert all("*" not in route for route in routes) + assert "POST /admin/remember" not in routes + assert "GET /health" not in routes + assert all(row["accepts"][0]["pay_to"] == "0xabc" for row in manifest) + + +def test_price_validation_keeps_x402_format_honest(): + with pytest.raises(ValueError, match="dollar prefix"): + X402Config(pay_to="0xabc", price="0.001") + + +def test_env_config_requires_pay_to_for_paid_mode(monkeypatch): + monkeypatch.delenv("LECORE_X402_PAY_TO", raising=False) + + with pytest.raises(ValueError, match="LECORE_X402_PAY_TO"): + X402Config.from_env(require_pay_to=True) + + assert X402Config.from_env(require_pay_to=False).pay_to == "0xYourAddress" + + +def test_optional_dependency_help_points_to_extra(): + assert 'pip install ".[x402]"' in optional_dependency_help() + + +def test_landing_page_explains_why_to_buy_the_api(): + html = landing_page_html(X402Config(pay_to="0x96e1604E92A8A1edD0701be3E67Bd4366e87BB84")) + + assert "leCore x402 API" in html + assert "Buy the small, useful surface of leCore" in html + assert "%s per call" % DEFAULT_PRICE in html + assert LEOS_TOKEN_PRICE in html + assert LEOS_TOKEN_CA in html + assert LEOS_SITE_URL in html + assert "Base Sepolia x402" in html + assert "/pricing" in html + assert "/v1/dashboard" in html + assert "0x96e1...BB84" in html + + +def test_leos_token_offer_is_ca_only_metadata(): + offer = leos_token_offer() + + assert offer["site"] == LEOS_SITE_URL + assert offer["ca"] == LEOS_TOKEN_CA + assert offer["price"] == LEOS_TOKEN_PRICE + assert "Only the CA is needed" in offer["note"] + + +def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app(config=X402Config(pay_to="0xabc"), paid=False) + ) + + landing = client.get("/") + assert landing.status_code == 200 + assert landing.headers["content-type"].startswith("text/html") + assert "leCore x402 API" in landing.text + + health = client.get("/health") + assert health.status_code == 200 + assert health.json()["ok"] is True + + pricing = client.get("/pricing") + assert pricing.status_code == 200 + assert pricing.json()["x402"]["price"] == DEFAULT_PRICE + assert pricing.json()["token_offer"]["ca"] == LEOS_TOKEN_CA + assert pricing.json()["token_offer"]["price"] == LEOS_TOKEN_PRICE diff --git a/tests/test_lecore.py b/tests/test_lecore.py index df56784f..80bd6baa 100644 --- a/tests/test_lecore.py +++ b/tests/test_lecore.py @@ -4,8 +4,9 @@ def test_facade_namespaces_exist(): - for area in ("scene", "model", "render", "sim", "transform"): + for area in ("product", "scene", "model", "render", "sim", "transform"): assert hasattr(lecore, area) + assert hasattr(lecore.product, "LocalAgentCore") assert hasattr(lecore.scene, "Scene") and hasattr(lecore.model, "ModifierStack") assert hasattr(lecore.render, "CancelToken") and hasattr(lecore.sim, "MPMSnow") assert hasattr(lecore.transform, "look_at") @@ -22,4 +23,4 @@ def test_facade_operations_work_end_to_end(): def test_areas_map(): a = lecore.areas() - assert set(a) == {"scene", "model", "render", "sim", "transform"} and all(len(v) for v in a.values()) + assert set(a) == {"product", "scene", "model", "render", "sim", "transform"} and all(len(v) for v in a.values()) From 564ad4ebf68a155e54eed2ba5f053756d16d9c2f Mon Sep 17 00:00:00 2001 From: atimics Date: Sun, 12 Jul 2026 07:54:00 -0700 Subject: [PATCH 02/16] Harden x402 tenant isolation --- AWS_X402_DEPLOY.md | 20 ++- X402_API.md | 35 ++++- holographic_product.py | 3 +- holographic_x402_api.py | 232 +++++++++++++++++++++++++++-- tests/test_holographic_x402_api.py | 133 ++++++++++++++++- 5 files changed, 398 insertions(+), 25 deletions(-) diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index 1c83108a..975d04a3 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -11,6 +11,7 @@ does **not** need a wallet private key in the container. It only needs: - the public receiving wallet address (`LECORE_X402_PAY_TO`) - x402/facilitator configuration - an admin token for seller-only memory writes +- a tenant-token secret if private customer memory is enabled The receiving wallet should be a cold wallet, hardware wallet, Safe/multisig, or a custody wallet. The API simply tells x402 where funds should go. @@ -24,9 +25,11 @@ or pay upstream APIs as a buyer. - **Application Load Balancer** terminates HTTPS and forwards to port `4021`. - **ECR** stores the container image. - **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN` and production - facilitator credentials. + facilitator credentials. Store `LECORE_X402_TENANT_SECRET` there too when + private tenants are enabled. - **SSM Parameter Store or plain task env** stores non-secret config like - `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, and `LECORE_X402_NETWORK`. + `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, `LECORE_X402_NETWORK`, and + `LECORE_X402_TENANT_STATE_DIR`. - **CloudWatch Logs** captures service logs. - **AWS WAF** can rate-limit and block bad traffic at the ALB. @@ -35,6 +38,9 @@ Protected paid routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` +- `POST /leos/v1/recall`, at the leOS CA offer price +- `POST /leos/v1/route`, at the leOS CA offer price +- `GET /leos/v1/dashboard`, at the leOS CA offer price Free routes: @@ -44,6 +50,7 @@ Free routes: Seller-only route: - `POST /admin/remember`, guarded by `X-Admin-Token` +- `POST /admin/tenant-token`, guarded by `X-Admin-Token` ## Build And Push @@ -67,15 +74,17 @@ Non-secret environment variables: ```text LECORE_X402_PAY_TO=0xYourReceivingWallet -LECORE_X402_PRICE=$0.001 +LECORE_X402_PRICE=$0.0011 LECORE_X402_NETWORK=eip155:8453 LECORE_X402_FACILITATOR_URL=https://api.cdp.coinbase.com/platform/v2/x402 +LECORE_X402_TENANT_STATE_DIR=/data/tenants ``` Secrets Manager values: ```text LECORE_X402_ADMIN_TOKEN= +LECORE_X402_TENANT_SECRET= CDP_API_KEY_ID= CDP_API_KEY_SECRET= ``` @@ -131,10 +140,12 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. - Use mainnet network id and production facilitator URL. - Put the ALB behind HTTPS only. - Keep `/admin/remember` private or blocked from the public ALB path. +- Keep `/admin/tenant-token` private or blocked from the public ALB path. - Keep paid route configs explicit; avoid wildcard paid routes at first. - Add WAF rate limits. - Add CloudWatch alarms on 5xx, 402 spikes, and admin write attempts. -- Keep customer memory isolated before offering paid writes. +- Use tenant tokens plus isolated tenant state before offering private customer + memory. - Do not put secrets or PII in x402 route descriptions or payment metadata. ## Local Smoke Before AWS @@ -143,6 +154,7 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. pip install ".[x402]" export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" +export LECORE_X402_TENANT_SECRET="local-tenant-secret" python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 ``` diff --git a/X402_API.md b/X402_API.md index 2226af2a..7f00ea80 100644 --- a/X402_API.md +++ b/X402_API.md @@ -9,6 +9,9 @@ the public read/compute routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` +- `POST /leos/v1/recall`, at the leOS CA offer price +- `POST /leos/v1/route`, at the leOS CA offer price +- `GET /leos/v1/dashboard`, at the leOS CA offer price Free routes: @@ -18,10 +21,12 @@ Free routes: Admin route: - `POST /admin/remember`, guarded by `X-Admin-Token` +- `POST /admin/tenant-token`, guarded by `X-Admin-Token` This split is deliberate. Paid customers can use the memory/router/dashboard, -but they cannot mutate the shared memory store unless they also hold the admin -token. +but they cannot mutate memory unless they also hold the admin token. Private +tenant memory also requires a tenant token; x402 proves payment, not tenant +authorization. ## Install @@ -39,8 +44,9 @@ is the signup-free x402.org testnet facilitator. ```bash export LECORE_X402_PAY_TO="0xYourReceivingWallet" -export LECORE_X402_PRICE="$0.001" +export LECORE_X402_PRICE="$0.0011" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" +export LECORE_X402_TENANT_SECRET="local-tenant-secret" python holographic_x402_api.py --host 127.0.0.1 --port 4021 ``` @@ -60,6 +66,25 @@ curl -X POST http://127.0.0.1:4021/admin/remember \ -d '{"text":"local agents need deterministic durable memory","label":"memory"}' ``` +Issue a private tenant token: + +```bash +curl -X POST http://127.0.0.1:4021/admin/tenant-token \ + -H "Content-Type: application/json" \ + -H "X-Admin-Token: local-admin-secret" \ + -d '{"tenant":"acme"}' +``` + +Use that token with paid calls for private tenant memory: + +```bash +curl -X POST http://127.0.0.1:4021/v1/recall \ + -H "Content-Type: application/json" \ + -H "X-leCore-Tenant: acme" \ + -H "X-leCore-Tenant-Token: " \ + -d '{"query":"deterministic local memory"}' +``` + Requests to paid routes return `402 Payment Required` unless the client retries with a valid x402 payment payload: @@ -83,8 +108,8 @@ python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 - Put the API behind HTTPS. - Keep route prices explicit; avoid wildcard paid route configs for this first product surface. -- Keep writes admin-only, or move customer writes into isolated per-customer - stores before charging for them. +- Keep writes admin-only. Use `LECORE_X402_TENANT_SECRET` and + `LECORE_X402_TENANT_STATE_DIR` for isolated private customer memory. - Treat x402 payment metadata as public enough to avoid putting secrets or PII in route descriptions. diff --git a/holographic_product.py b/holographic_product.py index ae85e16c..168996c6 100644 --- a/holographic_product.py +++ b/holographic_product.py @@ -22,6 +22,7 @@ from dataclasses import dataclass, field import html +import importlib import json from pathlib import Path import re @@ -325,7 +326,7 @@ def _deterministic_probe(self) -> bool: @staticmethod def _c_kernel_status() -> Dict[str, Any]: try: - import holographic_c + holographic_c = importlib.import_module("holographic_c") return { "available": bool(holographic_c.available()), diff --git a/holographic_x402_api.py b/holographic_x402_api.py index e1e1477a..f22ab9d4 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -18,11 +18,15 @@ from __future__ import annotations from dataclasses import dataclass +import hashlib +import hmac from html import escape import argparse import os from pathlib import Path +import re from string import Template +import threading from typing import Any, Dict, Iterable, List, Optional, Tuple from holographic_product import LocalAgentCore, demo @@ -34,6 +38,10 @@ LEOS_SITE_URL = "https://discoverleos.com/" LEOS_TOKEN_CA = "5xgsnby6P9zqGK71J7H4yJLxzqPvNbC7rDZxNzjHmj7e" LEOS_TOKEN_PRICE = "$0.0010" +DEFAULT_TENANT_ID = "public" +TENANT_HEADER = "X-leCore-Tenant" +TENANT_TOKEN_HEADER = "X-leCore-Tenant-Token" +_TENANT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.:-]{0,63}$") @dataclass(frozen=True) @@ -52,12 +60,20 @@ def key(self) -> str: return "%s %s" % (self.method.upper(), self.path) -DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = ( +REGULAR_PAID_ROUTES: Tuple[PaidRoute, ...] = ( PaidRoute("POST", "/v1/recall", "Recall nearest memories from a LocalAgentCore instance"), PaidRoute("POST", "/v1/route", "Route a plain-English task to a leCore capability"), PaidRoute("GET", "/v1/dashboard", "Read the LocalAgentCore evidence dashboard"), ) +LEOS_PAID_ROUTES: Tuple[PaidRoute, ...] = ( + PaidRoute("POST", "/leos/v1/recall", "Recall nearest memories at the leOS CA offer price", price=LEOS_TOKEN_PRICE), + PaidRoute("POST", "/leos/v1/route", "Route a task at the leOS CA offer price", price=LEOS_TOKEN_PRICE), + PaidRoute("GET", "/leos/v1/dashboard", "Read the dashboard at the leOS CA offer price", price=LEOS_TOKEN_PRICE), +) + +DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = REGULAR_PAID_ROUTES + LEOS_PAID_ROUTES + LANDING_PAGE_TEMPLATE = Template(""" @@ -191,13 +207,94 @@ def _network_name(network: str) -> str: return {"eip155:84532": "Base Sepolia", "eip155:8453": "Base"}.get(network, network) -def leos_token_offer() -> Dict[str, str]: +def normalize_tenant_id(value: Optional[Any]) -> str: + """Return a path-safe tenant id for private memory routing.""" + tenant_id = str(value or DEFAULT_TENANT_ID).strip().lower() + if not tenant_id: + tenant_id = DEFAULT_TENANT_ID + if not _TENANT_ID_RE.match(tenant_id): + raise ValueError("tenant id must be 1-64 chars of lowercase letters, numbers, '.', ':', '_' or '-'") + return tenant_id + + +def tenant_access_token(tenant_id: str, secret: str) -> str: + """Deterministic tenant bearer token derived from a server-side secret.""" + normalized = normalize_tenant_id(tenant_id) + return hmac.new(secret.encode("utf-8"), normalized.encode("utf-8"), hashlib.sha256).hexdigest() + + +class TenantCoreStore: + """Thread-safe LocalAgentCore registry with optional per-tenant persistence.""" + + def __init__( + self, + default_core: LocalAgentCore, + state_dir: Optional[Any] = None, + ): + self._default_dim = default_core.dim + self._default_seed = default_core.seed + self._default_route_threshold = default_core.route_threshold + self._cores: Dict[str, LocalAgentCore] = {DEFAULT_TENANT_ID: default_core} + self._lock = threading.RLock() + self._state_dir = Path(state_dir) if state_dir else None + if self._state_dir is not None: + self._state_dir.mkdir(parents=True, exist_ok=True) + + def loaded_tenants(self) -> List[str]: + """Return tenant ids currently loaded in memory.""" + with self._lock: + return sorted(self._cores) + + def read(self, tenant_id: str, fn: Any) -> Any: + """Run a read-style operation while holding the tenant lock.""" + with self._lock: + return fn(self._get_locked(tenant_id)) + + def write(self, tenant_id: str, fn: Any) -> Any: + """Run a mutating operation, then persist that tenant if configured.""" + with self._lock: + normalized = normalize_tenant_id(tenant_id) + core = self._get_locked(normalized) + result = fn(core) + self._save_locked(normalized, core) + return result + + def _get_locked(self, tenant_id: str) -> LocalAgentCore: + normalized = normalize_tenant_id(tenant_id) + core = self._cores.get(normalized) + if core is not None: + return core + path = self._path_for(normalized) + if path is not None and path.exists(): + core = LocalAgentCore.load(path) + else: + core = LocalAgentCore( + dim=self._default_dim, + seed=self._default_seed, + route_threshold=self._default_route_threshold, + ) + self._cores[normalized] = core + return core + + def _path_for(self, tenant_id: str) -> Optional[Path]: + if self._state_dir is None: + return None + return self._state_dir / ("%s.json" % normalize_tenant_id(tenant_id)) + + def _save_locked(self, tenant_id: str, core: LocalAgentCore) -> None: + path = self._path_for(tenant_id) + if path is not None: + core.save(path) + + +def leos_token_offer() -> Dict[str, Any]: """Public metadata for the leOS CA-only offer.""" return { "name": "leOS CA offer", "site": LEOS_SITE_URL, "ca": LEOS_TOKEN_CA, "price": LEOS_TOKEN_PRICE, + "discount_routes": [route.key for route in LEOS_PAID_ROUTES], "note": "Only the CA is needed for this token offer.", } @@ -224,7 +321,7 @@ def payment_manifest(config: X402Config) -> List[Dict[str, Any]]: out = [] for route in config.routes: price = route.price or config.price - out.append({ + row = { "route": route.key, "description": route.description, "mime_type": route.mime_type, @@ -234,7 +331,10 @@ def payment_manifest(config: X402Config) -> List[Dict[str, Any]]: "network": config.network, "pay_to": config.pay_to, }], - }) + } + if route.price: + row["offer"] = "leos_ca" + out.append(row) return out @@ -283,11 +383,16 @@ def create_app( config: Optional[X402Config] = None, paid: bool = True, admin_token: Optional[str] = None, + tenant_secret: Optional[str] = None, + tenant_state_dir: Optional[Any] = None, ) -> Any: """Create the FastAPI app. With `paid=True`, the public `/v1/*` read/compute routes are protected by x402 middleware. Set `paid=False` for local development smoke tests. + + x402 proves that a request paid. Private tenant memory is intentionally a + separate authorization layer using `X-leCore-Tenant-Token`. """ try: from fastapi import FastAPI, Header, HTTPException @@ -297,7 +402,9 @@ def create_app( app = FastAPI(title="leCore x402 API", version="0.1.0") core = core or demo() + store = TenantCoreStore(core, state_dir=tenant_state_dir) config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) + tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") if paid: try: @@ -316,17 +423,53 @@ def require_admin(header_value: Optional[str]) -> None: if header_value != admin_token: raise HTTPException(status_code=401, detail="invalid admin token") + def require_tenant_access(tenant_id: str, token: Optional[str]) -> None: + normalized = normalize_tenant_id(tenant_id) + if normalized == DEFAULT_TENANT_ID: + return + if not tenant_secret: + raise HTTPException(status_code=403, detail="private tenants require LECORE_X402_TENANT_SECRET") + expected = tenant_access_token(normalized, tenant_secret) + if not token or not hmac.compare_digest(token, expected): + raise HTTPException(status_code=401, detail="invalid tenant token") + + def tenant_from_header(header_value: Optional[str]) -> str: + try: + return normalize_tenant_id(header_value) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + def tenant_from_payload(payload: Dict[str, Any], header_value: Optional[str]) -> str: + try: + return normalize_tenant_id(payload.get("tenant") or header_value) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + def tenancy_public_dict() -> Dict[str, Any]: + return { + "default_tenant": DEFAULT_TENANT_ID, + "tenant_header": TENANT_HEADER, + "tenant_token_header": TENANT_TOKEN_HEADER, + "private_tenants_enabled": bool(tenant_secret), + } + @app.get("/", response_class=HTMLResponse, include_in_schema=False) async def landing() -> str: return landing_page_html(config) @app.get("/health") async def health() -> Dict[str, Any]: + default_evidence = store.read(DEFAULT_TENANT_ID, lambda tenant_core: tenant_core.evidence()) return { "ok": True, "name": "leCore x402 API", "paid": bool(paid), - "memory": core.evidence()["memory"], + "memory": default_evidence["memory"], + "tenancy": { + "default_tenant": DEFAULT_TENANT_ID, + "loaded_tenants": len(store.loaded_tenants()), + "private_tenants_enabled": bool(tenant_secret), + }, } @app.get("/pricing") @@ -335,40 +478,92 @@ async def pricing() -> Dict[str, Any]: "ok": True, "x402": config.to_public_dict(), "token_offer": leos_token_offer(), + "tenancy": tenancy_public_dict(), "routes": payment_manifest(config), } @app.post("/v1/recall") - async def recall(payload: Dict[str, Any]) -> Dict[str, Any]: + @app.post("/leos/v1/recall") + async def recall( + payload: Dict[str, Any], + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + tenant_id = tenant_from_payload(payload, x_lecore_tenant) + require_tenant_access(tenant_id, x_lecore_tenant_token) query = payload.get("query") if query is None: raise HTTPException(status_code=400, detail="POST /v1/recall needs {query}") + hits = store.read( + tenant_id, + lambda tenant_core: tenant_core.recall(query, k=int(payload.get("k", 3)), abstain=payload.get("abstain")), + ) return { "ok": True, + "tenant": tenant_id, "query": query, - "hits": core.recall(query, k=int(payload.get("k", 3)), abstain=payload.get("abstain")), + "hits": hits, } @app.post("/v1/route") - async def route(payload: Dict[str, Any]) -> Dict[str, Any]: + @app.post("/leos/v1/route") + async def route( + payload: Dict[str, Any], + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + tenant_id = tenant_from_payload(payload, x_lecore_tenant) + require_tenant_access(tenant_id, x_lecore_tenant_token) task = payload.get("task") if task is None: raise HTTPException(status_code=400, detail="POST /v1/route needs {task}") - return {"ok": True, "route": core.route(str(task))} + routed = store.read(tenant_id, lambda tenant_core: tenant_core.route(str(task))) + return {"ok": True, "tenant": tenant_id, "route": routed} @app.get("/v1/dashboard") - async def dashboard() -> Dict[str, Any]: - return {"ok": True, "dashboard": core.dashboard()} + @app.get("/leos/v1/dashboard") + async def dashboard( + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + tenant_id = tenant_from_header(x_lecore_tenant) + require_tenant_access(tenant_id, x_lecore_tenant_token) + data = store.read(tenant_id, lambda tenant_core: tenant_core.dashboard()) + return {"ok": True, "tenant": tenant_id, "dashboard": data} @app.post("/admin/remember") - async def remember(payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None)) -> Dict[str, Any]: + async def remember( + payload: Dict[str, Any], + x_admin_token: Optional[str] = Header(default=None), + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + ) -> Dict[str, Any]: require_admin(x_admin_token) + tenant_id = tenant_from_payload(payload, x_lecore_tenant) text = payload.get("text") if text is None: raise HTTPException(status_code=400, detail="POST /admin/remember needs {text}") + memory = store.write( + tenant_id, + lambda tenant_core: tenant_core.remember(text, label=payload.get("label"), metadata=payload.get("metadata")), + ) + return { + "ok": True, + "tenant": tenant_id, + "memory": memory, + } + + @app.post("/admin/tenant-token") + async def issue_tenant_token(payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None)) -> Dict[str, Any]: + require_admin(x_admin_token) + if not tenant_secret: + raise HTTPException(status_code=403, detail="tenant tokens require LECORE_X402_TENANT_SECRET") + tenant_id = tenant_from_payload(payload, None) return { "ok": True, - "memory": core.remember(text, label=payload.get("label"), metadata=payload.get("metadata")), + "tenant": tenant_id, + "tenant_header": TENANT_HEADER, + "tenant_token_header": TENANT_TOKEN_HEADER, + "tenant_token": tenant_access_token(tenant_id, tenant_secret), } return app @@ -392,6 +587,8 @@ def main(argv: Optional[Iterable[str]] = None) -> None: p.add_argument("--network", default=os.environ.get("LECORE_X402_NETWORK", DEFAULT_NETWORK)) p.add_argument("--facilitator-url", default=os.environ.get("LECORE_X402_FACILITATOR_URL", DEFAULT_FACILITATOR_URL)) p.add_argument("--admin-token", default=os.environ.get("LECORE_X402_ADMIN_TOKEN")) + p.add_argument("--tenant-secret", default=os.environ.get("LECORE_X402_TENANT_SECRET")) + p.add_argument("--tenant-state-dir", default=os.environ.get("LECORE_X402_TENANT_STATE_DIR")) p.add_argument("--unpaid-dev", action="store_true", help="Disable x402 middleware for local development only") args = p.parse_args(list(argv) if argv is not None else None) @@ -402,7 +599,14 @@ def main(argv: Optional[Iterable[str]] = None) -> None: network=args.network, facilitator_url=args.facilitator_url, ) - app = create_app(load_core(args.state), config=config, paid=paid, admin_token=args.admin_token) + app = create_app( + load_core(args.state), + config=config, + paid=paid, + admin_token=args.admin_token, + tenant_secret=args.tenant_secret, + tenant_state_dir=args.tenant_state_dir, + ) try: import uvicorn except ImportError as exc: diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index e9315035..f731c87c 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -5,15 +5,20 @@ from holographic_x402_api import ( DEFAULT_NETWORK, DEFAULT_PRICE, + DEFAULT_TENANT_ID, LEOS_SITE_URL, LEOS_TOKEN_CA, LEOS_TOKEN_PRICE, + TENANT_HEADER, + TENANT_TOKEN_HEADER, X402Config, create_app, landing_page_html, leos_token_offer, optional_dependency_help, payment_manifest, + tenant_access_token, + x402_route_configs, ) @@ -29,11 +34,28 @@ def test_payment_manifest_protects_specific_read_routes_only(): manifest = payment_manifest(X402Config(pay_to="0xabc")) routes = {row["route"] for row in manifest} - assert routes == {"POST /v1/recall", "POST /v1/route", "GET /v1/dashboard"} + assert routes == { + "POST /v1/recall", + "POST /v1/route", + "GET /v1/dashboard", + "POST /leos/v1/recall", + "POST /leos/v1/route", + "GET /leos/v1/dashboard", + } assert all("*" not in route for route in routes) assert "POST /admin/remember" not in routes + assert "POST /admin/tenant-token" not in routes assert "GET /health" not in routes assert all(row["accepts"][0]["pay_to"] == "0xabc" for row in manifest) + assert { + row["route"]: row["accepts"][0]["price"] + for row in manifest + if row["route"].startswith("POST /leos") or row["route"].startswith("GET /leos") + } == { + "POST /leos/v1/recall": LEOS_TOKEN_PRICE, + "POST /leos/v1/route": LEOS_TOKEN_PRICE, + "GET /leos/v1/dashboard": LEOS_TOKEN_PRICE, + } def test_price_validation_keeps_x402_format_honest(): @@ -41,6 +63,21 @@ def test_price_validation_keeps_x402_format_honest(): X402Config(pay_to="0xabc", price="0.001") +def test_x402_route_configs_build_against_optional_sdk(): + pytest.importorskip("x402") + + routes = x402_route_configs(X402Config(pay_to="0xabc")) + + assert sorted(routes) == [ + "GET /leos/v1/dashboard", + "GET /v1/dashboard", + "POST /leos/v1/recall", + "POST /leos/v1/route", + "POST /v1/recall", + "POST /v1/route", + ] + + def test_env_config_requires_pay_to_for_paid_mode(monkeypatch): monkeypatch.delenv("LECORE_X402_PAY_TO", raising=False) @@ -75,6 +112,7 @@ def test_leos_token_offer_is_ca_only_metadata(): assert offer["site"] == LEOS_SITE_URL assert offer["ca"] == LEOS_TOKEN_CA assert offer["price"] == LEOS_TOKEN_PRICE + assert "POST /leos/v1/recall" in offer["discount_routes"] assert "Only the CA is needed" in offer["note"] @@ -98,3 +136,96 @@ def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): assert pricing.json()["x402"]["price"] == DEFAULT_PRICE assert pricing.json()["token_offer"]["ca"] == LEOS_TOKEN_CA assert pricing.json()["token_offer"]["price"] == LEOS_TOKEN_PRICE + assert pricing.json()["tenancy"]["default_tenant"] == DEFAULT_TENANT_ID + + leos_route = client.post("/leos/v1/route", json={"task": "search local agent memory"}) + assert leos_route.status_code == 200 + assert leos_route.json()["tenant"] == DEFAULT_TENANT_ID + + +def test_private_tenant_memory_requires_a_tenant_token(): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + tenant_secret="tenant-secret", + ) + ) + + issued = client.post( + "/admin/tenant-token", + headers={"X-Admin-Token": "admin-secret"}, + json={"tenant": "acme"}, + ) + assert issued.status_code == 200 + tenant_token = issued.json()["tenant_token"] + assert tenant_token == tenant_access_token("acme", "tenant-secret") + + written = client.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret", TENANT_HEADER: "acme"}, + json={"text": "acme-private-omega memory", "label": "tenant-memory"}, + ) + assert written.status_code == 200 + assert written.json()["tenant"] == "acme" + + blocked = client.post( + "/v1/recall", + headers={TENANT_HEADER: "acme"}, + json={"query": "acme private omega"}, + ) + assert blocked.status_code == 401 + + recalled = client.post( + "/v1/recall", + headers={TENANT_HEADER: "acme", TENANT_TOKEN_HEADER: tenant_token}, + json={"query": "acme private omega"}, + ) + assert recalled.status_code == 200 + assert recalled.json()["tenant"] == "acme" + assert recalled.json()["hits"][0]["label"] == "tenant-memory" + + public_recall = client.post("/v1/recall", json={"query": "acme private omega"}) + assert public_recall.status_code == 200 + assert public_recall.json()["tenant"] == DEFAULT_TENANT_ID + assert all(hit["label"] != "tenant-memory" for hit in public_recall.json()["hits"]) + + +def test_tenant_memory_can_persist_to_state_dir(tmp_path): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + first = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + tenant_secret="tenant-secret", + tenant_state_dir=tmp_path, + ) + ) + first.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret"}, + json={"tenant": "acme", "text": "persisted tenant recall text", "label": "persisted"}, + ) + + second = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + tenant_secret="tenant-secret", + tenant_state_dir=tmp_path, + ) + ) + recalled = second.post( + "/v1/recall", + headers={ + TENANT_HEADER: "acme", + TENANT_TOKEN_HEADER: tenant_access_token("acme", "tenant-secret"), + }, + json={"query": "persisted tenant recall"}, + ) + + assert recalled.status_code == 200 + assert recalled.json()["hits"][0]["label"] == "persisted" From 51ffb4065dcc8ecb7331b51b6f265834b56160ca Mon Sep 17 00:00:00 2001 From: atimics Date: Sun, 12 Jul 2026 14:13:17 -0700 Subject: [PATCH 03/16] Fix x402 persistence and availability --- .github/workflows/ci.yml | 2 +- API_QUICKREF.md | 14 +- AWS_X402_DEPLOY.md | 19 +- Dockerfile.x402 | 3 +- README.md | 2 +- X402_API.md | 27 ++- holographic_product.py | 38 +++- holographic_x402_api.py | 344 +++++++++++++++++++++++------ requirements-x402.txt | 4 +- setup.py | 2 +- tests/test_holographic_product.py | 12 + tests/test_holographic_x402_api.py | 139 +++++++++++- 12 files changed, 508 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 795c577e..0b864536 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements.txt -r requirements-x402.txt - name: Run the test suite # Two speed levers work together here: diff --git a/API_QUICKREF.md b/API_QUICKREF.md index fa6b3f49..863fff08 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -12,6 +12,7 @@ - `from_dict(cls, data)` -- Build a memory entry from `to_dict` data. - **class `LocalAgentCore`** -- Product facade for local agent memory, skill routing, and evidence. - `entries(self)` -- A copy of the stored entries, in insertion order. + - `memory_summary(self)` -- Return constant-time memory status without running evidence probes. - `remember(self, text, label=None, metadata=None, id=None)` -- Store one local memory. - `remember_many(self, items)` -- Store several memories. - `recall(self, query, k=3, abstain=None)` -- Return the nearest stored memories for `query`, best first. @@ -22,7 +23,7 @@ - `dashboard_html(data)` -- Render an evidence snapshot as a dependency-free static HTML dashboard. - `to_state(self)` -- Serialize configuration and entries. - `from_state(cls, state)` -- Rebuild a core from `to_state` data. - - `save(self, path)` -- Write the product state to JSON and return the path. + - `save(self, path)` -- Atomically write the product state to JSON and return the path. - `load(cls, path)` -- Load a product state saved by `save`. - `demo()` -- Build a tiny ready-to-query product demo. @@ -35,12 +36,19 @@ - `from_env(cls, require_pay_to=True)` -- Build config from LECORE_X402_* environment variables. - `to_public_dict(self)` -- Public, JSON-safe view of the payment configuration. - `optional_dependency_help()` -- Install hint for the optional paid API dependencies. -- `leos_token_offer()` -- Public metadata for the leOS CA-only offer. +- `normalize_tenant_id(value)` -- Return a path-safe tenant id for private memory routing. +- `tenant_access_token(tenant_id, secret)` -- Deterministic tenant bearer token derived from a server-side secret. +- **class `TenantCoreStore`** -- Thread-safe LocalAgentCore registry with optional per-tenant persistence. + - `loaded_tenants(self)` -- Return tenant ids currently loaded in memory. + - `summary(self, tenant_id)` -- Return a cheap cached status summary without probing capabilities. + - `read(self, tenant_id, fn)` -- Run a read-style operation while holding the tenant lock. + - `write(self, tenant_id, fn)` -- Run a mutating operation, then persist that tenant if configured. +- `leos_token_offer(access_required=True, enabled=True)` -- Return public metadata for the credential-gated leOS offer. - `landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. - `payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. - `x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. - `x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. -- `create_app(core=None, config=None, paid=True, admin_token=None)` -- Create the FastAPI app. +- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None, leos_access_token=None)` -- Create the FastAPI application for paid or local serving. - `load_core(path)` -- Load a persisted core if present, otherwise return the demo core. - `main(argv=None)` -- CLI entry point for local x402 API serving. diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index 975d04a3..32540602 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -12,6 +12,7 @@ does **not** need a wallet private key in the container. It only needs: - x402/facilitator configuration - an admin token for seller-only memory writes - a tenant-token secret if private customer memory is enabled +- an offer-access token if the discounted leOS routes are enabled The receiving wallet should be a cold wallet, hardware wallet, Safe/multisig, or a custody wallet. The API simply tells x402 where funds should go. @@ -24,9 +25,9 @@ or pay upstream APIs as a buyer. - **ECS Fargate** runs the `Dockerfile.x402` container. - **Application Load Balancer** terminates HTTPS and forwards to port `4021`. - **ECR** stores the container image. -- **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN` and production - facilitator credentials. Store `LECORE_X402_TENANT_SECRET` there too when - private tenants are enabled. +- **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN`, + `LECORE_X402_LEOS_ACCESS_TOKEN`, and production facilitator credentials. + Store `LECORE_X402_TENANT_SECRET` there too when private tenants are enabled. - **SSM Parameter Store or plain task env** stores non-secret config like `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, `LECORE_X402_NETWORK`, and `LECORE_X402_TENANT_STATE_DIR`. @@ -38,9 +39,9 @@ Protected paid routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` -- `POST /leos/v1/recall`, at the leOS CA offer price -- `POST /leos/v1/route`, at the leOS CA offer price -- `GET /leos/v1/dashboard`, at the leOS CA offer price +- `POST /leos/v1/recall`, at the credential-gated leOS offer price +- `POST /leos/v1/route`, at the credential-gated leOS offer price +- `GET /leos/v1/dashboard`, at the credential-gated leOS offer price Free routes: @@ -85,6 +86,7 @@ Secrets Manager values: ```text LECORE_X402_ADMIN_TOKEN= LECORE_X402_TENANT_SECRET= +LECORE_X402_LEOS_ACCESS_TOKEN= CDP_API_KEY_ID= CDP_API_KEY_SECRET= ``` @@ -146,6 +148,10 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. - Add CloudWatch alarms on 5xx, 402 spikes, and admin write attempts. - Use tenant tokens plus isolated tenant state before offering private customer memory. +- Mount `LECORE_X402_TENANT_STATE_DIR` on shared durable storage. Tenant writes + reload under an OS-level lock and use atomic replacement, so rolling ECS tasks + do not overwrite one another. +- Keep the leOS offer credential in Secrets Manager and rotate it if disclosed. - Do not put secrets or PII in x402 route descriptions or payment metadata. ## Local Smoke Before AWS @@ -155,6 +161,7 @@ pip install ".[x402]" export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" export LECORE_X402_TENANT_SECRET="local-tenant-secret" +export LECORE_X402_LEOS_ACCESS_TOKEN="local-leos-buyer-secret" python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 ``` diff --git a/Dockerfile.x402 b/Dockerfile.x402 index e5332891..fc4d6e2f 100644 --- a/Dockerfile.x402 +++ b/Dockerfile.x402 @@ -8,7 +8,8 @@ WORKDIR /app COPY . /app RUN python -m pip install --no-cache-dir --upgrade pip \ - && python -m pip install --no-cache-dir ".[x402]" + && python -m pip install --no-cache-dir -r requirements-x402.txt \ + && python -m pip install --no-cache-dir . EXPOSE 4021 diff --git a/README.md b/README.md index fa40bfa1..ffbc9d82 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ pip install "leos-core[symbolic]" # pip install .[symbolic] design-ti pip install "leos-core[zig]" # pip install .[zig] native batch kernels, 2-5x (ziglang -- whole # toolchain in one wheel, bit-identical in safe mode) pip install "leos-core[images]" # pip install .[images] jpg/webp/... image I/O (Pillow, no Flask) -pip install "leos-core[x402]" # pip install .[x402] paid API publishing (x402, FastAPI) +pip install "leos-core[x402]" # pip install .[x402] paid API publishing, Python 3.10+ (x402, FastAPI) pip install "leos-core[dev]" # pip install .[dev] run the tests and make plots (pytest, matplotlib) pip install "leos-core[all]" # pip install .[all] everything portable, one shot pip install "leos-core[ui,jit]" # pip install .[ui,jit] ...or combine whichever you want diff --git a/X402_API.md b/X402_API.md index 7f00ea80..1fccf956 100644 --- a/X402_API.md +++ b/X402_API.md @@ -9,9 +9,9 @@ the public read/compute routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` -- `POST /leos/v1/recall`, at the leOS CA offer price -- `POST /leos/v1/route`, at the leOS CA offer price -- `GET /leos/v1/dashboard`, at the leOS CA offer price +- `POST /leos/v1/recall`, at the credential-gated leOS offer price +- `POST /leos/v1/route`, at the credential-gated leOS offer price +- `GET /leos/v1/dashboard`, at the credential-gated leOS offer price Free routes: @@ -28,6 +28,11 @@ but they cannot mutate memory unless they also hold the admin token. Private tenant memory also requires a tenant token; x402 proves payment, not tenant authorization. +The leOS CA identifies the discounted offer, but does not itself prove buyer +eligibility because it is public. Discounted calls must also include the +operator-issued `X-leCore-leOS-Access` credential. Failed authorization responses +are not settled by the x402 middleware. + ## Install ```bash @@ -47,6 +52,8 @@ export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_PRICE="$0.0011" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" export LECORE_X402_TENANT_SECRET="local-tenant-secret" +export LECORE_X402_LEOS_ACCESS_TOKEN="local-leos-buyer-secret" +export LECORE_X402_TENANT_STATE_DIR="./tenant-state" python holographic_x402_api.py --host 127.0.0.1 --port 4021 ``` @@ -85,6 +92,15 @@ curl -X POST http://127.0.0.1:4021/v1/recall \ -d '{"query":"deterministic local memory"}' ``` +Use the discounted leOS route only with an issued offer credential: + +```bash +curl -X POST http://127.0.0.1:4021/leos/v1/recall \ + -H "Content-Type: application/json" \ + -H "X-leCore-leOS-Access: local-leos-buyer-secret" \ + -d '{"query":"deterministic local memory"}' +``` + Requests to paid routes return `402 Payment Required` unless the client retries with a valid x402 payment payload: @@ -109,7 +125,10 @@ python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 - Keep route prices explicit; avoid wildcard paid route configs for this first product surface. - Keep writes admin-only. Use `LECORE_X402_TENANT_SECRET` and - `LECORE_X402_TENANT_STATE_DIR` for isolated private customer memory. + `LECORE_X402_TENANT_STATE_DIR` for durable public and private memory. Writes + use per-tenant process locks plus atomic replacement on shared storage. +- Store `LECORE_X402_LEOS_ACCESS_TOKEN` as a secret and distribute it only to + buyers eligible for the discounted routes. - Treat x402 payment metadata as public enough to avoid putting secrets or PII in route descriptions. diff --git a/holographic_product.py b/holographic_product.py index 168996c6..eaf6ac88 100644 --- a/holographic_product.py +++ b/holographic_product.py @@ -24,8 +24,10 @@ import html import importlib import json +import os from pathlib import Path import re +import tempfile from typing import Any, Dict, Iterable, List, Optional import numpy as np @@ -105,6 +107,15 @@ def entries(self) -> List[MemoryEntry]: """A copy of the stored entries, in insertion order.""" return list(self._entries) + def memory_summary(self) -> Dict[str, Any]: + """Return constant-time memory status without running evidence probes.""" + return { + "entries": len(self._entries), + "dim": self.dim, + "index_method": self._index.method if self._index is not None else None, + "query_mutates_store": False, + } + def remember( self, text: Any, @@ -144,6 +155,15 @@ def recall(self, query: Any, k: int = 3, abstain: Optional[float] = None) -> Lis """ if not self._entries or self._index is None: return [] + if isinstance(k, bool) or not isinstance(k, (int, np.integer)) or int(k) < 1: + raise ValueError("k must be a positive integer") + if abstain is not None: + if isinstance(abstain, bool) or not isinstance(abstain, (int, float, np.number)): + raise ValueError("abstain must be a number between 0 and 1") + if not 0.0 <= float(abstain) <= 1.0: + raise ValueError("abstain must be between 0 and 1") + if not _tokens(query): + return [] q = self._encode_text(query) hits = self._index.nearest(q, k=min(int(k), len(self._entries)), abstain=abstain) by_id = {entry.id: entry for entry in self._entries} @@ -284,9 +304,23 @@ def from_state(cls, state: Dict[str, Any]) -> "LocalAgentCore": return core def save(self, path: Any) -> str: - """Write the product state to JSON and return the path.""" + """Atomically write the product state to JSON and return the path.""" p = Path(path) - p.write_text(json.dumps(self.to_state(), indent=2, sort_keys=True), encoding="utf-8") + p.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps(self.to_state(), indent=2, sort_keys=True) + fd, temporary = tempfile.mkstemp(prefix=".%s." % p.name, suffix=".tmp", dir=str(p.parent)) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, p) + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise return str(p) @classmethod diff --git a/holographic_x402_api.py b/holographic_x402_api.py index f22ab9d4..af982561 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -17,7 +17,8 @@ from __future__ import annotations -from dataclasses import dataclass +from contextlib import contextmanager +from dataclasses import dataclass, replace import hashlib import hmac from html import escape @@ -38,10 +39,15 @@ LEOS_SITE_URL = "https://discoverleos.com/" LEOS_TOKEN_CA = "5xgsnby6P9zqGK71J7H4yJLxzqPvNbC7rDZxNzjHmj7e" LEOS_TOKEN_PRICE = "$0.0010" +LEOS_ACCESS_HEADER = "X-leCore-leOS-Access" DEFAULT_TENANT_ID = "public" TENANT_HEADER = "X-leCore-Tenant" TENANT_TOKEN_HEADER = "X-leCore-Tenant-Token" _TENANT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.:-]{0,63}$") +MAX_QUERY_CHARS = 8192 +MAX_TASK_CHARS = 8192 +MAX_MEMORY_CHARS = 65536 +MAX_RECALL_K = 100 @dataclass(frozen=True) @@ -123,7 +129,7 @@ def key(self) -> str:
Endpointhttps://lecore.rati.foundation
Paymentx402 exact scheme
Buyer shapeinspect, pay, call

Why you would buy it

Because most agents do not need a platform. They need a few reliable cognitive calls.

You pay for an answerable primitive, not a monthly seat.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: local agent memory plus capability routing.

The implementation is deployed, health-checked, and already returning x402 payment challenges.

-

leOS token offer

A slightly cheaper CA-only price for the leOS token.

Token price$leos_token_price
CA$leos_token_ca
leOS website
+

leOS token offer

A slightly cheaper price for eligible leOS buyers.

Offer price$leos_token_price
CA$leos_token_ca
leOS website

What the payment unlocks

Three paid routes, each small enough to understand.

POST

Recall

/v1/recall

Pull nearest memories from a compact local agent core without shipping a whole application stack.

POST

Route

/v1/route

Send a plain-language task and get the leCore capability it should use, with evidence attached.

GET

Dashboard

/v1/dashboard

Read the readiness surface: memory counts, capability map, abstention behavior, and route coverage.

Good first buyers

Teams who want the leCore idea without adopting the whole repo.

Agent memory for prototypes that should remember without a database rollout.

Capability routing for tools that need to pick the right leCore subsystem before doing work.

Evidence dashboards for teams deciding whether a local vector system is ready to productize.

A working x402 seller endpoint to copy when you want pay-per-call APIs instead of subscriptions.

Proof it is real

It is already deployed, priced, and protected.

The free endpoints show health and pricing. Paid endpoints return a real x402 payment challenge. The receiving address is public, while admin writes stay out of the paid customer path.

Price
$price
Network
$network_name
Receiver
$pay_to_short
Status
Healthy
@@ -209,7 +215,11 @@ def _network_name(network: str) -> str: def normalize_tenant_id(value: Optional[Any]) -> str: """Return a path-safe tenant id for private memory routing.""" - tenant_id = str(value or DEFAULT_TENANT_ID).strip().lower() + if value is None: + return DEFAULT_TENANT_ID + if not isinstance(value, str): + raise ValueError("tenant id must be a string") + tenant_id = value.strip().lower() if not tenant_id: tenant_id = DEFAULT_TENANT_ID if not _TENANT_ID_RE.match(tenant_id): @@ -223,6 +233,42 @@ def tenant_access_token(tenant_id: str, secret: str) -> str: return hmac.new(secret.encode("utf-8"), normalized.encode("utf-8"), hashlib.sha256).hexdigest() +@contextmanager +def _process_file_lock(path: Path) -> Any: + """Hold an exclusive process lock for one persisted tenant state file.""" + lock_path = path.with_suffix(path.suffix + ".lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = open(lock_path, "a+b") + try: + if os.name == "nt": # pragma: no cover - exercised on Windows CI/users + import msvcrt + + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + yield + finally: + try: + if os.name == "nt": # pragma: no cover - exercised on Windows CI/users + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + finally: + handle.close() + + class TenantCoreStore: """Thread-safe LocalAgentCore registry with optional per-tenant persistence.""" @@ -235,70 +281,148 @@ def __init__( self._default_seed = default_core.seed self._default_route_threshold = default_core.route_threshold self._cores: Dict[str, LocalAgentCore] = {DEFAULT_TENANT_ID: default_core} - self._lock = threading.RLock() + self._versions: Dict[str, Tuple[int, int, int]] = {} + self._tenant_locks: Dict[str, threading.RLock] = {DEFAULT_TENANT_ID: threading.RLock()} + self._registry_lock = threading.RLock() self._state_dir = Path(state_dir) if state_dir else None if self._state_dir is not None: self._state_dir.mkdir(parents=True, exist_ok=True) + public_path = self._path_for(DEFAULT_TENANT_ID) + if public_path is not None and public_path.exists(): + with _process_file_lock(public_path): + self._cores[DEFAULT_TENANT_ID] = LocalAgentCore.load(public_path) + self._versions[DEFAULT_TENANT_ID] = self._version(public_path) def loaded_tenants(self) -> List[str]: """Return tenant ids currently loaded in memory.""" - with self._lock: + with self._registry_lock: return sorted(self._cores) + def summary(self, tenant_id: str) -> Dict[str, Any]: + """Return a cheap cached status summary without probing capabilities.""" + normalized = normalize_tenant_id(tenant_id) + with self._lock_for(normalized): + core = self._get_cached(normalized) + return core.memory_summary() + def read(self, tenant_id: str, fn: Any) -> Any: """Run a read-style operation while holding the tenant lock.""" - with self._lock: - return fn(self._get_locked(tenant_id)) + normalized = normalize_tenant_id(tenant_id) + with self._lock_for(normalized): + return fn(self._get_fresh(normalized)) def write(self, tenant_id: str, fn: Any) -> Any: """Run a mutating operation, then persist that tenant if configured.""" - with self._lock: - normalized = normalize_tenant_id(tenant_id) - core = self._get_locked(normalized) - result = fn(core) - self._save_locked(normalized, core) - return result - - def _get_locked(self, tenant_id: str) -> LocalAgentCore: normalized = normalize_tenant_id(tenant_id) - core = self._cores.get(normalized) - if core is not None: + with self._lock_for(normalized): + path = self._path_for(normalized) + if path is None: + core = self._get_cached(normalized) + return fn(core) + with _process_file_lock(path): + core = ( + LocalAgentCore.load(path) + if path.exists() + else LocalAgentCore.from_state(self._get_cached(normalized).to_state()) + ) + result = fn(core) + core.save(path) + with self._registry_lock: + self._cores[normalized] = core + self._versions[normalized] = self._version(path) + return result + + def _lock_for(self, tenant_id: str) -> threading.RLock: + with self._registry_lock: + lock = self._tenant_locks.get(tenant_id) + if lock is None: + lock = threading.RLock() + self._tenant_locks[tenant_id] = lock + return lock + + def _get_cached(self, tenant_id: str) -> LocalAgentCore: + with self._registry_lock: + core = self._cores.get(tenant_id) + if core is None: + core = LocalAgentCore( + dim=self._default_dim, + seed=self._default_seed, + route_threshold=self._default_route_threshold, + ) + self._cores[tenant_id] = core return core - path = self._path_for(normalized) + + def _get_fresh(self, tenant_id: str) -> LocalAgentCore: + path = self._path_for(tenant_id) if path is not None and path.exists(): - core = LocalAgentCore.load(path) - else: - core = LocalAgentCore( - dim=self._default_dim, - seed=self._default_seed, - route_threshold=self._default_route_threshold, - ) - self._cores[normalized] = core - return core + version = self._version(path) + with self._registry_lock: + cached_version = self._versions.get(tenant_id) + if cached_version != version: + with _process_file_lock(path): + core = LocalAgentCore.load(path) + version = self._version(path) + with self._registry_lock: + self._cores[tenant_id] = core + self._versions[tenant_id] = version + return core + return self._get_cached(tenant_id) + + @staticmethod + def _version(path: Path) -> Tuple[int, int, int]: + stat = path.stat() + return stat.st_ino, stat.st_mtime_ns, stat.st_size def _path_for(self, tenant_id: str) -> Optional[Path]: if self._state_dir is None: return None return self._state_dir / ("%s.json" % normalize_tenant_id(tenant_id)) - def _save_locked(self, tenant_id: str, core: LocalAgentCore) -> None: - path = self._path_for(tenant_id) - if path is not None: - core.save(path) - - -def leos_token_offer() -> Dict[str, Any]: - """Public metadata for the leOS CA-only offer.""" +def leos_token_offer(access_required: bool = True, enabled: bool = True) -> Dict[str, Any]: + """Return public metadata for the credential-gated leOS offer.""" return { "name": "leOS CA offer", "site": LEOS_SITE_URL, "ca": LEOS_TOKEN_CA, "price": LEOS_TOKEN_PRICE, + "access_header": LEOS_ACCESS_HEADER, + "access_required": bool(access_required), + "enabled": bool(enabled), "discount_routes": [route.key for route in LEOS_PAID_ROUTES], - "note": "Only the CA is needed for this token offer.", + "note": "The CA identifies the offer; eligible buyers also receive an operator-issued access credential.", } +def _required_text(payload: Dict[str, Any], key: str, maximum: int) -> str: + value = payload.get(key) + if not isinstance(value, str) or not value.strip(): + raise ValueError("%s must be a non-empty string" % key) + if len(value) > maximum: + raise ValueError("%s must be at most %d characters" % (key, maximum)) + return value + + +def _recall_k(payload: Dict[str, Any]) -> int: + value = payload.get("k", 3) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("k must be an integer") + if not 1 <= value <= MAX_RECALL_K: + raise ValueError("k must be between 1 and %d" % MAX_RECALL_K) + return value + + +def _abstain_threshold(payload: Dict[str, Any]) -> Optional[float]: + value = payload.get("abstain") + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("abstain must be a number between 0 and 1") + threshold = float(value) + if not 0.0 <= threshold <= 1.0: + raise ValueError("abstain must be between 0 and 1") + return threshold + + def landing_page_html(config: X402Config) -> str: """Render the buyer-facing landing page served from `/`.""" network_name = _network_name(config.network) @@ -385,8 +509,9 @@ def create_app( admin_token: Optional[str] = None, tenant_secret: Optional[str] = None, tenant_state_dir: Optional[Any] = None, + leos_access_token: Optional[str] = None, ) -> Any: - """Create the FastAPI app. + """Create the FastAPI application for paid or local serving. With `paid=True`, the public `/v1/*` read/compute routes are protected by x402 middleware. Set `paid=False` for local development smoke tests. @@ -405,6 +530,13 @@ def create_app( store = TenantCoreStore(core, state_dir=tenant_state_dir) config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") + leos_access_token = leos_access_token or os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN") + active_config = config + if not leos_access_token: + active_config = replace( + config, + routes=tuple(route for route in config.routes if not route.path.startswith("/leos/")), + ) if paid: try: @@ -413,16 +545,22 @@ def create_app( raise RuntimeError(optional_dependency_help()) from exc app.add_middleware( PaymentMiddlewareASGI, - routes=x402_route_configs(config), - server=x402_resource_server(config), + routes=x402_route_configs(active_config), + server=x402_resource_server(active_config), ) def require_admin(header_value: Optional[str]) -> None: if not admin_token: raise HTTPException(status_code=403, detail="admin writes are disabled") - if header_value != admin_token: + if not header_value or not hmac.compare_digest(header_value, admin_token): raise HTTPException(status_code=401, detail="invalid admin token") + def require_leos_access(header_value: Optional[str]) -> None: + if not leos_access_token: + raise HTTPException(status_code=503, detail="leOS discount access is not configured") + if not header_value or not hmac.compare_digest(header_value, leos_access_token): + raise HTTPException(status_code=401, detail="invalid leOS discount credential") + def require_tenant_access(tenant_id: str, token: Optional[str]) -> None: normalized = normalize_tenant_id(tenant_id) if normalized == DEFAULT_TENANT_ID: @@ -441,7 +579,18 @@ def tenant_from_header(header_value: Optional[str]) -> str: def tenant_from_payload(payload: Dict[str, Any], header_value: Optional[str]) -> str: try: - return normalize_tenant_id(payload.get("tenant") or header_value) + payload_value = payload.get("tenant") + payload_tenant = normalize_tenant_id(payload_value) if payload_value is not None else None + header_tenant = normalize_tenant_id(header_value) if header_value is not None else None + if payload_tenant is not None and header_tenant is not None and payload_tenant != header_tenant: + raise ValueError("tenant id in payload does not match %s" % TENANT_HEADER) + return payload_tenant or header_tenant or DEFAULT_TENANT_ID + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + def validated(callable_: Any, *args: Any) -> Any: + try: + return callable_(*args) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc @@ -454,17 +603,16 @@ def tenancy_public_dict() -> Dict[str, Any]: } @app.get("/", response_class=HTMLResponse, include_in_schema=False) - async def landing() -> str: + def landing() -> str: return landing_page_html(config) @app.get("/health") - async def health() -> Dict[str, Any]: - default_evidence = store.read(DEFAULT_TENANT_ID, lambda tenant_core: tenant_core.evidence()) + def health() -> Dict[str, Any]: return { "ok": True, "name": "leCore x402 API", "paid": bool(paid), - "memory": default_evidence["memory"], + "memory": store.summary(DEFAULT_TENANT_ID), "tenancy": { "default_tenant": DEFAULT_TENANT_ID, "loaded_tenants": len(store.loaded_tenants()), @@ -473,30 +621,28 @@ async def health() -> Dict[str, Any]: } @app.get("/pricing") - async def pricing() -> Dict[str, Any]: + def pricing() -> Dict[str, Any]: return { "ok": True, "x402": config.to_public_dict(), - "token_offer": leos_token_offer(), + "token_offer": leos_token_offer(enabled=bool(leos_access_token)), "tenancy": tenancy_public_dict(), - "routes": payment_manifest(config), + "routes": payment_manifest(active_config), } - @app.post("/v1/recall") - @app.post("/leos/v1/recall") - async def recall( + def recall_response( payload: Dict[str, Any], - x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), - x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + x_lecore_tenant: Optional[str], + x_lecore_tenant_token: Optional[str], ) -> Dict[str, Any]: tenant_id = tenant_from_payload(payload, x_lecore_tenant) require_tenant_access(tenant_id, x_lecore_tenant_token) - query = payload.get("query") - if query is None: - raise HTTPException(status_code=400, detail="POST /v1/recall needs {query}") + query = validated(_required_text, payload, "query", MAX_QUERY_CHARS) + k = validated(_recall_k, payload) + abstain = validated(_abstain_threshold, payload) hits = store.read( tenant_id, - lambda tenant_core: tenant_core.recall(query, k=int(payload.get("k", 3)), abstain=payload.get("abstain")), + lambda tenant_core: tenant_core.recall(query, k=k, abstain=abstain), ) return { "ok": True, @@ -505,46 +651,96 @@ async def recall( "hits": hits, } - @app.post("/v1/route") - @app.post("/leos/v1/route") - async def route( + @app.post("/v1/recall") + def recall( + payload: Dict[str, Any], + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + return recall_response(payload, x_lecore_tenant, x_lecore_tenant_token) + + @app.post("/leos/v1/recall") + def leos_recall( payload: Dict[str, Any], + x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + require_leos_access(x_lecore_leos_access) + return recall_response(payload, x_lecore_tenant, x_lecore_tenant_token) + + def route_response( + payload: Dict[str, Any], + x_lecore_tenant: Optional[str], + x_lecore_tenant_token: Optional[str], ) -> Dict[str, Any]: tenant_id = tenant_from_payload(payload, x_lecore_tenant) require_tenant_access(tenant_id, x_lecore_tenant_token) - task = payload.get("task") - if task is None: - raise HTTPException(status_code=400, detail="POST /v1/route needs {task}") - routed = store.read(tenant_id, lambda tenant_core: tenant_core.route(str(task))) + task = validated(_required_text, payload, "task", MAX_TASK_CHARS) + routed = store.read(tenant_id, lambda tenant_core: tenant_core.route(task)) return {"ok": True, "tenant": tenant_id, "route": routed} - @app.get("/v1/dashboard") - @app.get("/leos/v1/dashboard") - async def dashboard( + @app.post("/v1/route") + def route( + payload: Dict[str, Any], + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + return route_response(payload, x_lecore_tenant, x_lecore_tenant_token) + + @app.post("/leos/v1/route") + def leos_route( + payload: Dict[str, Any], + x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + require_leos_access(x_lecore_leos_access) + return route_response(payload, x_lecore_tenant, x_lecore_tenant_token) + + def dashboard_response( + x_lecore_tenant: Optional[str], + x_lecore_tenant_token: Optional[str], ) -> Dict[str, Any]: tenant_id = tenant_from_header(x_lecore_tenant) require_tenant_access(tenant_id, x_lecore_tenant_token) data = store.read(tenant_id, lambda tenant_core: tenant_core.dashboard()) return {"ok": True, "tenant": tenant_id, "dashboard": data} + @app.get("/v1/dashboard") + def dashboard( + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + return dashboard_response(x_lecore_tenant, x_lecore_tenant_token) + + @app.get("/leos/v1/dashboard") + def leos_dashboard( + x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), + x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), + ) -> Dict[str, Any]: + require_leos_access(x_lecore_leos_access) + return dashboard_response(x_lecore_tenant, x_lecore_tenant_token) + @app.post("/admin/remember") - async def remember( + def remember( payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None), x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), ) -> Dict[str, Any]: require_admin(x_admin_token) tenant_id = tenant_from_payload(payload, x_lecore_tenant) - text = payload.get("text") - if text is None: - raise HTTPException(status_code=400, detail="POST /admin/remember needs {text}") + text = validated(_required_text, payload, "text", MAX_MEMORY_CHARS) + label = payload.get("label") + metadata = payload.get("metadata") + if label is not None and not isinstance(label, str): + raise HTTPException(status_code=400, detail="label must be a string") + if metadata is not None and not isinstance(metadata, dict): + raise HTTPException(status_code=400, detail="metadata must be an object") memory = store.write( tenant_id, - lambda tenant_core: tenant_core.remember(text, label=payload.get("label"), metadata=payload.get("metadata")), + lambda tenant_core: tenant_core.remember(text, label=label, metadata=metadata), ) return { "ok": True, @@ -553,7 +749,7 @@ async def remember( } @app.post("/admin/tenant-token") - async def issue_tenant_token(payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None)) -> Dict[str, Any]: + def issue_tenant_token(payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None)) -> Dict[str, Any]: require_admin(x_admin_token) if not tenant_secret: raise HTTPException(status_code=403, detail="tenant tokens require LECORE_X402_TENANT_SECRET") @@ -589,6 +785,7 @@ def main(argv: Optional[Iterable[str]] = None) -> None: p.add_argument("--admin-token", default=os.environ.get("LECORE_X402_ADMIN_TOKEN")) p.add_argument("--tenant-secret", default=os.environ.get("LECORE_X402_TENANT_SECRET")) p.add_argument("--tenant-state-dir", default=os.environ.get("LECORE_X402_TENANT_STATE_DIR")) + p.add_argument("--leos-access-token", default=os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN")) p.add_argument("--unpaid-dev", action="store_true", help="Disable x402 middleware for local development only") args = p.parse_args(list(argv) if argv is not None else None) @@ -606,6 +803,7 @@ def main(argv: Optional[Iterable[str]] = None) -> None: admin_token=args.admin_token, tenant_secret=args.tenant_secret, tenant_state_dir=args.tenant_state_dir, + leos_access_token=args.leos_access_token, ) try: import uvicorn diff --git a/requirements-x402.txt b/requirements-x402.txt index 45aae28b..0b20d634 100644 --- a/requirements-x402.txt +++ b/requirements-x402.txt @@ -1,2 +1,2 @@ -x402[fastapi,evm] -uvicorn +x402[fastapi,evm]==2.15.0 +uvicorn==0.51.0 diff --git a/setup.py b/setup.py index ca3f27bc..2d71126d 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def read_version(): # `cupy-cuda12x` instead, so it is best installed by hand (and left # out of `all`, which is why `wgsl` and `gpu` are separate extras # rather than one). - "x402": ["x402[fastapi,evm]", "uvicorn"], # paid API publishing (holographic_x402_api) + "x402": ["x402[fastapi,evm]>=2.15,<3", "uvicorn>=0.51,<1"], # paid API publishing # -- optional tooling -- "ui": ["flask", "pillow"], # the browser UI (app.py) + image load/save "images": ["pillow"], # image I/O beyond stdlib PNG (jpg/webp/... via mind.save_render) -- diff --git a/tests/test_holographic_product.py b/tests/test_holographic_product.py index e525ec81..f2348003 100644 --- a/tests/test_holographic_product.py +++ b/tests/test_holographic_product.py @@ -1,5 +1,7 @@ """Tests for the product-facing LocalAgentCore facade.""" +import pytest + from holographic_product import LocalAgentCore, demo @@ -24,6 +26,16 @@ def test_recall_is_query_safe_and_deterministic(): assert before == after +def test_recall_rejects_invalid_k_and_abstains_on_empty_queries(): + core = demo() + + with pytest.raises(ValueError, match="positive integer"): + core.recall("memory", k=0) + with pytest.raises(ValueError, match="between 0 and 1"): + core.recall("memory", abstain=2) + assert core.recall("") == [] + + def test_route_uses_existing_skill_catalog(): core = LocalAgentCore(dim=128, seed=0) routed = core.route("start pause resume cancel a job") diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index f731c87c..c38f427a 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -7,10 +7,12 @@ DEFAULT_PRICE, DEFAULT_TENANT_ID, LEOS_SITE_URL, + LEOS_ACCESS_HEADER, LEOS_TOKEN_CA, LEOS_TOKEN_PRICE, TENANT_HEADER, TENANT_TOKEN_HEADER, + TenantCoreStore, X402Config, create_app, landing_page_html, @@ -20,6 +22,7 @@ tenant_access_token, x402_route_configs, ) +from holographic_product import LocalAgentCore, demo def test_default_x402_config_uses_testnet_price_shape(): @@ -106,20 +109,22 @@ def test_landing_page_explains_why_to_buy_the_api(): assert "0x96e1...BB84" in html -def test_leos_token_offer_is_ca_only_metadata(): +def test_leos_token_offer_identifies_ca_and_requires_access(): offer = leos_token_offer() assert offer["site"] == LEOS_SITE_URL assert offer["ca"] == LEOS_TOKEN_CA assert offer["price"] == LEOS_TOKEN_PRICE assert "POST /leos/v1/recall" in offer["discount_routes"] - assert "Only the CA is needed" in offer["note"] + assert offer["access_header"] == LEOS_ACCESS_HEADER + assert offer["access_required"] is True + assert "eligible buyers" in offer["note"] def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): fastapi_testclient = pytest.importorskip("fastapi.testclient") client = fastapi_testclient.TestClient( - create_app(config=X402Config(pay_to="0xabc"), paid=False) + create_app(config=X402Config(pay_to="0xabc"), paid=False, leos_access_token="leos-secret") ) landing = client.get("/") @@ -136,13 +141,92 @@ def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): assert pricing.json()["x402"]["price"] == DEFAULT_PRICE assert pricing.json()["token_offer"]["ca"] == LEOS_TOKEN_CA assert pricing.json()["token_offer"]["price"] == LEOS_TOKEN_PRICE + assert pricing.json()["token_offer"]["enabled"] is True assert pricing.json()["tenancy"]["default_tenant"] == DEFAULT_TENANT_ID - leos_route = client.post("/leos/v1/route", json={"task": "search local agent memory"}) + blocked = client.post("/leos/v1/route", json={"task": "search local agent memory"}) + assert blocked.status_code == 401 + assert client.post("/leos/v1/recall", json={"query": "memory"}).status_code == 401 + assert client.get("/leos/v1/dashboard").status_code == 401 + + leos_route = client.post( + "/leos/v1/route", + headers={LEOS_ACCESS_HEADER: "leos-secret"}, + json={"task": "search local agent memory"}, + ) assert leos_route.status_code == 200 assert leos_route.json()["tenant"] == DEFAULT_TENANT_ID +def test_health_does_not_run_expensive_evidence_probe(): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + core = demo() + + def fail_evidence(): + raise AssertionError("health must not run evidence") + + core.evidence = fail_evidence + client = fastapi_testclient.TestClient( + create_app(core=core, config=X402Config(pay_to="0xabc"), paid=False) + ) + + response = client.get("/health") + pricing = client.get("/pricing").json() + + assert response.status_code == 200 + assert response.json()["memory"]["entries"] == 3 + assert pricing["token_offer"]["enabled"] is False + assert all(not row["route"].split(" ", 1)[1].startswith("/leos/") for row in pricing["routes"]) + assert client.get("/leos/v1/dashboard").status_code == 503 + + +@pytest.mark.parametrize( + "payload", + [ + {"query": "x", "k": "bad"}, + {"query": "x", "k": 0}, + {"query": "x", "k": -1}, + {"query": ""}, + {"query": "x", "abstain": "bad"}, + {"query": "x", "abstain": 1.1}, + ], +) +def test_recall_rejects_invalid_inputs(payload): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app(config=X402Config(pay_to="0xabc"), paid=False) + ) + + response = client.post("/v1/recall", json=payload) + + assert response.status_code == 400 + + +def test_tenant_id_must_be_a_string_and_match_the_header(): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + ) + ) + + numeric = client.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret"}, + json={"tenant": 0, "text": "must not reach public"}, + ) + mismatch = client.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret", TENANT_HEADER: "acme"}, + json={"tenant": "beta", "text": "must not cross tenants"}, + ) + + assert numeric.status_code == 400 + assert mismatch.status_code == 400 + + def test_private_tenant_memory_requires_a_tenant_token(): fastapi_testclient = pytest.importorskip("fastapi.testclient") client = fastapi_testclient.TestClient( @@ -229,3 +313,50 @@ def test_tenant_memory_can_persist_to_state_dir(tmp_path): assert recalled.status_code == 200 assert recalled.json()["hits"][0]["label"] == "persisted" + + +def test_public_memory_persists_across_app_restart(tmp_path): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + first = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + tenant_state_dir=tmp_path, + ) + ) + written = first.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret"}, + json={"text": "unique public persisted phrase", "label": "public-persisted"}, + ) + assert written.status_code == 200 + + second = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + tenant_state_dir=tmp_path, + ) + ) + recalled = second.post( + "/v1/recall", + json={"query": "unique public persisted phrase", "k": 10}, + ) + + assert recalled.status_code == 200 + assert "public-persisted" in [hit["label"] for hit in recalled.json()["hits"]] + + +def test_persisted_writes_reload_under_process_lock(tmp_path): + first = TenantCoreStore(LocalAgentCore(), tmp_path) + second = TenantCoreStore(LocalAgentCore(), tmp_path) + first.read("acme", lambda core: core.entries) + second.read("acme", lambda core: core.entries) + + first.write("acme", lambda core: core.remember("first writer", label="first")) + second.write("acme", lambda core: core.remember("second writer", label="second")) + + reloaded = TenantCoreStore(LocalAgentCore(), tmp_path) + labels = [entry.label for entry in reloaded.read("acme", lambda core: core.entries)] + assert labels == ["first", "second"] From b1d518c4e8cd97992d3ea0d4c5dd33b1769c8b23 Mon Sep 17 00:00:00 2001 From: atimics Date: Sun, 12 Jul 2026 17:43:43 -0700 Subject: [PATCH 04/16] Add optional NoSQLite memory backend --- AWS_X402_DEPLOY.md | 37 + Dockerfile.x402 | 16 +- X402_API.md | 36 + holographic_x402_api.py | 468 +++++++- tests/test_holographic_x402_api.py | 101 ++ vendor/nosqlite/Cargo.lock | 1177 +++++++++++++++++++ vendor/nosqlite/Cargo.toml | 34 + vendor/nosqlite/REVISION | 7 + vendor/nosqlite/benches/engine.rs | 276 +++++ vendor/nosqlite/build.rs | 10 + vendor/nosqlite/c/nosqlite_kernel.c | 20 + vendor/nosqlite/c/nosqlite_kernel.h | 10 + vendor/nosqlite/src/bin/nosqlite.rs | 93 ++ vendor/nosqlite/src/encoder.rs | 192 ++++ vendor/nosqlite/src/engine.rs | 1658 +++++++++++++++++++++++++++ vendor/nosqlite/src/index.rs | 1543 +++++++++++++++++++++++++ vendor/nosqlite/src/kernel.rs | 14 + vendor/nosqlite/src/lib.rs | 61 + vendor/nosqlite/src/mutation.rs | 200 ++++ vendor/nosqlite/src/neural.rs | 175 +++ vendor/nosqlite/src/query.rs | 267 +++++ vendor/nosqlite/src/storage.rs | 1017 ++++++++++++++++ 22 files changed, 7405 insertions(+), 7 deletions(-) create mode 100644 vendor/nosqlite/Cargo.lock create mode 100644 vendor/nosqlite/Cargo.toml create mode 100644 vendor/nosqlite/REVISION create mode 100644 vendor/nosqlite/benches/engine.rs create mode 100644 vendor/nosqlite/build.rs create mode 100644 vendor/nosqlite/c/nosqlite_kernel.c create mode 100644 vendor/nosqlite/c/nosqlite_kernel.h create mode 100644 vendor/nosqlite/src/bin/nosqlite.rs create mode 100644 vendor/nosqlite/src/encoder.rs create mode 100644 vendor/nosqlite/src/engine.rs create mode 100644 vendor/nosqlite/src/index.rs create mode 100644 vendor/nosqlite/src/kernel.rs create mode 100644 vendor/nosqlite/src/lib.rs create mode 100644 vendor/nosqlite/src/mutation.rs create mode 100644 vendor/nosqlite/src/neural.rs create mode 100644 vendor/nosqlite/src/query.rs create mode 100644 vendor/nosqlite/src/storage.rs diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index 32540602..a51ba7d0 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -23,6 +23,8 @@ or pay upstream APIs as a buyer. ## Recommended AWS Architecture - **ECS Fargate** runs the `Dockerfile.x402` container. +- The image includes a pinned NoSQLite CLI for an optional semantic-memory + backend; it is disabled by default. - **Application Load Balancer** terminates HTTPS and forwards to port `4021`. - **ECR** stores the container image. - **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN`, @@ -79,6 +81,7 @@ LECORE_X402_PRICE=$0.0011 LECORE_X402_NETWORK=eip155:8453 LECORE_X402_FACILITATOR_URL=https://api.cdp.coinbase.com/platform/v2/x402 LECORE_X402_TENANT_STATE_DIR=/data/tenants +LECORE_X402_MEMORY_BACKEND=core ``` Secrets Manager values: @@ -94,6 +97,38 @@ CDP_API_KEY_SECRET= Use ECS task definition `secrets` entries for secrets, not literal environment variables in the task definition. +## Optional NoSQLite Cutover + +The container has `/usr/local/bin/nosqlite` built from the vendored source +snapshot pinned at `8964da27670c752121b8e6d26d113577429b02f6`. To use it for +`/v1/recall`, add: + +```text +LECORE_X402_MEMORY_BACKEND=nosqlite +LECORE_X402_NOSQLITE_BIN=/usr/local/bin/nosqlite +LECORE_X402_NOSQLITE_DATA_DIR=/data/nosqlite +LECORE_X402_NOSQLITE_DURABILITY=sync +``` + +Mount `/data/nosqlite` on durable storage. NoSQLite deliberately takes a +nonblocking exclusive writer lock for the whole process, so a single data path +must have exactly one active ECS writer. Use a deliberate drain-and-replace +maintenance deployment for the cutover; do not rely on the normal overlapping +rolling deployment. The service currently stays on `core` until that operation +is scheduled. + +For a no-serving-impact validation phase, use: + +```text +LECORE_X402_MEMORY_BACKEND=core +LECORE_X402_NOSQLITE_SHADOW=1 +LECORE_X402_NOSQLITE_BIN=/usr/local/bin/nosqlite +LECORE_X402_NOSQLITE_DATA_DIR=/data/nosqlite +``` + +That mirrors admin writes and compares recall internally while preserving the +existing LocalAgentCore response as the source of truth. + ## Wallet Storage Decision ### Seller API, Recommended @@ -151,6 +186,8 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. - Mount `LECORE_X402_TENANT_STATE_DIR` on shared durable storage. Tenant writes reload under an OS-level lock and use atomic replacement, so rolling ECS tasks do not overwrite one another. +- Do not enable NoSQLite on the same EFS directory in overlapping ECS tasks; + schedule a single-writer drain-and-replace cutover instead. - Keep the leOS offer credential in Secrets Manager and rotate it if disclosed. - Do not put secrets or PII in x402 route descriptions or payment metadata. diff --git a/Dockerfile.x402 b/Dockerfile.x402 index fc4d6e2f..8bae2d3b 100644 --- a/Dockerfile.x402 +++ b/Dockerfile.x402 @@ -1,10 +1,24 @@ +FROM rust:1.85-slim AS nosqlite-builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src/nosqlite + +COPY vendor/nosqlite /src/nosqlite + +RUN cargo build --release --locked --bin nosqlite + FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + PYTHONDONTWRITEBYTECODE=1 \ + LECORE_X402_NOSQLITE_BIN=/usr/local/bin/nosqlite WORKDIR /app +COPY --from=nosqlite-builder /src/nosqlite/target/release/nosqlite /usr/local/bin/nosqlite COPY . /app RUN python -m pip install --no-cache-dir --upgrade pip \ diff --git a/X402_API.md b/X402_API.md index 1fccf956..cb1f9a51 100644 --- a/X402_API.md +++ b/X402_API.md @@ -118,6 +118,40 @@ Use this only for development: python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 ``` +## Optional NoSQLite Memory Backend + +`Dockerfile.x402` builds the vendored NoSQLite source snapshot pinned at +`8964da2` into the service image. The default remains `core`: +`LocalAgentCore` is the serving backend and the existing per-tenant JSON state +remains the durable control-plane mirror. + +To cut semantic recall over to NoSQLite, configure a durable mounted directory: + +```bash +export LECORE_X402_MEMORY_BACKEND=nosqlite +export LECORE_X402_NOSQLITE_BIN=/usr/local/bin/nosqlite +export LECORE_X402_NOSQLITE_DATA_DIR=/data/nosqlite +export LECORE_X402_NOSQLITE_DURABILITY=sync +export LECORE_X402_TENANT_STATE_DIR=/data/tenants +``` + +The API keeps each tenant in a separate hashed collection, writes the same +admin-created entry to `LocalAgentCore` for routing/dashboard continuity, and +uses NoSQLite's deterministic `holographic-hash-v1` encoder plus neural +candidate routing and cosine reranking for `/v1/recall`. Responses retain the +existing `id`, `text`, `label`, `metadata`, and `score` shape. + +Before cutover, set `LECORE_X402_NOSQLITE_SHADOW=1` while leaving +`LECORE_X402_MEMORY_BACKEND=core`. Admin writes are mirrored; recall continues +to serve from the core while differences are logged without query text or +tenant identifiers. + +NoSQLite filesystem mode holds one nonblocking exclusive writer lock for the +life of its process. Run exactly one active writer against a given data +directory. A rolling ECS replacement must drain the old writer before enabling +the new one, so the initial deployed configuration keeps this feature disabled +until that maintenance window is scheduled. + ## Production Notes - Use a real receiving wallet and a production facilitator. @@ -127,6 +161,8 @@ python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 - Keep writes admin-only. Use `LECORE_X402_TENANT_SECRET` and `LECORE_X402_TENANT_STATE_DIR` for durable public and private memory. Writes use per-tenant process locks plus atomic replacement on shared storage. +- If NoSQLite is enabled, mount `LECORE_X402_NOSQLITE_DATA_DIR` on the same + durable storage and keep the service at a single active writer for that path. - Store `LECORE_X402_LEOS_ACCESS_TOKEN` as a secret and distribute it only to buyers eligible for the discounted routes. - Treat x402 payment metadata as public enough to avoid putting secrets or PII diff --git a/holographic_x402_api.py b/holographic_x402_api.py index af982561..876e7b8b 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -17,17 +17,22 @@ from __future__ import annotations -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager from dataclasses import dataclass, replace import hashlib import hmac from html import escape import argparse +import json +import logging import os from pathlib import Path +import queue import re from string import Template +import subprocess import threading +import time from typing import Any, Dict, Iterable, List, Optional, Tuple from holographic_product import LocalAgentCore, demo @@ -48,6 +53,14 @@ MAX_TASK_CHARS = 8192 MAX_MEMORY_CHARS = 65536 MAX_RECALL_K = 100 +MEMORY_BACKEND_CORE = "core" +MEMORY_BACKEND_NOSQLITE = "nosqlite" +NOSQLITE_ENCODER = "lecore_text" +NOSQLITE_INDEX = "embedding_neural" +NOSQLITE_DIMENSIONS = 384 + + +LOG = logging.getLogger(__name__) @dataclass(frozen=True) @@ -378,6 +391,330 @@ def _path_for(self, tenant_id: str) -> Optional[Path]: return None return self._state_dir / ("%s.json" % normalize_tenant_id(tenant_id)) + +class NoSQLiteError(RuntimeError): + """Raised when the optional NoSQLite command process cannot serve a request.""" + + +class NoSQLiteProcess: + """Serialize JSON-line requests to one long-lived NoSQLite CLI process. + + NoSQLite's filesystem mode intentionally takes an exclusive writer lock for + the life of the process. The API therefore keeps exactly one child process + per application process and serializes its stdin/stdout protocol here. + """ + + def __init__( + self, + binary: str, + data_dir: Any, + durability: str = "sync", + timeout_seconds: float = 10.0, + ): + self._binary = str(binary) + self._data_dir = Path(data_dir) + self._durability = durability + self._timeout_seconds = float(timeout_seconds) + self._lock = threading.RLock() + self._process: Optional[Any] = None + self._stdout: Any = queue.Queue() + self._stderr: Any = queue.Queue() + self._generation = 0 + + @property + def generation(self) -> int: + with self._lock: + return self._generation + + @property + def running(self) -> bool: + with self._lock: + return self._process is not None and self._process.poll() is None + + def ensure_started(self) -> int: + """Start the child lazily and return its generation number.""" + with self._lock: + if self._process is not None and self._process.poll() is None: + return self._generation + self._stop_process_unlocked() + command = [self._binary, "--data-dir", str(self._data_dir), "--durability", self._durability] + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + except OSError as exc: + raise NoSQLiteError("could not start NoSQLite: %s" % exc) from exc + + self._process = process + self._stdout = queue.Queue() + self._stderr = queue.Queue() + self._start_reader(process.stdout, self._stdout) + self._start_reader(process.stderr, self._stderr) + try: + banner = self._read_line_unlocked("startup") + except NoSQLiteError: + self._stop_process_unlocked() + raise + if not banner.startswith("nosqlite ready;"): + self._stop_process_unlocked() + raise NoSQLiteError("unexpected NoSQLite startup response: %s" % banner.strip()) + self._generation += 1 + return self._generation + + def command(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """Send one command and return the object response from NoSQLite.""" + with self._lock: + self.ensure_started() + process = self._process + if process is None or process.stdin is None: + raise NoSQLiteError("NoSQLite process has no writable stdin") + try: + process.stdin.write(json.dumps(payload, separators=(",", ":")) + "\n") + process.stdin.flush() + except OSError as exc: + self._stop_process_unlocked() + raise NoSQLiteError("failed to send a command to NoSQLite: %s" % exc) from exc + try: + line = self._read_line_unlocked("command") + except NoSQLiteError: + self._stop_process_unlocked() + raise + try: + response = json.loads(line) + except json.JSONDecodeError as exc: + raise NoSQLiteError("invalid NoSQLite response: %s" % line.strip()) from exc + if not isinstance(response, dict): + raise NoSQLiteError("NoSQLite response must be an object") + if response.get("ok") == "error": + raise NoSQLiteError(str(response.get("message") or "unknown NoSQLite error")) + return response + + def close(self) -> None: + """Release the child process and its filesystem writer lock.""" + with self._lock: + process = self._process + if process is None: + return + try: + if process.poll() is None and process.stdin is not None: + process.stdin.write('{"shutdown":1}\n') + process.stdin.flush() + self._read_line_unlocked("shutdown", timeout_seconds=2.0) + process.wait(timeout=2.0) + except (OSError, subprocess.TimeoutExpired, NoSQLiteError): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + process.kill() + finally: + self._clear_process_unlocked() + + def _start_reader(self, stream: Any, output: Any) -> None: + def read_lines() -> None: + if stream is None: + return + for line in stream: + output.put(line) + + threading.Thread(target=read_lines, daemon=True).start() + + def _read_line_unlocked(self, phase: str, timeout_seconds: Optional[float] = None) -> str: + timeout = self._timeout_seconds if timeout_seconds is None else timeout_seconds + deadline = time.monotonic() + timeout + while True: + process = self._process + if process is not None and process.poll() is not None: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + return self._stdout.get(timeout=min(remaining, 0.1)) + except queue.Empty: + continue + process = self._process + state = "" + if process is not None and process.poll() is not None: + state = " (process exited with code %s)" % process.returncode + stderr = self._stderr_text_unlocked() + if stderr: + state += ": %s" % stderr + raise NoSQLiteError("NoSQLite %s timed out%s" % (phase, state)) + + def _stderr_text_unlocked(self) -> str: + lines = [] + while True: + try: + lines.append(self._stderr.get_nowait().strip()) + except queue.Empty: + break + return " ".join(line for line in lines if line)[:2000] + + def _clear_process_unlocked(self) -> None: + self._process = None + + def _stop_process_unlocked(self) -> None: + process = self._process + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + process.kill() + self._clear_process_unlocked() + + +class NoSQLiteMemoryStore: + """Tenant-isolated semantic memory backed by the pinned NoSQLite CLI.""" + + def __init__( + self, + binary: str, + data_dir: Any, + durability: str = "sync", + dimensions: int = NOSQLITE_DIMENSIONS, + ): + if durability not in {"sync", "buffered"}: + raise ValueError("NoSQLite durability must be 'sync' or 'buffered'") + self._dimensions = int(dimensions) + self._process = NoSQLiteProcess(binary, data_dir, durability=durability) + self._lock = threading.RLock() + self._encoder_generation: Optional[int] = None + self._ready_collections: set[str] = set() + self._synced_collections: set[Tuple[int, str]] = set() + + @property + def running(self) -> bool: + return self._process.running + + def remember(self, tenant_id: str, memory: Dict[str, Any]) -> None: + """Persist one LocalAgentCore-compatible memory entry in its tenant collection.""" + normalized = normalize_tenant_id(tenant_id) + with self._lock: + collection = self._ensure_collection(normalized) + self._insert_memory(collection, normalized, memory) + + def sync(self, tenant_id: str, memories: Iterable[Dict[str, Any]]) -> None: + """Backfill the durable core mirror once per tenant and CLI generation.""" + normalized = normalize_tenant_id(tenant_id) + with self._lock: + collection = self._ensure_collection(normalized) + key = (self._process.generation, collection) + if key in self._synced_collections: + return + for memory in memories: + self._insert_memory(collection, normalized, memory) + self._synced_collections.add(key) + + def recall( + self, + tenant_id: str, + query: str, + k: int, + abstain: Optional[float] = None, + ) -> List[Dict[str, Any]]: + """Return NoSQLite semantic hits in the LocalAgentCore response shape.""" + normalized = normalize_tenant_id(tenant_id) + with self._lock: + collection = self._ensure_collection(normalized) + response = self._process.command({ + "semanticSearch": collection, + "encoder": NOSQLITE_ENCODER, + "index": NOSQLITE_INDEX, + "text": query, + "k": k, + }) + documents = response.get("documents") + if not isinstance(documents, list): + raise NoSQLiteError("NoSQLite semantic search returned no documents array") + hits = [] + for document in documents: + if not isinstance(document, dict): + continue + score = document.get("_score") + if isinstance(score, bool) or not isinstance(score, (int, float)): + continue + if abstain is not None and float(score) < abstain: + continue + metadata = document.get("metadata") + label = document.get("label") + hits.append({ + "id": str(document.get("_id", "")), + "text": str(document.get("text", "")), + "label": label if isinstance(label, str) else None, + "metadata": dict(metadata) if isinstance(metadata, dict) else {}, + "score": float(score), + }) + return hits + + def close(self) -> None: + self._process.close() + + def _ensure_collection(self, tenant_id: str) -> str: + generation = self._process.ensure_started() + if self._encoder_generation != generation: + self._ready_collections.clear() + self._synced_collections.clear() + self._ignore_duplicate({ + "createEncoder": NOSQLITE_ENCODER, + "provider": "holographic-hash-v1", + "kind": "text", + "dimensions": self._dimensions, + "seed": 0, + }) + self._encoder_generation = generation + collection = self._collection_name(tenant_id) + if collection not in self._ready_collections: + self._ignore_duplicate({"create": collection}) + self._ignore_duplicate({ + "createIndexes": collection, + "indexes": [{ + "neural": "embedding", + "dimensions": self._dimensions, + "name": NOSQLITE_INDEX, + }], + }) + self._ready_collections.add(collection) + return collection + + def _ignore_duplicate(self, command: Dict[str, Any]) -> None: + try: + self._process.command(command) + except NoSQLiteError as exc: + if "already exists" not in str(exc): + raise + + def _insert_memory(self, collection: str, tenant_id: str, memory: Dict[str, Any]) -> None: + document = { + "_id": str(memory["id"]), + "text": str(memory["text"]), + "label": memory.get("label"), + "metadata": dict(memory.get("metadata") or {}), + "tenant": tenant_id, + } + try: + self._process.command({ + "insert": collection, + "encode": {"encoder": NOSQLITE_ENCODER, "field": "text", "into": "embedding"}, + "documents": [document], + }) + except NoSQLiteError as exc: + if "duplicate value for `_id`" not in str(exc): + raise + + @staticmethod + def _collection_name(tenant_id: str) -> str: + digest = hashlib.sha256(tenant_id.encode("utf-8")).hexdigest()[:24] + return "lecore_memory_%s" % digest + + def leos_token_offer(access_required: bool = True, enabled: bool = True) -> Dict[str, Any]: """Return public metadata for the credential-gated leOS offer.""" return { @@ -423,6 +760,21 @@ def _abstain_threshold(payload: Dict[str, Any]) -> Optional[float]: return threshold +def normalize_memory_backend(value: Any) -> str: + """Validate the memory backend selector without accepting silent fallbacks.""" + if not isinstance(value, str): + raise ValueError("memory backend must be a string") + backend = value.strip().lower() or MEMORY_BACKEND_CORE + if backend not in {MEMORY_BACKEND_CORE, MEMORY_BACKEND_NOSQLITE}: + raise ValueError("memory backend must be 'core' or 'nosqlite'") + return backend + + +def env_flag(value: Optional[str]) -> bool: + """Parse the small explicit boolean surface used by deployment settings.""" + return (value or "").strip().lower() in {"1", "true", "yes", "on"} + + def landing_page_html(config: X402Config) -> str: """Render the buyer-facing landing page served from `/`.""" network_name = _network_name(config.network) @@ -510,6 +862,11 @@ def create_app( tenant_secret: Optional[str] = None, tenant_state_dir: Optional[Any] = None, leos_access_token: Optional[str] = None, + memory_backend: Optional[str] = None, + nosqlite_binary: Optional[str] = None, + nosqlite_data_dir: Optional[Any] = None, + nosqlite_durability: Optional[str] = None, + nosqlite_shadow: Optional[bool] = None, ) -> Any: """Create the FastAPI application for paid or local serving. @@ -525,9 +882,41 @@ def create_app( except ImportError as exc: raise RuntimeError(optional_dependency_help()) from exc - app = FastAPI(title="leCore x402 API", version="0.1.0") core = core or demo() store = TenantCoreStore(core, state_dir=tenant_state_dir) + memory_backend = normalize_memory_backend( + memory_backend if memory_backend is not None else os.environ.get("LECORE_X402_MEMORY_BACKEND", MEMORY_BACKEND_CORE) + ) + nosqlite_shadow = ( + bool(nosqlite_shadow) + if nosqlite_shadow is not None + else env_flag(os.environ.get("LECORE_X402_NOSQLITE_SHADOW")) + ) + nosqlite_store: Optional[NoSQLiteMemoryStore] = None + if memory_backend == MEMORY_BACKEND_NOSQLITE or nosqlite_shadow: + if not tenant_state_dir: + raise ValueError("LECORE_X402_TENANT_STATE_DIR is required when NoSQLite is enabled") + data_dir = nosqlite_data_dir or os.environ.get("LECORE_X402_NOSQLITE_DATA_DIR") + if not data_dir: + raise ValueError("LECORE_X402_NOSQLITE_DATA_DIR is required when NoSQLite is enabled") + nosqlite_store = NoSQLiteMemoryStore( + nosqlite_binary or os.environ.get("LECORE_X402_NOSQLITE_BIN", "nosqlite"), + data_dir, + durability=nosqlite_durability or os.environ.get("LECORE_X402_NOSQLITE_DURABILITY", "sync"), + ) + + @asynccontextmanager + async def lifespan(_: Any) -> Any: + try: + yield + finally: + if nosqlite_store is not None: + nosqlite_store.close() + + app = FastAPI(title="leCore x402 API", version="0.1.0", lifespan=lifespan) + app.state.memory_backend = memory_backend + app.state.nosqlite_shadow = nosqlite_shadow + app.state.nosqlite_store = nosqlite_store config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") leos_access_token = leos_access_token or os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN") @@ -602,6 +991,35 @@ def tenancy_public_dict() -> Dict[str, Any]: "private_tenants_enabled": bool(tenant_secret), } + def memory_public_dict() -> Dict[str, Any]: + return { + "backend": memory_backend, + "nosqlite_shadow": bool(nosqlite_shadow), + "nosqlite_configured": nosqlite_store is not None, + } + + def nosqlite_unavailable(error: NoSQLiteError) -> HTTPException: + LOG.warning("NoSQLite memory backend is unavailable: %s", error) + return HTTPException(status_code=503, detail="NoSQLite memory backend is unavailable") + + def sync_nosqlite_tenant(tenant_id: str) -> None: + if nosqlite_store is None: + return + memories = store.read(tenant_id, lambda tenant_core: [entry.to_dict() for entry in tenant_core.entries]) + nosqlite_store.sync(tenant_id, memories) + + def shadow_recall(tenant_id: str, query: str, k: int, abstain: Optional[float], core_hits: List[Dict[str, Any]]) -> None: + if nosqlite_store is None: + return + try: + sync_nosqlite_tenant(tenant_id) + shadow_hits = nosqlite_store.recall(tenant_id, query, k=k, abstain=abstain) + except NoSQLiteError as exc: + LOG.warning("NoSQLite shadow recall failed: %s", exc) + return + if [hit.get("id") for hit in core_hits] != [hit.get("id") for hit in shadow_hits]: + LOG.info("NoSQLite shadow recall differs from LocalAgentCore") + @app.get("/", response_class=HTMLResponse, include_in_schema=False) def landing() -> str: return landing_page_html(config) @@ -613,6 +1031,7 @@ def health() -> Dict[str, Any]: "name": "leCore x402 API", "paid": bool(paid), "memory": store.summary(DEFAULT_TENANT_ID), + "memory_backend": memory_public_dict(), "tenancy": { "default_tenant": DEFAULT_TENANT_ID, "loaded_tenants": len(store.loaded_tenants()), @@ -627,6 +1046,7 @@ def pricing() -> Dict[str, Any]: "x402": config.to_public_dict(), "token_offer": leos_token_offer(enabled=bool(leos_access_token)), "tenancy": tenancy_public_dict(), + "memory_backend": memory_public_dict(), "routes": payment_manifest(active_config), } @@ -640,10 +1060,21 @@ def recall_response( query = validated(_required_text, payload, "query", MAX_QUERY_CHARS) k = validated(_recall_k, payload) abstain = validated(_abstain_threshold, payload) - hits = store.read( - tenant_id, - lambda tenant_core: tenant_core.recall(query, k=k, abstain=abstain), - ) + if memory_backend == MEMORY_BACKEND_NOSQLITE: + if nosqlite_store is None: # pragma: no cover - guarded during app setup + raise HTTPException(status_code=503, detail="NoSQLite memory backend is not configured") + try: + sync_nosqlite_tenant(tenant_id) + hits = nosqlite_store.recall(tenant_id, query, k=k, abstain=abstain) + except NoSQLiteError as exc: + raise nosqlite_unavailable(exc) from exc + else: + hits = store.read( + tenant_id, + lambda tenant_core: tenant_core.recall(query, k=k, abstain=abstain), + ) + if nosqlite_shadow: + shadow_recall(tenant_id, query, k, abstain, hits) return { "ok": True, "tenant": tenant_id, @@ -742,6 +1173,13 @@ def remember( tenant_id, lambda tenant_core: tenant_core.remember(text, label=label, metadata=metadata), ) + if nosqlite_store is not None: + try: + nosqlite_store.remember(tenant_id, memory) + except NoSQLiteError as exc: + if memory_backend == MEMORY_BACKEND_NOSQLITE: + raise nosqlite_unavailable(exc) from exc + LOG.warning("NoSQLite shadow write failed: %s", exc) return { "ok": True, "tenant": tenant_id, @@ -786,6 +1224,19 @@ def main(argv: Optional[Iterable[str]] = None) -> None: p.add_argument("--tenant-secret", default=os.environ.get("LECORE_X402_TENANT_SECRET")) p.add_argument("--tenant-state-dir", default=os.environ.get("LECORE_X402_TENANT_STATE_DIR")) p.add_argument("--leos-access-token", default=os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN")) + p.add_argument( + "--memory-backend", + choices=(MEMORY_BACKEND_CORE, MEMORY_BACKEND_NOSQLITE), + default=os.environ.get("LECORE_X402_MEMORY_BACKEND", MEMORY_BACKEND_CORE), + ) + p.add_argument("--nosqlite-bin", default=os.environ.get("LECORE_X402_NOSQLITE_BIN", "nosqlite")) + p.add_argument("--nosqlite-data-dir", default=os.environ.get("LECORE_X402_NOSQLITE_DATA_DIR")) + p.add_argument( + "--nosqlite-durability", + choices=("sync", "buffered"), + default=os.environ.get("LECORE_X402_NOSQLITE_DURABILITY", "sync"), + ) + p.add_argument("--nosqlite-shadow", action="store_true", default=None) p.add_argument("--unpaid-dev", action="store_true", help="Disable x402 middleware for local development only") args = p.parse_args(list(argv) if argv is not None else None) @@ -804,6 +1255,11 @@ def main(argv: Optional[Iterable[str]] = None) -> None: tenant_secret=args.tenant_secret, tenant_state_dir=args.tenant_state_dir, leos_access_token=args.leos_access_token, + memory_backend=args.memory_backend, + nosqlite_binary=args.nosqlite_bin, + nosqlite_data_dir=args.nosqlite_data_dir, + nosqlite_durability=args.nosqlite_durability, + nosqlite_shadow=args.nosqlite_shadow, ) try: import uvicorn diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index c38f427a..0cfdeb8b 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -1,5 +1,8 @@ """Tests for the optional x402-paid API publisher.""" +import os +from pathlib import Path + import pytest from holographic_x402_api import ( @@ -10,6 +13,7 @@ LEOS_ACCESS_HEADER, LEOS_TOKEN_CA, LEOS_TOKEN_PRICE, + MEMORY_BACKEND_NOSQLITE, TENANT_HEADER, TENANT_TOKEN_HEADER, TenantCoreStore, @@ -20,6 +24,7 @@ optional_dependency_help, payment_manifest, tenant_access_token, + normalize_memory_backend, x402_route_configs, ) from holographic_product import LocalAgentCore, demo @@ -94,6 +99,39 @@ def test_optional_dependency_help_points_to_extra(): assert 'pip install ".[x402]"' in optional_dependency_help() +def test_memory_backend_selection_is_explicit(): + assert normalize_memory_backend("core") == "core" + assert normalize_memory_backend("NoSQLite") == MEMORY_BACKEND_NOSQLITE + with pytest.raises(ValueError, match="'core' or 'nosqlite'"): + normalize_memory_backend("sqlite") + + +def test_nosqlite_backend_requires_durable_state_dirs(tmp_path): + pytest.importorskip("fastapi") + + with pytest.raises(ValueError, match="TENANT_STATE_DIR"): + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + memory_backend=MEMORY_BACKEND_NOSQLITE, + ) + + with pytest.raises(ValueError, match="NOSQLITE_DATA_DIR"): + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + memory_backend=MEMORY_BACKEND_NOSQLITE, + tenant_state_dir=tmp_path / "core", + ) + + +def _nosqlite_binary() -> str: + binary = os.environ.get("LECORE_X402_NOSQLITE_BIN") + if not binary or not Path(binary).is_file(): + pytest.skip("set LECORE_X402_NOSQLITE_BIN to run the optional NoSQLite integration test") + return binary + + def test_landing_page_explains_why_to_buy_the_api(): html = landing_page_html(X402Config(pay_to="0x96e1604E92A8A1edD0701be3E67Bd4366e87BB84")) @@ -348,6 +386,69 @@ def test_public_memory_persists_across_app_restart(tmp_path): assert "public-persisted" in [hit["label"] for hit in recalled.json()["hits"]] +def test_nosqlite_memory_backend_isolates_tenants_and_restarts(tmp_path): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + binary = _nosqlite_binary() + data_dir = tmp_path / "nosqlite" + tenant_token = tenant_access_token("acme", "tenant-secret") + common = { + "config": X402Config(pay_to="0xabc"), + "paid": False, + "admin_token": "admin-secret", + "tenant_secret": "tenant-secret", + "tenant_state_dir": tmp_path / "core", + "memory_backend": MEMORY_BACKEND_NOSQLITE, + "nosqlite_binary": binary, + "nosqlite_data_dir": data_dir, + } + + with fastapi_testclient.TestClient(create_app(**common)) as first: + health = first.get("/health") + assert health.status_code == 200 + assert health.json()["memory_backend"] == { + "backend": MEMORY_BACKEND_NOSQLITE, + "nosqlite_shadow": False, + "nosqlite_configured": True, + } + + public = first.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret"}, + json={"text": "unique public nosqlite comet memory", "label": "public-nosqlite"}, + ) + private = first.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret", TENANT_HEADER: "acme"}, + json={"text": "unique acme nosqlite lighthouse memory", "label": "private-nosqlite"}, + ) + assert public.status_code == 200 + assert private.status_code == 200 + + acme = first.post( + "/v1/recall", + headers={TENANT_HEADER: "acme", TENANT_TOKEN_HEADER: tenant_token}, + json={"query": "acme lighthouse", "k": 10}, + ) + public_recall = first.post( + "/v1/recall", + json={"query": "acme lighthouse", "k": 10}, + ) + assert acme.status_code == 200 + assert public_recall.status_code == 200 + assert [hit["label"] for hit in acme.json()["hits"]] == ["private-nosqlite"] + assert "private-nosqlite" not in [hit["label"] for hit in public_recall.json()["hits"]] + + restart_common = dict(common) + restart_common.pop("admin_token") + with fastapi_testclient.TestClient(create_app(**restart_common)) as second: + persisted = second.post( + "/v1/recall", + json={"query": "public comet", "k": 10}, + ) + assert persisted.status_code == 200 + assert "public-nosqlite" in [hit["label"] for hit in persisted.json()["hits"]] + + def test_persisted_writes_reload_under_process_lock(tmp_path): first = TenantCoreStore(LocalAgentCore(), tmp_path) second = TenantCoreStore(LocalAgentCore(), tmp_path) diff --git a/vendor/nosqlite/Cargo.lock b/vendor/nosqlite/Cargo.lock new file mode 100644 index 00000000..b19c8686 --- /dev/null +++ b/vendor/nosqlite/Cargo.lock @@ -0,0 +1,1177 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "criterion" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "nosqlite" +version = "0.1.0" +dependencies = [ + "cc", + "clap", + "criterion", + "fs2", + "parking_lot", + "serde", + "serde_json", + "tempfile", + "thiserror", + "zstd", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/vendor/nosqlite/Cargo.toml b/vendor/nosqlite/Cargo.toml new file mode 100644 index 00000000..cee8bf04 --- /dev/null +++ b/vendor/nosqlite/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "nosqlite" +version = "0.1.0" +edition = "2021" +description = "MongoDB-shaped embedded document database with a tiny C kernel and Rust orchestration." +license = "MIT" + +[lib] +name = "nosqlite" +path = "src/lib.rs" + +[[bin]] +name = "nosqlite" +path = "src/bin/nosqlite.rs" + +[dependencies] +clap = { version = "4.5", features = ["derive"] } +fs2 = "0.4" +parking_lot = "0.12" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +zstd = "0.13" + +[build-dependencies] +cc = "1" + +[dev-dependencies] +criterion = "0.7" +tempfile = "3" + +[[bench]] +name = "engine" +harness = false diff --git a/vendor/nosqlite/REVISION b/vendor/nosqlite/REVISION new file mode 100644 index 00000000..b1ae7774 --- /dev/null +++ b/vendor/nosqlite/REVISION @@ -0,0 +1,7 @@ +NoSQLite source snapshot + +Repository: https://github.com/atimics/nosqlite +Commit: 8964da27670c752121b8e6d26d113577429b02f6 + +This source is vendored so Docker builds do not require access to the private +repository. Update it only by replacing the snapshot with a reviewed commit. diff --git a/vendor/nosqlite/benches/engine.rs b/vendor/nosqlite/benches/engine.rs new file mode 100644 index 00000000..34032130 --- /dev/null +++ b/vendor/nosqlite/benches/engine.rs @@ -0,0 +1,276 @@ +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use nosqlite::{CommandResult, Engine, EngineOptions, StorageMode}; +use serde_json::{json, Map, Value}; +use tempfile::tempdir; + +const BATCH_SIZE: usize = 500; + +fn bench_memory(c: &mut Criterion) { + let mut group = c.benchmark_group("engine_memory"); + + for docs in [1_000usize, 10_000, 50_000] { + group.throughput(Throughput::Elements(docs as u64)); + group.bench_with_input(BenchmarkId::new("batch_insert", docs), &docs, |b, &docs| { + b.iter(|| { + let engine = Engine::new(EngineOptions::default()).unwrap(); + insert_docs(&engine, "events", docs); + black_box(engine); + }); + }); + + group.bench_with_input(BenchmarkId::new("point_find", docs), &docs, |b, &docs| { + let engine = seeded_memory_engine(docs); + b.iter(|| { + for index in (0..250).map(|n| (n * 37) % docs) { + let result = engine + .find( + "events", + Some(filter("external_id", event_id(index))), + Some(1), + None, + None, + ) + .unwrap(); + black_box(result); + } + }); + }); + + group.bench_with_input( + BenchmarkId::new("indexed_point_find", docs), + &docs, + |b, &docs| { + let engine = seeded_indexed_memory_engine(docs); + b.iter(|| { + for index in (0..250).map(|n| (n * 37) % docs) { + let result = engine + .find( + "events", + Some(filter("external_id", event_id(index))), + Some(1), + None, + None, + ) + .unwrap(); + black_box(result); + } + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("filtered_update", docs), + &docs, + |b, &docs| { + b.iter_batched( + || seeded_memory_engine(docs), + |engine| { + let result = engine + .update( + "events", + Some(filter("bucket", json!(7))), + set_update("state", json!("hot")), + ) + .unwrap(); + black_box(result); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); + + group.bench_with_input( + BenchmarkId::new("neural_vector_search", docs), + &docs, + |b, &docs| { + let engine = seeded_neural_memory_engine(docs); + b.iter(|| { + for index in (0..100).map(|n| (n * 37) % docs) { + let result = engine + .vector_search( + "vectors", + Some("embedding_neural"), + None, + embedding(index), + 10, + None, + ) + .unwrap(); + black_box(result); + } + }); + }, + ); + } + + group.finish(); +} + +fn bench_filesystem(c: &mut Criterion) { + let mut group = c.benchmark_group("engine_filesystem"); + + for docs in [1_000usize, 5_000] { + group.throughput(Throughput::Elements(docs as u64)); + group.bench_with_input( + BenchmarkId::new("persist_insert", docs), + &docs, + |b, &docs| { + b.iter_batched( + || tempdir().unwrap(), + |dir| { + let engine = Engine::new(EngineOptions { + storage: StorageMode::FileSystem(dir.path().to_path_buf()), + ..EngineOptions::default() + }) + .unwrap(); + insert_docs(&engine, "events", docs); + black_box(engine); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); + + group.bench_with_input(BenchmarkId::new("cold_load", docs), &docs, |b, &docs| { + b.iter_batched( + || { + let dir = tempdir().unwrap(); + let engine = Engine::new(EngineOptions { + storage: StorageMode::FileSystem(dir.path().to_path_buf()), + ..EngineOptions::default() + }) + .unwrap(); + insert_docs(&engine, "events", docs); + drop(engine); + dir + }, + |dir| { + let engine = Engine::new(EngineOptions { + storage: StorageMode::FileSystem(dir.path().to_path_buf()), + ..EngineOptions::default() + }) + .unwrap(); + let result = engine + .find( + "events", + Some(filter("bucket", json!(3))), + Some(10), + None, + None, + ) + .unwrap(); + black_box(result); + }, + criterion::BatchSize::SmallInput, + ); + }); + } + + group.finish(); +} + +fn seeded_memory_engine(docs: usize) -> Engine { + let engine = Engine::new(EngineOptions::default()).unwrap(); + insert_docs(&engine, "events", docs); + engine +} + +fn seeded_indexed_memory_engine(docs: usize) -> Engine { + let engine = seeded_memory_engine(docs); + engine + .create_indexes( + "events", + &[json!({ + "key": { "external_id": 1 }, + "name": "external_id_1", + "unique": true + })], + ) + .unwrap(); + engine +} + +fn seeded_neural_memory_engine(docs: usize) -> Engine { + let engine = Engine::new(EngineOptions::default()).unwrap(); + for start in (0..docs).step_by(BATCH_SIZE) { + let end = (start + BATCH_SIZE).min(docs); + let batch = (start..end) + .map(|index| { + json!({ + "_id": format!("vector-{index:08}"), + "embedding": embedding(index), + "bucket": index % 20 + }) + }) + .collect::>(); + let CommandResult::Inserted { count, .. } = engine.insert("vectors", &batch).unwrap() + else { + unreachable!("insert returns inserted result"); + }; + assert_eq!(count, end - start); + } + engine + .create_indexes( + "vectors", + &[json!({ + "neural": "embedding", + "dimensions": 16, + "name": "embedding_neural" + })], + ) + .unwrap(); + engine +} + +fn insert_docs(engine: &Engine, collection: &str, docs: usize) { + for start in (0..docs).step_by(BATCH_SIZE) { + let end = (start + BATCH_SIZE).min(docs); + let batch = (start..end).map(document).collect::>(); + let CommandResult::Inserted { count, .. } = engine.insert(collection, &batch).unwrap() + else { + unreachable!("insert returns inserted result"); + }; + assert_eq!(count, end - start); + } +} + +fn document(index: usize) -> Value { + json!({ + "external_id": event_id(index), + "bucket": index % 20, + "score": (index % 10_000) as f64 / 100.0, + "state": "warm", + "payload": { + "model": "embedding-v1", + "dimensions": 384, + "tokens": index % 8192 + } + }) +} + +fn event_id(index: usize) -> Value { + json!(format!("event-{index:08}")) +} + +fn filter(key: &str, value: Value) -> Map { + Map::from_iter([(key.to_string(), value)]) +} + +fn set_update(key: &str, value: Value) -> Map { + Map::from_iter([("$set".to_string(), json!({ key: value }))]) +} + +fn embedding(index: usize) -> Vec { + let phase = index as f64 * 0.017; + (0..16) + .map(|dim| { + let dim = dim as f64 + 1.0; + (phase * dim).sin() + (phase / dim).cos() * 0.25 + }) + .collect() +} + +criterion_group!(benches, bench_memory, bench_filesystem); +criterion_main!(benches); diff --git a/vendor/nosqlite/build.rs b/vendor/nosqlite/build.rs new file mode 100644 index 00000000..af550ef9 --- /dev/null +++ b/vendor/nosqlite/build.rs @@ -0,0 +1,10 @@ +fn main() { + cc::Build::new() + .file("c/nosqlite_kernel.c") + .include("c") + .warnings(true) + .compile("nosqlite_kernel"); + + println!("cargo:rerun-if-changed=c/nosqlite_kernel.c"); + println!("cargo:rerun-if-changed=c/nosqlite_kernel.h"); +} diff --git a/vendor/nosqlite/c/nosqlite_kernel.c b/vendor/nosqlite/c/nosqlite_kernel.c new file mode 100644 index 00000000..adfbbe04 --- /dev/null +++ b/vendor/nosqlite/c/nosqlite_kernel.c @@ -0,0 +1,20 @@ +#include "nosqlite_kernel.h" + +#include + +static atomic_uint_fast64_t NOSQLITE_ID = 1; + +uint64_t nosqlite_fnv1a64(const uint8_t *data, size_t len) { + uint64_t hash = 1469598103934665603ULL; + + for (size_t i = 0; i < len; i++) { + hash ^= (uint64_t)data[i]; + hash *= 1099511628211ULL; + } + + return hash; +} + +uint64_t nosqlite_next_id(void) { + return atomic_fetch_add_explicit(&NOSQLITE_ID, 1, memory_order_relaxed); +} diff --git a/vendor/nosqlite/c/nosqlite_kernel.h b/vendor/nosqlite/c/nosqlite_kernel.h new file mode 100644 index 00000000..9f56326a --- /dev/null +++ b/vendor/nosqlite/c/nosqlite_kernel.h @@ -0,0 +1,10 @@ +#ifndef NOSQLITE_KERNEL_H +#define NOSQLITE_KERNEL_H + +#include +#include + +uint64_t nosqlite_fnv1a64(const uint8_t *data, size_t len); +uint64_t nosqlite_next_id(void); + +#endif diff --git a/vendor/nosqlite/src/bin/nosqlite.rs b/vendor/nosqlite/src/bin/nosqlite.rs new file mode 100644 index 00000000..042b406f --- /dev/null +++ b/vendor/nosqlite/src/bin/nosqlite.rs @@ -0,0 +1,93 @@ +use std::{ + io::{self, BufRead, Write}, + path::PathBuf, +}; + +use clap::Parser; +use nosqlite::{CommandResult, Durability, Engine, EngineOptions, StorageMode}; + +#[derive(Debug, Parser)] +#[command(about = "MongoDB-shaped embedded document database shell")] +struct Args { + #[arg(long)] + data_dir: Option, + #[arg(long, default_value = "sync", value_parser = ["sync", "buffered"])] + durability: String, +} + +fn main() -> nosqlite::Result<()> { + let args = Args::parse(); + let storage = args + .data_dir + .map(StorageMode::FileSystem) + .unwrap_or(StorageMode::Memory); + let engine = Engine::new(EngineOptions { + storage, + durability: parse_durability(&args.durability), + ..EngineOptions::default() + })?; + + let stdin = io::stdin(); + let mut stdin = stdin.lock(); + let mut stdout = io::stdout(); + let mut line = String::new(); + + writeln!(stdout, "nosqlite ready; send one JSON command per line")?; + loop { + line.clear(); + if stdin.read_line(&mut line)? == 0 { + break; + } + if line.trim().is_empty() { + continue; + } + + let command = match serde_json::from_str(&line) { + Ok(command) => command, + Err(error) => { + writeln!( + stdout, + "{{\"ok\":\"error\",\"message\":{}}}", + serde_json::to_string(&error.to_string())? + )?; + stdout.flush()?; + continue; + } + }; + + if is_shutdown(&command) { + serde_json::to_writer(&mut stdout, &CommandResult::Shutdown)?; + stdout.write_all(b"\n")?; + stdout.flush()?; + break; + } + + match engine.execute(command) { + Ok(result) => { + serde_json::to_writer(&mut stdout, &result)?; + stdout.write_all(b"\n")?; + } + Err(error) => writeln!( + stdout, + "{{\"ok\":\"error\",\"message\":{}}}", + serde_json::to_string(&error.to_string())? + )?, + } + stdout.flush()?; + } + + Ok(()) +} + +fn parse_durability(value: &str) -> Durability { + match value { + "buffered" => Durability::Buffered, + _ => Durability::Sync, + } +} + +fn is_shutdown(command: &serde_json::Value) -> bool { + command + .as_object() + .is_some_and(|command| command.get("shutdown").is_some()) +} diff --git a/vendor/nosqlite/src/encoder.rs b/vendor/nosqlite/src/encoder.rs new file mode 100644 index 00000000..e566a1e4 --- /dev/null +++ b/vendor/nosqlite/src/encoder.rs @@ -0,0 +1,192 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{Error, Result}; + +pub const DEFAULT_PROVIDER: &str = "holographic-hash-v1"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncoderSpec { + #[serde(default = "default_kind")] + pub kind: String, + #[serde(default = "default_provider")] + pub provider: String, + #[serde(default = "default_dimensions")] + pub dimensions: usize, + #[serde(default)] + pub seed: u64, +} + +impl EncoderSpec { + pub fn validate(&self) -> Result<()> { + if self.kind != "text" { + return Err(Error::UnsupportedEncoder(self.kind.clone())); + } + if self.provider != DEFAULT_PROVIDER && self.provider != "builtin-hash-v1" { + return Err(Error::UnsupportedEncoder(self.provider.clone())); + } + if self.dimensions == 0 { + return Err(Error::VectorDimensionMismatch { + field: "encoder.dimensions".to_string(), + expected: 1, + actual: 0, + }); + } + Ok(()) + } +} + +pub fn encode_value(spec: &EncoderSpec, value: &Value) -> Result> { + spec.validate()?; + Ok(encode_text(spec, &semantic_text(value))) +} + +pub fn encode_text(spec: &EncoderSpec, text: &str) -> Vec { + let mut vector = vec![0.0; spec.dimensions]; + let tokens = tokenize(text); + + if tokens.is_empty() { + accumulate_feature(&mut vector, spec.seed, "empty", 1.0); + } + + for token in &tokens { + accumulate_feature(&mut vector, spec.seed, &format!("tok:{token}"), 1.0); + for ngram in char_ngrams(token, 3, 5) { + accumulate_feature(&mut vector, spec.seed, &format!("chr:{ngram}"), 0.35); + } + } + + for pair in tokens.windows(2) { + accumulate_feature( + &mut vector, + spec.seed, + &format!("pair:{}:{}", pair[0], pair[1]), + 0.75, + ); + } + + normalize(&mut vector); + vector +} + +fn semantic_text(value: &Value) -> String { + match value { + Value::Null => String::new(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + Value::String(value) => value.clone(), + Value::Array(values) => values + .iter() + .map(semantic_text) + .collect::>() + .join(" "), + Value::Object(values) => values + .iter() + .map(|(key, value)| format!("{key} {}", semantic_text(value))) + .collect::>() + .join(" "), + } +} + +fn tokenize(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + + for ch in text.chars().flat_map(char::to_lowercase) { + if ch.is_ascii_alphanumeric() { + current.push(ch); + } else if !current.is_empty() { + tokens.push(std::mem::take(&mut current)); + } + } + if !current.is_empty() { + tokens.push(current); + } + + tokens +} + +fn char_ngrams(token: &str, min: usize, max: usize) -> Vec { + let chars = token.chars().collect::>(); + let mut ngrams = Vec::new(); + for width in min..=max { + if chars.len() < width { + continue; + } + for start in 0..=chars.len() - width { + ngrams.push(chars[start..start + width].iter().collect()); + } + } + ngrams +} + +fn accumulate_feature(vector: &mut [f64], seed: u64, feature: &str, weight: f64) { + let feature_hash = hash_bytes(feature.as_bytes()); + for (dimension, value) in vector.iter_mut().enumerate() { + let mut hash = seed ^ feature_hash ^ (dimension as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); + hash = splitmix64(hash); + let signed = if hash & 1 == 0 { -1.0 } else { 1.0 }; + let magnitude = (((hash >> 11) as f64) / ((1_u64 << 53) as f64)).max(0.05); + *value += signed * magnitude * weight; + } +} + +fn normalize(vector: &mut [f64]) { + let norm = vector.iter().map(|value| value * value).sum::().sqrt(); + if norm == 0.0 { + return; + } + for value in vector { + *value /= norm; + } +} + +fn hash_bytes(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + +fn default_kind() -> String { + "text".to_string() +} + +fn default_provider() -> String { + DEFAULT_PROVIDER.to_string() +} + +fn default_dimensions() -> usize { + 384 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_encoder_is_deterministic_and_normalized() { + let spec = EncoderSpec { + kind: "text".to_string(), + provider: DEFAULT_PROVIDER.to_string(), + dimensions: 64, + seed: 42, + }; + let first = encode_text(&spec, "memory of a quiet blue room"); + let second = encode_text(&spec, "memory of a quiet blue room"); + assert_eq!(first, second); + + let norm = first.iter().map(|value| value * value).sum::().sqrt(); + assert!((norm - 1.0).abs() < 0.0000001); + } +} diff --git a/vendor/nosqlite/src/engine.rs b/vendor/nosqlite/src/engine.rs new file mode 100644 index 00000000..5449cd20 --- /dev/null +++ b/vendor/nosqlite/src/engine.rs @@ -0,0 +1,1658 @@ +use std::{cmp::Ordering, collections::BTreeMap}; + +use parking_lot::{Mutex, RwLock}; +use serde::Serialize; +use serde_json::{json, Number, Value}; + +use crate::{ + encoder::{encode_text, encode_value, EncoderSpec}, + index::{json_vector, CollectionState, IndexKind, IndexSpec}, + kernel, + mutation::{ + apply_cleanup_expired, apply_compiled_updates, apply_delete_many_compiled, CompiledUpdates, + }, + neural::NeuralSpace, + query::{compile_filter, compile_filter_excluding, CompiledFilter}, + storage::{Catalog, CollectionCatalog, Durability, Storage}, + Document, Error, Result, +}; + +const AUTO_INDEX_PROMOTION_THRESHOLD: usize = 3; +const MAX_AUTO_INDEXES_PER_COLLECTION: usize = 4; + +pub use crate::storage::StorageMode; + +#[derive(Debug, Clone)] +pub struct EngineOptions { + pub storage: StorageMode, + pub durability: Durability, + pub shard_count: u64, +} + +impl Default for EngineOptions { + fn default() -> Self { + Self { + storage: StorageMode::Memory, + durability: Durability::Sync, + shard_count: 64, + } + } +} + +#[derive(Debug, Serialize, PartialEq)] +#[serde(tag = "ok", rename_all = "camelCase")] +pub enum CommandResult { + Created { + collection: String, + }, + Dropped { + collection: String, + }, + Snapshotted { + collection: String, + count: usize, + }, + Compacted { + collections: usize, + #[serde(rename = "lastSeq")] + last_seq: u64, + }, + Inserted { + count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + ids: Option>, + }, + Found { + count: usize, + documents: Vec, + #[serde(rename = "lastEvaluatedKey")] + #[serde(skip_serializing_if = "Option::is_none")] + last_evaluated_key: Option, + }, + Counted { + count: usize, + }, + Updated { + matched: usize, + modified: usize, + }, + Deleted { + count: usize, + }, + Indexed { + collection: String, + indexes: Vec, + }, + EncoderCreated { + encoder: String, + }, + Encoded { + encoder: String, + dimensions: usize, + vector: Vec, + }, + NeuralSpaceCreated { + space: String, + }, + Learned { + space: String, + prototypes: usize, + }, + Pong { + protocol: u32, + }, + Shutdown, +} + +#[derive(Debug, Clone)] +struct ProjectionSpec { + fields: Vec, +} + +#[derive(Debug)] +pub struct Engine { + collections: RwLock>, + encoders: RwLock>, + neural_spaces: RwLock>, + auto_index_hits: Mutex, usize>>>, + storage: Storage, + shard_count: u64, +} + +impl Engine { + pub fn new(options: EngineOptions) -> Result { + let storage = Storage::new(options.storage, options.durability)?; + let catalog = storage.load_catalog()?; + let collections = storage + .load()? + .into_iter() + .map(|(name, documents)| { + let mut state = CollectionState::new(documents)?; + if let Some(collection_catalog) = catalog.collections.get(&name) { + state.create_indexes(collection_catalog.indexes.clone())?; + } + Ok((name, state)) + }) + .collect::>>()?; + + Ok(Self { + collections: RwLock::new(collections), + encoders: RwLock::new(catalog.encoders), + neural_spaces: RwLock::new(catalog.neural_spaces), + auto_index_hits: Mutex::new(BTreeMap::new()), + storage, + shard_count: options.shard_count.max(1), + }) + } + + pub fn execute_json(&self, command: &str) -> Result { + let value = serde_json::from_str(command)?; + self.execute(value) + } + + pub fn execute(&self, command: Value) -> Result { + let mut command = match command { + Value::Object(command) => command, + _ => return Err(Error::ExpectedObject("command")), + }; + + if command.get("ping").is_some() { + return Ok(CommandResult::Pong { protocol: 1 }); + } + if let Some(encoder) = command.get("createEncoder") { + return self.create_encoder( + required_str(encoder, "createEncoder")?, + parse_encoder_spec(&command)?, + ); + } + if let Some(encoder) = command.get("encodeText") { + return self.encode_text_command( + required_str(encoder, "encodeText")?, + required_str( + command.get("text").ok_or(Error::MissingField("text"))?, + "text", + )?, + ); + } + if let Some(space) = command.get("createNeuralSpace") { + return self.create_neural_space( + required_str(space, "createNeuralSpace")?, + required_str( + command + .get("encoder") + .ok_or(Error::MissingField("encoder"))?, + "encoder", + )?, + command + .get("dimensions") + .and_then(Value::as_u64) + .map(|dimensions| dimensions as usize), + command + .get("clusters") + .or_else(|| command.get("maxPrototypes")) + .and_then(Value::as_u64) + .map(|clusters| clusters as usize) + .unwrap_or(256), + ); + } + if let Some(space) = command.get("learnText") { + return self.learn_text( + required_str(space, "learnText")?, + required_str( + command.get("text").ok_or(Error::MissingField("text"))?, + "text", + )?, + command + .get("label") + .and_then(Value::as_str) + .map(str::to_string), + command.get("weight").and_then(Value::as_f64).unwrap_or(1.0), + ); + } + if let Some(space) = command.get("learnVector") { + return self.learn_vector( + required_str(space, "learnVector")?, + required_vector(command.get("vector"), "vector")?, + command + .get("label") + .and_then(Value::as_str) + .map(str::to_string), + command.get("weight").and_then(Value::as_f64).unwrap_or(1.0), + ); + } + if let Some(collection) = command.get("create") { + return self.create_collection(required_str(collection, "create")?); + } + if let Some(collection) = command.get("drop") { + return self.drop_collection(required_str(collection, "drop")?); + } + if let Some(collection) = command.get("snapshot") { + return self.snapshot_collection(required_str(collection, "snapshot")?); + } + if command.get("compact").is_some() { + return self.compact(); + } + if let Some(collection) = command.get("insert") { + let collection = required_str(collection, "insert")?.to_string(); + let encode = optional_encode_spec(command.get("encode"))?; + let return_ids = command + .get("returnIds") + .and_then(Value::as_bool) + .unwrap_or(true); + return self.insert_values_with_encode( + &collection, + required_array_value( + command + .remove("documents") + .ok_or(Error::MissingField("documents"))?, + "documents", + )?, + encode, + return_ids, + ); + } + if let Some(collection) = command.get("createIndexes") { + return self.create_indexes( + required_str(collection, "createIndexes")?, + required_array(command.get("indexes"), "indexes")?, + ); + } + if let Some(collection) = command.get("find") { + let collection = required_str(collection, "find")?.to_string(); + let filter = optional_object_value(command.remove("filter"), "filter")?; + let sort = optional_sort_value(command.remove("sort"))?; + let projection = optional_projection_value(command.remove("projection"))?; + return self.find_with_projection( + &collection, + filter, + command + .get("limit") + .and_then(Value::as_u64) + .map(|limit| limit as usize), + sort, + command + .get("pageToken") + .or_else(|| command.get("exclusiveStartKey")) + .and_then(page_offset), + projection, + ); + } + if let Some(collection) = command.get("count") { + let collection = required_str(collection, "count")?.to_string(); + let filter = optional_object_value(command.remove("filter"), "filter")?; + return self.count(&collection, filter); + } + if let Some(collection) = command.get("vectorSearch") { + let collection = required_str(collection, "vectorSearch")?.to_string(); + let filter = optional_object_value(command.remove("filter"), "filter")?; + return self.vector_search( + &collection, + command.get("index").and_then(Value::as_str), + command.get("field").and_then(Value::as_str), + required_vector(command.get("query"), "query")?, + command.get("k").and_then(Value::as_u64).unwrap_or(10) as usize, + filter, + ); + } + if let Some(collection) = command.get("semanticSearch") { + return self.semantic_search(parse_semantic_search_spec( + required_str(collection, "semanticSearch")?, + &command, + )?); + } + if let Some(collection) = command.get("update") { + let collection = required_str(collection, "update")?.to_string(); + let filter = optional_object_value(command.remove("filter"), "filter")?; + let updates = required_object_value( + command + .remove("updates") + .ok_or(Error::MissingField("updates"))?, + "updates", + )?; + return self.update(&collection, filter, updates); + } + if let Some(collection) = command.get("delete") { + let collection = required_str(collection, "delete")?.to_string(); + let filter = optional_object_value(command.remove("filter"), "filter")?; + return self.delete(&collection, filter); + } + if let Some(collection) = command.get("cleanupExpired") { + return self.cleanup_expired( + required_str(collection, "cleanupExpired")?, + command + .get("ttlField") + .and_then(Value::as_str) + .unwrap_or("ttl"), + command + .get("now") + .and_then(Value::as_i64) + .unwrap_or_else(now_epoch_seconds), + ); + } + + Err(Error::UnsupportedCommand) + } + + pub fn create_collection(&self, name: &str) -> Result { + let mut collections = self.collections.write(); + if collections.contains_key(name) { + return Err(Error::CollectionExists(name.to_string())); + } + + collections.insert(name.to_string(), CollectionState::empty()); + self.storage.save_collection(name, &[])?; + self.storage.append_create_collection(name)?; + + Ok(CommandResult::Created { + collection: name.to_string(), + }) + } + + pub fn drop_collection(&self, name: &str) -> Result { + let mut collections = self.collections.write(); + if collections.remove(name).is_none() { + return Err(Error::CollectionMissing(name.to_string())); + } + + self.storage.append_drop_collection(name)?; + self.storage.delete_collection(name)?; + self.save_catalog(&collections)?; + Ok(CommandResult::Dropped { + collection: name.to_string(), + }) + } + + pub fn snapshot_collection(&self, name: &str) -> Result { + let collections = self.collections.read(); + let collection = collections + .get(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + self.storage.append_snapshot(name, &collection.documents)?; + Ok(CommandResult::Snapshotted { + collection: name.to_string(), + count: collection.documents.len(), + }) + } + + pub fn compact(&self) -> Result { + let collections = self.collections.read(); + let views = collections + .iter() + .map(|(name, collection)| (name.clone(), collection.documents.clone())) + .collect::>(); + let last_seq = self.storage.compact_checkpoints(&views)?; + Ok(CommandResult::Compacted { + collections: views.len(), + last_seq, + }) + } + + pub fn create_encoder(&self, name: &str, spec: EncoderSpec) -> Result { + spec.validate()?; + let collections = self.collections.read(); + let mut encoders = self.encoders.write(); + if encoders.contains_key(name) { + return Err(Error::EncoderExists(name.to_string())); + } + encoders.insert(name.to_string(), spec); + let neural_spaces = self.neural_spaces.read(); + self.save_catalog_parts(&collections, &encoders, &neural_spaces)?; + Ok(CommandResult::EncoderCreated { + encoder: name.to_string(), + }) + } + + pub fn encode_text_command(&self, name: &str, text: &str) -> Result { + let encoders = self.encoders.read(); + let spec = encoders + .get(name) + .ok_or_else(|| Error::EncoderMissing(name.to_string()))?; + let vector = encode_text(spec, text); + Ok(CommandResult::Encoded { + encoder: name.to_string(), + dimensions: vector.len(), + vector, + }) + } + + pub fn create_neural_space( + &self, + name: &str, + encoder: &str, + dimensions: Option, + max_prototypes: usize, + ) -> Result { + let encoder_spec = self.encoder_spec(encoder)?; + let dimensions = dimensions.unwrap_or(encoder_spec.dimensions); + if dimensions != encoder_spec.dimensions { + return Err(Error::VectorDimensionMismatch { + field: "neuralSpace.dimensions".to_string(), + expected: encoder_spec.dimensions, + actual: dimensions, + }); + } + + let collections = self.collections.read(); + let encoders = self.encoders.read(); + let mut neural_spaces = self.neural_spaces.write(); + if neural_spaces.contains_key(name) { + return Err(Error::NeuralSpaceExists(name.to_string())); + } + neural_spaces.insert( + name.to_string(), + NeuralSpace::new(encoder, dimensions, max_prototypes), + ); + self.save_catalog_parts(&collections, &encoders, &neural_spaces)?; + Ok(CommandResult::NeuralSpaceCreated { + space: name.to_string(), + }) + } + + pub fn learn_text( + &self, + space: &str, + text: &str, + label: Option, + weight: f64, + ) -> Result { + let encoder = { + let spaces = self.neural_spaces.read(); + spaces + .get(space) + .ok_or_else(|| Error::NeuralSpaceMissing(space.to_string()))? + .encoder + .clone() + }; + let spec = self.encoder_spec(&encoder)?; + self.learn_vector(space, encode_text(&spec, text), label, weight) + } + + pub fn learn_vector( + &self, + space: &str, + vector: Vec, + label: Option, + weight: f64, + ) -> Result { + let collections = self.collections.read(); + let encoders = self.encoders.read(); + let mut neural_spaces = self.neural_spaces.write(); + let neural_space = neural_spaces + .get_mut(space) + .ok_or_else(|| Error::NeuralSpaceMissing(space.to_string()))?; + neural_space.learn(&vector, label, weight)?; + let prototypes = neural_space.prototypes.len(); + self.save_catalog_parts(&collections, &encoders, &neural_spaces)?; + Ok(CommandResult::Learned { + space: space.to_string(), + prototypes, + }) + } + + pub fn insert(&self, name: &str, documents: &[Value]) -> Result { + self.insert_with_encode(name, documents, None, true) + } + + fn insert_with_encode( + &self, + name: &str, + documents: &[Value], + encode: Option, + return_ids: bool, + ) -> Result { + self.insert_values_with_encode(name, documents.to_vec(), encode, return_ids) + } + + fn insert_values_with_encode( + &self, + name: &str, + documents: Vec, + encode: Option, + return_ids: bool, + ) -> Result { + let mut collections = self.collections.write(); + let collection = collections + .entry(name.to_string()) + .or_insert_with(CollectionState::empty); + let original_len = collection.documents.len(); + let document_count = documents.len(); + collection.documents.reserve(document_count); + let mut ids = return_ids.then(|| Vec::with_capacity(document_count)); + let encoder = encode + .as_ref() + .map(|encode| self.encoder_spec(&encode.encoder)) + .transpose()?; + + for document in documents { + let mut document = match document { + Value::Object(document) => document, + _ => return Err(Error::ExpectedObject("documents[]")), + }; + if let (Some(encode), Some(encoder)) = (&encode, &encoder) { + let source = document + .get(&encode.field) + .ok_or(Error::MissingField("encode.field source"))?; + let vector = encode_value(encoder, source)?; + document.insert( + encode.into.clone(), + Value::Array(vector.into_iter().map(Value::from).collect()), + ); + } + + let id = if let Some(id) = document.get("_id") { + let returned_id = ids.is_some().then(|| id.clone()); + let shard = self.shard_for(id); + document.insert("_shard".to_string(), Value::Number(Number::from(shard))); + returned_id + } else { + let generated = format!("{:016x}", kernel::next_id()); + let shard = self.shard_for_generated_string_id(&generated); + let returned_id = if ids.is_some() { + let id = Value::String(generated); + document.insert("_id".to_string(), id.clone()); + Some(id) + } else { + document.insert("_id".to_string(), Value::String(generated)); + None + }; + document.insert("_shard".to_string(), Value::Number(Number::from(shard))); + returned_id + }; + if let (Some(ids), Some(id)) = (&mut ids, id) { + ids.push(id); + } + let position = collection.documents.len(); + collection.documents.push(document); + if let Err(error) = collection.index_document(position) { + collection.documents.truncate(original_len); + collection.rebuild_indexes()?; + return Err(error); + } + } + + let inserted = &collection.documents[original_len..]; + self.storage.append_documents(name, inserted)?; + Ok(CommandResult::Inserted { + count: document_count, + ids, + }) + } + + pub fn create_indexes(&self, name: &str, specs: &[Value]) -> Result { + let mut collections = self.collections.write(); + let collection = collections + .get_mut(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + let mut names = Vec::with_capacity(specs.len()); + let mut parsed_specs = Vec::with_capacity(specs.len()); + + for spec in specs { + let spec = parse_index_spec(spec)?; + names.push(spec.name.clone()); + parsed_specs.push(spec); + } + collection.create_indexes(parsed_specs)?; + + self.save_catalog(&collections)?; + Ok(CommandResult::Indexed { + collection: name.to_string(), + indexes: names, + }) + } + + pub fn find( + &self, + name: &str, + filter: Option, + limit: Option, + sort: Option, + page_offset: Option, + ) -> Result { + self.find_with_projection(name, filter, limit, sort, page_offset, None) + } + + fn find_with_projection( + &self, + name: &str, + filter: Option, + limit: Option, + sort: Option, + page_offset: Option, + projection: Option, + ) -> Result { + let empty_filter = Document::new(); + let filter = filter.as_ref().unwrap_or(&empty_filter); + let limit = limit.unwrap_or(usize::MAX); + let projection = projection.as_ref(); + + let (result, used_index) = { + let collections = self.collections.read(); + let collection = collections + .get(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + + if let Some(sort) = &sort { + let requested_offset = page_offset.unwrap_or(0); + let requested_end = requested_offset.saturating_add(limit); + if let Some((positions, has_more, already_sorted)) = collection + .ordered_page_candidates(filter, &sort.field, sort.ascending, requested_end) + { + let mut sorted_positions = positions; + if !already_sorted { + sorted_positions.sort_by(|left, right| { + compare_positioned_documents(collection, *left, *right, sort) + }); + } + let offset = requested_offset.min(sorted_positions.len()); + let end = requested_end.min(sorted_positions.len()); + let last_evaluated_key = (has_more || end < sorted_positions.len()) + .then(|| json!({ "offset": end })); + let documents: Vec = sorted_positions[offset..end] + .iter() + .filter_map(|position| collection.documents.get(*position)) + .map(|document| project_document(document, projection)) + .collect(); + + ( + CommandResult::Found { + count: documents.len(), + documents, + last_evaluated_key, + }, + true, + ) + } else { + let candidates = collection.indexed_candidates_with_coverage(filter); + let filter_covered_by_index = candidates + .as_ref() + .is_some_and(|candidates| candidates.covered); + let compiled_filter = + (!filter_covered_by_index).then(|| compile_filter(filter)); + let positions: Box> = + if let Some(candidates) = candidates { + Box::new(candidates.positions.into_iter()) + } else { + Box::new(0..collection.documents.len()) + }; + let mut sorted_positions = positions + .filter(|position| { + collection.documents.get(*position).is_some_and(|document| { + compiled_filter + .as_ref() + .is_none_or(|filter| filter.matches(document)) + }) + }) + .collect::>(); + let offset = page_offset.unwrap_or(0).min(sorted_positions.len()); + let end = (offset + limit).min(sorted_positions.len()); + if end < sorted_positions.len() { + sorted_positions.select_nth_unstable_by(end, |left, right| { + compare_positioned_documents(collection, *left, *right, sort) + }); + } + sorted_positions[..end].sort_by(|left, right| { + compare_positioned_documents(collection, *left, *right, sort) + }); + let last_evaluated_key = + (end < sorted_positions.len()).then(|| json!({ "offset": end })); + let documents: Vec = sorted_positions[offset..end] + .iter() + .filter_map(|position| collection.documents.get(*position)) + .map(|document| project_document(document, projection)) + .collect(); + + ( + CommandResult::Found { + count: documents.len(), + documents, + last_evaluated_key, + }, + filter_covered_by_index, + ) + } + } else { + let offset = page_offset.unwrap_or(0); + let end = offset.saturating_add(limit); + if limit != usize::MAX { + if let Some((positions, has_more)) = + collection.indexed_page_candidates(filter, end) + { + let documents: Vec = positions[offset.min(positions.len())..] + .iter() + .filter_map(|position| collection.documents.get(*position)) + .map(|document| project_document(document, projection)) + .collect(); + let last_evaluated_key = has_more.then(|| json!({ "offset": end })); + return Ok(CommandResult::Found { + count: documents.len(), + documents, + last_evaluated_key, + }); + } + } + + let candidates = collection.indexed_candidates_with_coverage(filter); + let filter_covered_by_index = candidates + .as_ref() + .is_some_and(|candidates| candidates.covered); + let compiled_filter = (!filter_covered_by_index).then(|| compile_filter(filter)); + let positions: Box> = + if let Some(candidates) = candidates { + Box::new(candidates.positions.into_iter()) + } else { + Box::new(0..collection.documents.len()) + }; + let mut matched = 0; + let mut documents = Vec::new(); + let mut last_evaluated_key = None; + + for position in positions { + let Some(document) = collection.documents.get(position) else { + continue; + }; + if compiled_filter + .as_ref() + .is_some_and(|filter| !filter.matches(document)) + { + continue; + } + + let current = matched; + matched += 1; + if current >= offset && current < end { + documents.push(project_document(document, projection)); + } else if current >= end { + last_evaluated_key = Some(json!({ "offset": end })); + break; + } + } + + ( + CommandResult::Found { + count: documents.len(), + documents, + last_evaluated_key, + }, + filter_covered_by_index, + ) + } + }; + + if !used_index { + if let Some(fields) = auto_index_shape(filter) { + self.maybe_promote_auto_index(name, fields)?; + } + } + + Ok(result) + } + + pub fn count(&self, name: &str, filter: Option) -> Result { + let empty_filter = Document::new(); + let filter = filter.as_ref().unwrap_or(&empty_filter); + + let (count, used_index) = { + let collections = self.collections.read(); + let collection = collections + .get(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + + if filter.is_empty() { + (collection.documents.len(), true) + } else if let Some(count) = collection.indexed_candidate_count(filter) { + (count, true) + } else if let Some(candidates) = collection.indexed_candidates_with_coverage(filter) { + let compiled_filter = compile_filter(filter); + ( + candidates + .positions + .into_iter() + .filter(|position| { + collection + .documents + .get(*position) + .is_some_and(|document| compiled_filter.matches(document)) + }) + .count(), + false, + ) + } else { + let compiled_filter = compile_filter(filter); + ( + collection + .documents + .iter() + .filter(|document| compiled_filter.matches(document)) + .count(), + false, + ) + } + }; + + if !used_index { + if let Some(fields) = auto_index_shape(filter) { + self.maybe_promote_auto_index(name, fields)?; + } + } + + Ok(CommandResult::Counted { count }) + } + + pub fn vector_search( + &self, + name: &str, + index_name: Option<&str>, + field: Option<&str>, + query: Vec, + k: usize, + filter: Option, + ) -> Result { + let collections = self.collections.read(); + let collection = collections + .get(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + let empty_filter = Document::new(); + let filter = filter.as_ref().unwrap_or(&empty_filter); + let filter_is_empty = filter.is_empty(); + let compiled_filter = (!filter_is_empty).then(|| compile_filter(filter)); + let candidate_limit = filter_is_empty.then_some(k); + + let mut documents = Vec::with_capacity(k); + for (position, score) in + collection.neural_candidates(index_name, field, &query, candidate_limit)? + { + let Some(document) = collection.documents.get(position) else { + continue; + }; + if compiled_filter + .as_ref() + .is_some_and(|filter| !filter.matches(document)) + { + continue; + } + let mut document = document.clone(); + document.insert("_score".to_string(), json!(score)); + documents.push(document); + if documents.len() >= k { + break; + } + } + + Ok(CommandResult::Found { + count: documents.len(), + documents, + last_evaluated_key: None, + }) + } + + fn semantic_search(&self, request: SemanticSearchSpec) -> Result { + let query = self.adapted_query_vector(&request)?; + self.vector_search( + &request.collection, + request.index.as_deref(), + request.field.as_deref(), + query, + request.k, + request.filter, + ) + } + + pub fn update( + &self, + name: &str, + filter: Option, + updates: Document, + ) -> Result { + let mut collections = self.collections.write(); + let collection = collections + .get_mut(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + let empty_filter = Document::new(); + let filter = filter.as_ref().unwrap_or(&empty_filter); + let compiled_updates = CompiledUpdates::new(&updates); + let affects_indexes = collection.updates_affect_indexes(&compiled_updates); + if !affects_indexes { + let mut matched = 0; + let mut modified = 0; + if let Some(candidates) = collection.indexed_candidates_with_residual_hint(filter) { + let covered = candidates.covered; + let compiled_filter = (!covered).then(|| { + compile_filter_excluding(filter, candidates.residual_filter_excluded_field) + }); + for position in candidates.positions { + let Some(document) = collection.documents.get_mut(position) else { + continue; + }; + if compiled_filter + .as_ref() + .is_some_and(|filter| !filter.matches(document)) + { + continue; + } + matched += 1; + if apply_compiled_updates(document, &compiled_updates) { + modified += 1; + } + } + } else { + let compiled_filter = compile_filter(filter); + for document in &mut collection.documents { + if !compiled_filter.matches(document) { + continue; + } + matched += 1; + if apply_compiled_updates(document, &compiled_updates) { + modified += 1; + } + } + } + self.storage.append_update(name, filter, &updates)?; + return Ok(CommandResult::Updated { matched, modified }); + } + let can_update_indexes_in_place = + collection.updates_can_update_indexes_in_place(&compiled_updates); + let mut update_state = UpdateApplyState { + matched: 0, + modified: 0, + affects_indexes, + can_update_indexes_in_place, + in_place_changes: Vec::new(), + rollback_changes: Vec::new(), + }; + if let Some(candidates) = collection.indexed_candidates_with_residual_hint(filter) { + let covered = candidates.covered; + let candidate_count = candidates.positions.len(); + let compiled_filter = (!covered).then(|| { + compile_filter_excluding(filter, candidates.residual_filter_excluded_field) + }); + if update_state.affects_indexes { + if update_state.can_update_indexes_in_place { + update_state.in_place_changes.reserve(candidate_count); + } else { + update_state.rollback_changes.reserve(candidate_count); + } + } + for position in candidates.positions { + apply_update_at_position( + collection, + compiled_filter.as_ref(), + &compiled_updates, + position, + &mut update_state, + ); + } + } else { + let compiled_filter = compile_filter(filter); + for position in 0..collection.documents.len() { + apply_update_at_position( + collection, + Some(&compiled_filter), + &compiled_updates, + position, + &mut update_state, + ); + } + } + + if update_state.affects_indexes { + if update_state.can_update_indexes_in_place { + if let Err(error) = collection.update_documents_in_place( + update_state + .in_place_changes + .iter() + .map(|(position, old_document)| (*position, old_document)), + ) { + for (position, old_document) in update_state.in_place_changes { + if let Some(document) = collection.documents.get_mut(position) { + *document = old_document; + } + } + collection.rebuild_indexes()?; + return Err(error); + } + } else if let Err(error) = + collection.update_documents(update_state.rollback_changes.iter().map( + |(position, old_document, new_document)| { + (*position, old_document, new_document) + }, + )) + { + for (position, old_document, _) in update_state.rollback_changes { + if let Some(document) = collection.documents.get_mut(position) { + *document = old_document; + } + } + return Err(error); + } + } + self.storage.append_update(name, filter, &updates)?; + Ok(CommandResult::Updated { + matched: update_state.matched, + modified: update_state.modified, + }) + } + + pub fn delete(&self, name: &str, filter: Option) -> Result { + let mut collections = self.collections.write(); + let collection = collections + .get_mut(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + let empty_filter = Document::new(); + let filter = filter.as_ref().unwrap_or(&empty_filter); + let before = collection.documents.len(); + + let used_index = if filter.is_empty() { + collection.documents.clear(); + false + } else { + let compiled_filter = compile_filter(filter); + if let Some(candidates) = collection.indexed_candidates_with_coverage(filter) { + let covered = candidates.covered; + let mut deleted_positions = Vec::with_capacity(candidates.positions.len()); + for position in candidates.positions { + let Some(document) = collection.documents.get(position) else { + continue; + }; + if covered || compiled_filter.matches(document) { + deleted_positions.push(position); + } + } + deleted_positions.sort_unstable(); + deleted_positions.dedup(); + collection.delete_positions(&deleted_positions); + true + } else { + apply_delete_many_compiled(&mut collection.documents, &compiled_filter); + false + } + }; + let deleted = before - collection.documents.len(); + + if !used_index { + collection.rebuild_indexes()?; + } + self.storage.append_delete(name, filter)?; + Ok(CommandResult::Deleted { count: deleted }) + } + + pub fn cleanup_expired(&self, name: &str, ttl_field: &str, now: i64) -> Result { + let mut collections = self.collections.write(); + let collection = collections + .get_mut(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + let before = collection.documents.len(); + + apply_cleanup_expired(&mut collection.documents, ttl_field, now); + let deleted = before - collection.documents.len(); + + collection.rebuild_indexes()?; + self.storage.append_cleanup_expired(name, ttl_field, now)?; + Ok(CommandResult::Deleted { count: deleted }) + } + + #[doc(hidden)] + pub fn internal_auto_index_count(&self, name: &str) -> usize { + self.collections + .read() + .get(name) + .map(CollectionState::internal_exact_index_count) + .unwrap_or(0) + } + + fn maybe_promote_auto_index(&self, name: &str, fields: Vec) -> Result<()> { + let should_promote = { + let mut hits = self.auto_index_hits.lock(); + let count = hits + .entry(name.to_string()) + .or_default() + .entry(fields.clone()) + .or_default(); + *count += 1; + *count >= AUTO_INDEX_PROMOTION_THRESHOLD + }; + if !should_promote { + return Ok(()); + } + + let mut collections = self.collections.write(); + let collection = collections + .get_mut(name) + .ok_or_else(|| Error::CollectionMissing(name.to_string()))?; + if collection.has_exact_index_fields(&fields) + || collection.internal_exact_index_count() >= MAX_AUTO_INDEXES_PER_COLLECTION + { + return Ok(()); + } + collection.create_internal_exact_index(fields) + } + + fn shard_for(&self, id: &Value) -> u64 { + if let Some(id) = id.as_str() { + return self.shard_for_generated_string_id(id); + } + let raw = id.to_string(); + kernel::hash_bytes(raw.as_bytes()) % self.shard_count + } + + fn shard_for_generated_string_id(&self, id: &str) -> u64 { + hash_bytes_parts([b"\"", id.as_bytes(), b"\""]) % self.shard_count + } + + fn save_catalog(&self, collections: &BTreeMap) -> Result<()> { + let encoders = self.encoders.read(); + let neural_spaces = self.neural_spaces.read(); + self.save_catalog_parts(collections, &encoders, &neural_spaces) + } + + fn save_catalog_parts( + &self, + collections: &BTreeMap, + encoders: &BTreeMap, + neural_spaces: &BTreeMap, + ) -> Result<()> { + let catalog = Catalog { + collections: collections + .iter() + .map(|(name, collection)| { + ( + name.clone(), + CollectionCatalog { + indexes: collection.index_specs(), + }, + ) + }) + .collect(), + encoders: encoders.clone(), + neural_spaces: neural_spaces.clone(), + }; + self.storage.save_catalog(&catalog) + } + + fn encoder_spec(&self, name: &str) -> Result { + self.encoders + .read() + .get(name) + .cloned() + .ok_or_else(|| Error::EncoderMissing(name.to_string())) + } + + fn adapted_query_vector(&self, request: &SemanticSearchSpec) -> Result> { + if let Some(space_name) = &request.space { + let (encoder, vector) = { + let spaces = self.neural_spaces.read(); + let space = spaces + .get(space_name) + .ok_or_else(|| Error::NeuralSpaceMissing(space_name.clone()))?; + let encoder = space.encoder.clone(); + let spec = self.encoder_spec(&encoder)?; + let vector = space + .adapt_query(&encode_text(&spec, &request.text), request.label.as_deref())?; + (encoder, vector) + }; + if request + .encoder + .as_ref() + .is_some_and(|requested| requested != &encoder) + { + return Err(Error::UnsupportedEncoder(format!( + "semanticSearch encoder does not match neural space `{space_name}`" + ))); + } + return Ok(vector); + } + + let encoder = request + .encoder + .as_ref() + .ok_or(Error::MissingField("encoder"))?; + let spec = self.encoder_spec(encoder)?; + Ok(encode_text(&spec, &request.text)) + } +} + +#[derive(Debug, Clone)] +struct EncodeSpec { + encoder: String, + field: String, + into: String, +} + +#[derive(Debug, Clone)] +struct SemanticSearchSpec { + collection: String, + encoder: Option, + space: Option, + label: Option, + index: Option, + field: Option, + text: String, + k: usize, + filter: Option, +} + +struct UpdateApplyState { + matched: usize, + modified: usize, + affects_indexes: bool, + can_update_indexes_in_place: bool, + in_place_changes: Vec<(usize, Document)>, + rollback_changes: Vec<(usize, Document, Document)>, +} + +fn apply_update_at_position( + collection: &mut CollectionState, + compiled_filter: Option<&CompiledFilter<'_>>, + updates: &CompiledUpdates<'_>, + position: usize, + state: &mut UpdateApplyState, +) { + if position >= collection.documents.len() { + return; + } + if compiled_filter.is_some_and(|filter| !filter.matches(&collection.documents[position])) { + return; + } + + state.matched += 1; + if state.affects_indexes && !updates.would_change(&collection.documents[position]) { + return; + } + + let old_document = state + .affects_indexes + .then(|| collection.documents[position].clone()); + if !apply_compiled_updates(&mut collection.documents[position], updates) { + return; + } + + if state.affects_indexes { + let old_document = old_document.expect("old document captured for indexed update"); + if state.can_update_indexes_in_place { + state.in_place_changes.push((position, old_document)); + } else { + let new_document = collection.documents[position].clone(); + state + .rollback_changes + .push((position, old_document, new_document)); + } + } + state.modified += 1; +} + +#[derive(Debug, Clone)] +pub struct SortSpec { + pub field: String, + pub ascending: bool, +} + +fn required_str<'a>(value: &'a Value, field: &'static str) -> Result<&'a str> { + value.as_str().ok_or(Error::ExpectedObject(field)) +} + +fn required_array<'a>(value: Option<&'a Value>, field: &'static str) -> Result<&'a [Value]> { + value + .ok_or(Error::MissingField(field))? + .as_array() + .map(Vec::as_slice) + .ok_or(Error::ExpectedArray(field)) +} + +fn required_array_value(value: Value, field: &'static str) -> Result> { + match value { + Value::Array(values) => Ok(values), + _ => Err(Error::ExpectedArray(field)), + } +} + +fn required_object_value(value: Value, field: &'static str) -> Result { + match value { + Value::Object(object) => Ok(object), + _ => Err(Error::ExpectedObject(field)), + } +} + +fn required_vector(value: Option<&Value>, field: &'static str) -> Result> { + json_vector(value.ok_or(Error::MissingField(field))?).ok_or(Error::ExpectedArray(field)) +} + +fn optional_object(value: Option<&Value>, field: &'static str) -> Result> { + value + .map(|value| { + value + .as_object() + .cloned() + .ok_or(Error::ExpectedObject(field)) + }) + .transpose() +} + +fn optional_object_value(value: Option, field: &'static str) -> Result> { + value + .map(|value| required_object_value(value, field)) + .transpose() +} + +fn optional_sort(value: Option<&Value>) -> Result> { + let Some(value) = value else { + return Ok(None); + }; + + if let Some(object) = value.as_object() { + let (field, direction) = object.iter().next().ok_or(Error::MissingField("sort"))?; + return Ok(Some(SortSpec { + field: field.clone(), + ascending: direction.as_i64().unwrap_or(1) >= 0, + })); + } + + Err(Error::ExpectedObject("sort")) +} + +fn optional_sort_value(value: Option) -> Result> { + value + .as_ref() + .map_or(Ok(None), |value| optional_sort(Some(value))) +} + +fn optional_projection(value: Option<&Value>) -> Result> { + let Some(value) = value else { + return Ok(None); + }; + + let object = value + .as_object() + .ok_or(Error::ExpectedObject("projection"))?; + let fields = object + .iter() + .filter(|(_, include)| projection_includes_field(include)) + .map(|(field, _)| field.clone()) + .collect(); + Ok(Some(ProjectionSpec { fields })) +} + +fn optional_projection_value(value: Option) -> Result> { + value + .as_ref() + .map_or(Ok(None), |value| optional_projection(Some(value))) +} + +fn projection_includes_field(value: &Value) -> bool { + match value { + Value::Bool(value) => *value, + Value::Number(value) => value.as_i64().unwrap_or(0) != 0, + _ => false, + } +} + +fn project_document(document: &Document, projection: Option<&ProjectionSpec>) -> Document { + let Some(projection) = projection else { + return document.clone(); + }; + + let mut projected = Document::with_capacity(projection.fields.len()); + for field in &projection.fields { + if let Some(value) = document.get(field) { + projected.insert(field.clone(), value.clone()); + } + } + projected +} + +fn optional_encode_spec(value: Option<&Value>) -> Result> { + value.map(parse_encode_spec).transpose() +} + +fn parse_encode_spec(value: &Value) -> Result { + let spec = value.as_object().ok_or(Error::ExpectedObject("encode"))?; + Ok(EncodeSpec { + encoder: spec + .get("encoder") + .and_then(Value::as_str) + .ok_or(Error::MissingField("encode.encoder"))? + .to_string(), + field: spec + .get("field") + .and_then(Value::as_str) + .ok_or(Error::MissingField("encode.field"))? + .to_string(), + into: spec + .get("into") + .and_then(Value::as_str) + .ok_or(Error::MissingField("encode.into"))? + .to_string(), + }) +} + +fn auto_index_shape(filter: &Document) -> Option> { + if filter.is_empty() { + return None; + } + + if indexable_equality_value(filter, "pk").is_some() + && (indexable_equality_value(filter, "sk").is_some() + || indexable_prefix_value(filter, "sk").is_some()) + { + return Some(vec!["pk".to_string(), "sk".to_string()]); + } + + let mut fields = filter + .iter() + .filter_map(|(field, _)| indexable_equality_value(filter, field).map(|_| field.to_string())) + .collect::>(); + if fields.is_empty() { + return None; + } + fields.sort(); + Some(fields) +} + +fn indexable_equality_value<'a>(filter: &'a Document, field: &str) -> Option<&'a Value> { + let value = match filter.get(field)? { + Value::Object(operator) if operator.len() == 1 => operator.get("$eq")?, + Value::Object(_) => return None, + value => value, + }; + is_indexable_scalar(value).then_some(value) +} + +fn indexable_prefix_value<'a>(filter: &'a Document, field: &str) -> Option<&'a str> { + let prefix = filter.get(field)?.as_object()?.get("$prefix")?.as_str()?; + (prefix.len() <= 128).then_some(prefix) +} + +fn is_indexable_scalar(value: &Value) -> bool { + match value { + Value::Null | Value::Bool(_) | Value::Number(_) => true, + Value::String(value) => value.len() <= 128, + Value::Array(_) | Value::Object(_) => false, + } +} + +fn parse_semantic_search_spec( + collection: &str, + command: &serde_json::Map, +) -> Result { + let encoder = command + .get("encoder") + .and_then(Value::as_str) + .map(str::to_string); + let space = command + .get("space") + .and_then(Value::as_str) + .map(str::to_string); + if encoder.is_none() && space.is_none() { + return Err(Error::MissingField("encoder")); + } + + Ok(SemanticSearchSpec { + collection: collection.to_string(), + encoder, + space, + label: command + .get("label") + .and_then(Value::as_str) + .map(str::to_string), + index: command + .get("index") + .and_then(Value::as_str) + .map(str::to_string), + field: command + .get("field") + .and_then(Value::as_str) + .map(str::to_string), + text: required_str( + command.get("text").ok_or(Error::MissingField("text"))?, + "text", + )? + .to_string(), + k: command.get("k").and_then(Value::as_u64).unwrap_or(10) as usize, + filter: optional_object(command.get("filter"), "filter")?, + }) +} + +fn parse_encoder_spec(command: &serde_json::Map) -> Result { + let spec = EncoderSpec { + kind: command + .get("kind") + .and_then(Value::as_str) + .unwrap_or("text") + .to_string(), + provider: command + .get("provider") + .and_then(Value::as_str) + .unwrap_or(crate::encoder::DEFAULT_PROVIDER) + .to_string(), + dimensions: command + .get("dimensions") + .and_then(Value::as_u64) + .map(|dimensions| dimensions as usize) + .unwrap_or(384), + seed: command.get("seed").and_then(Value::as_u64).unwrap_or(0), + }; + spec.validate()?; + Ok(spec) +} + +fn page_offset(value: &Value) -> Option { + value.as_u64().map(|offset| offset as usize).or_else(|| { + value + .as_object()? + .get("offset")? + .as_u64() + .map(|offset| offset as usize) + }) +} + +fn compare_documents(left: &Document, right: &Document, sort: &SortSpec) -> Ordering { + let ordering = compare_values(left.get(&sort.field), right.get(&sort.field)) + .then_with(|| compare_values(left.get("_id"), right.get("_id"))); + if sort.ascending { + ordering + } else { + ordering.reverse() + } +} + +fn compare_positioned_documents( + collection: &CollectionState, + left: usize, + right: usize, + sort: &SortSpec, +) -> Ordering { + let left = collection + .documents + .get(left) + .expect("sorted position came from documents"); + let right = collection + .documents + .get(right) + .expect("sorted position came from documents"); + compare_documents(left, right, sort) +} + +fn compare_values(left: Option<&Value>, right: Option<&Value>) -> Ordering { + match (left, right) { + (None, None) => Ordering::Equal, + (None, Some(_)) => Ordering::Less, + (Some(_), None) => Ordering::Greater, + (Some(Value::Number(left)), Some(Value::Number(right))) => left + .as_f64() + .partial_cmp(&right.as_f64()) + .unwrap_or(Ordering::Equal), + (Some(Value::String(left)), Some(Value::String(right))) => left.cmp(right), + (Some(Value::Bool(left)), Some(Value::Bool(right))) => left.cmp(right), + (Some(left), Some(right)) => left.to_string().cmp(&right.to_string()), + } +} + +fn now_epoch_seconds() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} + +fn hash_bytes_parts(parts: [&[u8]; N]) -> u64 { + let mut hash = 1469598103934665603_u64; + for part in parts { + for byte in part { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(1099511628211_u64); + } + } + hash +} + +fn parse_index_spec(value: &Value) -> Result { + let spec = value + .as_object() + .ok_or(Error::ExpectedObject("indexes[]"))?; + + if let Some(key) = spec.get("key") { + let key = key.as_object().ok_or(Error::ExpectedObject("key"))?; + let fields = key.keys().cloned().collect::>(); + let field = fields.first().ok_or(Error::MissingField("key"))?; + let name = spec + .get("name") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or_else(|| { + fields + .iter() + .map(|field| format!("{field}_1")) + .collect::>() + .join("_") + }); + return Ok(IndexSpec { + name, + field: field.clone(), + fields: (fields.len() > 1).then_some(fields), + kind: IndexKind::Exact, + unique: spec.get("unique").and_then(Value::as_bool).unwrap_or(false), + dimensions: None, + }); + } + + if let Some(field) = spec.get("neural").and_then(Value::as_str) { + let dimensions = spec + .get("dimensions") + .and_then(Value::as_u64) + .map(|dimensions| dimensions as usize) + .ok_or(Error::MissingField("dimensions"))?; + let name = spec + .get("name") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or_else(|| format!("{field}_neural")); + return Ok(IndexSpec { + name, + field: field.to_string(), + fields: None, + kind: IndexKind::Neural, + unique: false, + dimensions: Some(dimensions), + }); + } + + Err(Error::MissingField("key")) +} diff --git a/vendor/nosqlite/src/index.rs b/vendor/nosqlite/src/index.rs new file mode 100644 index 00000000..7030236f --- /dev/null +++ b/vendor/nosqlite/src/index.rs @@ -0,0 +1,1543 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{mutation::CompiledUpdates, Document, Error, Result}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum ValueKey { + Null, + Bool(bool), + String(String), + I64(i64), + U64(u64), + F64(u64), + Json(String), +} + +impl ValueKey { + fn new(value: &Value) -> Self { + match value { + Value::Null => Self::Null, + Value::Bool(value) => Self::Bool(*value), + Value::String(value) => Self::String(value.clone()), + Value::Number(value) => { + if let Some(value) = value.as_i64() { + Self::I64(value) + } else if let Some(value) = value.as_u64() { + Self::U64(value) + } else if let Some(value) = value.as_f64() { + Self::F64(value.to_bits()) + } else { + Self::Json(value.to_string()) + } + } + Value::Array(_) | Value::Object(_) => { + Self::Json(serde_json::to_string(value).expect("serde_json::Value serializes")) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum IndexKind { + Exact, + Neural, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IndexSpec { + pub name: String, + pub field: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fields: Option>, + pub kind: IndexKind, + pub unique: bool, + pub dimensions: Option, +} + +#[derive(Debug)] +pub struct CollectionState { + pub documents: Vec, + indexes: IndexSet, +} + +pub struct IndexedCandidates<'a> { + pub positions: Vec, + pub covered: bool, + pub residual_filter_excluded_field: Option<&'a str>, +} + +impl CollectionState { + pub fn new(documents: Vec) -> Result { + let mut state = Self { + documents, + indexes: IndexSet::default(), + }; + state.indexes.add_primary(); + state.rebuild_indexes()?; + Ok(state) + } + + pub fn empty() -> Self { + Self { + documents: Vec::new(), + indexes: IndexSet::with_primary(), + } + } + + pub fn create_indexes(&mut self, specs: I) -> Result<()> + where + I: IntoIterator, + { + self.indexes + .create_and_append_indexed(specs, &self.documents) + } + + pub fn index_specs(&self) -> Vec { + self.indexes.specs() + } + + pub fn rebuild_indexes(&mut self) -> Result<()> { + self.indexes.rebuild(&self.documents) + } + + pub fn index_document(&mut self, position: usize) -> Result<()> { + let document = self + .documents + .get(position) + .ok_or(Error::MissingField("document"))?; + self.indexes.index_document(position, document) + } + + pub fn update_documents<'a, I>(&mut self, changes: I) -> Result<()> + where + I: IntoIterator, + { + let changes = changes.into_iter().collect::>(); + let mut indexes = self.indexes.clone(); + indexes.update_documents(changes)?; + self.indexes = indexes; + Ok(()) + } + + pub fn update_documents_in_place<'a, I>(&mut self, changes: I) -> Result<()> + where + I: IntoIterator, + { + let documents = &self.documents; + let indexes = &mut self.indexes; + for (position, old_document) in changes { + let Some(new_document) = documents.get(position) else { + continue; + }; + indexes.update_document_in_place(position, old_document, new_document)?; + } + Ok(()) + } + + pub fn updates_affect_indexes(&self, updates: &CompiledUpdates<'_>) -> bool { + self.indexes.updates_affect_indexes(updates) + } + + pub fn updates_can_update_indexes_in_place(&self, updates: &CompiledUpdates<'_>) -> bool { + self.indexes.updates_can_update_in_place(updates) + } + + pub fn indexed_candidates_with_coverage<'a>( + &'a self, + filter: &Document, + ) -> Option> { + self.indexes.exact_candidates_with_coverage(filter) + } + + pub fn indexed_candidates_with_residual_hint<'a>( + &'a self, + filter: &Document, + ) -> Option> { + self.indexes.exact_candidates_with_residual_hint(filter) + } + + pub fn indexed_candidate_count(&self, filter: &Document) -> Option { + self.indexes.exact_candidate_count(filter) + } + + pub fn indexed_page_candidates( + &self, + filter: &Document, + end: usize, + ) -> Option<(Vec, bool)> { + self.indexes.exact_page_candidates(filter, end) + } + + pub fn ordered_page_candidates( + &self, + filter: &Document, + sort_field: &str, + ascending: bool, + end: usize, + ) -> Option<(Vec, bool, bool)> { + self.indexes + .ordered_page_candidates(filter, sort_field, ascending, end) + } + + pub fn has_exact_index_fields(&self, fields: &[String]) -> bool { + self.indexes.has_exact_fields(fields) + } + + pub fn internal_exact_index_count(&self) -> usize { + self.indexes.internal_exact_count() + } + + pub fn create_internal_exact_index(&mut self, fields: Vec) -> Result<()> { + self.indexes + .create_internal_exact_indexed(fields, &self.documents) + } + + pub fn delete_positions(&mut self, positions: &[usize]) { + if positions.is_empty() { + return; + } + let remap = PositionRemap::new(self.documents.len(), positions); + if positions.len() <= 8 { + for position in positions.iter().rev() { + if *position < self.documents.len() { + self.documents.remove(*position); + } + } + self.indexes.remap_after_delete(&remap); + return; + } + + let mut deleted = positions.iter().copied().peekable(); + let mut position = 0; + self.documents.retain(|_| { + while deleted + .peek() + .is_some_and(|deleted_position| *deleted_position < position) + { + deleted.next(); + } + let remove = deleted + .peek() + .is_some_and(|deleted_position| *deleted_position == position); + if remove { + deleted.next(); + } + position += 1; + !remove + }); + self.indexes.remap_after_delete(&remap); + } + + pub fn neural_candidates( + &self, + index_name: Option<&str>, + field: Option<&str>, + query: &[f64], + limit: Option, + ) -> Result> { + self.indexes + .neural_candidates(index_name, field, query, limit) + } +} + +#[derive(Debug, Default, Clone)] +struct IndexSet { + exact: Vec, + neural: Vec, +} + +impl IndexSet { + fn with_primary() -> Self { + let mut indexes = Self::default(); + indexes.add_primary(); + indexes + } + + fn add_primary(&mut self) { + if self.exact.iter().any(|index| index.name == "_id_") { + return; + } + self.exact.push(ExactIndex::new("_id_", "_id", true)); + } + + fn create(&mut self, spec: IndexSpec) -> Result<()> { + if self.exact.iter().any(|index| index.name == spec.name) + || self.neural.iter().any(|index| index.name == spec.name) + { + return Err(Error::IndexExists(spec.name)); + } + + match spec.kind { + IndexKind::Exact => { + if let Some(fields) = spec.fields { + self.exact + .push(ExactIndex::new_compound(&spec.name, fields, spec.unique)); + } else { + self.exact + .push(ExactIndex::new(&spec.name, &spec.field, spec.unique)); + } + } + IndexKind::Neural => { + let dimensions = spec.dimensions.ok_or(Error::MissingField("dimensions"))?; + self.neural + .push(NeuralIndex::new(&spec.name, &spec.field, dimensions)); + } + } + + Ok(()) + } + + fn create_and_append_indexed(&mut self, specs: I, documents: &[Document]) -> Result<()> + where + I: IntoIterator, + { + let mut additions = IndexSet::default(); + for spec in specs { + if self.has_name(&spec.name) { + return Err(Error::IndexExists(spec.name)); + } + additions.create(spec)?; + } + additions.rebuild(documents)?; + self.exact.extend(additions.exact); + self.neural.extend(additions.neural); + Ok(()) + } + + fn has_name(&self, name: &str) -> bool { + self.exact.iter().any(|index| index.name == name) + || self.neural.iter().any(|index| index.name == name) + } + + fn rebuild(&mut self, documents: &[Document]) -> Result<()> { + for index in &mut self.exact { + index.clear(); + } + for index in &mut self.neural { + index.clear(); + } + + for (position, document) in documents.iter().enumerate() { + self.index_document(position, document)?; + } + + Ok(()) + } + + fn index_document(&mut self, position: usize, document: &Document) -> Result<()> { + for index in &mut self.exact { + index.index_document(position, document)?; + } + for index in &mut self.neural { + index.index_document(position, document)?; + } + Ok(()) + } + + fn update_documents<'a, I>(&mut self, changes: I) -> Result<()> + where + I: IntoIterator, + { + let changes = changes.into_iter().collect::>(); + for (position, old_document, _) in &changes { + for index in &mut self.exact { + index.remove_document(*position, old_document); + } + for index in &mut self.neural { + index.remove_document(*position, old_document); + } + } + + for (position, _, new_document) in changes { + for index in &mut self.exact { + index.index_document(position, new_document)?; + } + for index in &mut self.neural { + index.index_document(position, new_document)?; + } + } + Ok(()) + } + + fn update_document_in_place( + &mut self, + position: usize, + old_document: &Document, + new_document: &Document, + ) -> Result<()> { + for index in &mut self.exact { + index.update_document_in_place(position, old_document, new_document)?; + } + Ok(()) + } + + fn remap_after_delete(&mut self, remap: &PositionRemap<'_>) { + for index in &mut self.exact { + index.remap_after_delete(remap); + } + for index in &mut self.neural { + index.remap_after_delete(remap); + } + } + + fn specs(&self) -> Vec { + self.exact + .iter() + .filter(|index| index.name != "_id_" && !index.name.starts_with("__auto_")) + .map(ExactIndex::spec) + .chain(self.neural.iter().map(NeuralIndex::spec)) + .collect() + } + + fn exact_candidates_with_coverage<'a>( + &'a self, + filter: &Document, + ) -> Option> { + let index = self.best_exact_index(filter)?; + let positions = index.lookup_filter(filter)?; + let covered = page_filter_fields_covered(filter, &index.fields); + Some(IndexedCandidates { + positions, + covered, + residual_filter_excluded_field: None, + }) + } + + fn exact_candidates_with_residual_hint<'a>( + &'a self, + filter: &Document, + ) -> Option> { + let index = self.best_exact_index(filter)?; + let positions = index.lookup_filter(filter)?; + let covered = page_filter_fields_covered(filter, &index.fields); + let residual_filter_excluded_field = (!covered) + .then(|| index.single_equality_field(filter)) + .flatten(); + Some(IndexedCandidates { + positions, + covered, + residual_filter_excluded_field, + }) + } + + fn best_exact_index(&self, filter: &Document) -> Option<&ExactIndex> { + let mut best: Option<(usize, usize)> = None; + + for (index_position, index) in self.exact.iter().enumerate() { + let Some(candidate_len) = index.lookup_filter_len(filter) else { + continue; + }; + if best.is_none_or(|(_, existing_len)| candidate_len < existing_len) { + best = Some((index_position, candidate_len)); + if candidate_len == 0 { + break; + } + } + } + + let (index_position, _) = best?; + self.exact.get(index_position) + } + + fn exact_candidate_count(&self, filter: &Document) -> Option { + let mut best: Option = None; + + for index in &self.exact { + let Some(candidate_len) = index.lookup_covered_filter_len(filter) else { + continue; + }; + if best.is_none_or(|existing_len| candidate_len < existing_len) { + best = Some(candidate_len); + if candidate_len == 0 { + break; + } + } + } + + best + } + + fn exact_page_candidates(&self, filter: &Document, end: usize) -> Option<(Vec, bool)> { + for index in &self.exact { + if let Some(candidates) = index.lookup_filter_page(filter, end) { + return Some(candidates); + } + } + None + } + + fn ordered_page_candidates( + &self, + filter: &Document, + sort_field: &str, + ascending: bool, + end: usize, + ) -> Option<(Vec, bool, bool)> { + for index in &self.exact { + if let Some(candidates) = + index.ordered_page_candidates(filter, sort_field, ascending, end) + { + return Some(candidates); + } + } + None + } + + fn has_exact_fields(&self, fields: &[String]) -> bool { + self.exact.iter().any(|index| index.fields == fields) + } + + fn internal_exact_count(&self) -> usize { + self.exact + .iter() + .filter(|index| index.name.starts_with("__auto_")) + .count() + } + + fn create_internal_exact_indexed( + &mut self, + fields: Vec, + documents: &[Document], + ) -> Result<()> { + if fields.is_empty() || self.has_exact_fields(&fields) { + return Ok(()); + } + + let name = auto_index_name(&fields); + if self.exact.iter().any(|index| index.name == name) { + return Ok(()); + } + + let mut index = if fields.len() == 1 { + ExactIndex::new(&name, &fields[0], false) + } else { + ExactIndex::new_compound(&name, fields, false) + }; + index.rebuild(documents)?; + self.exact.push(index); + Ok(()) + } + + fn updates_affect_indexes(&self, updates: &CompiledUpdates<'_>) -> bool { + updates.fields().any(|field| { + self.exact + .iter() + .any(|index| index.fields.iter().any(|indexed| indexed == field)) + || self.neural.iter().any(|index| index.field == field) + }) + } + + fn updates_can_update_in_place(&self, updates: &CompiledUpdates<'_>) -> bool { + !updates.fields().any(|field| { + self.exact + .iter() + .any(|index| index.unique && index.fields.iter().any(|indexed| indexed == field)) + || self.neural.iter().any(|index| index.field == field) + }) + } + + fn neural_candidates( + &self, + index_name: Option<&str>, + field: Option<&str>, + query: &[f64], + limit: Option, + ) -> Result> { + let index = self + .neural + .iter() + .find(|index| { + index_name.is_none_or(|name| index.name == name) + && field.is_none_or(|field| index.field == field) + }) + .ok_or_else(|| { + Error::IndexMissing(index_name.or(field).unwrap_or("").to_string()) + })?; + + index.search(query, limit) + } +} + +fn auto_index_name(fields: &[String]) -> String { + let fields = fields + .iter() + .map(|field| { + field + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => ch, + _ => '_', + }) + .collect::() + }) + .collect::>() + .join("_"); + format!("__auto_{fields}") +} + +#[derive(Debug, Clone)] +struct ExactIndex { + name: String, + field: String, + fields: Vec, + unique: bool, + entries: ExactEntries, +} + +#[derive(Debug, Clone)] +enum ExactEntries { + Unique(HashMap), + Multi(HashMap>), + Compound(BTreeMap, Vec>), +} + +impl ExactIndex { + fn new(name: &str, field: &str, unique: bool) -> Self { + Self { + name: name.to_string(), + field: field.to_string(), + fields: vec![field.to_string()], + unique, + entries: if unique { + ExactEntries::Unique(HashMap::new()) + } else { + ExactEntries::Multi(HashMap::new()) + }, + } + } + + fn new_compound(name: &str, fields: Vec, unique: bool) -> Self { + let field = fields.first().cloned().unwrap_or_default(); + Self { + name: name.to_string(), + field, + fields, + unique, + entries: ExactEntries::Compound(BTreeMap::new()), + } + } + + fn rebuild(&mut self, documents: &[Document]) -> Result<()> { + self.clear(); + for (position, document) in documents.iter().enumerate() { + self.index_document(position, document)?; + } + Ok(()) + } + + fn clear(&mut self) { + self.entries.clear(); + } + + fn index_document(&mut self, position: usize, document: &Document) -> Result<()> { + let fields = &self.fields; + match &mut self.entries { + ExactEntries::Unique(entries) => { + let Some(key) = document.get(&self.field).map(ValueKey::new) else { + return Ok(()); + }; + if entries.insert(key, position).is_some() { + return Err(Error::UniqueIndexViolation { + index: self.name.clone(), + field: self.field.clone(), + }); + } + } + ExactEntries::Multi(entries) => { + let Some(key) = document.get(&self.field).map(ValueKey::new) else { + return Ok(()); + }; + insert_sorted_position(entries.entry(key).or_default(), position); + } + ExactEntries::Compound(entries) => { + let Some(key) = document_key(fields, document) else { + return Ok(()); + }; + let positions = entries.entry(key).or_default(); + if self.unique && !positions.is_empty() { + return Err(Error::UniqueIndexViolation { + index: self.name.clone(), + field: self.fields.join(","), + }); + } + insert_sorted_position(positions, position); + } + } + Ok(()) + } + + fn lookup_filter(&self, filter: &Document) -> Option> { + match &self.entries { + ExactEntries::Unique(entries) => { + let value = equality_filter_value(filter, &self.field)?; + Some( + entries + .get(&ValueKey::new(value)) + .copied() + .into_iter() + .collect(), + ) + } + ExactEntries::Multi(entries) => { + let value = equality_filter_value(filter, &self.field)?; + Some( + entries + .get(&ValueKey::new(value)) + .cloned() + .unwrap_or_default(), + ) + } + ExactEntries::Compound(entries) => self.lookup_compound(filter, entries), + } + } + + fn lookup_filter_len(&self, filter: &Document) -> Option { + match &self.entries { + ExactEntries::Unique(entries) => { + let value = equality_filter_value(filter, &self.field)?; + Some(usize::from(entries.contains_key(&ValueKey::new(value)))) + } + ExactEntries::Multi(entries) => { + let value = equality_filter_value(filter, &self.field)?; + Some(entries.get(&ValueKey::new(value)).map_or(0, Vec::len)) + } + ExactEntries::Compound(entries) => self.lookup_compound_len(filter, entries), + } + } + + fn single_equality_field<'a>(&'a self, filter: &Document) -> Option<&'a str> { + match &self.entries { + ExactEntries::Unique(_) | ExactEntries::Multi(_) => { + page_equality_filter_value(filter, &self.field)?; + Some(&self.field) + } + ExactEntries::Compound(_) => None, + } + } + + fn lookup_covered_filter_len(&self, filter: &Document) -> Option { + if !page_filter_fields_covered(filter, &self.fields) { + return None; + } + self.lookup_filter_len(filter) + } + + fn lookup_filter_page(&self, filter: &Document, end: usize) -> Option<(Vec, bool)> { + if !page_filter_fields_covered(filter, &self.fields) { + return None; + } + + match &self.entries { + ExactEntries::Unique(entries) => { + let value = page_equality_filter_value(filter, &self.field)?; + let positions = entries + .get(&ValueKey::new(value)) + .copied() + .into_iter() + .collect(); + Some((positions, false)) + } + ExactEntries::Multi(entries) => { + let value = page_equality_filter_value(filter, &self.field)?; + let Some(positions) = entries.get(&ValueKey::new(value)) else { + return Some((Vec::new(), false)); + }; + Some(take_positions_page(positions.iter().copied(), end)) + } + ExactEntries::Compound(entries) => self.lookup_compound_page(filter, entries, end), + } + } + + fn ordered_page_candidates( + &self, + filter: &Document, + sort_field: &str, + ascending: bool, + end: usize, + ) -> Option<(Vec, bool, bool)> { + if !ascending + || self.fields.last()? != sort_field + || !filter_fields_covered(filter, &self.fields) + { + return None; + } + let ExactEntries::Compound(entries) = &self.entries else { + return None; + }; + let (prefix, string_prefix) = compound_order_prefix(filter, &self.fields)?; + let mut positions = Vec::new(); + let mut tie_free = true; + for (key, group) in entries + .range(prefix.clone()..) + .take_while(|(key, _)| compound_key_has_prefix(key, &prefix)) + .filter(|(key, _)| { + string_prefix.as_deref().is_none_or(|string_prefix| { + compound_key_matches_string_prefix(key, &prefix, string_prefix) + }) + }) + { + let _ = key; + tie_free &= group.len() <= 1; + positions.extend(group.iter().copied()); + if positions.len() > end { + return Some((positions, true, tie_free)); + } + } + Some((positions, false, tie_free)) + } + + fn remove_document(&mut self, position: usize, document: &Document) { + let fields = &self.fields; + match &mut self.entries { + ExactEntries::Unique(entries) => { + let Some(key) = document.get(&self.field).map(ValueKey::new) else { + return; + }; + if entries.get(&key).copied() == Some(position) { + entries.remove(&key); + } + } + ExactEntries::Multi(entries) => { + let Some(key) = document.get(&self.field).map(ValueKey::new) else { + return; + }; + if let Some(positions) = entries.get_mut(&key) { + remove_sorted_position(positions, position); + if positions.is_empty() { + entries.remove(&key); + } + } + } + ExactEntries::Compound(entries) => { + let Some(key) = document_key(fields, document) else { + return; + }; + if let Some(positions) = entries.get_mut(&key) { + remove_sorted_position(positions, position); + if positions.is_empty() { + entries.remove(&key); + } + } + } + } + } + + fn update_document_in_place( + &mut self, + position: usize, + old_document: &Document, + new_document: &Document, + ) -> Result<()> { + if self.document_keys_equal(old_document, new_document) { + return Ok(()); + } + self.remove_document(position, old_document); + self.index_document(position, new_document) + } + + fn document_keys_equal(&self, left: &Document, right: &Document) -> bool { + match &self.entries { + ExactEntries::Unique(_) | ExactEntries::Multi(_) => { + scalar_document_key(&self.field, left) == scalar_document_key(&self.field, right) + } + ExactEntries::Compound(_) => { + document_key(&self.fields, left) == document_key(&self.fields, right) + } + } + } + + fn remap_after_delete(&mut self, remap: &PositionRemap<'_>) { + match &mut self.entries { + ExactEntries::Unique(entries) => { + let mut remove_keys = Vec::new(); + for (key, position) in entries.iter_mut() { + if let Some(remapped) = remap.position(*position) { + *position = remapped; + } else { + remove_keys.push(key.clone()); + } + } + for key in remove_keys { + entries.remove(&key); + } + } + ExactEntries::Multi(entries) => { + let mut remove_keys = Vec::new(); + for (key, positions) in entries.iter_mut() { + remap_positions_after_delete(positions, remap); + if positions.is_empty() { + remove_keys.push(key.clone()); + } + } + for key in remove_keys { + entries.remove(&key); + } + } + ExactEntries::Compound(entries) => { + let mut remove_keys = Vec::new(); + for (key, positions) in entries.iter_mut() { + remap_positions_after_delete(positions, remap); + if positions.is_empty() { + remove_keys.push(key.clone()); + } + } + for key in remove_keys { + entries.remove(&key); + } + } + } + } + + fn lookup_compound( + &self, + filter: &Document, + entries: &BTreeMap, Vec>, + ) -> Option> { + let (prefix, string_prefix) = compound_filter_prefix(filter, &self.fields)?; + if let Some(string_prefix) = string_prefix { + return Some( + entries + .range(prefix.clone()..) + .take_while(|(key, _)| compound_key_has_prefix(key, &prefix)) + .filter(|(key, _)| { + compound_key_matches_string_prefix(key, &prefix, &string_prefix) + }) + .flat_map(|(_, positions)| positions.iter().copied()) + .collect(), + ); + } + + Some(entries.get(&prefix).cloned().unwrap_or_default()) + } + + fn lookup_compound_len( + &self, + filter: &Document, + entries: &BTreeMap, Vec>, + ) -> Option { + let (prefix, string_prefix) = compound_filter_prefix(filter, &self.fields)?; + if let Some(string_prefix) = string_prefix { + return Some( + entries + .range(prefix.clone()..) + .take_while(|(key, _)| compound_key_has_prefix(key, &prefix)) + .filter(|(key, _)| { + compound_key_matches_string_prefix(key, &prefix, &string_prefix) + }) + .map(|(_, positions)| positions.len()) + .sum(), + ); + } + + Some(entries.get(&prefix).map_or(0, Vec::len)) + } + + fn lookup_compound_page( + &self, + filter: &Document, + entries: &BTreeMap, Vec>, + end: usize, + ) -> Option<(Vec, bool)> { + let (prefix, string_prefix) = compound_page_filter_prefix(filter, &self.fields)?; + if let Some(string_prefix) = string_prefix { + let mut positions = Vec::new(); + for (_, group) in entries + .range(prefix.clone()..) + .take_while(|(key, _)| compound_key_has_prefix(key, &prefix)) + .filter(|(key, _)| compound_key_matches_string_prefix(key, &prefix, &string_prefix)) + { + positions.extend(group.iter().copied()); + if positions.len() > end { + positions.truncate(end); + return Some((positions, true)); + } + } + return Some((positions, false)); + } + + let Some(positions) = entries.get(&prefix) else { + return Some((Vec::new(), false)); + }; + Some(take_positions_page(positions.iter().copied(), end)) + } + + fn spec(&self) -> IndexSpec { + IndexSpec { + name: self.name.clone(), + field: self.field.clone(), + fields: (self.fields.len() > 1).then(|| self.fields.clone()), + kind: IndexKind::Exact, + unique: self.unique, + dimensions: None, + } + } +} + +impl ExactEntries { + fn clear(&mut self) { + match self { + ExactEntries::Unique(entries) => entries.clear(), + ExactEntries::Multi(entries) => entries.clear(), + ExactEntries::Compound(entries) => entries.clear(), + } + } +} + +#[derive(Debug, Clone)] +struct NeuralIndex { + name: String, + field: String, + dimensions: usize, + planes: Vec>, + buckets: BTreeMap>, + vectors: Vec>>, +} + +impl NeuralIndex { + const PLANES: usize = 16; + + fn new(name: &str, field: &str, dimensions: usize) -> Self { + Self { + name: name.to_string(), + field: field.to_string(), + dimensions, + planes: projection_planes(name.as_bytes(), dimensions), + buckets: BTreeMap::new(), + vectors: Vec::new(), + } + } + + fn clear(&mut self) { + self.buckets.clear(); + self.vectors.clear(); + } + + fn index_document(&mut self, position: usize, document: &Document) -> Result<()> { + let Some(value) = document.get(&self.field) else { + return Ok(()); + }; + let vector = json_vector(value).ok_or(Error::ExpectedArray("vector field"))?; + if vector.len() != self.dimensions { + return Err(Error::VectorDimensionMismatch { + field: self.field.clone(), + expected: self.dimensions, + actual: vector.len(), + }); + } + let signature = self.signature(&vector); + self.buckets.entry(signature).or_default().insert(position); + if position >= self.vectors.len() { + self.vectors.resize_with(position + 1, || None); + } + self.vectors[position] = Some(normalized_vector(vector)); + Ok(()) + } + + fn remove_document(&mut self, position: usize, document: &Document) { + let Some(value) = document.get(&self.field) else { + return; + }; + let Some(vector) = json_vector(value) else { + if let Some(vector) = self.vectors.get_mut(position) { + *vector = None; + } + return; + }; + if vector.len() == self.dimensions { + let signature = self.signature(&vector); + if let Some(bucket) = self.buckets.get_mut(&signature) { + bucket.remove(&position); + if bucket.is_empty() { + self.buckets.remove(&signature); + } + } + } + if let Some(vector) = self.vectors.get_mut(position) { + *vector = None; + } + } + + fn remap_after_delete(&mut self, remap: &PositionRemap<'_>) { + self.vectors = std::mem::take(&mut self.vectors) + .into_iter() + .enumerate() + .filter_map(|(position, vector)| remap.position(position).map(|_| vector)) + .collect(); + self.buckets = self + .buckets + .iter() + .filter_map(|(signature, positions)| { + let positions = positions + .iter() + .filter_map(|position| remap.position(*position)) + .collect::>(); + (!positions.is_empty()).then_some((*signature, positions)) + }) + .collect(); + } + + fn search(&self, query: &[f64], limit: Option) -> Result> { + if query.len() != self.dimensions { + return Err(Error::VectorDimensionMismatch { + field: self.field.clone(), + expected: self.dimensions, + actual: query.len(), + }); + } + + let query_norm = vector_norm(query); + let signature = self.signature(query); + let mut candidates = self + .buckets + .get(&signature) + .map(|bucket| bucket.iter().copied().collect::>()) + .unwrap_or_default(); + let mut needs_dedup = false; + + // Projection buckets are a route, not a truth oracle. If the exact bucket is sparse, + // widen by Hamming distance so small datasets still behave predictably. + if candidates.len() < 32 { + for bit in 0..Self::PLANES { + if candidates.len() >= 32 { + break; + } + if let Some(bucket) = self.buckets.get(&(signature ^ (1 << bit))) { + candidates.extend(bucket.iter().copied()); + needs_dedup = true; + } + } + } + + if candidates.len() < 32 { + let mut scored = Vec::with_capacity(limit.unwrap_or(self.vectors.len())); + for (position, vector) in self.vectors.iter().enumerate() { + let Some(vector) = vector else { + continue; + }; + let candidate = (position, normalized_query_dot(query, query_norm, vector)); + if let Some(limit) = limit { + push_top_scored_candidate(&mut scored, candidate, limit); + } else { + scored.push(candidate); + } + } + return Ok(rank_scored_candidates(scored, limit)); + } + + if needs_dedup { + candidates.sort_unstable(); + candidates.dedup(); + } + + let scored = candidates + .into_iter() + .filter_map(|position| { + self.vectors + .get(position) + .and_then(Option::as_ref) + .map(|vector| (position, normalized_query_dot(query, query_norm, vector))) + }) + .collect::>(); + Ok(rank_scored_candidates(scored, limit)) + } + + fn signature(&self, vector: &[f64]) -> u64 { + let mut signature = 0_u64; + for (plane, projection) in self.planes.iter().enumerate() { + let dot = dot_product(vector, projection); + if dot >= 0.0 { + signature |= 1 << plane; + } + } + signature + } + + fn spec(&self) -> IndexSpec { + IndexSpec { + name: self.name.clone(), + field: self.field.clone(), + fields: None, + kind: IndexKind::Neural, + unique: false, + dimensions: Some(self.dimensions), + } + } +} + +fn equality_filter_value<'a>(filter: &'a Document, field: &str) -> Option<&'a Value> { + match filter.get(field)? { + Value::Object(operator) => operator.get("$eq"), + value => Some(value), + } +} + +fn prefix_filter_value<'a>(filter: &'a Document, field: &str) -> Option<&'a str> { + filter.get(field)?.as_object()?.get("$prefix")?.as_str() +} + +fn page_equality_filter_value<'a>(filter: &'a Document, field: &str) -> Option<&'a Value> { + match filter.get(field)? { + Value::Object(operator) if operator.len() == 1 => operator.get("$eq"), + Value::Object(_) => None, + value => Some(value), + } +} + +fn page_prefix_filter_value<'a>(filter: &'a Document, field: &str) -> Option<&'a str> { + let operator = filter.get(field)?.as_object()?; + (operator.len() == 1) + .then(|| operator.get("$prefix")?.as_str()) + .flatten() +} + +fn scalar_document_key(field: &str, document: &Document) -> Option { + document.get(field).map(ValueKey::new) +} + +fn document_key(fields: &[String], document: &Document) -> Option> { + fields + .iter() + .map(|field| document.get(field).map(ValueKey::new)) + .collect() +} + +fn compound_filter_prefix( + filter: &Document, + fields: &[String], +) -> Option<(Vec, Option)> { + if fields.is_empty() { + return None; + } + + let mut prefix = Vec::with_capacity(fields.len()); + for (index, field) in fields.iter().enumerate() { + if let Some(value) = equality_filter_value(filter, field) { + prefix.push(ValueKey::new(value)); + continue; + } + if index == fields.len() - 1 { + return Some(( + prefix, + Some(prefix_filter_value(filter, field)?.to_string()), + )); + } + return None; + } + + Some((prefix, None)) +} + +fn compound_page_filter_prefix( + filter: &Document, + fields: &[String], +) -> Option<(Vec, Option)> { + if fields.is_empty() { + return None; + } + + let mut prefix = Vec::with_capacity(fields.len()); + for (index, field) in fields.iter().enumerate() { + if let Some(value) = page_equality_filter_value(filter, field) { + prefix.push(ValueKey::new(value)); + continue; + } + if index == fields.len() - 1 { + return Some(( + prefix, + Some(page_prefix_filter_value(filter, field)?.to_string()), + )); + } + return None; + } + + Some((prefix, None)) +} + +fn compound_order_prefix( + filter: &Document, + fields: &[String], +) -> Option<(Vec, Option)> { + if fields.len() < 2 { + return None; + } + + let mut prefix = Vec::with_capacity(fields.len() - 1); + for field in &fields[..fields.len() - 1] { + prefix.push(ValueKey::new(equality_filter_value(filter, field)?)); + } + + let sort_field = fields.last()?; + if let Some(value) = equality_filter_value(filter, sort_field) { + prefix.push(ValueKey::new(value)); + return Some((prefix, None)); + } + let string_prefix = prefix_filter_value(filter, sort_field).map(str::to_string); + if filter.get(sort_field).is_some() && string_prefix.is_none() { + return None; + } + Some((prefix, string_prefix)) +} + +fn filter_fields_covered(filter: &Document, fields: &[String]) -> bool { + filter + .keys() + .all(|field| fields.iter().any(|indexed| indexed == field)) +} + +fn page_filter_fields_covered(filter: &Document, fields: &[String]) -> bool { + filter.iter().all(|(field, _)| { + fields.iter().any(|indexed| indexed == field) + && (page_equality_filter_value(filter, field).is_some() + || page_prefix_filter_value(filter, field).is_some()) + }) +} + +fn compound_key_has_prefix(key: &[ValueKey], prefix: &[ValueKey]) -> bool { + key.starts_with(prefix) +} + +fn compound_key_matches_string_prefix( + key: &[ValueKey], + prefix: &[ValueKey], + string_prefix: &str, +) -> bool { + if key.len() != prefix.len() + 1 || !compound_key_has_prefix(key, prefix) { + return false; + } + matches!( + key.last(), + Some(ValueKey::String(value)) if value.starts_with(string_prefix) + ) +} + +fn take_positions_page(positions: impl Iterator, end: usize) -> (Vec, bool) { + let mut page = Vec::with_capacity(end); + for position in positions { + if page.len() == end { + return (page, true); + } + page.push(position); + } + (page, false) +} + +enum PositionRemap<'a> { + Single(usize), + Search(&'a [usize]), + Dense(Vec), +} + +impl<'a> PositionRemap<'a> { + const DELETED: usize = usize::MAX; + + fn new(old_len: usize, deleted_positions: &'a [usize]) -> Self { + if deleted_positions.len() == 1 { + return Self::Single(deleted_positions[0]); + } + if deleted_positions.len() <= 8 { + return Self::Search(deleted_positions); + } + + let mut deleted = deleted_positions.iter().copied().peekable(); + let mut shift = 0; + let mut positions = Vec::with_capacity(old_len); + for position in 0..old_len { + while deleted + .peek() + .is_some_and(|deleted_position| *deleted_position < position) + { + deleted.next(); + shift += 1; + } + if deleted + .peek() + .is_some_and(|deleted_position| *deleted_position == position) + { + deleted.next(); + shift += 1; + positions.push(Self::DELETED); + } else { + positions.push(position - shift); + } + } + Self::Dense(positions) + } + + fn position(&self, position: usize) -> Option { + match self { + Self::Single(deleted) => { + if position == *deleted { + None + } else if position > *deleted { + Some(position - 1) + } else { + Some(position) + } + } + Self::Search(deleted_positions) => match deleted_positions.binary_search(&position) { + Ok(_) => None, + Err(shift) => Some(position - shift), + }, + Self::Dense(positions) => positions + .get(position) + .copied() + .filter(|position| *position != Self::DELETED), + } + } +} + +fn remap_positions_after_delete(positions: &mut Vec, remap: &PositionRemap<'_>) { + positions.retain_mut(|position| { + let Some(remapped) = remap.position(*position) else { + return false; + }; + *position = remapped; + true + }); +} + +fn remove_sorted_position(positions: &mut Vec, position: usize) { + if let Ok(index) = positions.binary_search(&position) { + positions.remove(index); + } +} + +fn insert_sorted_position(positions: &mut Vec, position: usize) { + if positions.last().is_none_or(|last| *last < position) { + positions.push(position); + return; + } + + match positions.binary_search(&position) { + Ok(_) => {} + Err(index) => positions.insert(index, position), + } +} + +fn compare_scored_candidates(left: &(usize, f64), right: &(usize, f64)) -> std::cmp::Ordering { + right + .1 + .partial_cmp(&left.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| left.0.cmp(&right.0)) +} + +fn rank_scored_candidates( + mut scored: Vec<(usize, f64)>, + limit: Option, +) -> Vec<(usize, f64)> { + if let Some(limit) = limit { + if limit < scored.len() { + scored.select_nth_unstable_by(limit, compare_scored_candidates); + scored.truncate(limit); + } + } + scored.sort_by(compare_scored_candidates); + scored +} + +fn push_top_scored_candidate( + scored: &mut Vec<(usize, f64)>, + candidate: (usize, f64), + limit: usize, +) { + if limit == 0 { + return; + } + if scored.len() < limit { + scored.push(candidate); + return; + } + let Some((worst_index, worst)) = scored + .iter() + .enumerate() + .max_by(|(_, left), (_, right)| compare_scored_candidates(left, right)) + else { + return; + }; + if compare_scored_candidates(&candidate, worst).is_lt() { + scored[worst_index] = candidate; + } +} + +pub fn json_vector(value: &Value) -> Option> { + value + .as_array()? + .iter() + .map(Value::as_f64) + .collect::>>() +} + +fn normalized_vector(mut vector: Vec) -> Vec { + let norm = vector_norm(&vector); + if norm == 0.0 { + return vector; + } + for value in &mut vector { + *value /= norm; + } + vector +} + +fn vector_norm(vector: &[f64]) -> f64 { + let mut sum = 0.0; + for value in vector { + sum += value * value; + } + sum.sqrt() +} + +fn normalized_query_dot(query: &[f64], query_norm: f64, stored_normalized: &[f64]) -> f64 { + if query_norm == 0.0 { + return 0.0; + } + dot_product(query, stored_normalized) / query_norm +} + +fn dot_product(left: &[f64], right: &[f64]) -> f64 { + let len = left.len().min(right.len()); + let mut sum = 0.0; + let mut index = 0; + while index < len { + sum += left[index] * right[index]; + index += 1; + } + sum +} + +fn projection_planes(index_seed: &[u8], dimensions: usize) -> Vec> { + (0..NeuralIndex::PLANES) + .map(|plane| { + (0..dimensions) + .map(|dimension| projection(index_seed, plane, dimension)) + .collect() + }) + .collect() +} + +fn projection(index_seed: &[u8], plane: usize, dimension: usize) -> f64 { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + for byte in index_seed { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash ^= (plane as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); + hash ^= (dimension as u64).wrapping_mul(0xbf58_476d_1ce4_e5b9); + hash = splitmix64(hash); + if hash & 1 == 0 { + -1.0 + } else { + 1.0 + } +} + +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} diff --git a/vendor/nosqlite/src/kernel.rs b/vendor/nosqlite/src/kernel.rs new file mode 100644 index 00000000..a3b5e5e9 --- /dev/null +++ b/vendor/nosqlite/src/kernel.rs @@ -0,0 +1,14 @@ +use std::ffi::c_uchar; + +extern "C" { + fn nosqlite_fnv1a64(data: *const c_uchar, len: usize) -> u64; + fn nosqlite_next_id() -> u64; +} + +pub fn hash_bytes(bytes: &[u8]) -> u64 { + unsafe { nosqlite_fnv1a64(bytes.as_ptr(), bytes.len()) } +} + +pub fn next_id() -> u64 { + unsafe { nosqlite_next_id() } +} diff --git a/vendor/nosqlite/src/lib.rs b/vendor/nosqlite/src/lib.rs new file mode 100644 index 00000000..2826c006 --- /dev/null +++ b/vendor/nosqlite/src/lib.rs @@ -0,0 +1,61 @@ +mod encoder; +mod engine; +mod index; +mod kernel; +mod mutation; +mod neural; +mod query; +mod storage; + +pub use encoder::{encode_text, EncoderSpec}; +pub use engine::{CommandResult, Engine, EngineOptions, SortSpec, StorageMode}; +pub use mutation::apply_updates; +pub use neural::{NeuralSpace, Prototype}; +pub use query::matches_filter; +pub use storage::Durability; + +pub type Document = serde_json::Map; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("collection `{0}` already exists")] + CollectionExists(String), + #[error("collection `{0}` does not exist")] + CollectionMissing(String), + #[error("command is missing required field `{0}`")] + MissingField(&'static str), + #[error("unsupported command")] + UnsupportedCommand, + #[error("expected JSON object for `{0}`")] + ExpectedObject(&'static str), + #[error("expected JSON array for `{0}`")] + ExpectedArray(&'static str), + #[error("index `{0}` already exists")] + IndexExists(String), + #[error("index `{0}` does not exist")] + IndexMissing(String), + #[error("encoder `{0}` already exists")] + EncoderExists(String), + #[error("encoder `{0}` does not exist")] + EncoderMissing(String), + #[error("unsupported encoder `{0}`")] + UnsupportedEncoder(String), + #[error("neural space `{0}` already exists")] + NeuralSpaceExists(String), + #[error("neural space `{0}` does not exist")] + NeuralSpaceMissing(String), + #[error("unique index `{index}` rejected duplicate value for `{field}`")] + UniqueIndexViolation { index: String, field: String }, + #[error("vector field `{field}` expected {expected} dimensions, got {actual}")] + VectorDimensionMismatch { + field: String, + expected: usize, + actual: usize, + }, + #[error("storage error: {0}")] + Storage(#[from] std::io::Error), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} + +pub type Result = std::result::Result; diff --git a/vendor/nosqlite/src/mutation.rs b/vendor/nosqlite/src/mutation.rs new file mode 100644 index 00000000..07b38627 --- /dev/null +++ b/vendor/nosqlite/src/mutation.rs @@ -0,0 +1,200 @@ +use serde_json::{json, Map, Value}; + +use crate::{ + query::{compile_filter, CompiledFilter}, + Document, +}; + +pub struct CompiledUpdates<'a> { + set: Option<&'a Map>, + unset: Vec<&'a str>, + inc: Option<&'a Map>, + single_set: Option<(&'a str, &'a Value)>, +} + +impl<'a> CompiledUpdates<'a> { + pub fn new(updates: &'a Document) -> Self { + let set = updates.get("$set").and_then(Value::as_object); + let unset = updates + .get("$unset") + .and_then(Value::as_array) + .into_iter() + .flat_map(|fields| fields.iter().filter_map(Value::as_str)) + .collect::>(); + let inc = updates.get("$inc").and_then(Value::as_object); + let single_set = match (set, unset.is_empty(), inc) { + (Some(set), true, None) if set.len() == 1 => { + set.iter().next().map(|(key, value)| (key.as_str(), value)) + } + _ => None, + }; + Self { + set, + unset, + inc, + single_set, + } + } + + pub fn fields(&self) -> impl Iterator { + let set = self + .set + .into_iter() + .flat_map(|fields| fields.keys().map(String::as_str)); + let inc = self + .inc + .into_iter() + .flat_map(|fields| fields.keys().map(String::as_str)); + + set.chain(inc).chain(self.unset.iter().copied()) + } + + pub fn would_change(&self, document: &Document) -> bool { + if let Some((key, value)) = self.single_set { + return document.get(key) != Some(value); + } + + if let Some(set) = self.set { + if set + .iter() + .any(|(key, value)| document.get(key) != Some(value)) + { + return true; + } + } + + if self + .unset + .iter() + .copied() + .any(|field| document.contains_key(field)) + { + return true; + } + + if let Some(inc) = self.inc { + return inc.iter().any(|(key, value)| { + let delta = value.as_f64().unwrap_or(0.0); + let actual = document.get(key); + let current = actual.and_then(Value::as_f64).unwrap_or(0.0); + !number_value_equals_f64(actual, current + delta) + }); + } + + false + } +} + +pub fn apply_updates(document: &mut Document, updates: &Document) -> bool { + apply_compiled_updates(document, &CompiledUpdates::new(updates)) +} + +pub fn apply_compiled_updates(document: &mut Document, updates: &CompiledUpdates<'_>) -> bool { + if let Some((key, value)) = updates.single_set { + let changed = document.get(key) != Some(value); + set_document_value(document, key, value); + return changed; + } + + let mut changed = false; + + if let Some(set) = updates.set { + for (key, value) in set { + changed |= document.get(key) != Some(value); + set_document_value(document, key, value); + } + } + + if !updates.unset.is_empty() { + for field in updates.unset.iter().copied() { + changed |= document.remove(field).is_some(); + } + } + + if let Some(inc) = updates.inc { + for (key, value) in inc { + let delta = value.as_f64().unwrap_or(0.0); + let current = document.get(key).and_then(Value::as_f64).unwrap_or(0.0); + let next = current + delta; + let next = if next.fract() == 0.0 { + json!(next as i64) + } else { + json!(next) + }; + changed |= document.get(key) != Some(&next); + set_document_value(document, key, &next); + } + } + + changed +} + +fn number_value_equals_f64(value: Option<&Value>, expected: f64) -> bool { + let Some(Value::Number(number)) = value else { + return false; + }; + + if expected.fract() == 0.0 { + let expected = expected as i64; + return number.as_i64() == Some(expected) + || (expected >= 0 + && number + .as_u64() + .is_some_and(|actual| actual == expected as u64)); + } + + number + .as_f64() + .is_some_and(|actual| actual.to_bits() == expected.to_bits()) +} + +fn set_document_value(document: &mut Document, key: &str, value: &Value) { + if let Some(existing) = document.get_mut(key) { + *existing = value.clone(); + } else { + document.insert(key.to_string(), value.clone()); + } +} + +pub fn apply_update_many(documents: &mut [Document], filter: &Document, updates: &Document) { + let filter = compile_filter(filter); + let updates = CompiledUpdates::new(updates); + for document in documents { + if filter.matches(document) { + apply_compiled_updates(document, &updates); + } + } +} + +pub fn apply_delete_many(documents: &mut Vec, filter: &Document) { + let filter = compile_filter(filter); + apply_delete_many_compiled(documents, &filter); +} + +pub fn apply_delete_many_compiled(documents: &mut Vec, filter: &CompiledFilter<'_>) { + documents.retain(|document| !filter.matches(document)); +} + +pub fn apply_cleanup_expired(documents: &mut Vec, ttl_field: &str, now: i64) { + documents.retain(|document| { + document + .get(ttl_field) + .and_then(Value::as_i64) + .is_none_or(|ttl| ttl > now) + }); +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn increment_preflight_compares_numbers_without_allocating_values() { + assert!(number_value_equals_f64(Some(&json!(3)), 3.0)); + assert!(number_value_equals_f64(Some(&json!(3.5)), 3.5)); + assert!(!number_value_equals_f64(Some(&json!(3)), 4.0)); + assert!(!number_value_equals_f64(Some(&json!("3")), 3.0)); + } +} diff --git a/vendor/nosqlite/src/neural.rs b/vendor/nosqlite/src/neural.rs new file mode 100644 index 00000000..1a034121 --- /dev/null +++ b/vendor/nosqlite/src/neural.rs @@ -0,0 +1,175 @@ +use serde::{Deserialize, Serialize}; + +use crate::{Error, Result}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NeuralSpace { + pub encoder: String, + pub dimensions: usize, + #[serde(default = "default_max_prototypes")] + pub max_prototypes: usize, + #[serde(default)] + pub prototypes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Prototype { + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + pub vector: Vec, + pub weight: f64, +} + +impl NeuralSpace { + pub fn new(encoder: &str, dimensions: usize, max_prototypes: usize) -> Self { + Self { + encoder: encoder.to_string(), + dimensions, + max_prototypes: max_prototypes.max(1), + prototypes: Vec::new(), + } + } + + pub fn validate_vector(&self, vector: &[f64]) -> Result<()> { + if vector.len() != self.dimensions { + return Err(Error::VectorDimensionMismatch { + field: "neuralSpace.vector".to_string(), + expected: self.dimensions, + actual: vector.len(), + }); + } + Ok(()) + } + + pub fn learn(&mut self, vector: &[f64], label: Option, weight: f64) -> Result<()> { + self.validate_vector(vector)?; + let mut vector = vector.to_vec(); + normalize(&mut vector); + let weight = if weight == 0.0 { 1.0 } else { weight }; + + if let Some(position) = self.matching_prototype(&vector, label.as_deref()) { + update_prototype(&mut self.prototypes[position], &vector, weight); + } else if weight > 0.0 && self.prototypes.len() < self.max_prototypes { + self.prototypes.push(Prototype { + label, + vector, + weight, + }); + } else if let Some(position) = self.nearest_prototype(&vector, label.as_deref()) { + update_prototype(&mut self.prototypes[position], &vector, weight); + } + + self.prototypes + .sort_by(|left, right| right.weight.total_cmp(&left.weight)); + Ok(()) + } + + pub fn adapt_query(&self, query: &[f64], label: Option<&str>) -> Result> { + self.validate_vector(query)?; + let mut adapted = query.to_vec(); + normalize(&mut adapted); + + let mut matches = self + .prototypes + .iter() + .filter(|prototype| label.is_none_or(|label| prototype.label.as_deref() == Some(label))) + .map(|prototype| (cosine(&adapted, &prototype.vector), prototype)) + .filter(|(score, prototype)| *score > 0.0 && prototype.weight > 0.0) + .collect::>(); + matches.sort_by(|left, right| right.0.total_cmp(&left.0)); + + for (rank, (score, prototype)) in matches.into_iter().take(3).enumerate() { + let blend = (0.30 / (rank as f64 + 1.0)) * score.min(1.0); + for (value, proto_value) in adapted.iter_mut().zip(&prototype.vector) { + *value = (*value * (1.0 - blend)) + (proto_value * blend); + } + } + + normalize(&mut adapted); + Ok(adapted) + } + + fn matching_prototype(&self, vector: &[f64], label: Option<&str>) -> Option { + if let Some(label) = label { + return self + .prototypes + .iter() + .position(|prototype| prototype.label.as_deref() == Some(label)); + } + + self.nearest_prototype(vector, None) + .filter(|position| cosine(vector, &self.prototypes[*position].vector) >= 0.92) + } + + fn nearest_prototype(&self, vector: &[f64], label: Option<&str>) -> Option { + self.prototypes + .iter() + .enumerate() + .filter(|(_, prototype)| { + label.is_none_or(|label| prototype.label.as_deref() == Some(label)) + }) + .max_by(|(_, left), (_, right)| { + cosine(vector, &left.vector).total_cmp(&cosine(vector, &right.vector)) + }) + .map(|(position, _)| position) + } +} + +fn update_prototype(prototype: &mut Prototype, vector: &[f64], weight: f64) { + let total = (prototype.weight.abs() + weight.abs()).max(1.0); + let direction = if weight >= 0.0 { 1.0 } else { -1.0 }; + let rate = (weight.abs() / total).clamp(0.05, 0.50); + + for (value, learned) in prototype.vector.iter_mut().zip(vector) { + *value = (*value * (1.0 - rate)) + (learned * rate * direction); + } + normalize(&mut prototype.vector); + prototype.weight = (prototype.weight + weight).max(0.0); +} + +fn normalize(vector: &mut [f64]) { + let norm = vector.iter().map(|value| value * value).sum::().sqrt(); + if norm == 0.0 { + return; + } + for value in vector { + *value /= norm; + } +} + +fn cosine(left: &[f64], right: &[f64]) -> f64 { + let mut dot = 0.0; + let mut left_norm = 0.0; + let mut right_norm = 0.0; + for (left, right) in left.iter().zip(right) { + dot += left * right; + left_norm += left * left; + right_norm += right * right; + } + if left_norm == 0.0 || right_norm == 0.0 { + return 0.0; + } + dot / (left_norm.sqrt() * right_norm.sqrt()) +} + +fn default_max_prototypes() -> usize { + 256 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn learns_and_adapts_toward_matching_prototypes() { + let mut space = NeuralSpace::new("local_text", 3, 8); + space + .learn(&[1.0, 0.0, 0.0], Some("east".to_string()), 1.0) + .unwrap(); + let adapted = space.adapt_query(&[0.8, 0.2, 0.0], Some("east")).unwrap(); + + assert!(adapted[0] > 0.8); + } +} diff --git a/vendor/nosqlite/src/query.rs b/vendor/nosqlite/src/query.rs new file mode 100644 index 00000000..cbbe3541 --- /dev/null +++ b/vendor/nosqlite/src/query.rs @@ -0,0 +1,267 @@ +use serde_json::Value; + +use crate::Document; + +pub struct CompiledFilter<'a> { + fields: Vec>, +} + +struct CompiledField<'a> { + key: &'a str, + predicate: FieldPredicate<'a>, +} + +enum FieldPredicate<'a> { + Equality(&'a Value), + Operators(Vec>), +} + +enum FieldOperator<'a> { + Eq(&'a Value), + Ne(&'a Value), + Gt(&'a Value), + Gte(&'a Value), + Lt(&'a Value), + Lte(&'a Value), + Prefix(&'a Value), + Exists(&'a Value), + Unknown, +} + +pub fn matches_filter(document: &Document, filter: &Document) -> bool { + compile_filter(filter).matches(document) +} + +pub fn compile_filter(filter: &Document) -> CompiledFilter<'_> { + let fields = filter + .iter() + .map(|(key, expected)| CompiledField { + key, + predicate: compile_field_predicate(expected), + }) + .collect(); + CompiledFilter { fields } +} + +pub fn compile_filter_excluding<'a>( + filter: &'a Document, + excluded_field: Option<&str>, +) -> CompiledFilter<'a> { + let fields = filter + .iter() + .filter(|(key, _)| excluded_field != Some(key.as_str())) + .map(|(key, expected)| CompiledField { + key, + predicate: compile_field_predicate(expected), + }) + .collect(); + CompiledFilter { fields } +} + +impl CompiledFilter<'_> { + pub fn matches(&self, document: &Document) -> bool { + if self.fields.is_empty() { + return true; + } + if self.fields.len() == 1 { + let field = &self.fields[0]; + if let FieldPredicate::Equality(expected) = field.predicate { + return document.get(field.key) == Some(expected); + } + } + + self.fields + .iter() + .all(|field| field.predicate.matches(document.get(field.key))) + } +} + +impl FieldPredicate<'_> { + fn matches(&self, actual: Option<&Value>) -> bool { + match self { + FieldPredicate::Equality(expected) => actual == Some(*expected), + FieldPredicate::Operators(operators) => { + operators.iter().all(|operator| operator.matches(actual)) + } + } + } +} + +impl FieldOperator<'_> { + fn matches(&self, actual: Option<&Value>) -> bool { + match self { + FieldOperator::Eq(expected) => actual == Some(*expected), + FieldOperator::Ne(expected) => actual != Some(*expected), + FieldOperator::Gt(expected) => { + compare_numbers(actual, expected, |left, right| left > right) + } + FieldOperator::Gte(expected) => { + compare_numbers(actual, expected, |left, right| left >= right) + } + FieldOperator::Lt(expected) => { + compare_numbers(actual, expected, |left, right| left < right) + } + FieldOperator::Lte(expected) => { + compare_numbers(actual, expected, |left, right| left <= right) + } + FieldOperator::Prefix(expected) => actual + .and_then(Value::as_str) + .zip(expected.as_str()) + .is_some_and(|(actual, expected)| actual.starts_with(expected)), + FieldOperator::Exists(expected) => expected + .as_bool() + .is_some_and(|should_exist| actual.is_some() == should_exist), + FieldOperator::Unknown => false, + } + } +} + +fn compile_field_predicate(expected: &Value) -> FieldPredicate<'_> { + match expected { + Value::Object(operator) if operator.len() == 1 => { + if let Some(expected) = operator.get("$eq") { + FieldPredicate::Equality(expected) + } else { + operator + .iter() + .next() + .filter(|(op, _)| op.starts_with('$')) + .map(|(op, value)| FieldPredicate::Operators(vec![compile_operator(op, value)])) + .unwrap_or(FieldPredicate::Equality(expected)) + } + } + Value::Object(operator) if operator.keys().any(|key| key.starts_with('$')) => { + FieldPredicate::Operators( + operator + .iter() + .map(|(op, value)| compile_operator(op.as_str(), value)) + .collect(), + ) + } + _ => FieldPredicate::Equality(expected), + } +} + +fn compile_operator<'a>(op: &str, expected: &'a Value) -> FieldOperator<'a> { + match op { + "$eq" => FieldOperator::Eq(expected), + "$ne" => FieldOperator::Ne(expected), + "$gt" => FieldOperator::Gt(expected), + "$gte" => FieldOperator::Gte(expected), + "$lt" => FieldOperator::Lt(expected), + "$lte" => FieldOperator::Lte(expected), + "$prefix" => FieldOperator::Prefix(expected), + "$exists" => FieldOperator::Exists(expected), + _ => FieldOperator::Unknown, + } +} + +#[cfg(test)] +fn matches_filter_interpreted(document: &Document, filter: &Document) -> bool { + if filter.len() == 1 { + let (key, expected) = filter.iter().next().expect("filter has one entry"); + if let Some(expected) = equality_expected(expected) { + return document.get(key) == Some(expected); + } + } + + filter + .iter() + .all(|(key, expected)| matches_field(document.get(key), expected)) +} + +#[cfg(test)] +fn equality_expected(expected: &Value) -> Option<&Value> { + match expected { + Value::Object(operator) if operator.len() == 1 => operator.get("$eq"), + Value::Object(operator) if operator.keys().any(|key| key.starts_with('$')) => None, + _ => Some(expected), + } +} + +#[cfg(test)] +fn matches_field(actual: Option<&Value>, expected: &Value) -> bool { + match expected { + Value::Object(operator) if operator.keys().any(|key| key.starts_with('$')) => operator + .iter() + .all(|(op, value)| matches_operator(actual, op.as_str(), value)), + _ => actual == Some(expected), + } +} + +#[cfg(test)] +fn matches_operator(actual: Option<&Value>, op: &str, expected: &Value) -> bool { + match op { + "$eq" => actual == Some(expected), + "$ne" => actual != Some(expected), + "$gt" => compare_numbers(actual, expected, |left, right| left > right), + "$gte" => compare_numbers(actual, expected, |left, right| left >= right), + "$lt" => compare_numbers(actual, expected, |left, right| left < right), + "$lte" => compare_numbers(actual, expected, |left, right| left <= right), + "$prefix" => actual + .and_then(Value::as_str) + .zip(expected.as_str()) + .is_some_and(|(actual, expected)| actual.starts_with(expected)), + "$exists" => expected + .as_bool() + .is_some_and(|should_exist| actual.is_some() == should_exist), + _ => false, + } +} + +fn compare_numbers( + actual: Option<&Value>, + expected: &Value, + predicate: impl FnOnce(f64, f64) -> bool, +) -> bool { + let Some(actual) = actual.and_then(Value::as_f64) else { + return false; + }; + let Some(expected) = expected.as_f64() else { + return false; + }; + + predicate(actual, expected) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn compiled_filter_matches_interpreted_filter() { + let document = json!({ + "_id": "a", + "pk": "room", + "sk": "chat#001", + "count": 7, + "active": true, + "nested": { "x": 1 } + }) + .as_object() + .expect("object") + .clone(); + let filters = [ + json!({}), + json!({ "pk": "room" }), + json!({ "pk": { "$eq": "room" } }), + json!({ "pk": { "$ne": "other" } }), + json!({ "count": { "$gt": 6, "$lte": 7 } }), + json!({ "sk": { "$prefix": "chat#" } }), + json!({ "missing": { "$exists": false } }), + json!({ "active": true, "pk": "room" }), + json!({ "nested": { "x": 1 } }), + json!({ "pk": { "$unknown": "room" } }), + ]; + + for filter in filters { + let filter = filter.as_object().expect("filter object"); + assert_eq!( + compile_filter(filter).matches(&document), + matches_filter_interpreted(&document, filter) + ); + } + } +} diff --git a/vendor/nosqlite/src/storage.rs b/vendor/nosqlite/src/storage.rs new file mode 100644 index 00000000..9d276f1b --- /dev/null +++ b/vendor/nosqlite/src/storage.rs @@ -0,0 +1,1017 @@ +use std::{ + collections::BTreeMap, + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, ErrorKind, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use fs2::FileExt; +use parking_lot::Mutex; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +use crate::{ + encoder::EncoderSpec, + index::IndexSpec, + kernel, + mutation::{apply_cleanup_expired, apply_delete_many, apply_update_many}, + neural::NeuralSpace, + Document, Error, Result, +}; + +const CATALOG_FILE: &str = "__nosqlite_catalog.json"; +const EVENTS_FILE: &str = "__nosqlite_events.jsonl"; +const MANIFEST_FILE: &str = "__nosqlite_manifest.json"; +const LOCK_FILE: &str = ".nosqlite.lock"; + +type CollectionViews = BTreeMap>; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct Catalog { + #[serde(default)] + pub collections: BTreeMap, + #[serde(default)] + pub encoders: BTreeMap, + #[serde(default)] + pub neural_spaces: BTreeMap, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct CollectionCatalog { + #[serde(default)] + pub indexes: Vec, +} + +#[derive(Debug, Clone)] +pub enum StorageMode { + Memory, + FileSystem(PathBuf), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Durability { + Sync, + Buffered, +} + +#[derive(Debug)] +pub struct Storage { + mode: StorageMode, + _lock: Option, + next_seq: Mutex>, + event_file: Mutex>, + manifest: Mutex>, + durability: Durability, +} + +impl Storage { + pub fn new(mode: StorageMode, durability: Durability) -> Result { + let mut lock = None; + if let StorageMode::FileSystem(path) = &mode { + fs::create_dir_all(path)?; + let lock_file = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path.join(LOCK_FILE))?; + lock_file.try_lock_exclusive()?; + lock = Some(lock_file); + } + + Ok(Self { + mode, + _lock: lock, + next_seq: Mutex::new(None), + event_file: Mutex::new(None), + manifest: Mutex::new(None), + durability, + }) + } + + pub fn load(&self) -> Result>> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(BTreeMap::new()); + }; + + let (mut collections, checkpoint_seq) = load_collection_views(path)?; + + for entry in fs::read_dir(path)? { + let entry = entry?; + let file_path = entry.path(); + if file_path.extension().and_then(|ext| ext.to_str()) != Some("wal") { + continue; + } + let Some(name) = file_path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + let documents = collections.entry(name.to_string()).or_default(); + apply_wal(&file_path, documents)?; + } + let manifest = load_manifest(path)?; + let last_event_seq = + apply_sync_events(path, &mut collections, checkpoint_seq, manifest.as_ref())?; + *self.manifest.lock() = manifest; + *self.next_seq.lock() = Some(last_event_seq + 1); + + Ok(collections) + } + + pub fn load_catalog(&self) -> Result { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(Catalog::default()); + }; + + let catalog_path = path.join(CATALOG_FILE); + match fs::read_to_string(catalog_path) { + Ok(raw) if raw.trim().is_empty() => Ok(Catalog::default()), + Ok(raw) => Ok(serde_json::from_str(&raw)?), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(Catalog::default()), + Err(error) => Err(Error::Storage(error)), + } + } + + pub fn save_catalog(&self, catalog: &Catalog) -> Result<()> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(()); + }; + + fs::create_dir_all(path)?; + let file_path = path.join(CATALOG_FILE); + let tmp_path = file_path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(catalog)?; + + fs::write(&tmp_path, body)?; + fs::rename(tmp_path, file_path)?; + Ok(()) + } + + pub fn save_collection(&self, name: &str, documents: &[Document]) -> Result<()> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(()); + }; + + fs::create_dir_all(path)?; + let file_path = collection_path(path, name); + let tmp_path = file_path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(documents)?; + + fs::write(&tmp_path, body)?; + fs::rename(tmp_path, file_path)?; + truncate_wal(path, name)?; + Ok(()) + } + + pub fn save_checkpoint(&self, name: &str, documents: &[Document], last_seq: u64) -> Result<()> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(()); + }; + + fs::create_dir_all(path)?; + write_json_zstd_atomic(&collection_compressed_path(path, name), documents)?; + remove_if_exists(&collection_path(path, name))?; + write_json_atomic( + &collection_meta_path(path, name), + &ProjectionMeta { + last_seq, + document_count: documents.len(), + compression: Some("zstd".to_string()), + }, + )?; + truncate_wal(path, name)?; + Ok(()) + } + + pub fn compact_checkpoints( + &self, + collections: &BTreeMap>, + ) -> Result { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(0); + }; + let last_seq = self.current_event_seq(path)?; + for (name, documents) in collections { + self.save_checkpoint(name, documents, last_seq)?; + } + self.rotate_event_segment_after_compact(path, collections.keys(), last_seq)?; + Ok(last_seq) + } + + pub fn append_create_collection(&self, name: &str) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::Create) + } + + pub fn append_snapshot(&self, name: &str, documents: &[Document]) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::Snapshot { documents }) + } + + pub fn append_documents(&self, name: &str, documents: &[Document]) -> Result<()> { + if documents.is_empty() { + return Ok(()); + } + + self.append_sync_event_ref(name, WalRecordRef::Insert { documents }) + } + + pub fn append_update(&self, name: &str, filter: &Document, updates: &Document) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::Update { filter, updates }) + } + + pub fn append_delete(&self, name: &str, filter: &Document) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::Delete { filter }) + } + + pub fn append_cleanup_expired(&self, name: &str, ttl_field: &str, now: i64) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::CleanupExpired { ttl_field, now }) + } + + pub fn append_drop_collection(&self, name: &str) -> Result<()> { + self.append_sync_event_ref(name, WalRecordRef::Drop) + } + + fn append_sync_event_ref(&self, collection: &str, op: WalRecordRef<'_>) -> Result<()> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(()); + }; + + let mut next_seq = self.next_seq.lock(); + let seq = match *next_seq { + Some(seq) => seq, + None => last_sync_event_seq(path)? + 1, + }; + let mut manifest = self.manifest.lock(); + if manifest.is_none() { + *manifest = Some(load_or_default_manifest(path)?); + } + let event_path = event_segment_path( + path, + manifest + .as_ref() + .expect("manifest initialized") + .active_segment, + ); + let ts = now_epoch_seconds(); + let event = SyncEventRef { + seq: Some(seq), + ts: Some(ts), + collection, + op, + checksum: None, + }; + let event_body = serde_json::to_string(&event)?; + let mut event_file = self.event_file.lock(); + if event_file.is_none() { + *event_file = Some( + OpenOptions::new() + .create(true) + .append(true) + .open(event_path)?, + ); + } + let file = event_file.as_mut().expect("event file initialized"); + write_sync_event_line(file, &event_body)?; + if self.durability == Durability::Sync { + file.sync_data()?; + } + *next_seq = Some(seq + 1); + Ok(()) + } + + pub fn delete_collection(&self, name: &str) -> Result<()> { + let StorageMode::FileSystem(path) = &self.mode else { + return Ok(()); + }; + + remove_if_exists(&collection_path(path, name))?; + remove_if_exists(&collection_compressed_path(path, name))?; + remove_if_exists(&collection_meta_path(path, name))?; + remove_if_exists(&wal_path(path, name)) + } + + fn current_event_seq(&self, path: &Path) -> Result { + if let Some(next_seq) = *self.next_seq.lock() { + return Ok(next_seq.saturating_sub(1)); + } + last_sync_event_seq(path) + } + + fn rotate_event_segment_after_compact<'a>( + &self, + root: &Path, + collections: impl Iterator, + last_seq: u64, + ) -> Result<()> { + let mut guard = self.manifest.lock(); + let mut manifest = match guard.take() { + Some(manifest) => manifest, + None => load_manifest(root)?.unwrap_or_else(default_manifest), + }; + for name in collections { + manifest.checkpoints.insert(name.clone(), last_seq); + } + + if let Some(max_seq) = event_file_max_seq(&events_path(root))? { + manifest.legacy_max_seq = Some(max_seq); + } + + if last_seq > 0 { + let active_segment = manifest.active_segment.max(1); + if let Some((min_seq, max_seq)) = + event_file_bounds(&event_segment_path(root, active_segment))? + { + upsert_segment_meta( + &mut manifest, + EventSegmentMeta { + id: active_segment, + min_seq, + max_seq, + prunable: false, + }, + ); + } + manifest.active_segment = active_segment + 1; + *self.event_file.lock() = None; + } + + mark_prunable_segments(&mut manifest); + save_manifest(root, &manifest)?; + *guard = Some(manifest); + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "camelCase")] +enum WalRecord { + Create, + Drop, + Snapshot { documents: Vec }, + Insert { documents: Vec }, + Update { filter: Document, updates: Document }, + Delete { filter: Document }, + CleanupExpired { ttl_field: String, now: i64 }, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "op", rename_all = "camelCase")] +enum WalRecordRef<'a> { + Create, + Drop, + Snapshot { + documents: &'a [Document], + }, + Insert { + documents: &'a [Document], + }, + Update { + filter: &'a Document, + updates: &'a Document, + }, + Delete { + filter: &'a Document, + }, + CleanupExpired { + ttl_field: &'a str, + now: i64, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SyncEvent { + #[serde(skip_serializing_if = "Option::is_none")] + seq: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ts: Option, + collection: String, + #[serde(flatten)] + op: WalRecord, + #[serde(skip_serializing_if = "Option::is_none")] + checksum: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SyncEventEnvelope { + seq: Option, + ts: Option, + checksum: Option, +} + +#[derive(Debug, Serialize)] +struct SyncEventRef<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + seq: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ts: Option, + collection: &'a str, + #[serde(flatten)] + op: WalRecordRef<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + checksum: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ProjectionMeta { + last_seq: u64, + document_count: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + compression: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StorageManifest { + #[serde(default = "default_active_segment")] + active_segment: u64, + #[serde(default)] + segments: Vec, + #[serde(default)] + checkpoints: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + legacy_max_seq: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct EventSegmentMeta { + id: u64, + min_seq: u64, + max_seq: u64, + #[serde(default)] + prunable: bool, +} + +impl StorageManifest { + fn segment(&self, id: u64) -> Option<&EventSegmentMeta> { + self.segments.iter().find(|segment| segment.id == id) + } +} + +fn default_active_segment() -> u64 { + 1 +} + +fn default_manifest() -> StorageManifest { + StorageManifest { + active_segment: default_active_segment(), + segments: Vec::new(), + checkpoints: BTreeMap::new(), + legacy_max_seq: None, + } +} + +fn load_manifest(root: &Path) -> Result> { + match fs::read_to_string(manifest_path(root)) { + Ok(raw) if raw.trim().is_empty() => Ok(None), + Ok(raw) => Ok(Some(serde_json::from_str(&raw)?)), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(Error::Storage(error)), + } +} + +fn load_or_default_manifest(root: &Path) -> Result { + if let Some(manifest) = load_manifest(root)? { + return Ok(manifest); + } + + let mut manifest = default_manifest(); + let discovered = discover_event_segment_ids(root)?; + if let Some(max_segment) = discovered.iter().max() { + manifest.active_segment = max_segment + 1; + } + Ok(manifest) +} + +fn save_manifest(root: &Path, manifest: &StorageManifest) -> Result<()> { + write_json_atomic(&manifest_path(root), manifest) +} + +fn event_segment_ids(root: &Path, manifest: Option<&StorageManifest>) -> Result> { + let mut ids = discover_event_segment_ids(root)?; + if let Some(manifest) = manifest { + ids.push(manifest.active_segment); + ids.extend(manifest.segments.iter().map(|segment| segment.id)); + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) +} + +fn discover_event_segment_ids(root: &Path) -> Result> { + let mut ids = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if let Some(id) = event_segment_id(file_name) { + ids.push(id); + } + } + ids.sort_unstable(); + ids.dedup(); + Ok(ids) +} + +fn event_file_max_seq(path: &Path) -> Result> { + Ok(event_file_bounds(path)?.map(|(_, max_seq)| max_seq)) +} + +fn event_file_bounds(path: &Path) -> Result> { + let file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Storage(error)), + }; + + let mut reader = BufReader::new(file); + let mut first = None; + let mut last = 0; + let mut line = String::new(); + loop { + line.clear(); + if reader.read_line(&mut line)? == 0 { + break; + } + if line.trim().is_empty() { + continue; + } + let Ok(envelope) = serde_json::from_str::(&line) else { + break; + }; + let seq = envelope.seq.unwrap_or(last + 1); + first.get_or_insert(seq); + last = seq; + } + + Ok(first.map(|first| (first, last))) +} + +fn last_sync_event_seq(root: &Path) -> Result { + let mut last = event_file_max_seq(&events_path(root))?.unwrap_or(0); + for id in discover_event_segment_ids(root)? { + last = last.max(event_file_max_seq(&event_segment_path(root, id))?.unwrap_or(0)); + } + Ok(last) +} + +fn upsert_segment_meta(manifest: &mut StorageManifest, meta: EventSegmentMeta) { + if let Some(existing) = manifest + .segments + .iter_mut() + .find(|segment| segment.id == meta.id) + { + *existing = meta; + } else { + manifest.segments.push(meta); + manifest.segments.sort_by_key(|segment| segment.id); + } +} + +fn mark_prunable_segments(manifest: &mut StorageManifest) { + let Some(min_checkpoint) = manifest.checkpoints.values().copied().min() else { + return; + }; + for segment in &mut manifest.segments { + segment.prunable = segment.max_seq <= min_checkpoint; + } +} + +fn load_collection_views(root: &Path) -> Result<(CollectionViews, Option)> { + let mut collections = BTreeMap::new(); + let mut checkpoint_seq: Option = None; + + for entry in fs::read_dir(root)? { + let entry = entry?; + let file_path = entry.path(); + let Some(file_name) = file_path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let Some(name) = file_name.strip_suffix(".json.zst") else { + continue; + }; + + let documents = read_json_zstd(&file_path)?; + collections.insert(name.to_string(), documents); + if let Some(meta) = load_projection_meta(root, name)? { + checkpoint_seq = + Some(checkpoint_seq.map_or(meta.last_seq, |seq| seq.min(meta.last_seq))); + } + } + + for entry in fs::read_dir(root)? { + let entry = entry?; + let file_path = entry.path(); + let file_name = file_path.file_name().and_then(|name| name.to_str()); + if matches!( + file_name, + Some(CATALOG_FILE) | Some(EVENTS_FILE) | Some(MANIFEST_FILE) | Some(LOCK_FILE) + ) { + continue; + } + if file_name.is_some_and(|name| name.ends_with(".meta.json")) { + continue; + } + if file_path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + + let Some(name) = file_path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + if collections.contains_key(name) { + continue; + } + + let raw = fs::read_to_string(&file_path)?; + let documents = if raw.trim().is_empty() { + Vec::new() + } else { + serde_json::from_str(&raw)? + }; + collections.insert(name.to_string(), documents); + if let Some(meta) = load_projection_meta(root, name)? { + checkpoint_seq = + Some(checkpoint_seq.map_or(meta.last_seq, |seq| seq.min(meta.last_seq))); + } + } + + Ok((collections, checkpoint_seq)) +} + +fn apply_sync_events( + root: &Path, + collections: &mut BTreeMap>, + after_seq: Option, + manifest: Option<&StorageManifest>, +) -> Result { + let mut last_seq = 0; + let legacy_path = events_path(root); + + if manifest + .and_then(|manifest| manifest.legacy_max_seq) + .zip(after_seq) + .is_some_and(|(max_seq, after_seq)| max_seq <= after_seq) + { + last_seq = last_seq.max( + manifest + .and_then(|manifest| manifest.legacy_max_seq) + .unwrap_or(0), + ); + } else if legacy_path.exists() { + last_seq = last_seq.max(apply_sync_event_file(&legacy_path, collections, after_seq)?); + } + + for segment_id in event_segment_ids(root, manifest)? { + let meta = manifest.and_then(|manifest| manifest.segment(segment_id)); + if meta + .as_ref() + .zip(after_seq) + .is_some_and(|(meta, after_seq)| meta.max_seq <= after_seq) + { + last_seq = last_seq.max(meta.expect("segment meta").max_seq); + continue; + } + last_seq = last_seq.max(apply_sync_event_file( + &event_segment_path(root, segment_id), + collections, + after_seq, + )?); + } + + Ok(last_seq) +} + +fn apply_sync_event_file( + path: &Path, + collections: &mut BTreeMap>, + after_seq: Option, +) -> Result { + let mut file = match OpenOptions::new().read(true).write(true).open(path) { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(Error::Storage(error)), + }; + let mut reader = BufReader::new(file.try_clone()?); + let mut valid_len = 0_u64; + let mut expected_seq = None; + let mut last_seq = 0; + let mut line = String::new(); + + loop { + line.clear(); + let bytes = reader.read_line(&mut line)?; + if bytes == 0 { + break; + } + if line.trim().is_empty() { + valid_len += bytes as u64; + continue; + } + if let Some(after_seq) = after_seq { + let Ok(envelope) = serde_json::from_str::(&line) else { + truncate_to_valid_tail(&mut file, valid_len)?; + break; + }; + if let Some(seq) = envelope.seq { + if !event_envelope_is_valid(&envelope, &line, &mut expected_seq) { + truncate_to_valid_tail(&mut file, valid_len)?; + break; + } + last_seq = seq; + if seq <= after_seq { + valid_len += bytes as u64; + continue; + } + let Ok(event) = serde_json::from_str::(&line) else { + truncate_to_valid_tail(&mut file, valid_len)?; + break; + }; + apply_record(collections, &event.collection, event.op); + valid_len += bytes as u64; + continue; + } + } + let Ok(event) = serde_json::from_str::(&line) else { + truncate_to_valid_tail(&mut file, valid_len)?; + break; + }; + if !event_is_valid(&event, &line, &mut expected_seq) { + truncate_to_valid_tail(&mut file, valid_len)?; + break; + } + last_seq = event.seq.unwrap_or(last_seq + 1); + if event + .seq + .is_none_or(|seq| after_seq.is_none_or(|after| seq > after)) + { + apply_record(collections, &event.collection, event.op); + } + valid_len += bytes as u64; + } + + Ok(last_seq) +} + +fn event_envelope_is_valid( + event: &SyncEventEnvelope, + raw_line: &str, + expected_seq: &mut Option, +) -> bool { + let Some(seq) = event.seq else { + return true; + }; + if event.ts.is_none() { + return false; + }; + let Some(checksum) = event.checksum else { + return false; + }; + if expected_seq.is_some_and(|expected| seq != expected) { + return false; + } + let Some(actual) = checksum_from_raw_line(raw_line) else { + return false; + }; + if actual != checksum { + return false; + } + *expected_seq = Some(seq + 1); + true +} + +fn event_is_valid(event: &SyncEvent, raw_line: &str, expected_seq: &mut Option) -> bool { + let Some(seq) = event.seq else { + return true; + }; + if event.ts.is_none() { + return false; + }; + let Some(checksum) = event.checksum else { + return false; + }; + if expected_seq.is_some_and(|expected| seq != expected) { + return false; + } + let Some(actual) = checksum_from_raw_line(raw_line) else { + return false; + }; + if actual != checksum { + return false; + } + *expected_seq = Some(seq + 1); + true +} + +fn truncate_to_valid_tail(file: &mut File, valid_len: u64) -> Result<()> { + file.set_len(valid_len)?; + file.seek(SeekFrom::Start(valid_len))?; + Ok(()) +} + +fn write_sync_event_line(file: &mut File, event_body: &str) -> Result<()> { + let checksum = kernel::hash_bytes(event_body.as_bytes()); + let body = event_body.strip_suffix('}').unwrap_or(event_body); + let mut line = Vec::with_capacity(body.len() + 34); + line.extend_from_slice(body.as_bytes()); + line.extend_from_slice(b",\"checksum\":"); + write!(&mut line, "{checksum}")?; + line.extend_from_slice(b"}\n"); + file.write_all(&line)?; + Ok(()) +} + +fn checksum_from_raw_line(line: &str) -> Option { + let line = line.trim_end_matches(['\n', '\r']); + let checksum_start = line.rfind(",\"checksum\":")?; + Some(fnv1a64_parts([&line.as_bytes()[..checksum_start], b"}"])) +} + +fn fnv1a64_parts(parts: [&[u8]; N]) -> u64 { + let mut hash = 1469598103934665603_u64; + for part in parts { + for byte in part { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(1099511628211_u64); + } + } + hash +} + +fn apply_wal(path: &Path, documents: &mut Vec) -> Result<()> { + let file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Error::Storage(error)), + }; + + let mut reader = BufReader::new(file); + let mut line = String::new(); + loop { + line.clear(); + if reader.read_line(&mut line)? == 0 { + break; + } + if line.trim().is_empty() { + continue; + } + match serde_json::from_str::(&line)? { + WalRecord::Create => {} + WalRecord::Drop => documents.clear(), + WalRecord::Snapshot { + documents: snapshot, + } => *documents = snapshot, + WalRecord::Insert { + documents: inserted, + } => documents.extend(inserted), + WalRecord::Update { filter, updates } => { + apply_update_many(documents, &filter, &updates) + } + WalRecord::Delete { filter } => apply_delete_many(documents, &filter), + WalRecord::CleanupExpired { ttl_field, now } => { + apply_cleanup_expired(documents, &ttl_field, now) + } + } + } + + Ok(()) +} + +fn apply_record( + collections: &mut BTreeMap>, + collection: &str, + record: WalRecord, +) { + match record { + WalRecord::Create => { + collections.entry(collection.to_string()).or_default(); + } + WalRecord::Drop => { + collections.remove(collection); + } + WalRecord::Snapshot { documents } => { + collections.insert(collection.to_string(), documents); + } + WalRecord::Insert { documents } => { + collections + .entry(collection.to_string()) + .or_default() + .extend(documents); + } + WalRecord::Update { filter, updates } => { + let documents = collections.entry(collection.to_string()).or_default(); + apply_update_many(documents, &filter, &updates); + } + WalRecord::Delete { filter } => { + let documents = collections.entry(collection.to_string()).or_default(); + apply_delete_many(documents, &filter); + } + WalRecord::CleanupExpired { ttl_field, now } => { + let documents = collections.entry(collection.to_string()).or_default(); + apply_cleanup_expired(documents, &ttl_field, now); + } + } +} + +fn truncate_wal(root: &Path, name: &str) -> Result<()> { + match fs::remove_file(wal_path(root, name)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(Error::Storage(error)), + } +} + +fn load_projection_meta(root: &Path, name: &str) -> Result> { + let path = collection_meta_path(root, name); + match fs::read_to_string(path) { + Ok(raw) if raw.trim().is_empty() => Ok(None), + Ok(raw) => Ok(Some(serde_json::from_str(&raw)?)), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(Error::Storage(error)), + } +} + +fn write_json_atomic(path: &Path, value: &T) -> Result<()> { + let tmp_path = path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(value)?; + fs::write(&tmp_path, body)?; + fs::rename(tmp_path, path)?; + Ok(()) +} + +fn write_json_zstd_atomic(path: &Path, value: &T) -> Result<()> { + let tmp_path = path.with_extension("zst.tmp"); + let body = serde_json::to_vec(value)?; + let compressed = zstd::stream::encode_all(body.as_slice(), 3)?; + fs::write(&tmp_path, compressed)?; + fs::rename(tmp_path, path)?; + Ok(()) +} + +fn read_json_zstd(path: &Path) -> Result { + let compressed = fs::read(path)?; + let body = zstd::stream::decode_all(compressed.as_slice())?; + Ok(serde_json::from_slice(&body)?) +} + +fn remove_if_exists(path: &Path) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(Error::Storage(error)), + } +} + +fn collection_path(root: &Path, name: &str) -> PathBuf { + root.join(format!("{}.json", safe_collection_name(name))) +} + +fn collection_compressed_path(root: &Path, name: &str) -> PathBuf { + root.join(format!("{}.json.zst", safe_collection_name(name))) +} + +fn collection_meta_path(root: &Path, name: &str) -> PathBuf { + root.join(format!("{}.meta.json", safe_collection_name(name))) +} + +fn safe_collection_name(name: &str) -> String { + let safe_name: String = name + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => ch, + _ => '_', + }) + .collect(); + safe_name +} + +fn wal_path(root: &Path, name: &str) -> PathBuf { + collection_path(root, name).with_extension("wal") +} + +fn events_path(root: &Path) -> PathBuf { + root.join(EVENTS_FILE) +} + +fn manifest_path(root: &Path) -> PathBuf { + root.join(MANIFEST_FILE) +} + +fn event_segment_path(root: &Path, id: u64) -> PathBuf { + root.join(format!("__nosqlite_events.{id:06}.jsonl")) +} + +fn event_segment_id(file_name: &str) -> Option { + file_name + .strip_prefix("__nosqlite_events.")? + .strip_suffix(".jsonl")? + .parse() + .ok() +} + +fn now_epoch_seconds() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} From cdb021d717a935ef37fb9f132111bc5ae792e34e Mon Sep 17 00:00:00 2001 From: atimics Date: Sun, 12 Jul 2026 21:04:33 -0700 Subject: [PATCH 05/16] Make x402 memory writes replay-safe --- AWS_X402_DEPLOY.md | 4 + X402_API.md | 17 ++ holographic_x402_api.py | 323 ++++++++++++++++++++++++++++- tests/test_holographic_x402_api.py | 143 +++++++++++++ 4 files changed, 481 insertions(+), 6 deletions(-) diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index a51ba7d0..53b1fc6a 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -186,6 +186,10 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. - Mount `LECORE_X402_TENANT_STATE_DIR` on shared durable storage. Tenant writes reload under an OS-level lock and use atomic replacement, so rolling ECS tasks do not overwrite one another. +- Preserve the `.x402-memory-transactions` directory inside tenant state. It is + the durable outbox for core-to-NoSQLite writes; callers should send an + `Idempotency-Key` on `/admin/remember` retries so a timeout cannot duplicate + a memory. - Do not enable NoSQLite on the same EFS directory in overlapping ECS tasks; schedule a single-writer drain-and-replace cutover instead. - Keep the leOS offer credential in Secrets Manager and rotate it if disclosed. diff --git a/X402_API.md b/X402_API.md index cb1f9a51..6069b4ab 100644 --- a/X402_API.md +++ b/X402_API.md @@ -70,9 +70,21 @@ Add memories locally as the seller: curl -X POST http://127.0.0.1:4021/admin/remember \ -H "Content-Type: application/json" \ -H "X-Admin-Token: local-admin-secret" \ + -H "Idempotency-Key: initial-memory-001" \ -d '{"text":"local agents need deterministic durable memory","label":"memory"}' ``` +When `LECORE_X402_TENANT_STATE_DIR` is configured, admin writes use a small +durable transaction journal. Reuse the same `Idempotency-Key` after a timeout: +the API returns the original memory rather than creating another entry. Reusing +one key with a different request is rejected with `409 Conflict`. + +For an enabled NoSQLite mirror, the journal records the core commit before +projecting the same stable memory id to NoSQLite. A temporary NoSQLite failure +leaves that projection pending; the same idempotent retry, or the next app +startup, resumes it without duplicating core memory. The implementation does +not advertise cross-store rollback it cannot provide. + Issue a private tenant token: ```bash @@ -146,6 +158,11 @@ Before cutover, set `LECORE_X402_NOSQLITE_SHADOW=1` while leaving to serve from the core while differences are logged without query text or tenant identifiers. +NoSQLite-enabled writes require `LECORE_X402_TENANT_STATE_DIR`, which is also +where the transaction journal lives. Keep that directory on durable shared +storage with the tenant state; do not delete `.x402-memory-transactions` during +normal deployment cleanup. + NoSQLite filesystem mode holds one nonblocking exclusive writer lock for the life of its process. Run exactly one active writer against a given data directory. A rolling ECS replacement must drain the old writer before enabling diff --git a/holographic_x402_api.py b/holographic_x402_api.py index 876e7b8b..65cebd72 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -48,7 +48,9 @@ DEFAULT_TENANT_ID = "public" TENANT_HEADER = "X-leCore-Tenant" TENANT_TOKEN_HEADER = "X-leCore-Tenant-Token" +IDEMPOTENCY_HEADER = "Idempotency-Key" _TENANT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.:-]{0,63}$") +_IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,256}$") MAX_QUERY_CHARS = 8192 MAX_TASK_CHARS = 8192 MAX_MEMORY_CHARS = 65536 @@ -246,6 +248,15 @@ def tenant_access_token(tenant_id: str, secret: str) -> str: return hmac.new(secret.encode("utf-8"), normalized.encode("utf-8"), hashlib.sha256).hexdigest() +def normalize_idempotency_key(value: Optional[Any]) -> Optional[str]: + """Validate an optional caller-provided retry key without persisting the raw value.""" + if value is None: + return None + if not isinstance(value, str) or not _IDEMPOTENCY_KEY_RE.match(value): + raise ValueError("Idempotency-Key must be 1-256 letters, numbers, '.', '_', ':', or '-'") + return value + + @contextmanager def _process_file_lock(path: Path) -> Any: """Hold an exclusive process lock for one persisted tenant state file.""" @@ -282,6 +293,25 @@ def _process_file_lock(path: Path) -> Any: handle.close() +def _atomic_write_json(path: Path, value: Dict[str, Any]) -> None: + """Durably replace a small JSON control record without exposing a partial file.""" + path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps(value, sort_keys=True, separators=(",", ":")) + temporary = path.with_name(".%s.%s.tmp" % (path.name, os.urandom(8).hex())) + try: + with open(temporary, "x", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except Exception: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + + class TenantCoreStore: """Thread-safe LocalAgentCore registry with optional per-tenant persistence.""" @@ -715,6 +745,246 @@ def _collection_name(tenant_id: str) -> str: return "lecore_memory_%s" % digest +class MemoryTransactionError(RuntimeError): + """The durable memory write journal could not be read or completed safely.""" + + +class MemoryTransactionConflict(MemoryTransactionError): + """One idempotency key was reused for a different memory write.""" + + +class MemoryMirrorPending(NoSQLiteError): + """A durable core commit needs the same transaction projected to NoSQLite.""" + + def __init__(self, tenant_id: str, transaction_id: str, cause: NoSQLiteError): + super().__init__(str(cause)) + self.tenant_id = tenant_id + self.transaction_id = transaction_id + + +class TenantMemoryTransactions: + """Durable, idempotent memory writes spanning LocalAgentCore and NoSQLite. + + A query-layer transaction can roll in-memory tables back. This API crosses a + durable JSON core and an external NoSQLite process, so it instead records an + intent first, commits the core entry with a stable id, then projects that + entry to NoSQLite. If the process dies between steps, the journal replays the + same idempotent projection on the next request or app start. + """ + + _VERSION = 1 + _PLANNED = "planned" + _CORE_COMMITTED = "core_committed" + _COMPLETE = "complete" + + def __init__(self, core_store: TenantCoreStore, state_dir: Any): + self._core_store = core_store + self._root = Path(state_dir) / ".x402-memory-transactions" + self._root.mkdir(parents=True, exist_ok=True) + + def remember( + self, + tenant_id: str, + text: str, + label: Optional[str], + metadata: Optional[Dict[str, Any]], + idempotency_key: Optional[str], + mirror: Optional[NoSQLiteMemoryStore], + ) -> Dict[str, Any]: + """Commit one memory and return its stable transaction status. + + Supplying the same `idempotency_key` with the same request returns the + original memory id. Reusing that key for a different request is refused. + """ + tenant = normalize_tenant_id(tenant_id) + key = normalize_idempotency_key(idempotency_key) + request = { + "tenant": tenant, + "text": str(text), + "label": label, + "metadata": dict(metadata or {}), + } + transaction_id = self._transaction_id(tenant, key) + path = self._path_for(tenant, transaction_id) + with _process_file_lock(path): + record = self._load_or_create(path, transaction_id, request, key, mirror is not None) + return self._apply_locked(path, record, mirror) + + def resume( + self, + tenant_id: str, + transaction_id: str, + mirror: Optional[NoSQLiteMemoryStore], + ) -> Dict[str, Any]: + """Resume a known journal record without minting a second transaction.""" + tenant = normalize_tenant_id(tenant_id) + if not re.fullmatch(r"[0-9a-f]{64}", transaction_id): + raise MemoryTransactionError("invalid memory transaction id") + path = self._path_for(tenant, transaction_id) + with _process_file_lock(path): + record = self._load(path) + self._validate_record(record, path) + if record["tenant"] != tenant or record["transaction_id"] != transaction_id: + raise MemoryTransactionError("memory transaction does not match its tenant") + return self._apply_locked(path, record, mirror) + + def recover_pending(self, mirror: Optional[NoSQLiteMemoryStore]) -> Dict[str, int]: + """Replay incomplete durable writes, leaving unavailable mirrors pending.""" + recovered = 0 + pending = 0 + invalid = 0 + for path in sorted(self._root.glob("*/*.json")): + with _process_file_lock(path): + try: + record = self._load(path) + if record.get("state") == self._COMPLETE: + continue + result = self._apply_locked(path, record, mirror) + if result["transaction"]["state"] == self._COMPLETE: + recovered += 1 + else: + pending += 1 + except NoSQLiteError as exc: + pending += 1 + LOG.warning("NoSQLite transaction recovery remains pending: %s", exc) + except MemoryTransactionError as exc: + invalid += 1 + LOG.error("could not recover memory transaction %s: %s", path.name, exc) + return {"recovered": recovered, "pending": pending, "invalid": invalid} + + def _apply_locked( + self, + path: Path, + record: Dict[str, Any], + mirror: Optional[NoSQLiteMemoryStore], + ) -> Dict[str, Any]: + self._validate_record(record, path) + memory = dict(record["memory"]) + stored = self._core_store.write( + record["tenant"], + lambda core: self._ensure_core_memory(core, memory), + ) + if record["state"] == self._PLANNED: + record["state"] = self._CORE_COMMITTED + _atomic_write_json(path, record) + + if record["requires_mirror"]: + if mirror is None: + return self._result(record, stored) + try: + mirror.remember(record["tenant"], stored) + except NoSQLiteError as exc: + raise MemoryMirrorPending(record["tenant"], record["transaction_id"], exc) from exc + + if record["state"] != self._COMPLETE: + record["state"] = self._COMPLETE + _atomic_write_json(path, record) + return self._result(record, stored) + + @staticmethod + def _ensure_core_memory(core: LocalAgentCore, memory: Dict[str, Any]) -> Dict[str, Any]: + for entry in core.entries: + if entry.id != memory["id"]: + continue + stored = entry.to_dict() + if stored != memory: + raise MemoryTransactionConflict("memory id %s already holds different content" % memory["id"]) + return stored + return core.remember( + memory["text"], + label=memory.get("label"), + metadata=memory.get("metadata"), + id=memory["id"], + ) + + def _load_or_create( + self, + path: Path, + transaction_id: str, + request: Dict[str, Any], + key: Optional[str], + requires_mirror: bool, + ) -> Dict[str, Any]: + if path.exists(): + record = self._load(path) + self._validate_record(record, path) + if record["request_fingerprint"] != self._fingerprint(request): + raise MemoryTransactionConflict("Idempotency-Key was already used for a different memory write") + return record + record = { + "version": self._VERSION, + "transaction_id": transaction_id, + "tenant": request["tenant"], + "request_fingerprint": self._fingerprint(request), + "idempotency_key_hash": self._hash(key) if key is not None else None, + "requires_mirror": bool(requires_mirror), + "state": self._PLANNED, + "memory": { + "id": "tx_%s" % transaction_id[:32], + "text": request["text"], + "label": request["label"], + "metadata": request["metadata"], + }, + } + _atomic_write_json(path, record) + return record + + @staticmethod + def _load(path: Path) -> Dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise MemoryTransactionError("invalid transaction journal %s" % path.name) from exc + if not isinstance(value, dict): + raise MemoryTransactionError("transaction journal %s is not an object" % path.name) + return value + + def _validate_record(self, record: Dict[str, Any], path: Path) -> None: + required = {"version", "transaction_id", "tenant", "request_fingerprint", "requires_mirror", "state", "memory"} + if not required.issubset(record) or record.get("version") != self._VERSION: + raise MemoryTransactionError("unsupported transaction journal %s" % path.name) + if record["state"] not in {self._PLANNED, self._CORE_COMMITTED, self._COMPLETE}: + raise MemoryTransactionError("unknown transaction state in %s" % path.name) + memory = record["memory"] + if not isinstance(memory, dict) or set(memory) != {"id", "text", "label", "metadata"}: + raise MemoryTransactionError("invalid memory transaction payload in %s" % path.name) + if not isinstance(memory["id"], str) or not isinstance(memory["text"], str): + raise MemoryTransactionError("invalid memory transaction value in %s" % path.name) + if memory["label"] is not None and not isinstance(memory["label"], str): + raise MemoryTransactionError("invalid memory transaction label in %s" % path.name) + if not isinstance(memory["metadata"], dict): + raise MemoryTransactionError("invalid memory transaction metadata in %s" % path.name) + if normalize_tenant_id(record["tenant"]) != record["tenant"]: + raise MemoryTransactionError("invalid transaction tenant in %s" % path.name) + + def _path_for(self, tenant_id: str, transaction_id: str) -> Path: + tenant_digest = self._hash(tenant_id)[:24] + return self._root / tenant_digest / (transaction_id + ".json") + + @staticmethod + def _hash(value: Optional[str]) -> str: + return hashlib.sha256((value or "").encode("utf-8")).hexdigest() + + def _transaction_id(self, tenant_id: str, key: Optional[str]) -> str: + material = key if key is not None else os.urandom(32).hex() + return self._hash("%s\0%s" % (tenant_id, material)) + + @classmethod + def _fingerprint(cls, value: Dict[str, Any]) -> str: + return cls._hash(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)) + + @staticmethod + def _result(record: Dict[str, Any], memory: Dict[str, Any]) -> Dict[str, Any]: + return { + "memory": memory, + "transaction": { + "id": record["transaction_id"], + "state": record["state"], + "idempotent": record.get("idempotency_key_hash") is not None, + }, + } + + def leos_token_offer(access_required: bool = True, enabled: bool = True) -> Dict[str, Any]: """Return public metadata for the credential-gated leOS offer.""" return { @@ -904,10 +1174,15 @@ def create_app( data_dir, durability=nosqlite_durability or os.environ.get("LECORE_X402_NOSQLITE_DURABILITY", "sync"), ) + memory_transactions = TenantMemoryTransactions(store, tenant_state_dir) if tenant_state_dir else None @asynccontextmanager async def lifespan(_: Any) -> Any: try: + if memory_transactions is not None: + recovery = memory_transactions.recover_pending(nosqlite_store) + if recovery["recovered"] or recovery["pending"] or recovery["invalid"]: + LOG.info("memory transaction recovery: %s", recovery) yield finally: if nosqlite_store is not None: @@ -917,6 +1192,7 @@ async def lifespan(_: Any) -> Any: app.state.memory_backend = memory_backend app.state.nosqlite_shadow = nosqlite_shadow app.state.nosqlite_store = nosqlite_store + app.state.memory_transactions = memory_transactions config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") leos_access_token = leos_access_token or os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN") @@ -996,6 +1272,7 @@ def memory_public_dict() -> Dict[str, Any]: "backend": memory_backend, "nosqlite_shadow": bool(nosqlite_shadow), "nosqlite_configured": nosqlite_store is not None, + "durable_transactions": memory_transactions is not None, } def nosqlite_unavailable(error: NoSQLiteError) -> HTTPException: @@ -1159,31 +1436,65 @@ def remember( payload: Dict[str, Any], x_admin_token: Optional[str] = Header(default=None), x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), + idempotency_key: Optional[str] = Header(default=None, alias=IDEMPOTENCY_HEADER), ) -> Dict[str, Any]: require_admin(x_admin_token) tenant_id = tenant_from_payload(payload, x_lecore_tenant) text = validated(_required_text, payload, "text", MAX_MEMORY_CHARS) + key = validated(normalize_idempotency_key, idempotency_key) label = payload.get("label") metadata = payload.get("metadata") if label is not None and not isinstance(label, str): raise HTTPException(status_code=400, detail="label must be a string") if metadata is not None and not isinstance(metadata, dict): raise HTTPException(status_code=400, detail="metadata must be an object") - memory = store.write( - tenant_id, - lambda tenant_core: tenant_core.remember(text, label=label, metadata=metadata), - ) - if nosqlite_store is not None: + + transaction = None + if memory_transactions is not None: try: - nosqlite_store.remember(tenant_id, memory) + committed = memory_transactions.remember( + tenant_id, + text, + label, + metadata, + key, + nosqlite_store, + ) + except MemoryTransactionConflict as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except MemoryTransactionError as exc: + raise HTTPException(status_code=500, detail="memory transaction could not be completed") from exc except NoSQLiteError as exc: if memory_backend == MEMORY_BACKEND_NOSQLITE: raise nosqlite_unavailable(exc) from exc LOG.warning("NoSQLite shadow write failed: %s", exc) + if not isinstance(exc, MemoryMirrorPending): # pragma: no cover - mirror errors are wrapped above + raise nosqlite_unavailable(exc) from exc + committed = memory_transactions.resume(exc.tenant_id, exc.transaction_id, None) + memory = committed["memory"] + transaction = committed["transaction"] + else: + if key is not None: + raise HTTPException( + status_code=400, + detail="Idempotency-Key requires LECORE_X402_TENANT_STATE_DIR for durable retries", + ) + memory = store.write( + tenant_id, + lambda tenant_core: tenant_core.remember(text, label=label, metadata=metadata), + ) + if nosqlite_store is not None: # pragma: no cover - NoSQLite requires durable tenant state + try: + nosqlite_store.remember(tenant_id, memory) + except NoSQLiteError as exc: + if memory_backend == MEMORY_BACKEND_NOSQLITE: + raise nosqlite_unavailable(exc) from exc + LOG.warning("NoSQLite shadow write failed: %s", exc) return { "ok": True, "tenant": tenant_id, "memory": memory, + "transaction": transaction, } @app.post("/admin/tenant-token") diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index 0cfdeb8b..1b9ac50c 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -9,14 +9,19 @@ DEFAULT_NETWORK, DEFAULT_PRICE, DEFAULT_TENANT_ID, + IDEMPOTENCY_HEADER, LEOS_SITE_URL, LEOS_ACCESS_HEADER, LEOS_TOKEN_CA, LEOS_TOKEN_PRICE, MEMORY_BACKEND_NOSQLITE, + MemoryTransactionConflict, + MemoryMirrorPending, + NoSQLiteError, TENANT_HEADER, TENANT_TOKEN_HEADER, TenantCoreStore, + TenantMemoryTransactions, X402Config, create_app, landing_page_html, @@ -386,6 +391,143 @@ def test_public_memory_persists_across_app_restart(tmp_path): assert "public-persisted" in [hit["label"] for hit in recalled.json()["hits"]] +def test_durable_memory_transaction_reuses_one_memory_for_retries(tmp_path): + store = TenantCoreStore(LocalAgentCore(), tmp_path) + transactions = TenantMemoryTransactions(store, tmp_path) + + first = transactions.remember( + "acme", + "one durable transaction memory", + "journal", + {"source": "test"}, + "retry-001", + None, + ) + second = transactions.remember( + "acme", + "one durable transaction memory", + "journal", + {"source": "test"}, + "retry-001", + None, + ) + + assert first["memory"] == second["memory"] + assert first["transaction"]["state"] == "complete" + entries = store.read("acme", lambda core: core.entries) + assert [entry.id for entry in entries] == [first["memory"]["id"]] + + with pytest.raises(MemoryTransactionConflict, match="different memory write"): + transactions.remember("acme", "different payload", "journal", {}, "retry-001", None) + + +def test_durable_memory_transaction_recovers_a_failed_mirror(tmp_path): + class FlakyMirror: + def __init__(self): + self.fail = True + self.memories = [] + + def remember(self, tenant_id, memory): + if self.fail: + raise NoSQLiteError("mirror offline") + self.memories.append((tenant_id, dict(memory))) + + store = TenantCoreStore(LocalAgentCore(), tmp_path) + transactions = TenantMemoryTransactions(store, tmp_path) + mirror = FlakyMirror() + + with pytest.raises(NoSQLiteError, match="mirror offline") as failed: + transactions.remember("acme", "recover this mirror write", "journal", {}, "retry-002", mirror) + + committed = store.read("acme", lambda core: [entry.to_dict() for entry in core.entries]) + assert len(committed) == 1 + assert isinstance(failed.value, MemoryMirrorPending) + pending = transactions.resume("acme", failed.value.transaction_id, None) + assert pending["transaction"]["state"] == "core_committed" + assert len(store.read("acme", lambda core: core.entries)) == 1 + + restarted = TenantMemoryTransactions(TenantCoreStore(LocalAgentCore(), tmp_path), tmp_path) + mirror.fail = False + recovery = restarted.recover_pending(mirror) + + assert recovery == {"recovered": 1, "pending": 0, "invalid": 0} + assert mirror.memories == [("acme", committed[0])] + retried = restarted.remember("acme", "recover this mirror write", "journal", {}, "retry-002", mirror) + assert retried["memory"] == committed[0] + assert len(TenantCoreStore(LocalAgentCore(), tmp_path).read("acme", lambda core: core.entries)) == 1 + + +def test_admin_remember_idempotency_header_is_durable_and_conflicts_cleanly(tmp_path): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app( + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + tenant_state_dir=tmp_path, + ) + ) + headers = {"X-Admin-Token": "admin-secret", IDEMPOTENCY_HEADER: "api-retry-001"} + payload = {"text": "idempotent API memory", "label": "idempotent"} + + first = client.post("/admin/remember", headers=headers, json=payload) + second = client.post("/admin/remember", headers=headers, json=payload) + conflict = client.post( + "/admin/remember", + headers=headers, + json={"text": "idempotent API memory but changed", "label": "idempotent"}, + ) + + assert first.status_code == 200 and second.status_code == 200 + assert first.json()["memory"] == second.json()["memory"] + assert first.json()["transaction"]["state"] == "complete" + assert conflict.status_code == 409 + + +def test_idempotency_header_requires_a_durable_tenant_state_dir(): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app(config=X402Config(pay_to="0xabc"), paid=False, admin_token="admin-secret") + ) + + response = client.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret", IDEMPOTENCY_HEADER: "requires-state"}, + json={"text": "this key needs durable state"}, + ) + + assert response.status_code == 400 + assert "TENANT_STATE_DIR" in response.json()["detail"] + + +def test_shadow_mirror_failure_keeps_the_original_unkeyed_transaction(tmp_path): + fastapi_testclient = pytest.importorskip("fastapi.testclient") + client = fastapi_testclient.TestClient( + create_app( + core=LocalAgentCore(), + config=X402Config(pay_to="0xabc"), + paid=False, + admin_token="admin-secret", + tenant_state_dir=tmp_path / "core", + nosqlite_shadow=True, + nosqlite_binary=str(tmp_path / "missing-nosqlite"), + nosqlite_data_dir=tmp_path / "nosqlite", + ) + ) + + response = client.post( + "/admin/remember", + headers={"X-Admin-Token": "admin-secret"}, + json={"text": "shadow write survives its first mirror failure", "label": "shadow"}, + ) + + assert response.status_code == 200 + assert response.json()["transaction"]["state"] == "core_committed" + persisted = TenantCoreStore(LocalAgentCore(), tmp_path / "core") + entries = persisted.read(DEFAULT_TENANT_ID, lambda core: core.entries) + assert len(entries) == 1 and entries[0].label == "shadow" + + def test_nosqlite_memory_backend_isolates_tenants_and_restarts(tmp_path): fastapi_testclient = pytest.importorskip("fastapi.testclient") binary = _nosqlite_binary() @@ -409,6 +551,7 @@ def test_nosqlite_memory_backend_isolates_tenants_and_restarts(tmp_path): "backend": MEMORY_BACKEND_NOSQLITE, "nosqlite_shadow": False, "nosqlite_configured": True, + "durable_transactions": True, } public = first.post( From 36e3fa26bda3a54c407eec01f0697ddb10fc0b33 Mon Sep 17 00:00:00 2001 From: atimics Date: Mon, 13 Jul 2026 06:12:03 -0700 Subject: [PATCH 06/16] Exclude NoSQLite build output from Docker context --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 4b883791..1e45e17f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ .ruff_cache __pycache__ node_modules +vendor/nosqlite/target dist .next .vinext From 094dcdd90860c6993fab80b09976c7b6fd82d18d Mon Sep 17 00:00:00 2001 From: atimics Date: Tue, 14 Jul 2026 08:37:07 -0700 Subject: [PATCH 07/16] Simplify x402 preview pricing --- API_QUICKREF.md | 4 +- AWS_X402_DEPLOY.md | 11 +-- X402_API.md | 24 +---- holographic_x402_api.py | 141 +++++++++++------------------ tests/test_holographic_x402_api.py | 90 +++++++----------- 5 files changed, 93 insertions(+), 177 deletions(-) diff --git a/API_QUICKREF.md b/API_QUICKREF.md index 863fff08..7a5bd46a 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -43,12 +43,12 @@ - `summary(self, tenant_id)` -- Return a cheap cached status summary without probing capabilities. - `read(self, tenant_id, fn)` -- Run a read-style operation while holding the tenant lock. - `write(self, tenant_id, fn)` -- Run a mutating operation, then persist that tenant if configured. -- `leos_token_offer(access_required=True, enabled=True)` -- Return public metadata for the credential-gated leOS offer. +- `pricing_summary(config)` -- Return the customer-facing request and per-1,000-request price plus preview/production status. - `landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. - `payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. - `x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. - `x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. -- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None, leos_access_token=None)` -- Create the FastAPI application for paid or local serving. +- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None)` -- Create the FastAPI application for paid or local serving. - `load_core(path)` -- Load a persisted core if present, otherwise return the demo core. - `main(argv=None)` -- CLI entry point for local x402 API serving. diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index 53b1fc6a..fa3f7040 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -12,7 +12,6 @@ does **not** need a wallet private key in the container. It only needs: - x402/facilitator configuration - an admin token for seller-only memory writes - a tenant-token secret if private customer memory is enabled -- an offer-access token if the discounted leOS routes are enabled The receiving wallet should be a cold wallet, hardware wallet, Safe/multisig, or a custody wallet. The API simply tells x402 where funds should go. @@ -27,8 +26,8 @@ or pay upstream APIs as a buyer. backend; it is disabled by default. - **Application Load Balancer** terminates HTTPS and forwards to port `4021`. - **ECR** stores the container image. -- **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN`, - `LECORE_X402_LEOS_ACCESS_TOKEN`, and production facilitator credentials. +- **Secrets Manager** stores `LECORE_X402_ADMIN_TOKEN` and production + facilitator credentials. Store `LECORE_X402_TENANT_SECRET` there too when private tenants are enabled. - **SSM Parameter Store or plain task env** stores non-secret config like `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, `LECORE_X402_NETWORK`, and @@ -41,9 +40,6 @@ Protected paid routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` -- `POST /leos/v1/recall`, at the credential-gated leOS offer price -- `POST /leos/v1/route`, at the credential-gated leOS offer price -- `GET /leos/v1/dashboard`, at the credential-gated leOS offer price Free routes: @@ -89,7 +85,6 @@ Secrets Manager values: ```text LECORE_X402_ADMIN_TOKEN= LECORE_X402_TENANT_SECRET= -LECORE_X402_LEOS_ACCESS_TOKEN= CDP_API_KEY_ID= CDP_API_KEY_SECRET= ``` @@ -192,7 +187,6 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. a memory. - Do not enable NoSQLite on the same EFS directory in overlapping ECS tasks; schedule a single-writer drain-and-replace cutover instead. -- Keep the leOS offer credential in Secrets Manager and rotate it if disclosed. - Do not put secrets or PII in x402 route descriptions or payment metadata. ## Local Smoke Before AWS @@ -202,7 +196,6 @@ pip install ".[x402]" export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" export LECORE_X402_TENANT_SECRET="local-tenant-secret" -export LECORE_X402_LEOS_ACCESS_TOKEN="local-leos-buyer-secret" python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 ``` diff --git a/X402_API.md b/X402_API.md index 6069b4ab..10c0f871 100644 --- a/X402_API.md +++ b/X402_API.md @@ -9,9 +9,6 @@ the public read/compute routes: - `POST /v1/recall` - `POST /v1/route` - `GET /v1/dashboard` -- `POST /leos/v1/recall`, at the credential-gated leOS offer price -- `POST /leos/v1/route`, at the credential-gated leOS offer price -- `GET /leos/v1/dashboard`, at the credential-gated leOS offer price Free routes: @@ -28,11 +25,6 @@ but they cannot mutate memory unless they also hold the admin token. Private tenant memory also requires a tenant token; x402 proves payment, not tenant authorization. -The leOS CA identifies the discounted offer, but does not itself prove buyer -eligibility because it is public. Discounted calls must also include the -operator-issued `X-leCore-leOS-Access` credential. Failed authorization responses -are not settled by the x402 middleware. - ## Install ```bash @@ -45,14 +37,15 @@ FastAPI/x402/uvicorn stack. ## Testnet Run The default network is Base Sepolia (`eip155:84532`) and the default facilitator -is the signup-free x402.org testnet facilitator. +is the signup-free x402.org testnet facilitator. This is a **developer preview**: +the listed `$0.0011` request price is displayed as `$1.10 per 1,000 requests`, +uses testnet USDC, and does not accept production payments. ```bash export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_PRICE="$0.0011" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" export LECORE_X402_TENANT_SECRET="local-tenant-secret" -export LECORE_X402_LEOS_ACCESS_TOKEN="local-leos-buyer-secret" export LECORE_X402_TENANT_STATE_DIR="./tenant-state" python holographic_x402_api.py --host 127.0.0.1 --port 4021 @@ -104,15 +97,6 @@ curl -X POST http://127.0.0.1:4021/v1/recall \ -d '{"query":"deterministic local memory"}' ``` -Use the discounted leOS route only with an issued offer credential: - -```bash -curl -X POST http://127.0.0.1:4021/leos/v1/recall \ - -H "Content-Type: application/json" \ - -H "X-leCore-leOS-Access: local-leos-buyer-secret" \ - -d '{"query":"deterministic local memory"}' -``` - Requests to paid routes return `402 Payment Required` unless the client retries with a valid x402 payment payload: @@ -180,8 +164,6 @@ until that maintenance window is scheduled. use per-tenant process locks plus atomic replacement on shared storage. - If NoSQLite is enabled, mount `LECORE_X402_NOSQLITE_DATA_DIR` on the same durable storage and keep the service at a single active writer for that path. -- Store `LECORE_X402_LEOS_ACCESS_TOKEN` as a secret and distribute it only to - buyers eligible for the discounted routes. - Treat x402 payment metadata as public enough to avoid putting secrets or PII in route descriptions. diff --git a/holographic_x402_api.py b/holographic_x402_api.py index 65cebd72..bb5c7a64 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -18,7 +18,8 @@ from __future__ import annotations from contextlib import asynccontextmanager, contextmanager -from dataclasses import dataclass, replace +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation import hashlib import hmac from html import escape @@ -41,10 +42,6 @@ DEFAULT_FACILITATOR_URL = "https://x402.org/facilitator" DEFAULT_NETWORK = "eip155:84532" # Base Sepolia, safe default for testnet publishing. DEFAULT_PRICE = "$0.0011" -LEOS_SITE_URL = "https://discoverleos.com/" -LEOS_TOKEN_CA = "5xgsnby6P9zqGK71J7H4yJLxzqPvNbC7rDZxNzjHmj7e" -LEOS_TOKEN_PRICE = "$0.0010" -LEOS_ACCESS_HEADER = "X-leCore-leOS-Access" DEFAULT_TENANT_ID = "public" TENANT_HEADER = "X-leCore-Tenant" TENANT_TOKEN_HEADER = "X-leCore-Tenant-Token" @@ -87,13 +84,19 @@ def key(self) -> str: PaidRoute("GET", "/v1/dashboard", "Read the LocalAgentCore evidence dashboard"), ) -LEOS_PAID_ROUTES: Tuple[PaidRoute, ...] = ( - PaidRoute("POST", "/leos/v1/recall", "Recall nearest memories at the leOS CA offer price", price=LEOS_TOKEN_PRICE), - PaidRoute("POST", "/leos/v1/route", "Route a task at the leOS CA offer price", price=LEOS_TOKEN_PRICE), - PaidRoute("GET", "/leos/v1/dashboard", "Read the dashboard at the leOS CA offer price", price=LEOS_TOKEN_PRICE), -) +DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = REGULAR_PAID_ROUTES +TESTNET_NETWORKS = frozenset({"eip155:84532"}) + -DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = REGULAR_PAID_ROUTES + LEOS_PAID_ROUTES +def _price_amount(price: str) -> Decimal: + """Parse a dollar-denominated x402 price without a floating-point round trip.""" + try: + amount = Decimal(price[1:]) + except (InvalidOperation, ValueError) as exc: + raise ValueError("x402 price must be a positive dollar amount, e.g. '$0.001'") from exc + if not amount.is_finite() or amount <= 0: + raise ValueError("x402 price must be a positive dollar amount, e.g. '$0.001'") + return amount LANDING_PAGE_TEMPLATE = Template(""" @@ -102,7 +105,7 @@ def key(self) -> str: leCore x402 API - + @@ -130,8 +132,8 @@ def key(self) -> str:
- -

Live on AWS$network_label$price per callleOS CA $leos_token_price

leCore x402 API

Buy the small, useful surface of leCore: local agent memory, capability routing, and a readiness dashboard, sold as paid HTTP primitives instead of another subscription dashboard.

View pricingSee x402 challengeleOS site
+ +

$environment_label$network_label$price_per_thousand

leCore x402 API

Try local agent memory, capability routing, and a readiness dashboard as paid HTTP primitives. $payment_notice

View preview termsSee x402 challenge
+asset: $payment_asset
-
Endpointhttps://lecore.rati.foundation
Paymentx402 exact scheme
Buyer shapeinspect, pay, call
-

Why you would buy it

Because most agents do not need a platform. They need a few reliable cognitive calls.

You pay for an answerable primitive, not a monthly seat.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: local agent memory plus capability routing.

The implementation is deployed, health-checked, and already returning x402 payment challenges.

-

leOS token offer

A slightly cheaper price for eligible leOS buyers.

Offer price$leos_token_price
CA$leos_token_ca
leOS website
+
Endpointhttps://lecore.rati.foundation
Stage$environment_label
Buyer shapeinspect, pay, call
+

Why try it

Most agents do not need a platform. They need a few reliable cognitive calls.

Test the x402 payment flow against one answerable primitive at a time.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: local agent memory plus capability routing.

$payment_notice

What the payment unlocks

Three paid routes, each small enough to understand.

POST

Recall

/v1/recall

Pull nearest memories from a compact local agent core without shipping a whole application stack.

POST

Route

/v1/route

Send a plain-language task and get the leCore capability it should use, with evidence attached.

GET

Dashboard

/v1/dashboard

Read the readiness surface: memory counts, capability map, abstention behavior, and route coverage.

Good first buyers

Teams who want the leCore idea without adopting the whole repo.

Agent memory for prototypes that should remember without a database rollout.

Capability routing for tools that need to pick the right leCore subsystem before doing work.

Evidence dashboards for teams deciding whether a local vector system is ready to productize.

A working x402 seller endpoint to copy when you want pay-per-call APIs instead of subscriptions.

-

Proof it is real

It is already deployed, priced, and protected.

The free endpoints show health and pricing. Paid endpoints return a real x402 payment challenge. The receiving address is public, while admin writes stay out of the paid customer path.

Price
$price
Network
$network_name
Receiver
$pay_to_short
Status
Healthy
-

The pitch

Buy it when you want a local-memory agent primitive that can pay for itself one request at a time.

Inspect the offerCheck live health
+

Preview status

It is deployed, health-checked, and ready to integrate.

The free endpoints show health and preview terms. Paid endpoints return an x402 challenge on $network_name. The receiving address is public, while admin writes stay out of the customer path.

Preview price
$price_per_thousand
Network
$network_name
Receiver
$pay_to_short
Status
Healthy
+

The pitch

Try it when you want a local-memory agent primitive without adopting the whole repo.

Inspect preview termsCheck live health
""") @@ -170,6 +171,7 @@ def __post_init__(self) -> None: raise ValueError("pay_to is required") if not self.price.startswith("$"): raise ValueError("x402 price must include a dollar prefix, e.g. '$0.001'") + _price_amount(self.price) if not self.network: raise ValueError("network is required") if not self.facilitator_url: @@ -985,18 +987,26 @@ def _result(record: Dict[str, Any], memory: Dict[str, Any]) -> Dict[str, Any]: } -def leos_token_offer(access_required: bool = True, enabled: bool = True) -> Dict[str, Any]: - """Return public metadata for the credential-gated leOS offer.""" +def pricing_summary(config: X402Config) -> Dict[str, Any]: + """Describe the customer-facing price and whether it is a production charge.""" + per_thousand = _price_amount(config.price) * Decimal("1000") + per_thousand_display = "$%s" % per_thousand.quantize(Decimal("0.01")) + testnet = config.network in TESTNET_NETWORKS + environment = "testnet_preview" if testnet else "production" + payment_asset = "testnet USDC" if testnet else "USDC" + payment_notice = ( + "This Base Sepolia developer preview uses testnet USDC and does not accept production payments." + if testnet + else "Payments settle in USDC through x402." + ) return { - "name": "leOS CA offer", - "site": LEOS_SITE_URL, - "ca": LEOS_TOKEN_CA, - "price": LEOS_TOKEN_PRICE, - "access_header": LEOS_ACCESS_HEADER, - "access_required": bool(access_required), - "enabled": bool(enabled), - "discount_routes": [route.key for route in LEOS_PAID_ROUTES], - "note": "The CA identifies the offer; eligible buyers also receive an operator-issued access credential.", + "environment": environment, + "environment_label": "Testnet developer preview" if testnet else "Production API", + "payment_asset": payment_asset, + "per_request": config.price, + "per_1000_requests": per_thousand_display, + "display_price": "%s per 1,000 requests" % per_thousand_display, + "payment_notice": payment_notice, } @@ -1048,17 +1058,17 @@ def env_flag(value: Optional[str]) -> bool: def landing_page_html(config: X402Config) -> str: """Render the buyer-facing landing page served from `/`.""" network_name = _network_name(config.network) - offer = leos_token_offer() + summary = pricing_summary(config) return LANDING_PAGE_TEMPLATE.substitute( nodes=_landing_nodes(), - price=escape(config.price), network=escape(config.network), network_label=escape("%s x402" % network_name), network_name=escape(network_name), pay_to_short=escape(_short_address(config.pay_to)), - leos_site_url=escape(offer["site"], quote=True), - leos_token_ca=escape(offer["ca"]), - leos_token_price=escape(offer["price"]), + environment_label=escape(summary["environment_label"]), + payment_asset=escape(summary["payment_asset"]), + payment_notice=escape(summary["payment_notice"]), + price_per_thousand=escape(summary["display_price"]), ) @@ -1078,8 +1088,6 @@ def payment_manifest(config: X402Config) -> List[Dict[str, Any]]: "pay_to": config.pay_to, }], } - if route.price: - row["offer"] = "leos_ca" out.append(row) return out @@ -1131,7 +1139,6 @@ def create_app( admin_token: Optional[str] = None, tenant_secret: Optional[str] = None, tenant_state_dir: Optional[Any] = None, - leos_access_token: Optional[str] = None, memory_backend: Optional[str] = None, nosqlite_binary: Optional[str] = None, nosqlite_data_dir: Optional[Any] = None, @@ -1195,13 +1202,6 @@ async def lifespan(_: Any) -> Any: app.state.memory_transactions = memory_transactions config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") - leos_access_token = leos_access_token or os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN") - active_config = config - if not leos_access_token: - active_config = replace( - config, - routes=tuple(route for route in config.routes if not route.path.startswith("/leos/")), - ) if paid: try: @@ -1210,8 +1210,8 @@ async def lifespan(_: Any) -> Any: raise RuntimeError(optional_dependency_help()) from exc app.add_middleware( PaymentMiddlewareASGI, - routes=x402_route_configs(active_config), - server=x402_resource_server(active_config), + routes=x402_route_configs(config), + server=x402_resource_server(config), ) def require_admin(header_value: Optional[str]) -> None: @@ -1220,12 +1220,6 @@ def require_admin(header_value: Optional[str]) -> None: if not header_value or not hmac.compare_digest(header_value, admin_token): raise HTTPException(status_code=401, detail="invalid admin token") - def require_leos_access(header_value: Optional[str]) -> None: - if not leos_access_token: - raise HTTPException(status_code=503, detail="leOS discount access is not configured") - if not header_value or not hmac.compare_digest(header_value, leos_access_token): - raise HTTPException(status_code=401, detail="invalid leOS discount credential") - def require_tenant_access(tenant_id: str, token: Optional[str]) -> None: normalized = normalize_tenant_id(tenant_id) if normalized == DEFAULT_TENANT_ID: @@ -1321,10 +1315,10 @@ def pricing() -> Dict[str, Any]: return { "ok": True, "x402": config.to_public_dict(), - "token_offer": leos_token_offer(enabled=bool(leos_access_token)), + "pricing": pricing_summary(config), "tenancy": tenancy_public_dict(), "memory_backend": memory_public_dict(), - "routes": payment_manifest(active_config), + "routes": payment_manifest(config), } def recall_response( @@ -1367,16 +1361,6 @@ def recall( ) -> Dict[str, Any]: return recall_response(payload, x_lecore_tenant, x_lecore_tenant_token) - @app.post("/leos/v1/recall") - def leos_recall( - payload: Dict[str, Any], - x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), - x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), - x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), - ) -> Dict[str, Any]: - require_leos_access(x_lecore_leos_access) - return recall_response(payload, x_lecore_tenant, x_lecore_tenant_token) - def route_response( payload: Dict[str, Any], x_lecore_tenant: Optional[str], @@ -1396,16 +1380,6 @@ def route( ) -> Dict[str, Any]: return route_response(payload, x_lecore_tenant, x_lecore_tenant_token) - @app.post("/leos/v1/route") - def leos_route( - payload: Dict[str, Any], - x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), - x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), - x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), - ) -> Dict[str, Any]: - require_leos_access(x_lecore_leos_access) - return route_response(payload, x_lecore_tenant, x_lecore_tenant_token) - def dashboard_response( x_lecore_tenant: Optional[str], x_lecore_tenant_token: Optional[str], @@ -1422,15 +1396,6 @@ def dashboard( ) -> Dict[str, Any]: return dashboard_response(x_lecore_tenant, x_lecore_tenant_token) - @app.get("/leos/v1/dashboard") - def leos_dashboard( - x_lecore_leos_access: Optional[str] = Header(default=None, alias=LEOS_ACCESS_HEADER), - x_lecore_tenant: Optional[str] = Header(default=None, alias=TENANT_HEADER), - x_lecore_tenant_token: Optional[str] = Header(default=None, alias=TENANT_TOKEN_HEADER), - ) -> Dict[str, Any]: - require_leos_access(x_lecore_leos_access) - return dashboard_response(x_lecore_tenant, x_lecore_tenant_token) - @app.post("/admin/remember") def remember( payload: Dict[str, Any], @@ -1534,7 +1499,6 @@ def main(argv: Optional[Iterable[str]] = None) -> None: p.add_argument("--admin-token", default=os.environ.get("LECORE_X402_ADMIN_TOKEN")) p.add_argument("--tenant-secret", default=os.environ.get("LECORE_X402_TENANT_SECRET")) p.add_argument("--tenant-state-dir", default=os.environ.get("LECORE_X402_TENANT_STATE_DIR")) - p.add_argument("--leos-access-token", default=os.environ.get("LECORE_X402_LEOS_ACCESS_TOKEN")) p.add_argument( "--memory-backend", choices=(MEMORY_BACKEND_CORE, MEMORY_BACKEND_NOSQLITE), @@ -1565,7 +1529,6 @@ def main(argv: Optional[Iterable[str]] = None) -> None: admin_token=args.admin_token, tenant_secret=args.tenant_secret, tenant_state_dir=args.tenant_state_dir, - leos_access_token=args.leos_access_token, memory_backend=args.memory_backend, nosqlite_binary=args.nosqlite_bin, nosqlite_data_dir=args.nosqlite_data_dir, diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index 1b9ac50c..eb3b5d7c 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -10,10 +10,6 @@ DEFAULT_PRICE, DEFAULT_TENANT_ID, IDEMPOTENCY_HEADER, - LEOS_SITE_URL, - LEOS_ACCESS_HEADER, - LEOS_TOKEN_CA, - LEOS_TOKEN_PRICE, MEMORY_BACKEND_NOSQLITE, MemoryTransactionConflict, MemoryMirrorPending, @@ -25,9 +21,9 @@ X402Config, create_app, landing_page_html, - leos_token_offer, optional_dependency_help, payment_manifest, + pricing_summary, tenant_access_token, normalize_memory_backend, x402_route_configs, @@ -51,29 +47,19 @@ def test_payment_manifest_protects_specific_read_routes_only(): "POST /v1/recall", "POST /v1/route", "GET /v1/dashboard", - "POST /leos/v1/recall", - "POST /leos/v1/route", - "GET /leos/v1/dashboard", } assert all("*" not in route for route in routes) assert "POST /admin/remember" not in routes assert "POST /admin/tenant-token" not in routes assert "GET /health" not in routes assert all(row["accepts"][0]["pay_to"] == "0xabc" for row in manifest) - assert { - row["route"]: row["accepts"][0]["price"] - for row in manifest - if row["route"].startswith("POST /leos") or row["route"].startswith("GET /leos") - } == { - "POST /leos/v1/recall": LEOS_TOKEN_PRICE, - "POST /leos/v1/route": LEOS_TOKEN_PRICE, - "GET /leos/v1/dashboard": LEOS_TOKEN_PRICE, - } def test_price_validation_keeps_x402_format_honest(): with pytest.raises(ValueError, match="dollar prefix"): X402Config(pay_to="0xabc", price="0.001") + with pytest.raises(ValueError, match="positive dollar amount"): + X402Config(pay_to="0xabc", price="$0") def test_x402_route_configs_build_against_optional_sdk(): @@ -82,10 +68,7 @@ def test_x402_route_configs_build_against_optional_sdk(): routes = x402_route_configs(X402Config(pay_to="0xabc")) assert sorted(routes) == [ - "GET /leos/v1/dashboard", "GET /v1/dashboard", - "POST /leos/v1/recall", - "POST /leos/v1/route", "POST /v1/recall", "POST /v1/route", ] @@ -137,38 +120,41 @@ def _nosqlite_binary() -> str: return binary -def test_landing_page_explains_why_to_buy_the_api(): +def test_landing_page_marks_the_testnet_api_as_a_preview(): html = landing_page_html(X402Config(pay_to="0x96e1604E92A8A1edD0701be3E67Bd4366e87BB84")) assert "leCore x402 API" in html - assert "Buy the small, useful surface of leCore" in html - assert "%s per call" % DEFAULT_PRICE in html - assert LEOS_TOKEN_PRICE in html - assert LEOS_TOKEN_CA in html - assert LEOS_SITE_URL in html + assert "Testnet developer preview" in html + assert "$1.10 per 1,000 requests" in html + assert "does not accept production payments" in html assert "Base Sepolia x402" in html assert "/pricing" in html assert "/v1/dashboard" in html assert "0x96e1...BB84" in html + assert "leOS" not in html -def test_leos_token_offer_identifies_ca_and_requires_access(): - offer = leos_token_offer() +def test_pricing_summary_distinguishes_testnet_preview_from_production(): + preview = pricing_summary(X402Config(pay_to="0xabc")) + production = pricing_summary(X402Config(pay_to="0xabc", network="eip155:8453")) - assert offer["site"] == LEOS_SITE_URL - assert offer["ca"] == LEOS_TOKEN_CA - assert offer["price"] == LEOS_TOKEN_PRICE - assert "POST /leos/v1/recall" in offer["discount_routes"] - assert offer["access_header"] == LEOS_ACCESS_HEADER - assert offer["access_required"] is True - assert "eligible buyers" in offer["note"] + assert preview == { + "environment": "testnet_preview", + "environment_label": "Testnet developer preview", + "payment_asset": "testnet USDC", + "per_request": DEFAULT_PRICE, + "per_1000_requests": "$1.10", + "display_price": "$1.10 per 1,000 requests", + "payment_notice": "This Base Sepolia developer preview uses testnet USDC and does not accept production payments.", + } + assert production["environment"] == "production" + assert production["payment_asset"] == "USDC" + assert production["payment_notice"] == "Payments settle in USDC through x402." def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): fastapi_testclient = pytest.importorskip("fastapi.testclient") - client = fastapi_testclient.TestClient( - create_app(config=X402Config(pay_to="0xabc"), paid=False, leos_access_token="leos-secret") - ) + client = fastapi_testclient.TestClient(create_app(config=X402Config(pay_to="0xabc"), paid=False)) landing = client.get("/") assert landing.status_code == 200 @@ -182,23 +168,15 @@ def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): pricing = client.get("/pricing") assert pricing.status_code == 200 assert pricing.json()["x402"]["price"] == DEFAULT_PRICE - assert pricing.json()["token_offer"]["ca"] == LEOS_TOKEN_CA - assert pricing.json()["token_offer"]["price"] == LEOS_TOKEN_PRICE - assert pricing.json()["token_offer"]["enabled"] is True + assert pricing.json()["pricing"]["environment"] == "testnet_preview" + assert pricing.json()["pricing"]["per_1000_requests"] == "$1.10" assert pricing.json()["tenancy"]["default_tenant"] == DEFAULT_TENANT_ID - - blocked = client.post("/leos/v1/route", json={"task": "search local agent memory"}) - assert blocked.status_code == 401 - assert client.post("/leos/v1/recall", json={"query": "memory"}).status_code == 401 - assert client.get("/leos/v1/dashboard").status_code == 401 - - leos_route = client.post( - "/leos/v1/route", - headers={LEOS_ACCESS_HEADER: "leos-secret"}, - json={"task": "search local agent memory"}, - ) - assert leos_route.status_code == 200 - assert leos_route.json()["tenant"] == DEFAULT_TENANT_ID + assert {row["route"] for row in pricing.json()["routes"]} == { + "POST /v1/recall", + "POST /v1/route", + "GET /v1/dashboard", + } + assert client.get("/leos/v1/dashboard").status_code == 404 def test_health_does_not_run_expensive_evidence_probe(): @@ -218,9 +196,9 @@ def fail_evidence(): assert response.status_code == 200 assert response.json()["memory"]["entries"] == 3 - assert pricing["token_offer"]["enabled"] is False + assert pricing["pricing"]["environment"] == "testnet_preview" assert all(not row["route"].split(" ", 1)[1].startswith("/leos/") for row in pricing["routes"]) - assert client.get("/leos/v1/dashboard").status_code == 503 + assert client.get("/leos/v1/dashboard").status_code == 404 @pytest.mark.parametrize( From cbb4aedf954f84bb70ecac2bdb8f8922eccdfa30 Mon Sep 17 00:00:00 2001 From: atimics Date: Sat, 8 Aug 2026 11:24:38 -0700 Subject: [PATCH 08/16] Fix canonical x402 resource URLs --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + API_QUICKREF.md | 27 ++++- AWS_X402_DEPLOY.md | 143 ++++++++++++++++++++++++-- CAPABILITIES.md | 18 +++- REFERENCE.md | 156 +++++++++++++++++++++-------- X402_API.md | 6 +- capabilities.json | 51 +++++++++- docs/PIPELINE_MAP.md | 2 +- holographic_x402_api.py | 52 +++++++++- pipelines.json | 2 +- setup.py | 2 +- tests/test_holographic_x402_api.py | 102 ++++++++++++++++++- 13 files changed, 501 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b864536..b3df1fbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,7 +309,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements.txt -r requirements-x402.txt - name: Sanity-check the partition (exact cover, disjoint, deterministic) run: python tools/shard_tests.py --selfcheck --num-shards 4 diff --git a/.gitignore b/.gitignore index b84605c2..47a53bde 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ holographic_vsa_complete.zip /dist /build /build_pkg +vendor/nosqlite/target/ *.egg-info repo.zip /temp diff --git a/API_QUICKREF.md b/API_QUICKREF.md index 7a5bd46a..64f2a493 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -38,17 +38,40 @@ - `optional_dependency_help()` -- Install hint for the optional paid API dependencies. - `normalize_tenant_id(value)` -- Return a path-safe tenant id for private memory routing. - `tenant_access_token(tenant_id, secret)` -- Deterministic tenant bearer token derived from a server-side secret. +- `normalize_idempotency_key(value)` -- Validate an optional caller-provided retry key without persisting the raw value. - **class `TenantCoreStore`** -- Thread-safe LocalAgentCore registry with optional per-tenant persistence. - `loaded_tenants(self)` -- Return tenant ids currently loaded in memory. - `summary(self, tenant_id)` -- Return a cheap cached status summary without probing capabilities. - `read(self, tenant_id, fn)` -- Run a read-style operation while holding the tenant lock. - `write(self, tenant_id, fn)` -- Run a mutating operation, then persist that tenant if configured. -- `pricing_summary(config)` -- Return the customer-facing request and per-1,000-request price plus preview/production status. +- **class `NoSQLiteError`** -- Raised when the optional NoSQLite command process cannot serve a request. +- **class `NoSQLiteProcess`** -- Serialize JSON-line requests to one long-lived NoSQLite CLI process. + - `generation(self)` -- Return the number of successful NoSQLite process starts. + - `running(self)` -- Return whether the managed NoSQLite process is currently alive. + - `ensure_started(self)` -- Start the child lazily and return its generation number. + - `command(self, payload)` -- Send one command and return the object response from NoSQLite. + - `close(self)` -- Release the child process and its filesystem writer lock. +- **class `NoSQLiteMemoryStore`** -- Tenant-isolated semantic memory backed by the pinned NoSQLite CLI. + - `running(self)` -- Return whether the underlying NoSQLite process is currently alive. + - `remember(self, tenant_id, memory)` -- Persist one LocalAgentCore-compatible memory entry in its tenant collection. + - `sync(self, tenant_id, memories)` -- Backfill the durable core mirror once per tenant and CLI generation. + - `recall(self, tenant_id, query, k, abstain=None)` -- Return NoSQLite semantic hits in the LocalAgentCore response shape. + - `close(self)` -- Release the underlying NoSQLite process and writer lock. +- **class `MemoryTransactionError`** -- The durable memory write journal could not be read or completed safely. +- **class `MemoryTransactionConflict`** -- One idempotency key was reused for a different memory write. +- **class `MemoryMirrorPending`** -- A durable core commit needs the same transaction projected to NoSQLite. +- **class `TenantMemoryTransactions`** -- Durable, idempotent memory writes spanning LocalAgentCore and NoSQLite. + - `remember(self, tenant_id, text, label, metadata, idempotency_key, mirror)` -- Commit one memory and return its stable transaction status. + - `resume(self, tenant_id, transaction_id, mirror)` -- Resume a known journal record without minting a second transaction. + - `recover_pending(self, mirror)` -- Replay incomplete durable writes, leaving unavailable mirrors pending. +- `pricing_summary(config)` -- Describe the customer-facing price and whether it is a production charge. +- `normalize_memory_backend(value)` -- Validate the memory backend selector without accepting silent fallbacks. +- `env_flag(value)` -- Parse the small explicit boolean surface used by deployment settings. - `landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. - `payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. - `x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. - `x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. -- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None)` -- Create the FastAPI application for paid or local serving. +- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None, memory_backend=None, nosqlite_binary=None, nosqlite_data_dir=None, nosqlite_durability=None, nosqlite_shadow=None)` -- Create the FastAPI application for paid or local serving. - `load_core(path)` -- Load a persisted core if present, otherwise return the demo core. - `main(argv=None)` -- CLI entry point for local x402 API serving. diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index fa3f7040..4a1e8e91 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -30,8 +30,8 @@ or pay upstream APIs as a buyer. facilitator credentials. Store `LECORE_X402_TENANT_SECRET` there too when private tenants are enabled. - **SSM Parameter Store or plain task env** stores non-secret config like - `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, `LECORE_X402_NETWORK`, and - `LECORE_X402_TENANT_STATE_DIR`. + `LECORE_X402_PAY_TO`, `LECORE_X402_PRICE`, `LECORE_X402_NETWORK`, + `LECORE_X402_PUBLIC_URL`, and `LECORE_X402_TENANT_STATE_DIR`. - **CloudWatch Logs** captures service logs. - **AWS WAF** can rate-limit and block bad traffic at the ALB. @@ -53,20 +53,42 @@ Seller-only route: ## Build And Push -```bash -aws ecr create-repository --repository-name lecore-x402 +The deployed preview runs in `us-east-1` from the existing +`lecore-x402-api` ECR repository. Build a unique ARM64 image, then pin the +deployment to its digest. Never deploy `latest`. +```bash ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" -REGION="${AWS_REGION:-us-west-2}" -IMAGE="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/lecore-x402:latest" +REGION="us-east-1" +REPOSITORY="lecore-x402-api" +REGISTRY="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" +REVISION="$(git rev-parse HEAD)" +IMAGE_TAG="${REVISION:0:12}-$(date -u +%Y%m%dT%H%M%SZ)" +IMAGE="$REGISTRY/$REPOSITORY:$IMAGE_TAG" aws ecr get-login-password --region "$REGION" \ - | docker login --username AWS --password-stdin "$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" + | docker login --username AWS --password-stdin "$REGISTRY" -docker build -f Dockerfile.x402 -t "$IMAGE" . +docker build --platform linux/arm64 \ + --label "org.opencontainers.image.revision=$REVISION" \ + -f Dockerfile.x402 -t "$IMAGE" . docker push "$IMAGE" + +DIGEST="$(aws ecr describe-images --region "$REGION" \ + --repository-name "$REPOSITORY" --image-ids imageTag="$IMAGE_TAG" \ + --query 'imageDetails[0].imageDigest' --output text)" +PINNED_IMAGE="$REGISTRY/$REPOSITORY@$DIGEST" + +aws ecr wait image-scan-complete --region "$REGION" \ + --repository-name "$REPOSITORY" --image-id imageDigest="$DIGEST" +aws ecr describe-image-scan-findings --region "$REGION" \ + --repository-name "$REPOSITORY" --image-id imageDigest="$DIGEST" \ + --query 'imageScanFindings.findingSeverityCounts' ``` +Review the scan before registration. Do not deploy if the new image regresses +against the active image or violates the release vulnerability policy. + ## Runtime Environment Non-secret environment variables: @@ -76,6 +98,7 @@ LECORE_X402_PAY_TO=0xYourReceivingWallet LECORE_X402_PRICE=$0.0011 LECORE_X402_NETWORK=eip155:8453 LECORE_X402_FACILITATOR_URL=https://api.cdp.coinbase.com/platform/v2/x402 +LECORE_X402_PUBLIC_URL=https://lecore.rati.foundation LECORE_X402_TENANT_STATE_DIR=/data/tenants LECORE_X402_MEMORY_BACKEND=core ``` @@ -92,6 +115,108 @@ CDP_API_KEY_SECRET= Use ECS task definition `secrets` entries for secrets, not literal environment variables in the task definition. +## ECS Rollout + +The live service is `lonely-forest-cluster/lecore-x402-api`. Treat its current +task definition as the rollback target and change only the `app` container +image. This preserves the task roles, ARM64 runtime, CPU and memory, logging, +EFS volume, environment, and secret references. + +The commands below assume `REGION`, `PINNED_IMAGE`, and `DIGEST` are still set +from the build step and that `jq` is installed. + +```bash +CLUSTER="lonely-forest-cluster" +SERVICE="lecore-x402-api" +umask 077 +DEPLOY_DIR="$(mktemp -d /tmp/lecore-x402-deploy.XXXXXX)" +trap 'rm -rf -- "$DEPLOY_DIR"' EXIT +ROLLBACK_TASK_DEF="$(aws ecs describe-services --region "$REGION" \ + --cluster "$CLUSTER" --services "$SERVICE" \ + --query 'services[0].taskDefinition' --output text)" + +aws ecs describe-task-definition --region "$REGION" \ + --task-definition "$ROLLBACK_TASK_DEF" --include TAGS \ + --output json > "$DEPLOY_DIR/described-task.json" +jq '.taskDefinition' "$DEPLOY_DIR/described-task.json" \ + > "$DEPLOY_DIR/base-task.json" +TASK_TAGS="$(jq -c '.tags // []' "$DEPLOY_DIR/described-task.json")" + +jq --arg image "$PINNED_IMAGE" ' + if ([.containerDefinitions[] | select(.name == "app")] | length) != 1 + then error("expected exactly one app container") else . end + | + del( + .taskDefinitionArn, .revision, .status, .requiresAttributes, + .compatibilities, .registeredAt, .registeredBy, .deregisteredAt + ) + | (.containerDefinitions[] | select(.name == "app").image) = $image +' "$DEPLOY_DIR/base-task.json" > "$DEPLOY_DIR/next-task.json" + +jq -S ' + del( + .taskDefinitionArn, .revision, .status, .requiresAttributes, + .compatibilities, .registeredAt, .registeredBy, .deregisteredAt + ) + | (.containerDefinitions[] | select(.name == "app").image) = "__IMAGE__" +' "$DEPLOY_DIR/base-task.json" > "$DEPLOY_DIR/base.normalized.json" +jq -S ' + (.containerDefinitions[] | select(.name == "app").image) = "__IMAGE__" +' "$DEPLOY_DIR/next-task.json" > "$DEPLOY_DIR/next.normalized.json" +cmp "$DEPLOY_DIR/base.normalized.json" "$DEPLOY_DIR/next.normalized.json" + +CURRENT_TASK_DEF="$(aws ecs describe-services --region "$REGION" \ + --cluster "$CLUSTER" --services "$SERVICE" \ + --query 'services[0].taskDefinition' --output text)" +test "$CURRENT_TASK_DEF" = "$ROLLBACK_TASK_DEF" + +NEW_TASK_DEF="$(aws ecs register-task-definition --region "$REGION" \ + --cli-input-json "file://$DEPLOY_DIR/next-task.json" --tags "$TASK_TAGS" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + +aws ecs update-service --region "$REGION" --cluster "$CLUSTER" \ + --service "$SERVICE" --task-definition "$NEW_TASK_DEF" +aws ecs wait services-stable --region "$REGION" \ + --cluster "$CLUSTER" --services "$SERVICE" +``` + +If the equality check fails, another operator changed the service after this +rollout began. Stop, inspect that task definition, and rebuild the candidate +from the new base rather than overwriting it. + +Normal rolling deployment is safe while `LECORE_X402_MEMORY_BACKEND=core`. +Do not turn on the single-writer NoSQLite backend in the same rollout. + +## Verify And Roll Back + +Verify all of the following before considering the rollout complete: + +- ECS reports one running task, none pending, and the service references + `NEW_TASK_DEF`. +- The running `app` container's `imageDigest` equals `DIGEST`. +- The ALB target is healthy. +- `GET /health` and `GET /pricing` return `200`. +- `GET /v1/dashboard` returns `402`, and its decoded `payment-required` header + advertises exactly + `https://lecore.rati.foundation/v1/dashboard`. +- `/health` still reports private tenancy and durable transactions, and the + EFS-backed memory state is present. +- CloudWatch logs since the rollout contain no new errors, tracebacks, or + exceptions. + +Roll back on a stability wait failure, an unhealthy target, a wrong image +digest, any `5xx`, missing durable state, or an incorrect payment resource URL: + +```bash +aws ecs update-service --region "$REGION" --cluster "$CLUSTER" \ + --service "$SERVICE" --task-definition "$ROLLBACK_TASK_DEF" +aws ecs wait services-stable --region "$REGION" \ + --cluster "$CLUSTER" --services "$SERVICE" +``` + +Re-run the endpoint and digest checks after rollback. Do not deregister the +rollback task definition or delete its ECR digest. + ## Optional NoSQLite Cutover The container has `/usr/local/bin/nosqlite` built from the vendored source @@ -171,6 +296,8 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. - Use mainnet network id and production facilitator URL. - Put the ALB behind HTTPS only. +- Set `LECORE_X402_PUBLIC_URL` to the canonical HTTPS endpoint so payment + challenges never depend on forwarded request headers. - Keep `/admin/remember` private or blocked from the public ALB path. - Keep `/admin/tenant-token` private or blocked from the public ALB path. - Keep paid route configs explicit; avoid wildcard paid routes at first. diff --git a/CAPABILITIES.md b/CAPABILITIES.md index b4368462..c5dfcfd1 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -448,6 +448,14 @@ import lecore; m=lecore.UnifiedMind(dim=256,seed=0); print(m.suggest_pipeline('t ``` *Find it by:* how do I get from points to a mesh, chain capabilities, build a pipeline, route between datatypes, what steps turn X into Y +### x402 paid API publisher +publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, task routing, and the evidence dashboard protected by x402 middleware, with free health/pricing routes and admin-token-gated memory writes.. + +```python +from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...')) +``` +*Find it by:* x402, paid api, payment required, 402, monetize api, micropayment, agent payments, pay per request + ## Memory, search & recall *store things and get them back by CONTENT, not by exact key.* @@ -664,6 +672,14 @@ mind.navigator_benchmark() # recall + the fixed-beam baseline ``` *Find it by:* navigator, adaptive search, learned search, search a tree, nearest neighbour search, beam search, spend less effort on easy queries, reflex cache +### Local agent core (memory + routing) +the PRODUCT-FACING wedge: LocalAgentCore gives a local agent deterministic text memory (remember/recall), skill routing over the live capability catalog, JSON persistence, and a readiness dashboard with C-kernel status. This is the small stable door for embedding leCore without learning the whole research surface first.. + +```python +from holographic_product import LocalAgentCore; core = LocalAgentCore(); core.remember('local agent memory'); core.recall('agent memory') +``` +*Find it by:* product, productization, agent memory, local memory, durable memory, recall, skill routing, dashboard + ### Memoize a pure function (the purity gate is the point) skip re-execution of PURE work whose inputs repeat. mind.memoize_pure(fn) keys on (the function's EXACT canonical source, its arguments) and REFUSES a function that is not pure -- is_pure rejects the clock, RNG, IO, global writes, and transitive impurity through a call-graph fixpoint, while accepting a locally-allocated container. A cache over an impure function returns a stale answer silently, so the gate raises instead. MEASURED: 36x on a repeated 256x256 SVD, bit-identical. THE BACKLOG CALLS THIS 'shape-keyed memoization', AND THAT NAME IS A BUG: a canonical shape erases identifiers and constants, so `def f(x): return x + 1` and `def g(x): return x + 2` have the SAME shape and would share a cache entry. mind.canonical_shape(fn) exists, and is a COMPRESSION primitive, never a cache key. KEPT NEGATIVE: the key costs O(input bytes) -- fingerprinting a 512x512 array costs 1.747 ms while A.sum() costs 0.084 ms, so a cheap function of a large array loses 21x; ask mind.machine_place with the function's own cost as the baseline. TWO BACKLOG NUMBERS DID NOT REPRODUCE: shape reuse is 1.13x (node type + depth) or 1.87x (control flow), not 2.36x -- it is a property of the equivalence relation, not the code; and tree purity is 35.4% (781 of 2,188 module-level functions), not 76%. HONEST SCOPE: the gate resolves callees within ONE module, so a function that calls an IMPORTED helper is refused as unresolved (sound, and why tucker.rank_gate is rejected -- it reaches fix_eigvec_signs from another module). Cross-module resolution wants types.. @@ -4989,4 +5005,4 @@ import lecore; m=lecore.UnifiedMind(); print([n for n,_ in m.workflow_neighbors( --- -*638 capability homes. Regenerate this file with `python capdoc.py` (it reads the live catalog, so it stays in step with the engine).* +*640 capability homes. Regenerate this file with `python capdoc.py` (it reads the live catalog, so it stays in step with the engine).* diff --git a/REFERENCE.md b/REFERENCE.md index 84893a00..f6bdf347 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -1,7 +1,7 @@ # leCore -- Code Reference *Auto-generated by `docgen.py` -- do not edit by hand; edit the module docstrings instead and re-run it.* -*620 modules, 216,511 lines of engine code.* +*622 modules, 218,516 lines of engine code.* > **New here? Read this first.** leCore represents *everything* -- memory, geometry, physics, rendering -- as > points in one very high-dimensional space (hypervectors), and combines them with a tiny algebra: **bind** @@ -81,7 +81,7 @@ | [`holographic_splatprune.py`](#holographic-splatprune) | Splat prune / merge + a quality-budget LOD chain (holographic_splatprune). | 187 | | [`holographic_splatsharpen.py`](#holographic-splatsharpen) | C4 probe (cross-cutting: XDATA-3 negative-lobe sharpening -> splat/archive reconstruction). KEPT NEGATIVE. | 87 | -### Core & standalone (578) +### Core & standalone (580) | module | what it is | lines | |---|---|---| @@ -144,7 +144,7 @@ | [`holographic_catalog_p01.py`](#holographic-catalog-p01) | holographic_catalog_p01 -- part 1/6 of the capability registry (split from holographic_catalog). | 785 | | [`holographic_catalog_p02.py`](#holographic-catalog-p02) | holographic_catalog_p02 -- part 2/6 of the capability registry (split from holographic_catalog). | 570 | | [`holographic_catalog_p03.py`](#holographic-catalog-p03) | holographic_catalog_p03 -- part 3/6 of the capability registry (split from holographic_catalog). | 1478 | -| [`holographic_catalog_p04.py`](#holographic-catalog-p04) | holographic_catalog_p04 -- part 4/6 of the capability registry (split from holographic_catalog). | 1541 | +| [`holographic_catalog_p04.py`](#holographic-catalog-p04) | holographic_catalog_p04 -- part 4/6 of the capability registry (split from holographic_catalog). | 1561 | | [`holographic_catalog_p05.py`](#holographic-catalog-p05) | holographic_catalog_p05 -- part 5/6 of the capability registry (split from holographic_catalog). | 966 | | [`holographic_catalog_p06.py`](#holographic-catalog-p06) | holographic_catalog_p06 -- part 6/6 of the capability registry (split from holographic_catalog). | 2345 | | [`holographic_ccrun.py`](#holographic-ccrun) | holographic_ccrun.py -- compile emitted C kernels with the system C compiler and batch-run them. | 149 | @@ -452,6 +452,7 @@ | [`holographic_procbridge.py`](#holographic-procbridge) | Procedural bridges (S3): where the SDF / procedural layer connects to the rest of the stack -- MEASURED. | 158 | | [`holographic_procgen.py`](#holographic-procgen) | Procedural generation (S2): 3D objects from a seed, greebled & fractal models, vegetated terrain. | 223 | | [`holographic_proctex.py`](#holographic-proctex) | Procedural textures (the standard 3D-app set, 2D and 3D) + the mask-edge REFRACTION effect. | 626 | +| [`holographic_product.py`](#holographic-product) | holographic_product.py -- the small product-facing leCore facade. | 392 | | [`holographic_projectivetower.py`](#holographic-projectivetower) | holographic_projectivetower.py -- the ceiling of the transform tower, and where the "word" analogy breaks. | 258 | | [`holographic_protocol.py`](#holographic-protocol) | Protocol-as-data auditing (backlog D1): the honesty discipline as a STRUCTURAL property of a program | 197 | | [`holographic_provenance.py`](#holographic-provenance) | holographic_provenance.py -- tag a vector with WHERE it came from, one model for the whole stack. | 73 | @@ -554,8 +555,8 @@ | [`holographic_skymodel.py`](#holographic-skymodel) | holographic_skymodel.py -- a PARAMETRIC sky: time of day, sun, moon, stars, and HIGH cloud layers, as | 450 | | [`holographic_slime.py`](#holographic-slime) | Slime-mold path-finding over a HOLOGRAPHIC associative graph. | 391 | | [`holographic_smokepresets.py`](#holographic-smokepresets) | holographic_smokepresets.py -- SMOKE PRESETS (fluids/matter backlog, content item 1). | 159 | -| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). | 141 | | [`holographic_snap.py`](#holographic-snap) | holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragge | 184 | +| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). | 141 | | [`holographic_softbody.py`](#holographic-softbody) | Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. | 680 | | [`holographic_sparsefield.py`](#holographic-sparsefield) | FS-2 -- the narrow-band sparse field (holographic_sparsefield), array-backed for parallelism. | 530 | | [`holographic_spatial.py`](#holographic-spatial) | holographic_spatial.py -- ONE shared spatial index. Bin points into a uniform grid of cells so radius, | 191 | @@ -576,8 +577,8 @@ | [`holographic_structure.py`](#holographic-structure) | Proof of meaning: verify that a sequence carries structure, rather than trust | 174 | | [`holographic_subdivcurve.py`](#holographic-subdivcurve) | Subdivision curves on hypervector sequences (ARCH-5): Loop subdivision (FWD-8), turned inward. | 148 | | [`holographic_supermemory.py`](#holographic-supermemory) | Superposed key-value memory with a CLOSED-FORM capacity law, a single-shot allocator, | 634 | -| [`holographic_superposed.py`](#holographic-superposed) | COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. | 51 | | [`holographic_superposed.py`](#holographic-superposed) | holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). | 363 | +| [`holographic_superposed.py`](#holographic-superposed) | COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. | 51 | | [`holographic_superres.py`](#holographic-superres) | holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). | 83 | | [`holographic_superschedule.py`](#holographic-superschedule) | holographic_superschedule.py -- Fill 3: AUTO-SUPERPOSITION + SPILL. The latency-hiding move: hold N | 225 | | [`holographic_surface.py`](#holographic-surface) | holographic_surface.py -- the FIRST-CLASS render material: every channel is a Param socket, resolved PER HIT. | 293 | @@ -661,6 +662,7 @@ | [`holographic_worstview.py`](#holographic-worstview) | M16 -- find the GLOBAL worst view of a mesh over the sphere of directions, without a dense turntable sweep. | 194 | | [`holographic_wos.py`](#holographic-wos) | holographic_wos.py -- #7 / M1 from the SIGGRAPH list: WALK ON SPHERES. Solve PDEs on ANY geometry, no mesh. | 174 | | [`holographic_wost.py`](#holographic-wost) | holographic_wost.py -- Walk on Spheres / Walk on *Stars*: a grid-free Laplace/Poisson solver on an SDF. | 253 | +| [`holographic_x402_api.py`](#holographic-x402-api) | holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. | 1593 | | [`holographic_zigmarch.py`](#holographic-zigmarch) | holographic_zigmarch.py -- the one-kernel-two-runtimes raymarch demo, EXECUTED (backlog Z4). | 230 | | [`holographic_zigrun.py`](#holographic-zigrun) | holographic_zigrun.py -- compile emitted Zig kernels to shared libraries and batch-run them (backlog Z2 + Z3). | 354 | @@ -17300,6 +17302,33 @@ - `def values_to_texture(values, normalize)` -- ASSIGN arbitrary numbers to a texture: an (H,W) / (H,W,C) / (N,) / (N,C) array becomes a - `def mask_refraction(image, mask, strength, ior, profile, edge_width, chromatic, ripple, seed)` -- Refract `image` through a 2D shape given by `mask` (H,W bool/0-1): the LENS reading of a mask. +### holographic_product.py + +> holographic_product.py -- the small product-facing leCore facade. +> +> WHY THIS EXISTS +> --------------- +> The research engine is intentionally broad: memory, geometry, rendering, +> simulation, jobs, skills, and more all share the same holographic substrate. +> That is useful for research, but a first-time product user needs one narrow, +> reliable door. +> +> `LocalAgentCore` is that door. It packages the current production wedge: +> +> * local deterministic text memory (`remember` / `recall`) +> * agent skill routing through the existing capability catalog (`route`) +> * an evidence snapshot and static HTML dashboard (`dashboard`) +> +> It does not replace `UnifiedMind` or hide the research surface. It is a small, +> boring facade over the stable pieces, meant to be easy to install, test, demo, +> and embed. + +**Public API:** + +- `class MemoryEntry` -- One stored memory item. +- `class LocalAgentCore` -- Product facade for local agent memory, skill routing, and evidence. +- `def demo()` -- Build a tiny ready-to-query product demo. + ### holographic_projectivetower.py > holographic_projectivetower.py -- the ceiling of the transform tower, and where the "word" analogy breaks. @@ -22358,28 +22387,6 @@ ### holographic_snap.py -> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). -> -> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in -> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an -> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup -> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, -> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. -> -> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; -> deterministic. - -**Public API:** - -- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the -- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns -- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. -- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. -- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. -- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever - -### holographic_snap.py - > holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragged > point / transform delta actually go?' in the shapes the interactive edit spine wants (dict hit records, a corrected > transform delta), DELEGATING all the actual snap math to the canonical snap primitives in @@ -22405,6 +22412,28 @@ - `def snap_to_edge(point, vertices, edges, max_dist)` -- Snap a point to the nearest point ON any edge, returned as {edge, position, distance, t}, or None if beyond - `def snap_transform_delta(delta, target, increment, moved_point, vertices, edges, origin, max_dist)` -- Snap a TRANSFORM DELTA so the moved point lands on a snap target, and return the corrected delta. This is the +### holographic_snap.py + +> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). +> +> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in +> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an +> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup +> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, +> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. +> +> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; +> deterministic. + +**Public API:** + +- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the +- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns +- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. +- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. +- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. +- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever + ### holographic_softbody.py > Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. @@ -23593,20 +23622,6 @@ ### holographic_superposed.py -> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. -> -> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module -> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing -> in superposition"); this one's build audit queried capability phrasings but never -> grepped the basename, so two unrelated modules shared a name across families -- a -> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's -> name. The capacity-law memory now lives under its own name; this shim keeps every -> existing import working forever (additive, backward-compatible only). - -*(no public functions or classes -- internal or data-only)* - -### holographic_superposed.py - > holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). > > PORTED FROM leOS (`superposed_compute.py`, "one processor, many states simultaneously"). @@ -23644,6 +23659,20 @@ - `def hierarchical_recall(S, group_key, leaf_key, chunk_codebook, item_codebook, min_chunk_similarity)` -- Descend one hierarchical superposition with a CLEANUP at the middle level. - `def flat_recall(S, group_key, leaf_key, item_codebook)` -- The BASELINE hierarchical_recall must beat, and the strongest honest one: unbind both roles from the single +### holographic_superposed.py + +> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. +> +> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module +> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing +> in superposition"); this one's build audit queried capability phrasings but never +> grepped the basename, so two unrelated modules shared a name across families -- a +> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's +> name. The capacity-law memory now lives under its own name; this shim keeps every +> existing import working forever (additive, backward-compatible only). + +*(no public functions or classes -- internal or data-only)* + ### holographic_superres.py > holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). @@ -26787,6 +26816,51 @@ - `def solve_laplace(sdf_eval, points, boundary_value, walks, max_steps, eps, seed, source, dirichlet_sdf, dim)` -- Solve the Laplace (or Poisson) equation at `points`, grid-free, by Walk on Spheres / Stars. +### holographic_x402_api.py + +> holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. +> +> WHY THIS EXISTS +> --------------- +> `LocalAgentCore` is the narrow product wedge. This module makes it sellable as +> an HTTP API without making x402, FastAPI, or uvicorn core dependencies. +> +> The boundary is intentionally conservative: +> +> * public read/compute routes are x402-paid +> * health/pricing routes are free +> * memory writes are admin-token gated, not pay-to-write +> +> That keeps the paid surface useful while preventing customers from poisoning a +> shared memory store just because they paid for one request. + +**Public API:** + +- `class PaidRoute` -- One x402-protected route. +- `class X402Config` -- Seller configuration for the x402-paid API. +- `def optional_dependency_help()` -- Install hint for the optional paid API dependencies. +- `def normalize_tenant_id(value)` -- Return a path-safe tenant id for private memory routing. +- `def tenant_access_token(tenant_id, secret)` -- Deterministic tenant bearer token derived from a server-side secret. +- `def normalize_idempotency_key(value)` -- Validate an optional caller-provided retry key without persisting the raw value. +- `class TenantCoreStore` -- Thread-safe LocalAgentCore registry with optional per-tenant persistence. +- `class NoSQLiteError` -- Raised when the optional NoSQLite command process cannot serve a request. +- `class NoSQLiteProcess` -- Serialize JSON-line requests to one long-lived NoSQLite CLI process. +- `class NoSQLiteMemoryStore` -- Tenant-isolated semantic memory backed by the pinned NoSQLite CLI. +- `class MemoryTransactionError` -- The durable memory write journal could not be read or completed safely. +- `class MemoryTransactionConflict` -- One idempotency key was reused for a different memory write. +- `class MemoryMirrorPending` -- A durable core commit needs the same transaction projected to NoSQLite. +- `class TenantMemoryTransactions` -- Durable, idempotent memory writes spanning LocalAgentCore and NoSQLite. +- `def pricing_summary(config)` -- Describe the customer-facing price and whether it is a production charge. +- `def normalize_memory_backend(value)` -- Validate the memory backend selector without accepting silent fallbacks. +- `def env_flag(value)` -- Parse the small explicit boolean surface used by deployment settings. +- `def landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. +- `def payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. +- `def x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. +- `def x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. +- `def create_app(core, config, paid, admin_token, tenant_secret, tenant_state_dir, memory_backend, nosqlite_binary, nosqlite_data_dir, nosqlite_durability, nosqlite_shadow)` -- Create the FastAPI application for paid or local serving. +- `def load_core(path)` -- Load a persisted core if present, otherwise return the demo core. +- `def main(argv)` -- CLI entry point for local x402 API serving. + ### holographic_zigmarch.py > holographic_zigmarch.py -- the one-kernel-two-runtimes raymarch demo, EXECUTED (backlog Z4). diff --git a/X402_API.md b/X402_API.md index 10c0f871..3fcfbf00 100644 --- a/X402_API.md +++ b/X402_API.md @@ -44,6 +44,7 @@ uses testnet USDC, and does not accept production payments. ```bash export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_PRICE="$0.0011" +export LECORE_X402_PUBLIC_URL="http://127.0.0.1:4021" export LECORE_X402_ADMIN_TOKEN="local-admin-secret" export LECORE_X402_TENANT_SECRET="local-tenant-secret" export LECORE_X402_TENANT_STATE_DIR="./tenant-state" @@ -156,7 +157,10 @@ until that maintenance window is scheduled. ## Production Notes - Use a real receiving wallet and a production facilitator. -- Put the API behind HTTPS. +- Put the API behind HTTPS and set `LECORE_X402_PUBLIC_URL` to its canonical + public base URL, for example `https://lecore.rati.foundation`. Each payment + challenge advertises that configured URL rather than trusting forwarded + request headers. - Keep route prices explicit; avoid wildcard paid route configs for this first product surface. - Keep writes admin-only. Use `LECORE_X402_TENANT_SECRET` and diff --git a/capabilities.json b/capabilities.json index 8c3d3c99..b7df1012 100644 --- a/capabilities.json +++ b/capabilities.json @@ -5841,6 +5841,30 @@ "semantic": null, "theme": "Geometry, modeling & rendering" }, + { + "aliases": [ + "product", + "productization", + "agent memory", + "local memory", + "durable memory", + "recall", + "skill routing", + "dashboard", + "first user", + "facade", + "local agent core" + ], + "consumes": [], + "does": "the PRODUCT-FACING wedge: LocalAgentCore gives a local agent deterministic text memory (remember/recall), skill routing over the live capability catalog, JSON persistence, and a readiness dashboard with C-kernel status. This is the small stable door for embedding leCore without learning the whole research surface first.", + "example": "from holographic_product import LocalAgentCore; core = LocalAgentCore(); core.remember('local agent memory'); core.recall('agent memory')", + "method": null, + "name": "Local agent core (memory + routing)", + "native": true, + "produces": [], + "semantic": null, + "theme": "Memory, search & recall" + }, { "aliases": [ "look ahead linter", @@ -14921,9 +14945,34 @@ "produces": [], "semantic": null, "theme": "Scenes you can describe & adjust" + }, + { + "aliases": [ + "x402", + "paid api", + "payment required", + "402", + "monetize api", + "micropayment", + "agent payments", + "pay per request", + "fastapi", + "api publishing", + "sell api", + "paid route" + ], + "consumes": [], + "does": "publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, task routing, and the evidence dashboard protected by x402 middleware, with free health/pricing routes and admin-token-gated memory writes.", + "example": "from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...'))", + "method": null, + "name": "x402 paid API publisher", + "native": false, + "produces": [], + "semantic": null, + "theme": "Discover & drive it (for agents)" } ], - "count": 638, + "count": 640, "schema_version": "1.0", "scope": "curated capability homes only -- the full live catalog is served at runtime by mind.find_capability / mind.pipeline_map / GET /tools" } diff --git a/docs/PIPELINE_MAP.md b/docs/PIPELINE_MAP.md index 9fcd573c..4f509a1a 100644 --- a/docs/PIPELINE_MAP.md +++ b/docs/PIPELINE_MAP.md @@ -2,7 +2,7 @@ *The workflow graph, auto-derived by `pipelinemap.py` from the catalog's `consumes`/`produces` tags. Nodes are io-kinds; an edge means some capability turns the source kind into the target kind. This is a VIEW of the live tags -- to change it, tag capabilities, not this file.* -> **Coverage: 110 of 2919 capabilities carry io-kind tags (3%).** The graph below is that tagged subset. Untagged capabilities are real but do not yet declare a typed edge -- backfilling tags grows the map. +> **Coverage: 110 of 2921 capabilities carry io-kind tags (3%).** The graph below is that tagged subset. Untagged capabilities are real but do not yet declare a typed edge -- backfilling tags grows the map. ```mermaid graph LR diff --git a/holographic_x402_api.py b/holographic_x402_api.py index bb5c7a64..1ec3ba38 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -35,13 +35,16 @@ import threading import time from typing import Any, Dict, Iterable, List, Optional, Tuple +from urllib.parse import urlsplit from holographic_product import LocalAgentCore, demo +from lecore import __version__ as LECORE_VERSION DEFAULT_FACILITATOR_URL = "https://x402.org/facilitator" DEFAULT_NETWORK = "eip155:84532" # Base Sepolia, safe default for testnet publishing. DEFAULT_PRICE = "$0.0011" +DEFAULT_PUBLIC_URL = "https://lecore.rati.foundation" DEFAULT_TENANT_ID = "public" TENANT_HEADER = "X-leCore-Tenant" TENANT_TOKEN_HEADER = "X-leCore-Tenant-Token" @@ -99,6 +102,33 @@ def _price_amount(price: str) -> Decimal: return amount +def _normalize_public_url(value: str) -> str: + """Return a canonical public base URL safe to advertise in x402 challenges.""" + if not isinstance(value, str): + raise ValueError("public_url must be a string") + value = value.strip().rstrip("/") + if not value or any(char.isspace() or char == "\\" or ord(char) == 127 for char in value): + raise ValueError("public_url must be an absolute http(s) URL") + try: + parts = urlsplit(value) + _ = parts.port + except ValueError as exc: + raise ValueError("public_url must be an absolute http(s) URL") from exc + if ( + parts.scheme not in {"http", "https"} + or not parts.netloc + or parts.netloc.endswith(":") + or parts.hostname is None + or not parts.hostname.strip(".") + ): + raise ValueError("public_url must be an absolute http(s) URL") + if parts.username is not None or parts.password is not None: + raise ValueError("public_url must not contain credentials") + if parts.query or parts.fragment or "?" in value or "#" in value: + raise ValueError("public_url must not contain a query or fragment") + return value + + LANDING_PAGE_TEMPLATE = Template(""" @@ -144,7 +174,7 @@ def _price_amount(price: str) -> Decimal: network: $network asset: $payment_asset -
Endpointhttps://lecore.rati.foundation
Stage$environment_label
Buyer shapeinspect, pay, call
+
Endpoint$public_url
Stage$environment_label
Buyer shapeinspect, pay, call

Why try it

Most agents do not need a platform. They need a few reliable cognitive calls.

Test the x402 payment flow against one answerable primitive at a time.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: local agent memory plus capability routing.

$payment_notice

What the payment unlocks

Three paid routes, each small enough to understand.

POST

Recall

/v1/recall

Pull nearest memories from a compact local agent core without shipping a whole application stack.

POST

Route

/v1/route

Send a plain-language task and get the leCore capability it should use, with evidence attached.

GET

Dashboard

/v1/dashboard

Read the readiness surface: memory counts, capability map, abstention behavior, and route coverage.

Good first buyers

Teams who want the leCore idea without adopting the whole repo.

Agent memory for prototypes that should remember without a database rollout.

Capability routing for tools that need to pick the right leCore subsystem before doing work.

Evidence dashboards for teams deciding whether a local vector system is ready to productize.

A working x402 seller endpoint to copy when you want pay-per-call APIs instead of subscriptions.

@@ -165,6 +195,7 @@ class X402Config: facilitator_url: str = DEFAULT_FACILITATOR_URL scheme: str = "exact" routes: Tuple[PaidRoute, ...] = DEFAULT_PAID_ROUTES + public_url: str = DEFAULT_PUBLIC_URL def __post_init__(self) -> None: if not self.pay_to: @@ -176,6 +207,7 @@ def __post_init__(self) -> None: raise ValueError("network is required") if not self.facilitator_url: raise ValueError("facilitator_url is required") + object.__setattr__(self, "public_url", _normalize_public_url(self.public_url)) @classmethod def from_env(cls, require_pay_to: bool = True) -> "X402Config": @@ -188,6 +220,7 @@ def from_env(cls, require_pay_to: bool = True) -> "X402Config": price=os.environ.get("LECORE_X402_PRICE", DEFAULT_PRICE), network=os.environ.get("LECORE_X402_NETWORK", DEFAULT_NETWORK), facilitator_url=os.environ.get("LECORE_X402_FACILITATOR_URL", DEFAULT_FACILITATOR_URL), + public_url=os.environ.get("LECORE_X402_PUBLIC_URL", DEFAULT_PUBLIC_URL), ) def to_public_dict(self) -> Dict[str, Any]: @@ -198,6 +231,7 @@ def to_public_dict(self) -> Dict[str, Any]: "network": self.network, "facilitator_url": self.facilitator_url, "scheme": self.scheme, + "public_url": self.public_url, } @@ -455,11 +489,13 @@ def __init__( @property def generation(self) -> int: + """Return the number of successful NoSQLite process starts.""" with self._lock: return self._generation @property def running(self) -> bool: + """Return whether the managed NoSQLite process is currently alive.""" with self._lock: return self._process is not None and self._process.poll() is None @@ -624,6 +660,7 @@ def __init__( @property def running(self) -> bool: + """Return whether the underlying NoSQLite process is currently alive.""" return self._process.running def remember(self, tenant_id: str, memory: Dict[str, Any]) -> None: @@ -687,6 +724,7 @@ def recall( return hits def close(self) -> None: + """Release the underlying NoSQLite process and writer lock.""" self._process.close() def _ensure_collection(self, tenant_id: str) -> str: @@ -1062,6 +1100,7 @@ def landing_page_html(config: X402Config) -> str: return LANDING_PAGE_TEMPLATE.substitute( nodes=_landing_nodes(), network=escape(config.network), + public_url=escape(config.public_url), network_label=escape("%s x402" % network_name), network_name=escape(network_name), pay_to_short=escape(_short_address(config.pay_to)), @@ -1111,6 +1150,7 @@ def x402_route_configs(config: X402Config) -> Dict[str, Any]: network=config.network, ) ], + resource=config.public_url + route.path, mime_type=route.mime_type, description=route.description, ) @@ -1159,6 +1199,11 @@ def create_app( except ImportError as exc: raise RuntimeError(optional_dependency_help()) from exc + config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) + public = urlsplit(config.public_url) + if paid and public.scheme != "https" and public.hostname not in {"127.0.0.1", "::1", "localhost"}: + raise ValueError("paid mode public_url must use https outside localhost") + core = core or demo() store = TenantCoreStore(core, state_dir=tenant_state_dir) memory_backend = normalize_memory_backend( @@ -1195,12 +1240,11 @@ async def lifespan(_: Any) -> Any: if nosqlite_store is not None: nosqlite_store.close() - app = FastAPI(title="leCore x402 API", version="0.1.0", lifespan=lifespan) + app = FastAPI(title="leCore x402 API", version=LECORE_VERSION, lifespan=lifespan) app.state.memory_backend = memory_backend app.state.nosqlite_shadow = nosqlite_shadow app.state.nosqlite_store = nosqlite_store app.state.memory_transactions = memory_transactions - config = config or (X402Config.from_env(require_pay_to=paid) if paid else X402Config.from_env(require_pay_to=False)) tenant_secret = tenant_secret or os.environ.get("LECORE_X402_TENANT_SECRET") if paid: @@ -1496,6 +1540,7 @@ def main(argv: Optional[Iterable[str]] = None) -> None: p.add_argument("--price", default=os.environ.get("LECORE_X402_PRICE", DEFAULT_PRICE)) p.add_argument("--network", default=os.environ.get("LECORE_X402_NETWORK", DEFAULT_NETWORK)) p.add_argument("--facilitator-url", default=os.environ.get("LECORE_X402_FACILITATOR_URL", DEFAULT_FACILITATOR_URL)) + p.add_argument("--public-url", default=os.environ.get("LECORE_X402_PUBLIC_URL", DEFAULT_PUBLIC_URL)) p.add_argument("--admin-token", default=os.environ.get("LECORE_X402_ADMIN_TOKEN")) p.add_argument("--tenant-secret", default=os.environ.get("LECORE_X402_TENANT_SECRET")) p.add_argument("--tenant-state-dir", default=os.environ.get("LECORE_X402_TENANT_STATE_DIR")) @@ -1521,6 +1566,7 @@ def main(argv: Optional[Iterable[str]] = None) -> None: price=args.price, network=args.network, facilitator_url=args.facilitator_url, + public_url=args.public_url, ) app = create_app( load_core(args.state), diff --git a/pipelines.json b/pipelines.json index ac3e1e5c..111d0e73 100644 --- a/pipelines.json +++ b/pipelines.json @@ -157,7 +157,7 @@ "coverage": { "percent": 3, "tagged": 110, - "total": 2919 + "total": 2921 }, "edges": [ { diff --git a/setup.py b/setup.py index 2d71126d..3d378e37 100644 --- a/setup.py +++ b/setup.py @@ -96,6 +96,6 @@ def read_version(): # note above); wgpu is INCLUDED, because it ships prebuilt wheels for every platform and needs no # system toolchain -- the reason to leave CuPy out simply does not apply to it. -- "all": ["numba", "pyfftw", "sympy", "flask", "pillow", "pytest", "matplotlib", "ziglang", "nltk", - "wgpu"], + "wgpu", "x402[fastapi,evm]>=2.15,<3", "uvicorn>=0.51,<1"], }, ) diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index eb3b5d7c..173da153 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -8,6 +8,7 @@ from holographic_x402_api import ( DEFAULT_NETWORK, DEFAULT_PRICE, + DEFAULT_PUBLIC_URL, DEFAULT_TENANT_ID, IDEMPOTENCY_HEADER, MEMORY_BACKEND_NOSQLITE, @@ -29,6 +30,7 @@ x402_route_configs, ) from holographic_product import LocalAgentCore, demo +from lecore import __version__ as LECORE_VERSION def test_default_x402_config_uses_testnet_price_shape(): @@ -37,6 +39,7 @@ def test_default_x402_config_uses_testnet_price_shape(): assert cfg.network == DEFAULT_NETWORK assert cfg.price == DEFAULT_PRICE and cfg.price.startswith("$") assert cfg.facilitator_url == "https://x402.org/facilitator" + assert cfg.public_url == DEFAULT_PUBLIC_URL def test_payment_manifest_protects_specific_read_routes_only(): @@ -65,13 +68,18 @@ def test_price_validation_keeps_x402_format_honest(): def test_x402_route_configs_build_against_optional_sdk(): pytest.importorskip("x402") - routes = x402_route_configs(X402Config(pay_to="0xabc")) + routes = x402_route_configs( + X402Config(pay_to="0xabc", public_url="https://api.example.test/") + ) assert sorted(routes) == [ "GET /v1/dashboard", "POST /v1/recall", "POST /v1/route", ] + assert routes["GET /v1/dashboard"].resource == "https://api.example.test/v1/dashboard" + assert routes["POST /v1/recall"].resource == "https://api.example.test/v1/recall" + assert routes["POST /v1/route"].resource == "https://api.example.test/v1/route" def test_env_config_requires_pay_to_for_paid_mode(monkeypatch): @@ -80,7 +88,49 @@ def test_env_config_requires_pay_to_for_paid_mode(monkeypatch): with pytest.raises(ValueError, match="LECORE_X402_PAY_TO"): X402Config.from_env(require_pay_to=True) - assert X402Config.from_env(require_pay_to=False).pay_to == "0xYourAddress" + monkeypatch.setenv("LECORE_X402_PUBLIC_URL", "https://api.example.test/") + config = X402Config.from_env(require_pay_to=False) + + assert config.pay_to == "0xYourAddress" + assert config.public_url == "https://api.example.test" + + +@pytest.mark.parametrize( + "public_url, message", + [ + ("api.example.test", "absolute http"), + ("ftp://api.example.test", "absolute http"), + ("https://", "absolute http"), + ("https://user:secret@api.example.test", "credentials"), + ("https://api.example.test?tenant=public", "query or fragment"), + ("https://api.example.test#pricing", "query or fragment"), + ("https://api.example.test:bad", "absolute http"), + ("https://api.example.test:", "absolute http"), + ("https://.", "absolute http"), + ("https://api.example.test /base", "absolute http"), + ("https://api.example.test\\base", "absolute http"), + ], +) +def test_public_url_rejects_unsafe_or_ambiguous_values(public_url, message): + with pytest.raises(ValueError, match=message): + X402Config(pay_to="0xabc", public_url=public_url) + + +def test_paid_mode_requires_https_outside_localhost(): + pytest.importorskip("fastapi") + pytest.importorskip("x402") + + with pytest.raises(ValueError, match="must use https"): + create_app( + config=X402Config(pay_to="0xabc", public_url="http://api.example.test"), + paid=True, + ) + + local = create_app( + config=X402Config(pay_to="0xabc", public_url="http://127.0.0.1:4021"), + paid=True, + ) + assert local is not None def test_optional_dependency_help_points_to_extra(): @@ -131,9 +181,56 @@ def test_landing_page_marks_the_testnet_api_as_a_preview(): assert "/pricing" in html assert "/v1/dashboard" in html assert "0x96e1...BB84" in html + assert DEFAULT_PUBLIC_URL in html assert "leOS" not in html +def test_landing_page_uses_the_configured_public_url(): + html = landing_page_html( + X402Config(pay_to="0xabc", public_url="https://api.example.test/base/") + ) + + assert "https://api.example.test/base" in html + + +def test_paid_challenge_uses_canonical_resource_not_request_headers(monkeypatch): + pytest.importorskip("x402") + fastapi_testclient = pytest.importorskip("fastapi.testclient") + from x402 import SupportedKind, SupportedResponse + from x402.http import HTTPFacilitatorClient, decode_payment_required_header + + monkeypatch.setattr( + HTTPFacilitatorClient, + "get_supported", + lambda _client: SupportedResponse( + kinds=[ + SupportedKind( + x402_version=2, + scheme="exact", + network=DEFAULT_NETWORK, + ) + ] + ), + ) + client = fastapi_testclient.TestClient( + create_app( + config=X402Config( + pay_to="0x96e1604E92A8A1edD0701be3E67Bd4366e87BB84", + public_url=DEFAULT_PUBLIC_URL, + ), + paid=True, + ) + ) + + response = client.get( + "/v1/dashboard", + headers={"host": "attacker.invalid", "x-forwarded-proto": "http"}, + ) + assert response.status_code == 402 + challenge = decode_payment_required_header(response.headers["payment-required"]) + assert challenge.resource.url == DEFAULT_PUBLIC_URL + "/v1/dashboard" + + def test_pricing_summary_distinguishes_testnet_preview_from_production(): preview = pricing_summary(X402Config(pay_to="0xabc")) production = pricing_summary(X402Config(pay_to="0xabc", network="eip155:8453")) @@ -156,6 +253,7 @@ def test_unpaid_dev_app_serves_landing_page_and_keeps_api_routes_free(): fastapi_testclient = pytest.importorskip("fastapi.testclient") client = fastapi_testclient.TestClient(create_app(config=X402Config(pay_to="0xabc"), paid=False)) + assert client.app.version == LECORE_VERSION landing = client.get("/") assert landing.status_code == 200 assert landing.headers["content-type"].startswith("text/html") From d750f6d67956fc5db8019e75bbffb0f1b8c0c98e Mon Sep 17 00:00:00 2001 From: docs-bot Date: Sat, 8 Aug 2026 18:25:40 +0000 Subject: [PATCH 09/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 76 ++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index f6bdf347..38a2b403 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -555,8 +555,8 @@ | [`holographic_skymodel.py`](#holographic-skymodel) | holographic_skymodel.py -- a PARAMETRIC sky: time of day, sun, moon, stars, and HIGH cloud layers, as | 450 | | [`holographic_slime.py`](#holographic-slime) | Slime-mold path-finding over a HOLOGRAPHIC associative graph. | 391 | | [`holographic_smokepresets.py`](#holographic-smokepresets) | holographic_smokepresets.py -- SMOKE PRESETS (fluids/matter backlog, content item 1). | 159 | -| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragge | 184 | | [`holographic_snap.py`](#holographic-snap) | holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). | 141 | +| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragge | 184 | | [`holographic_softbody.py`](#holographic-softbody) | Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. | 680 | | [`holographic_sparsefield.py`](#holographic-sparsefield) | FS-2 -- the narrow-band sparse field (holographic_sparsefield), array-backed for parallelism. | 530 | | [`holographic_spatial.py`](#holographic-spatial) | holographic_spatial.py -- ONE shared spatial index. Bin points into a uniform grid of cells so radius, | 191 | @@ -577,8 +577,8 @@ | [`holographic_structure.py`](#holographic-structure) | Proof of meaning: verify that a sequence carries structure, rather than trust | 174 | | [`holographic_subdivcurve.py`](#holographic-subdivcurve) | Subdivision curves on hypervector sequences (ARCH-5): Loop subdivision (FWD-8), turned inward. | 148 | | [`holographic_supermemory.py`](#holographic-supermemory) | Superposed key-value memory with a CLOSED-FORM capacity law, a single-shot allocator, | 634 | -| [`holographic_superposed.py`](#holographic-superposed) | holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). | 363 | | [`holographic_superposed.py`](#holographic-superposed) | COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. | 51 | +| [`holographic_superposed.py`](#holographic-superposed) | holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). | 363 | | [`holographic_superres.py`](#holographic-superres) | holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). | 83 | | [`holographic_superschedule.py`](#holographic-superschedule) | holographic_superschedule.py -- Fill 3: AUTO-SUPERPOSITION + SPILL. The latency-hiding move: hold N | 225 | | [`holographic_surface.py`](#holographic-surface) | holographic_surface.py -- the FIRST-CLASS render material: every channel is a Param socket, resolved PER HIT. | 293 | @@ -22387,6 +22387,28 @@ ### holographic_snap.py +> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). +> +> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in +> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an +> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup +> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, +> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. +> +> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; +> deterministic. + +**Public API:** + +- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the +- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns +- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. +- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. +- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. +- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever + +### holographic_snap.py + > holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragged > point / transform delta actually go?' in the shapes the interactive edit spine wants (dict hit records, a corrected > transform delta), DELEGATING all the actual snap math to the canonical snap primitives in @@ -22412,28 +22434,6 @@ - `def snap_to_edge(point, vertices, edges, max_dist)` -- Snap a point to the nearest point ON any edge, returned as {edge, position, distance, t}, or None if beyond - `def snap_transform_delta(delta, target, increment, moved_point, vertices, edges, origin, max_dist)` -- Snap a TRANSFORM DELTA so the moved point lands on a snap target, and return the corrected delta. This is the -### holographic_snap.py - -> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). -> -> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in -> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an -> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup -> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, -> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. -> -> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; -> deterministic. - -**Public API:** - -- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the -- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns -- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. -- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. -- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. -- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever - ### holographic_softbody.py > Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. @@ -23622,6 +23622,20 @@ ### holographic_superposed.py +> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. +> +> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module +> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing +> in superposition"); this one's build audit queried capability phrasings but never +> grepped the basename, so two unrelated modules shared a name across families -- a +> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's +> name. The capacity-law memory now lives under its own name; this shim keeps every +> existing import working forever (additive, backward-compatible only). + +*(no public functions or classes -- internal or data-only)* + +### holographic_superposed.py + > holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). > > PORTED FROM leOS (`superposed_compute.py`, "one processor, many states simultaneously"). @@ -23659,20 +23673,6 @@ - `def hierarchical_recall(S, group_key, leaf_key, chunk_codebook, item_codebook, min_chunk_similarity)` -- Descend one hierarchical superposition with a CLEANUP at the middle level. - `def flat_recall(S, group_key, leaf_key, item_codebook)` -- The BASELINE hierarchical_recall must beat, and the strongest honest one: unbind both roles from the single -### holographic_superposed.py - -> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. -> -> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module -> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing -> in superposition"); this one's build audit queried capability phrasings but never -> grepped the basename, so two unrelated modules shared a name across families -- a -> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's -> name. The capacity-law memory now lives under its own name; this shim keeps every -> existing import working forever (additive, backward-compatible only). - -*(no public functions or classes -- internal or data-only)* - ### holographic_superres.py > holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). From d65448256f5227956f42194d39e920216f010e2b Mon Sep 17 00:00:00 2001 From: atimics Date: Sat, 8 Aug 2026 18:39:42 -0700 Subject: [PATCH 10/16] Make x402 API docs discoverable --- API_QUICKREF.md | 10 +- AWS_X402_DEPLOY.md | 13 +- CAPABILITIES.md | 2 +- README.md | 10 +- REFERENCE.md | 95 +++--- X402_API.md | 37 ++- capabilities.json | 2 +- .../holographic_catalog_p04.py | 6 +- holographic_x402_api.py | 309 ++++++++++++++++-- tests/test_holographic_x402_api.py | 88 ++++- 10 files changed, 459 insertions(+), 113 deletions(-) diff --git a/API_QUICKREF.md b/API_QUICKREF.md index 64f2a493..86ffe72f 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -28,10 +28,12 @@ - `demo()` -- Build a tiny ready-to-query product demo. ### `holographic_x402_api` -*holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API.* +*holographic_x402_api.py -- publish the leCore Agent Memory & Routing API.* - **class `PaidRoute`** -- One x402-protected route. - `key(self)` -- The route key shape expected by x402 middleware, e.g. +- `x402_payment_required_responses()` -- OpenAPI response metadata shared by every x402-protected operation. +- `paid_request_openapi(required, properties, example, example_summary)` -- Return an accurate OpenAPI request body while runtime validation stays compatible. - **class `X402Config`** -- Seller configuration for the x402-paid API. - `from_env(cls, require_pay_to=True)` -- Build config from LECORE_X402_* environment variables. - `to_public_dict(self)` -- Public, JSON-safe view of the payment configuration. @@ -68,12 +70,14 @@ - `normalize_memory_backend(value)` -- Validate the memory backend selector without accepting silent fallbacks. - `env_flag(value)` -- Parse the small explicit boolean surface used by deployment settings. - `landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. +- `documentation_manifest(config)` -- Return canonical public documentation URLs for discovery responses. +- `public_dashboard(data)` -- Translate the embedded SDK dashboard into the hosted API vocabulary. - `payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. - `x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. - `x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. -- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None, memory_backend=None, nosqlite_binary=None, nosqlite_data_dir=None, nosqlite_durability=None, nosqlite_shadow=None)` -- Create the FastAPI application for paid or local serving. +- `create_app(core=None, config=None, paid=True, admin_token=None, tenant_secret=None, tenant_state_dir=None, memory_backend=None, nosqlite_binary=None, nosqlite_data_dir=None, nosqlite_durability=None, nosqlite_shadow=None)` -- Create the FastAPI application for paid or unpaid development serving. - `load_core(path)` -- Load a persisted core if present, otherwise return the demo core. -- `main(argv=None)` -- CLI entry point for local x402 API serving. +- `main(argv=None)` -- CLI entry point for running the x402 API service. ## Scene authoring diff --git a/AWS_X402_DEPLOY.md b/AWS_X402_DEPLOY.md index 4a1e8e91..286b256f 100644 --- a/AWS_X402_DEPLOY.md +++ b/AWS_X402_DEPLOY.md @@ -1,7 +1,8 @@ # AWS x402 Deployment -This is the production shape for serving `LocalAgentCore` as an x402-paid API -on AWS. +This is the production shape for serving the hosted leCore Agent Memory & +Routing API with x402 payments on AWS. The service is backed internally by +`LocalAgentCore`. ## Short Answer @@ -316,13 +317,13 @@ plans, spend limits, CloudTrail alarms, and a tiny blast radius. schedule a single-writer drain-and-replace cutover instead. - Do not put secrets or PII in x402 route descriptions or payment metadata. -## Local Smoke Before AWS +## Pre-deployment Smoke Test ```bash pip install ".[x402]" export LECORE_X402_PAY_TO="0xYourReceivingWallet" -export LECORE_X402_ADMIN_TOKEN="local-admin-secret" -export LECORE_X402_TENANT_SECRET="local-tenant-secret" +export LECORE_X402_ADMIN_TOKEN="dev-admin-secret" +export LECORE_X402_TENANT_SECRET="dev-tenant-secret" python holographic_x402_api.py --unpaid-dev --host 127.0.0.1 --port 4021 ``` @@ -333,5 +334,5 @@ curl http://127.0.0.1:4021/health curl http://127.0.0.1:4021/pricing curl -X POST http://127.0.0.1:4021/v1/route \ -H "Content-Type: application/json" \ - -d '{"task":"search local agent memory"}' + -d '{"task":"search tenant-scoped agent memory"}' ``` diff --git a/CAPABILITIES.md b/CAPABILITIES.md index c5dfcfd1..dea05e3e 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -449,7 +449,7 @@ import lecore; m=lecore.UnifiedMind(dim=256,seed=0); print(m.suggest_pipeline('t *Find it by:* how do I get from points to a mesh, chain capabilities, build a pipeline, route between datatypes, what steps turn X into Y ### x402 paid API publisher -publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, task routing, and the evidence dashboard protected by x402 middleware, with free health/pricing routes and admin-token-gated memory writes.. +publish the leCore Agent Memory & Routing API as a hosted HTTP service: FastAPI routes for tenant-scoped recall, task routing, and the readiness dashboard protected by x402 middleware, with free health/pricing/docs routes and admin-token-gated memory writes.. ```python from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...')) diff --git a/README.md b/README.md index ffbc9d82..a07d7be1 100644 --- a/README.md +++ b/README.md @@ -188,11 +188,11 @@ Like leOS, leCore is **free and open source**, and the work that keeps it free i `find_capability` first, wire every capability to a mind faculty (so it is `/invoke`-able), register it in the catalog so it is discoverable, and run the reachability/gap audits — the discipline that keeps the codebase from growing gaps or isolating code in tests. Read this before making code changes. -- **[`PRODUCT.md`](PRODUCT.md)** — the **narrow product wedge**: `LocalAgentCore`, a small stable facade for local - agent memory, capability routing, persistence, and the readiness dashboard. Start here if you want the five-minute - "use it in an agent" path rather than the whole research surface. -- **[`X402_API.md`](X402_API.md)** — the **paid API publishing guide**: serve the product wedge over FastAPI with - optional x402 middleware, per-route pricing, and admin-gated memory writes. +- **[`PRODUCT.md`](PRODUCT.md)** — the **narrow embedded SDK wedge**: `LocalAgentCore`, a small stable in-process + facade for agent memory, capability routing, persistence, and the readiness dashboard. Start here if you want the + five-minute "embed it in an agent" path rather than the whole research surface. +- **[`X402_API.md`](X402_API.md)** — the **hosted leCore Agent Memory & Routing API guide**: serve tenant-scoped + memory and routing over FastAPI with x402 payment, per-route pricing, and admin-gated memory writes. - **[`AWS_X402_DEPLOY.md`](AWS_X402_DEPLOY.md)** — the **AWS launch guide**: ECS/Fargate deployment, Secrets Manager config, and when to use KMS or Nitro Enclaves for wallet signing. - **[`CAPABILITIES.md`](CAPABILITIES.md)** — the **front-door menu**: a plain-language, grouped list of what leCore can diff --git a/REFERENCE.md b/REFERENCE.md index 38a2b403..c59d2d70 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -1,7 +1,7 @@ # leCore -- Code Reference *Auto-generated by `docgen.py` -- do not edit by hand; edit the module docstrings instead and re-run it.* -*622 modules, 218,516 lines of engine code.* +*622 modules, 218,759 lines of engine code.* > **New here? Read this first.** leCore represents *everything* -- memory, geometry, physics, rendering -- as > points in one very high-dimensional space (hypervectors), and combines them with a tiny algebra: **bind** @@ -555,8 +555,8 @@ | [`holographic_skymodel.py`](#holographic-skymodel) | holographic_skymodel.py -- a PARAMETRIC sky: time of day, sun, moon, stars, and HIGH cloud layers, as | 450 | | [`holographic_slime.py`](#holographic-slime) | Slime-mold path-finding over a HOLOGRAPHIC associative graph. | 391 | | [`holographic_smokepresets.py`](#holographic-smokepresets) | holographic_smokepresets.py -- SMOKE PRESETS (fluids/matter backlog, content item 1). | 159 | -| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). | 141 | | [`holographic_snap.py`](#holographic-snap) | holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragge | 184 | +| [`holographic_snap.py`](#holographic-snap) | holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). | 141 | | [`holographic_softbody.py`](#holographic-softbody) | Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. | 680 | | [`holographic_sparsefield.py`](#holographic-sparsefield) | FS-2 -- the narrow-band sparse field (holographic_sparsefield), array-backed for parallelism. | 530 | | [`holographic_spatial.py`](#holographic-spatial) | holographic_spatial.py -- ONE shared spatial index. Bin points into a uniform grid of cells so radius, | 191 | @@ -577,8 +577,8 @@ | [`holographic_structure.py`](#holographic-structure) | Proof of meaning: verify that a sequence carries structure, rather than trust | 174 | | [`holographic_subdivcurve.py`](#holographic-subdivcurve) | Subdivision curves on hypervector sequences (ARCH-5): Loop subdivision (FWD-8), turned inward. | 148 | | [`holographic_supermemory.py`](#holographic-supermemory) | Superposed key-value memory with a CLOSED-FORM capacity law, a single-shot allocator, | 634 | -| [`holographic_superposed.py`](#holographic-superposed) | COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. | 51 | | [`holographic_superposed.py`](#holographic-superposed) | holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). | 363 | +| [`holographic_superposed.py`](#holographic-superposed) | COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. | 51 | | [`holographic_superres.py`](#holographic-superres) | holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). | 83 | | [`holographic_superschedule.py`](#holographic-superschedule) | holographic_superschedule.py -- Fill 3: AUTO-SUPERPOSITION + SPILL. The latency-hiding move: hold N | 225 | | [`holographic_surface.py`](#holographic-surface) | holographic_surface.py -- the FIRST-CLASS render material: every channel is a Param socket, resolved PER HIT. | 293 | @@ -662,7 +662,7 @@ | [`holographic_worstview.py`](#holographic-worstview) | M16 -- find the GLOBAL worst view of a mesh over the sphere of directions, without a dense turntable sweep. | 194 | | [`holographic_wos.py`](#holographic-wos) | holographic_wos.py -- #7 / M1 from the SIGGRAPH list: WALK ON SPHERES. Solve PDEs on ANY geometry, no mesh. | 174 | | [`holographic_wost.py`](#holographic-wost) | holographic_wost.py -- Walk on Spheres / Walk on *Stars*: a grid-free Laplace/Poisson solver on an SDF. | 253 | -| [`holographic_x402_api.py`](#holographic-x402-api) | holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. | 1593 | +| [`holographic_x402_api.py`](#holographic-x402-api) | holographic_x402_api.py -- publish the leCore Agent Memory & Routing API. | 1836 | | [`holographic_zigmarch.py`](#holographic-zigmarch) | holographic_zigmarch.py -- the one-kernel-two-runtimes raymarch demo, EXECUTED (backlog Z4). | 230 | | [`holographic_zigrun.py`](#holographic-zigrun) | holographic_zigrun.py -- compile emitted Zig kernels to shared libraries and batch-run them (backlog Z2 + Z3). | 354 | @@ -22387,28 +22387,6 @@ ### holographic_snap.py -> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). -> -> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in -> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an -> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup -> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, -> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. -> -> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; -> deterministic. - -**Public API:** - -- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the -- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns -- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. -- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. -- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. -- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever - -### holographic_snap.py - > holographic_snap.py (mesh_and_geometry) -- the MODELING-GIZMO snap adapter: it answers 'where does this dragged > point / transform delta actually go?' in the shapes the interactive edit spine wants (dict hit records, a corrected > transform delta), DELEGATING all the actual snap math to the canonical snap primitives in @@ -22434,6 +22412,28 @@ - `def snap_to_edge(point, vertices, edges, max_dist)` -- Snap a point to the nearest point ON any edge, returned as {edge, position, distance, t}, or None if beyond - `def snap_transform_delta(delta, target, increment, moved_point, vertices, edges, origin, max_dist)` -- Snap a TRANSFORM DELTA so the moved point lands on a snap target, and return the corrected delta. This is the +### holographic_snap.py + +> holographic_snap.py -- SNAPPING = cleanup, applied to geometry (modeling-app feature layer). +> +> Thinking holographically: snapping IS cleanup. VSA cleanup projects a noisy vector onto the nearest CLEAN atom in +> a codebook; snapping projects a dragged, continuous position onto the nearest ALLOWED place -- a grid node, an +> existing vertex, a point on an edge, an angle increment. Same operation, geometric codebook. And just as cleanup +> can REFUSE a weak match (return "no confident atom"), a snap has a TOLERANCE: if nothing allowed is close enough, +> the point is left where it is. That confidence gate is what stops a cursor from teleporting across the screen. +> +> These read raw coordinates (the honest way -- no lossy encoding for something this exact). NumPy + stdlib only; +> deterministic. + +**Public API:** + +- `def snap_to_grid(p, spacing, origin)` -- Snap a point to the nearest grid node -- round each coordinate to the lattice. The simplest cleanup: the +- `def snap_to_points(p, points, tol)` -- Snap to the NEAREST point in a set -- this is literally cleanup (nearest codebook entry). Returns +- `def snap_to_segment(p, a, b)` -- The nearest point on the line SEGMENT a-b (clamped to the endpoints) -- snapping to an edge. +- `def snap_value(x, increment, origin)` -- Snap a scalar to the nearest multiple of `increment` from `origin` -- e.g. a length to 0.25 m steps. +- `def snap_angle(theta, increment)` -- Snap an angle (radians) to the nearest multiple of `increment` -- e.g. rotate in 15-degree steps. +- `class Snapper` -- Snaps a point to the nearest snap target within a tolerance, combining a GRID and a VERTEX set. Whichever + ### holographic_softbody.py > Position-Based Dynamics -- softbody & hardbody simulation, exposed to VSA. @@ -23622,20 +23622,6 @@ ### holographic_superposed.py -> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. -> -> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module -> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing -> in superposition"); this one's build audit queried capability phrasings but never -> grepped the basename, so two unrelated modules shared a name across families -- a -> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's -> name. The capacity-law memory now lives under its own name; this shim keeps every -> existing import working forever (additive, backward-compatible only). - -*(no public functions or classes -- internal or data-only)* - -### holographic_superposed.py - > holographic_superposed.py -- parallel computation in superposition (the WIDTH faculty). > > PORTED FROM leOS (`superposed_compute.py`, "one processor, many states simultaneously"). @@ -23673,6 +23659,20 @@ - `def hierarchical_recall(S, group_key, leaf_key, chunk_codebook, item_codebook, min_chunk_similarity)` -- Descend one hierarchical superposition with a CLEANUP at the middle level. - `def flat_recall(S, group_key, leaf_key, item_codebook)` -- The BASELINE hierarchical_recall must beat, and the strongest honest one: unbind both roles from the single +### holographic_superposed.py + +> COMPATIBILITY SHIM -- this module moved to `holographic_supermemory`. +> +> WHY THE RENAME (Rule-0 lesson, on record in NOTES): a week-old, DIFFERENT module +> already lived at holographic/misc/holographic_superposed.py (leOS-ported "computing +> in superposition"); this one's build audit queried capability phrasings but never +> grepped the basename, so two unrelated modules shared a name across families -- a +> discoverability tax caught by the fuzzy-ask demo answering 'misc' for this module's +> name. The capacity-law memory now lives under its own name; this shim keeps every +> existing import working forever (additive, backward-compatible only). + +*(no public functions or classes -- internal or data-only)* + ### holographic_superres.py > holographic_superres.py -- EXAMPLE-BASED SUPER-RESOLUTION / GUIDED UPSAMPLING (inverse-rendering ST3). @@ -26818,12 +26818,13 @@ ### holographic_x402_api.py -> holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. +> holographic_x402_api.py -- publish the leCore Agent Memory & Routing API. > > WHY THIS EXISTS > --------------- -> `LocalAgentCore` is the narrow product wedge. This module makes it sellable as -> an HTTP API without making x402, FastAPI, or uvicorn core dependencies. +> `LocalAgentCore` remains the embedded implementation facade. This module +> translates it into a hosted HTTP API without making x402, FastAPI, or uvicorn +> core dependencies. > > The boundary is intentionally conservative: > @@ -26837,6 +26838,8 @@ **Public API:** - `class PaidRoute` -- One x402-protected route. +- `def x402_payment_required_responses()` -- OpenAPI response metadata shared by every x402-protected operation. +- `def paid_request_openapi(required, properties, example, example_summary)` -- Return an accurate OpenAPI request body while runtime validation stays compatible. - `class X402Config` -- Seller configuration for the x402-paid API. - `def optional_dependency_help()` -- Install hint for the optional paid API dependencies. - `def normalize_tenant_id(value)` -- Return a path-safe tenant id for private memory routing. @@ -26854,12 +26857,14 @@ - `def normalize_memory_backend(value)` -- Validate the memory backend selector without accepting silent fallbacks. - `def env_flag(value)` -- Parse the small explicit boolean surface used by deployment settings. - `def landing_page_html(config)` -- Render the buyer-facing landing page served from `/`. +- `def documentation_manifest(config)` -- Return canonical public documentation URLs for discovery responses. +- `def public_dashboard(data)` -- Translate the embedded SDK dashboard into the hosted API vocabulary. - `def payment_manifest(config)` -- Plain JSON route manifest, useful for docs, `/pricing`, and tests. - `def x402_route_configs(config)` -- Build x402 SDK RouteConfig objects for the protected routes. - `def x402_resource_server(config)` -- Create an x402 resource server wired to the configured facilitator. -- `def create_app(core, config, paid, admin_token, tenant_secret, tenant_state_dir, memory_backend, nosqlite_binary, nosqlite_data_dir, nosqlite_durability, nosqlite_shadow)` -- Create the FastAPI application for paid or local serving. +- `def create_app(core, config, paid, admin_token, tenant_secret, tenant_state_dir, memory_backend, nosqlite_binary, nosqlite_data_dir, nosqlite_durability, nosqlite_shadow)` -- Create the FastAPI application for paid or unpaid development serving. - `def load_core(path)` -- Load a persisted core if present, otherwise return the demo core. -- `def main(argv)` -- CLI entry point for local x402 API serving. +- `def main(argv)` -- CLI entry point for running the x402 API service. ### holographic_zigmarch.py diff --git a/X402_API.md b/X402_API.md index 3fcfbf00..3a14265c 100644 --- a/X402_API.md +++ b/X402_API.md @@ -1,10 +1,19 @@ -# x402 API Publishing +# leCore Agent Memory & Routing API -Yes: leCore can be published as a paid API with x402. +leCore is available as a hosted memory and capability-routing API with x402 +payment on the protected routes. -The implementation lives in `holographic_x402_api.py`. It wraps -`LocalAgentCore` with a small FastAPI app and applies x402 middleware only to -the public read/compute routes: +Public reference: + +- [Swagger UI](https://lecore.rati.foundation/docs) +- [ReDoc reference](https://lecore.rati.foundation/redoc) +- [OpenAPI 3.1 schema](https://lecore.rati.foundation/openapi.json) +- [Pricing and route manifest](https://lecore.rati.foundation/pricing) + +The implementation lives in `holographic_x402_api.py`. It exposes +tenant-scoped agent memory and routing through FastAPI, backed internally by +`LocalAgentCore`, and applies x402 middleware only to the public read/compute +routes: - `POST /v1/recall` - `POST /v1/route` @@ -45,8 +54,8 @@ uses testnet USDC, and does not accept production payments. export LECORE_X402_PAY_TO="0xYourReceivingWallet" export LECORE_X402_PRICE="$0.0011" export LECORE_X402_PUBLIC_URL="http://127.0.0.1:4021" -export LECORE_X402_ADMIN_TOKEN="local-admin-secret" -export LECORE_X402_TENANT_SECRET="local-tenant-secret" +export LECORE_X402_ADMIN_TOKEN="dev-admin-secret" +export LECORE_X402_TENANT_SECRET="dev-tenant-secret" export LECORE_X402_TENANT_STATE_DIR="./tenant-state" python holographic_x402_api.py --host 127.0.0.1 --port 4021 @@ -58,14 +67,14 @@ Inspect pricing: curl http://127.0.0.1:4021/pricing ``` -Add memories locally as the seller: +Add memories through the operator endpoint: ```bash curl -X POST http://127.0.0.1:4021/admin/remember \ -H "Content-Type: application/json" \ - -H "X-Admin-Token: local-admin-secret" \ + -H "X-Admin-Token: dev-admin-secret" \ -H "Idempotency-Key: initial-memory-001" \ - -d '{"text":"local agents need deterministic durable memory","label":"memory"}' + -d '{"text":"agents need deterministic durable memory","label":"memory"}' ``` When `LECORE_X402_TENANT_STATE_DIR` is configured, admin writes use a small @@ -84,7 +93,7 @@ Issue a private tenant token: ```bash curl -X POST http://127.0.0.1:4021/admin/tenant-token \ -H "Content-Type: application/json" \ - -H "X-Admin-Token: local-admin-secret" \ + -H "X-Admin-Token: dev-admin-secret" \ -d '{"tenant":"acme"}' ``` @@ -95,7 +104,7 @@ curl -X POST http://127.0.0.1:4021/v1/recall \ -H "Content-Type: application/json" \ -H "X-leCore-Tenant: acme" \ -H "X-leCore-Tenant-Token: " \ - -d '{"query":"deterministic local memory"}' + -d '{"query":"deterministic agent memory"}' ``` Requests to paid routes return `402 Payment Required` unless the client retries @@ -104,10 +113,10 @@ with a valid x402 payment payload: ```bash curl -X POST http://127.0.0.1:4021/v1/recall \ -H "Content-Type: application/json" \ - -d '{"query":"deterministic local memory"}' + -d '{"query":"deterministic agent memory"}' ``` -## Local Unpaid Smoke Test +## Unpaid Development Smoke Test Use this only for development: diff --git a/capabilities.json b/capabilities.json index b7df1012..4cb8d5d9 100644 --- a/capabilities.json +++ b/capabilities.json @@ -14962,7 +14962,7 @@ "paid route" ], "consumes": [], - "does": "publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, task routing, and the evidence dashboard protected by x402 middleware, with free health/pricing routes and admin-token-gated memory writes.", + "does": "publish the leCore Agent Memory & Routing API as a hosted HTTP service: FastAPI routes for tenant-scoped recall, task routing, and the readiness dashboard protected by x402 middleware, with free health/pricing/docs routes and admin-token-gated memory writes.", "example": "from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...'))", "method": null, "name": "x402 paid API publisher", diff --git a/holographic/caching_and_storage/holographic_catalog_p04.py b/holographic/caching_and_storage/holographic_catalog_p04.py index 2eab132b..f83d57c7 100644 --- a/holographic/caching_and_storage/holographic_catalog_p04.py +++ b/holographic/caching_and_storage/holographic_catalog_p04.py @@ -1215,9 +1215,9 @@ def register_p04(c): "skill routing", "dashboard", "first user", "facade", "local agent core")) c.register_capability( "x402 paid API publisher", - "publish the LocalAgentCore product wedge as a paid HTTP API: FastAPI routes for recall, " - "task routing, and the evidence dashboard protected by x402 middleware, with free " - "health/pricing routes and admin-token-gated memory writes.", + "publish the leCore Agent Memory & Routing API as a hosted HTTP service: FastAPI routes " + "for tenant-scoped recall, task routing, and the readiness dashboard protected by x402 " + "middleware, with free health/pricing/docs routes and admin-token-gated memory writes.", example="from holographic_x402_api import create_app, X402Config; app = create_app(config=X402Config(pay_to='0x...'))", native=False, aliases=("x402", "paid api", "payment required", "402", "monetize api", "micropayment", diff --git a/holographic_x402_api.py b/holographic_x402_api.py index 1ec3ba38..679fd870 100644 --- a/holographic_x402_api.py +++ b/holographic_x402_api.py @@ -1,9 +1,10 @@ -"""holographic_x402_api.py -- publish LocalAgentCore as an x402-paid API. +"""holographic_x402_api.py -- publish the leCore Agent Memory & Routing API. WHY THIS EXISTS --------------- -`LocalAgentCore` is the narrow product wedge. This module makes it sellable as -an HTTP API without making x402, FastAPI, or uvicorn core dependencies. +`LocalAgentCore` remains the embedded implementation facade. This module +translates it into a hosted HTTP API without making x402, FastAPI, or uvicorn +core dependencies. The boundary is intentionally conservative: @@ -64,6 +65,33 @@ LOG = logging.getLogger(__name__) +SERVICE_NAME = "leCore Agent Memory & Routing API" +API_DESCRIPTION = """Hosted, tenant-scoped agent memory, capability routing, and +readiness data over HTTPS, with x402 payment on each protected request. + +## Request flow + +1. Read `GET /pricing` for the network, asset, price, and protected-route manifest. +2. Call a protected `/v1/*` route. An unsigned request returns `402 Payment Required`. +3. Decode the `Payment-Required` response header with an x402 v2 client. +4. Sign the selected payment option and retry with the resulting `Payment-Signature` header. + +The interactive reference describes the contract but does not sign payments. +`GET /health`, `GET /pricing`, `/docs`, `/redoc`, and `/openapi.json` +are free. Private tenant calls additionally require `X-leCore-Tenant` and +`X-leCore-Tenant-Token`; payment proves payment, not tenant authorization. +""" +OPENAPI_TAGS = [ + { + "name": "Discovery", + "description": "Free service health, pricing, network, and route discovery.", + }, + { + "name": "Paid API", + "description": "Hosted read and compute operations protected by the x402 v2 payment flow.", + }, +] + @dataclass(frozen=True) class PaidRoute: @@ -82,9 +110,9 @@ def key(self) -> str: REGULAR_PAID_ROUTES: Tuple[PaidRoute, ...] = ( - PaidRoute("POST", "/v1/recall", "Recall nearest memories from a LocalAgentCore instance"), + PaidRoute("POST", "/v1/recall", "Recall nearest memories from tenant-scoped agent memory"), PaidRoute("POST", "/v1/route", "Route a plain-English task to a leCore capability"), - PaidRoute("GET", "/v1/dashboard", "Read the LocalAgentCore evidence dashboard"), + PaidRoute("GET", "/v1/dashboard", "Read the service readiness dashboard"), ) DEFAULT_PAID_ROUTES: Tuple[PaidRoute, ...] = REGULAR_PAID_ROUTES @@ -129,13 +157,69 @@ def _normalize_public_url(value: str) -> str: return value +def x402_payment_required_responses() -> Dict[int, Dict[str, Any]]: + """OpenAPI response metadata shared by every x402-protected operation.""" + return { + 402: { + "description": ( + "Payment required. Decode the Payment-Required header with an " + "x402 v2 client, sign one accepted option, and retry with " + "Payment-Signature." + ), + "headers": { + "Payment-Required": { + "description": "Base64-encoded x402 v2 PaymentRequired challenge.", + "schema": {"type": "string", "format": "byte"}, + }, + }, + "content": { + "application/json": { + "schema": {"type": "object", "maxProperties": 0}, + "example": {}, + }, + }, + }, + } + + +def paid_request_openapi( + required: List[str], + properties: Dict[str, Dict[str, Any]], + example: Dict[str, Any], + example_summary: str, +) -> Dict[str, Any]: + """Return an accurate OpenAPI request body while runtime validation stays compatible.""" + return { + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": required, + "properties": properties, + "additionalProperties": True, + }, + "examples": { + "public": { + "summary": example_summary, + "value": example, + }, + }, + }, + }, + }, + } + + LANDING_PAGE_TEMPLATE = Template(""" -leCore x402 API - +$service_name + + +
- -

$environment_label$network_label$price_per_thousand

$service_name

Call hosted, tenant-scoped agent memory, capability routing, and readiness endpoints over HTTPS. Each protected request uses x402 payment. $payment_notice

Explore API docsRead the referenceView pricing
- + +

$price_per_request$environment_label$network_label

$hero_title

A hosted HTTPS API for querying seeded preview memory, routing tasks to leCore capabilities, and reading service readiness. $payment_notice

Make the first requestExplore API docsView pricing
+
-
Endpoint$public_url
Stage$environment_label
Buyer shapeinspect, pay, call
-

Why try it

Most agents do not need a platform. They need a few reliable cognitive calls.

Test the x402 payment flow against one answerable primitive at a time.

The API is narrow enough to trust: read/compute routes are paid, memory writes stay admin-gated.

It exposes the useful part of leCore first: tenant-scoped agent memory plus capability routing.

$payment_notice

-

What the payment unlocks

Three paid routes, each small enough to understand.

POST

Recall

/v1/recall

Query tenant-scoped agent memory over HTTPS while leCore handles storage and retrieval behind the API.

POST

Route

/v1/route

Send a plain-language task and get the leCore capability it should use, with evidence attached.

GET

Dashboard

/v1/dashboard

Read the readiness surface: memory counts, capability map, abstention behavior, and route coverage.

-

Good first buyers

Teams who want the leCore idea without adopting the whole repo.

Agent memory for prototypes that should remember without a database rollout.

Capability routing for tools that need to pick the right leCore subsystem before doing work.

Readiness dashboards for teams deciding whether a deterministic memory service fits their product.

A working x402 seller endpoint to copy when you want pay-per-call APIs instead of subscriptions.

-

Preview status

It is deployed, health-checked, and ready to integrate.

The free endpoints show health and preview terms. Paid endpoints return an x402 challenge on $network_name. The receiving address is public, while admin writes stay out of the customer path.

Preview price
$price_per_thousand
Network
$network_name
Receiver
$pay_to_short
Status
Healthy
-

Start integrating

Use memory and routing over HTTPS without adopting the whole repo.

Open API docsDownload OpenAPICheck live health
+
Endpoint$public_url
Stage$environment_label
Protocolx402 v2
+

Four-step quickstart

Inspect the terms before signing anything.

  1. Read the free manifest

    GET /pricing returns the exact route, network, asset, receiver, and price.

  2. Make an unsigned request

    The protected route returns 402 with a base64 Payment-Required challenge.

  3. Sign with an x402 v2 client

    Use the x402 buyer guide to configure a testnet wallet and payment client.

  4. Retry and verify settlement

    Send Payment-Signature; a successful response includes Payment-Response.

First request: no wallet requiredOpen route docs
curl -i $public_url/v1/dashboard

Expected: HTTP 402 plus Payment-Required. This safely exposes the payment contract without moving testnet funds.

Inspect exact preview termsOpen live JSON
curl -sS $public_url/pricing
+

Paid API surface

Three explicit operations, with no account or subscription.

POST

Recall

/v1/recall

Query the seeded public preview memory or an operator-provisioned private tenant.

POST

Route

/v1/route

Send a plain-language task and receive an explicit act, choose, or unknown decision with evidence.

GET

Dashboard

/v1/dashboard

Read memory, capability-routing, and deterministic-engine readiness for one tenant.

+

Preview boundaries

Deployed, health-checked, and ready to test.

Base Sepolia testnet only. The public memory dataset is read-only; private tenants and memory writes are currently operator-provisioned. Operator routes require separate authorization and are absent from the public OpenAPI schema.

Per request
$price_per_request
Per 1,000
$price_per_thousand
Network
$network_name
API version
$api_version
+

Start testing

See the full request and response contract.

Open API docsView OpenAPI schemaCheck live health
+ """) @@ -336,13 +898,6 @@ def _landing_nodes() -> str: return "".join(nodes) -def _short_address(address: str) -> str: - """Compact public wallet display.""" - if len(address) <= 12: - return address - return "%s...%s" % (address[:6], address[-4:]) - - def _network_name(network: str) -> str: """Human label for known x402 network ids.""" return {"eip155:84532": "Base Sepolia", "eip155:8453": "Base"}.get(network, network) @@ -1183,15 +1738,16 @@ def landing_page_html(config: X402Config) -> str: summary = pricing_summary(config) return LANDING_PAGE_TEMPLATE.substitute( service_name=escape(SERVICE_NAME), + hero_title=escape(HERO_TITLE), + api_version=escape(LECORE_VERSION), + buyer_guide_url=escape(X402_BUYER_GUIDE_URL), nodes=_landing_nodes(), - network=escape(config.network), public_url=escape(config.public_url), network_label=escape("%s x402" % network_name), network_name=escape(network_name), - pay_to_short=escape(_short_address(config.pay_to)), environment_label=escape(summary["environment_label"]), - payment_asset=escape(summary["payment_asset"]), payment_notice=escape(summary["payment_notice"]), + price_per_request=escape("%s per request" % summary["per_request"]), price_per_thousand=escape(summary["display_price"]), ) @@ -1354,6 +1910,10 @@ async def lifespan(_: Any) -> Any: openapi_url="/openapi.json", openapi_tags=OPENAPI_TAGS, servers=[{"url": config.public_url, "description": "Public API"}], + openapi_external_docs={ + "description": "x402 buyer quickstart", + "url": X402_BUYER_GUIDE_URL, + }, lifespan=lifespan, ) app.state.memory_backend = memory_backend @@ -1373,6 +1933,19 @@ async def lifespan(_: Any) -> Any: server=x402_resource_server(config), ) + @app.middleware("http") + async def apply_public_response_policy(request: Any, call_next: Any) -> Any: + response = await call_next(request) + for name, value in public_response_headers( + request.url.path, + response.status_code, + config.public_url, + response.headers.get("content-type", ""), + config.network, + ).items(): + response.headers[name] = value + return response + def require_admin(header_value: Optional[str]) -> None: if not admin_token: raise HTTPException(status_code=403, detail="admin writes are disabled") @@ -1457,8 +2030,19 @@ def landing() -> str: @app.get( "/health", tags=["Discovery"], + operation_id="getHealth", summary="Check service health", description="Free liveness, memory-state, backend, and tenancy summary. No x402 payment is required.", + responses={ + 200: health_success_openapi( + paid=bool(paid), + private_tenants_enabled=bool(tenant_secret), + memory_backend=memory_backend, + nosqlite_shadow=bool(nosqlite_shadow), + nosqlite_configured=nosqlite_store is not None, + durable_transactions=memory_transactions is not None, + ), + }, ) def health() -> Dict[str, Any]: return { @@ -1477,11 +2061,22 @@ def health() -> Dict[str, Any]: @app.get( "/pricing", tags=["Discovery"], + operation_id="getPricing", summary="Discover pricing and protected routes", description=( "Free discovery document for the x402 network, payment asset, price, " "tenant headers, documentation URLs, and protected-route manifest." ), + responses={ + 200: pricing_success_openapi( + config, + private_tenants_enabled=bool(tenant_secret), + memory_backend=memory_backend, + nosqlite_shadow=bool(nosqlite_shadow), + nosqlite_configured=nosqlite_store is not None, + durable_transactions=memory_transactions is not None, + ), + }, ) def pricing() -> Dict[str, Any]: return { @@ -1529,12 +2124,17 @@ def recall_response( @app.post( "/v1/recall", tags=["Paid API"], + operation_id="recallMemory", summary="Recall agent memory", description=( "Recall the nearest entries from tenant-scoped agent memory. An " "unsigned request returns the x402 challenge documented in the 402 response." ), - responses=x402_payment_required_responses(), + responses=paid_operation_responses( + recall_success_openapi(), + invalid_detail="query must be a non-empty string", + backend_unavailable=True, + ), openapi_extra=paid_request_openapi( required=["query"], properties={ @@ -1542,6 +2142,7 @@ def recall_response( "type": "string", "minLength": 1, "maxLength": MAX_QUERY_CHARS, + "pattern": r"\S", "description": "Text to match against stored agent memory.", }, "k": { @@ -1557,8 +2158,10 @@ def recall_response( }, "tenant": { "type": "string", - "pattern": _TENANT_ID_RE.pattern, - "description": "Tenant id; must match X-leCore-Tenant when both are supplied.", + "description": ( + "Tenant id. Leading/trailing whitespace is removed and letters are " + "lowercased; the normalized id must match X-leCore-Tenant when both are supplied." + ), }, }, example={"query": "deterministic agent memory", "k": 3}, @@ -1570,17 +2173,24 @@ def recall( x_lecore_tenant: Optional[str] = Header( default=None, alias=TENANT_HEADER, - description="Tenant id. Omit for the public tenant.", + description="Tenant id, trimmed and lowercased by the service. Omit for the public tenant.", ), x_lecore_tenant_token: Optional[str] = Header( default=None, alias=TENANT_TOKEN_HEADER, - description="Required with X-leCore-Tenant for private tenant memory.", + description=( + "Required whenever the resolved tenant is private, whether selected " + "by header or JSON body." + ), ), _payment_signature: Optional[str] = Header( default=None, alias="Payment-Signature", - description="x402 v2 payment signature produced from the Payment-Required challenge.", + description=( + "Omit to receive the x402 challenge; include the base64 x402 v2 " + "payment payload when retrying." + ), + json_schema_extra={"format": "byte"}, ), ) -> Dict[str, Any]: return recall_response(payload, x_lecore_tenant, x_lecore_tenant_token) @@ -1599,12 +2209,16 @@ def route_response( @app.post( "/v1/route", tags=["Paid API"], + operation_id="routeTask", summary="Route a task to a capability", description=( "Route a plain-English task to the best matching leCore capability. " "An unsigned request returns the x402 challenge documented in the 402 response." ), - responses=x402_payment_required_responses(), + responses=paid_operation_responses( + route_success_openapi(), + invalid_detail="task must be a non-empty string", + ), openapi_extra=paid_request_openapi( required=["task"], properties={ @@ -1612,12 +2226,15 @@ def route_response( "type": "string", "minLength": 1, "maxLength": MAX_TASK_CHARS, + "pattern": r"\S", "description": "Plain-English task to route.", }, "tenant": { "type": "string", - "pattern": _TENANT_ID_RE.pattern, - "description": "Tenant id; must match X-leCore-Tenant when both are supplied.", + "description": ( + "Tenant id. Leading/trailing whitespace is removed and letters are " + "lowercased; the normalized id must match X-leCore-Tenant when both are supplied." + ), }, }, example={"task": "find the best capability for semantic memory retrieval"}, @@ -1629,17 +2246,24 @@ def route( x_lecore_tenant: Optional[str] = Header( default=None, alias=TENANT_HEADER, - description="Tenant id. Omit for the public tenant.", + description="Tenant id, trimmed and lowercased by the service. Omit for the public tenant.", ), x_lecore_tenant_token: Optional[str] = Header( default=None, alias=TENANT_TOKEN_HEADER, - description="Required with X-leCore-Tenant for private tenant routing.", + description=( + "Required whenever the resolved tenant is private, whether selected " + "by header or JSON body." + ), ), _payment_signature: Optional[str] = Header( default=None, alias="Payment-Signature", - description="x402 v2 payment signature produced from the Payment-Required challenge.", + description=( + "Omit to receive the x402 challenge; include the base64 x402 v2 " + "payment payload when retrying." + ), + json_schema_extra={"format": "byte"}, ), ) -> Dict[str, Any]: return route_response(payload, x_lecore_tenant, x_lecore_tenant_token) @@ -1657,28 +2281,36 @@ def dashboard_response( @app.get( "/v1/dashboard", tags=["Paid API"], + operation_id="getDashboard", summary="Read the readiness dashboard", description=( "Read memory, routing, native-kernel, and deterministic-engine readiness " "for one tenant. An unsigned request returns the documented x402 challenge." ), - responses=x402_payment_required_responses(), + responses=paid_operation_responses( + dashboard_success_openapi(), + invalid_detail="tenant id is invalid", + ), ) def dashboard( x_lecore_tenant: Optional[str] = Header( default=None, alias=TENANT_HEADER, - description="Tenant id. Omit for the public tenant.", + description="Tenant id, trimmed and lowercased by the service. Omit for the public tenant.", ), x_lecore_tenant_token: Optional[str] = Header( default=None, alias=TENANT_TOKEN_HEADER, - description="Required with X-leCore-Tenant for a private tenant dashboard.", + description="Required whenever the resolved tenant is private.", ), _payment_signature: Optional[str] = Header( default=None, alias="Payment-Signature", - description="x402 v2 payment signature produced from the Payment-Required challenge.", + description=( + "Omit to receive the x402 challenge; include the base64 x402 v2 " + "payment payload when retrying." + ), + json_schema_extra={"format": "byte"}, ), ) -> Dict[str, Any]: return dashboard_response(x_lecore_tenant, x_lecore_tenant_token) @@ -1828,7 +2460,7 @@ def main(argv: Optional[Iterable[str]] = None) -> None: import uvicorn except ImportError as exc: raise RuntimeError(optional_dependency_help()) from exc - uvicorn.run(app, host=args.host, port=args.port) + uvicorn.run(app, host=args.host, port=args.port, server_header=False) if __name__ == "__main__": diff --git a/tests/test_holographic_x402_api.py b/tests/test_holographic_x402_api.py index ca65886c..a4173254 100644 --- a/tests/test_holographic_x402_api.py +++ b/tests/test_holographic_x402_api.py @@ -12,6 +12,7 @@ DEFAULT_PRICE, DEFAULT_PUBLIC_URL, DEFAULT_TENANT_ID, + HERO_TITLE, IDEMPOTENCY_HEADER, MEMORY_BACKEND_NOSQLITE, MemoryTransactionConflict, @@ -22,6 +23,7 @@ TENANT_TOKEN_HEADER, TenantCoreStore, TenantMemoryTransactions, + X402_BUYER_GUIDE_URL, X402Config, create_app, landing_page_html, @@ -182,6 +184,8 @@ def test_landing_page_marks_the_testnet_api_as_a_preview(): assert f"{escape(SERVICE_NAME)}" in html assert "Testnet developer preview" in html + assert f"

{escape(HERO_TITLE)}

" in html + assert "$0.0011 per request" in html assert "$1.10 per 1,000 requests" in html assert "does not accept production payments" in html assert "Base Sepolia x402" in html @@ -190,8 +194,26 @@ def test_landing_page_marks_the_testnet_api_as_a_preview(): assert 'href="/docs"' in html assert 'href="/redoc"' in html assert 'href="/openapi.json"' in html - assert "hosted, tenant-scoped agent memory" in html - assert "0x96e1...BB84" in html + assert "A hosted HTTPS API" in html + assert "querying seeded preview memory" in html + assert "The public memory dataset is read-only" in html + assert "operator-provisioned" in html + assert "ready to test" in html + assert "ready to integrate" not in html + assert "memory.entries" not in html + assert "curl -i %s/v1/dashboard" % DEFAULT_PUBLIC_URL in html + assert "Payment-Required" in html + assert "Payment-Signature" in html + assert "Payment-Response" in html + assert X402_BUYER_GUIDE_URL in html + assert ":focus-visible" in html + assert "prefers-reduced-motion" in html + assert "min-height:44px" in html + assert "outline:3px solid currentColor" in html + assert 'href="/docs#/Paid%20API/getDashboard"' in html + assert "dashboard_v1_dashboard_get" not in html + assert 'id="quickstart" class="section quickstart" tabindex="-1"' in html + assert "--coral-text:#b6402f" in html assert DEFAULT_PUBLIC_URL in html assert "leOS" not in html assert "local agent" not in html.lower() @@ -240,11 +262,38 @@ def test_paid_challenge_uses_canonical_resource_not_request_headers(monkeypatch) headers={"host": "attacker.invalid", "x-forwarded-proto": "http"}, ) assert response.status_code == 402 + assert response.headers["cache-control"] == "no-store" + assert response.headers["x-content-type-options"] == "nosniff" + assert response.headers["x-frame-options"] == "DENY" + assert response.headers["strict-transport-security"] == "max-age=31536000" + assert "frame-ancestors 'none'" in response.headers["content-security-policy"] + assert "unsafe-inline" not in response.headers["content-security-policy"] + assert "sepolia.base.org" not in response.headers["content-security-policy"] challenge = decode_payment_required_header(response.headers["payment-required"]) assert challenge.resource.url == DEFAULT_PUBLIC_URL + "/v1/dashboard" assert challenge.resource.description == "Read the service readiness dashboard" assert "LocalAgentCore" not in challenge.resource.description + browser_response = client.get( + "/v1/dashboard", + headers={"accept": "text/html", "user-agent": "Mozilla/5.0"}, + ) + assert browser_response.status_code == 402 + assert browser_response.headers["content-type"].startswith("text/html") + assert browser_response.headers["cache-control"] == "no-store" + assert browser_response.headers["x-content-type-options"] == "nosniff" + paywall_csp = browser_response.headers["content-security-policy"] + assert "script-src 'unsafe-inline'" in paywall_csp + assert "style-src 'unsafe-inline'" in paywall_csp + assert "connect-src 'self' https://sepolia.base.org" in paywall_csp + assert "https://rpc.wallet.coinbase.com" in paywall_csp + assert "object-src 'none'" in paywall_csp + assert "frame-src 'none'" in paywall_csp + assert "frame-ancestors 'none'" in paywall_csp + assert '