diff --git a/.env.example b/.env.example index 624fddc7..df102131 100644 --- a/.env.example +++ b/.env.example @@ -192,13 +192,13 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # # ── Client side (end-user machines) ───────────────────────────────────────── # Paid keys require a revocable, machine-bound lease from the managed service. The -# built-in default is https://engraphis-production.up.railway.app; override only for a different vendor +# built-in default is https://team.engraphis.com; override only for a different vendor # deployment. Free-tier use remains local and does not require the service. -# ENGRAPHIS_CLOUD_URL=https://engraphis-production.up.railway.app +# ENGRAPHIS_CLOUD_URL=https://team.engraphis.com # Server-side license enforcement (vendor/fulfillment server only): when set, every key # minted by the Polar webhook carries a signed enforce:"cloud" claim + this URL, and the # client requires a live lease from it (revocable, seat-counted, useless offline). # Set this to the canonical managed-service URL for newly issued keys. Older keys carrying -# the retired custom-domain hostname are migrated by the client without changing their signature. -# ENGRAPHIS_KEY_CLOUD_URL=https://engraphis-production.up.railway.app +# the retired Railway hostname are migrated by the client without changing their signature. +# ENGRAPHIS_KEY_CLOUD_URL=https://team.engraphis.com diff --git a/README.md b/README.md index b1c71bef..a696ed7c 100644 --- a/README.md +++ b/README.md @@ -571,7 +571,7 @@ All via environment (or `.env`): | `ENGRAPHIS_LOOP_INTERVAL` | `60` | Background consolidation loop interval in seconds (0 = disabled) | | `ENGRAPHIS_DECAY_HALFLIFE_DAYS` | `7` | Ebbinghaus decay half-life (higher = memories persist longer) | | `ENGRAPHIS_FORWARDED_ALLOW_IPS` | `127.0.0.1` | Trusted reverse-proxy IPs for TLS termination (`*` = trust all) | -| `ENGRAPHIS_RELAY_URL` | `https://engraphis-production.up.railway.app` | Managed sync, license, trial, and invite relay (Pro/Team); the retired custom-domain URL is migrated automatically | +| `ENGRAPHIS_RELAY_URL` | `https://team.engraphis.com` | Managed sync, license, trial, and invite relay (Pro/Team); the retired Railway URL is migrated automatically | | `ENGRAPHIS_AUTOSYNC_LOOP` | `1` | Kill switch for the in-process auto-sync loop (0 = off) | See `.env.example` for the full list including commercial/vendor, email delivery, and diff --git a/docker-compose.yml b/docker-compose.yml index 50760a44..a67a1b78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,10 @@ services: ports: - "8700:8700" env_file: - - .env + # Optional: a fresh clone has no .env (it's gitignored), and `docker compose up` + # must still work using the `environment:` block below plus the shell env. + - path: .env + required: false environment: ENGRAPHIS_HOST: 0.0.0.0 ENGRAPHIS_DB_PATH: /data/engraphis.db @@ -34,10 +37,14 @@ services: ports: - "8701:8700" env_file: - - .env + - path: .env + required: false environment: ENGRAPHIS_HOST: 0.0.0.0 - ENGRAPHIS_DB_PATH: /data/engraphis.db + # The v1 server uses a DIFFERENT, incompatible memory schema from the v2 dashboard, + # so it MUST NOT share the dashboard's engraphis.db (doing so corrupts both). Give it + # its own file on the shared volume. + ENGRAPHIS_DB_PATH: /data/engraphis_v1.db ENGRAPHIS_STATE_DIR: /data/.engraphis volumes: - engraphis-data:/data diff --git a/engraphis/autosync.py b/engraphis/autosync.py index f177148f..06b59714 100644 --- a/engraphis/autosync.py +++ b/engraphis/autosync.py @@ -131,7 +131,18 @@ def due(policy: dict, *, now: Optional[float] = None) -> bool: def _record(summary: dict, *, now: Optional[float] = None) -> None: """Stamp last_run + a compact last_result onto the persisted policy (best-effort).""" now = time.time() if now is None else now - existing = _read() + # Best-effort telemetry must NEVER clobber the policy. Distinguish a fresh install (no + # file — safe to create with the default) from a TRANSIENT read failure of an existing + # file: normalize_policy({}) is the DISABLED default, so blindly writing it back after a + # read hiccup would silently turn auto-sync off (this runs after every pass). + path = policy_path() + if path.exists(): + try: + existing = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return # exists but unreadable → preserve the saved policy, skip this record + else: + existing = {} try: _write({"policy": normalize_policy(existing.get("policy", existing)), "last_run": float(now), diff --git a/engraphis/billing.py b/engraphis/billing.py index 9ede3a37..55c1f7bf 100644 --- a/engraphis/billing.py +++ b/engraphis/billing.py @@ -115,6 +115,16 @@ def _dedup_conn() -> Optional[sqlite3.Connection]: conn.execute( "CREATE TABLE IF NOT EXISTS subscription_seats (" "subscription_id TEXT PRIMARY KEY, seats INTEGER NOT NULL, updated_at REAL)") + seat_columns = { + row[1] for row in conn.execute( + "PRAGMA table_info(subscription_seats)").fetchall()} + if "event_ts" not in seat_columns: + # The source event's own modification time (Polar ``modified_at``), so an + # out-of-order redelivery of an OLDER subscription.updated can't regress a + # newer seat count. Nullable: payloads without a timestamp fall back to the + # seat-count-diff logic exactly as before. + conn.execute( + "ALTER TABLE subscription_seats ADD COLUMN event_ts REAL") conn.commit() try: os.chmod(path, 0o600) @@ -127,21 +137,34 @@ def _dedup_conn() -> Optional[sqlite3.Connection]: raise WebhookStateError("durable webhook state store unavailable") from exc -def reserve_webhook(webhook_id: str) -> bool: - """Atomically claim *webhook_id* for processing. +def claim_webhook(webhook_id: str) -> str: + """Atomically determine this delivery's state and claim it if free. - Completed claims remain permanent. An unfinished claim can be reclaimed after - :data:`_RESERVATION_TTL_SECONDS`, preventing a process crash before fulfillment - from suppressing the purchase forever. In-memory state is used only when no - durable store is configured; a configured store failure is retryable and raises. + Returns one of three states so the caller can answer correctly instead of + conflating "already done" with "still running": + ``"claimed"`` — we now own a fresh (or reclaimed-after-TTL) processing slot; + proceed with fulfillment. + ``"in_flight"`` — another attempt holds an UNFINISHED claim younger than + :data:`_RESERVATION_TTL_SECONDS`. The caller must return a + RETRYABLE (non-2xx) response so Polar retries later: answering + 2xx here would cancel Polar's retries, and if the in-flight + attempt crashed before minting the key the purchase would be + lost forever (no future delivery to reclaim the slot at TTL). + ``"fulfilled"`` — a prior attempt completed; a true duplicate — answer 2xx. + + Completed claims remain permanent. In-memory state is used only when no durable + store is configured; a configured store failure is retryable and raises. """ conn = _dedup_conn() if conn is None: + # Single-worker fallback: state dies with the process, so a post-crash retry + # simply re-claims. A present entry means this process already handled (or is + # handling) it and WILL complete, so "fulfilled" (→ 2xx duplicate) is correct. with _mem_lock: if webhook_id in _mem_seen: - return False + return "fulfilled" _mem_seen.add(webhook_id) - return True + return "claimed" try: now = time.time() with conn: @@ -149,18 +172,34 @@ def reserve_webhook(webhook_id: str) -> bool: "INSERT OR IGNORE INTO processed(webhook_id, ts, state) " "VALUES (?, ?, 'processing')", (webhook_id, now)) if cur.rowcount == 1: - return True + return "claimed" cur = conn.execute( "UPDATE processed SET ts=? WHERE webhook_id=? " "AND state='processing' AND ts<=?", (now, webhook_id, now - _RESERVATION_TTL_SECONDS)) - return cur.rowcount == 1 + if cur.rowcount == 1: + return "claimed" + row = conn.execute( + "SELECT state FROM processed WHERE webhook_id=?", + (webhook_id,)).fetchone() + return "in_flight" if (row is not None and row[0] == "processing") else "fulfilled" except sqlite3.Error as exc: raise WebhookStateError("could not reserve durable webhook claim") from exc finally: conn.close() +def reserve_webhook(webhook_id: str) -> bool: + """Back-compat bool wrapper over :func:`claim_webhook`. + + True only when this call took ownership of a fresh or reclaimed processing slot + (i.e. the caller should proceed to fulfill). False for both an already-fulfilled + claim and an in-flight one — callers that must distinguish those use + :func:`claim_webhook` directly. + """ + return claim_webhook(webhook_id) == "claimed" + + def complete_webhook(webhook_id: str) -> None: """Mark a claimed webhook or fulfillment as durably complete.""" conn = _dedup_conn() @@ -206,32 +245,46 @@ def release_webhook(webhook_id: str) -> None: # store configured this fails CLOSED (never re-issues) rather than open. def get_known_seats(subscription_id: str) -> Optional[int]: """Last seat count recorded for *subscription_id*, or ``None`` if never seen.""" + baseline = get_seat_baseline(subscription_id) + return baseline[0] if baseline is not None else None + + +def get_seat_baseline(subscription_id: str) -> Optional[tuple[int, Optional[float]]]: + """Last ``(seats, event_ts)`` recorded for *subscription_id*, or ``None``. + + ``event_ts`` is the source event's own modification time (may be ``None`` for + baselines recorded from payloads that carried no timestamp).""" conn = _dedup_conn() if conn is None: return None try: row = conn.execute( - "SELECT seats FROM subscription_seats WHERE subscription_id = ?", + "SELECT seats, event_ts FROM subscription_seats WHERE subscription_id = ?", (subscription_id,)).fetchone() - return int(row[0]) if row else None + if not row: + return None + return int(row[0]), (float(row[1]) if row[1] is not None else None) except sqlite3.Error as exc: raise WebhookStateError("could not read durable seat baseline") from exc finally: conn.close() -def record_known_seats(subscription_id: str, seats: int) -> bool: - """Persist *seats* as the new baseline. Return false without a durable store.""" +def record_known_seats(subscription_id: str, seats: int, + event_ts: Optional[float] = None) -> bool: + """Persist *seats* (and the event's *event_ts*) as the new baseline. Return false + without a durable store.""" conn = _dedup_conn() if conn is None: return False try: with conn: conn.execute( - "INSERT INTO subscription_seats(subscription_id, seats, updated_at) " - "VALUES (?, ?, ?) ON CONFLICT(subscription_id) DO UPDATE SET " - "seats=excluded.seats, updated_at=excluded.updated_at", - (subscription_id, seats, time.time())) + "INSERT INTO subscription_seats(subscription_id, seats, updated_at, " + "event_ts) VALUES (?, ?, ?, ?) ON CONFLICT(subscription_id) DO UPDATE SET " + "seats=excluded.seats, updated_at=excluded.updated_at, " + "event_ts=excluded.event_ts", + (subscription_id, seats, time.time(), event_ts)) return True except sqlite3.Error as exc: raise WebhookStateError("could not persist durable seat baseline") from exc @@ -239,8 +292,12 @@ def record_known_seats(subscription_id: str, seats: int) -> bool: conn.close() def _finalize_webhook(delivery_id: str, fulfillment_id: str, - seat_baseline: Optional[tuple[str, int]] = None) -> None: - """Persist the seat baseline and complete both claims in one transaction.""" + seat_baseline: Optional[tuple] = None) -> None: + """Persist the seat baseline and complete both claims in one transaction. + + ``seat_baseline`` is ``(subscription_id, seats)`` or ``(subscription_id, seats, + event_ts)``; ``event_ts`` (the source event's modification time) defaults to ``None``. + """ conn = _dedup_conn() if conn is None: return @@ -248,11 +305,14 @@ def _finalize_webhook(delivery_id: str, fulfillment_id: str, with conn: now = time.time() if seat_baseline is not None: + sub_id, seats = seat_baseline[0], seat_baseline[1] + event_ts = seat_baseline[2] if len(seat_baseline) > 2 else None conn.execute( - "INSERT INTO subscription_seats(subscription_id, seats, updated_at) " - "VALUES (?, ?, ?) ON CONFLICT(subscription_id) DO UPDATE SET " - "seats=excluded.seats, updated_at=excluded.updated_at", - (*seat_baseline, now)) + "INSERT INTO subscription_seats(subscription_id, seats, updated_at, " + "event_ts) VALUES (?, ?, ?, ?) ON CONFLICT(subscription_id) DO UPDATE " + "SET seats=excluded.seats, updated_at=excluded.updated_at, " + "event_ts=excluded.event_ts", + (sub_id, seats, now, event_ts)) for claim_id in (fulfillment_id, delivery_id): cur = conn.execute( "UPDATE processed SET state='fulfilled', ts=? " @@ -282,6 +342,65 @@ def _subscription_id(data: dict) -> str: return str(raw or "").strip()[:128] +def _event_modified_at(data: dict) -> Optional[float]: + """Epoch of the event object's own last-modification time, if the payload carries + one (Polar sends ``modified_at`` on Subscription objects). Used to reject an + out-of-order redelivery of an older subscription.updated. Returns ``None`` when the + payload has no usable timestamp, in which case ordering can't be established and the + caller falls back to seat-count comparison.""" + from engraphis.inspector.webhooks import _parse_ts + return _parse_ts(data.get("modified_at") or data.get("updated_at")) + + +def _event_organization_id(event: dict, data: dict) -> str: + """Best-effort extraction of the Polar organization id from an event, checked in the + locations Polar populates across order/subscription payloads.""" + product = data.get("product") or {} + subscription = data.get("subscription") or {} + for candidate in ( + data.get("organization_id"), + event.get("organization_id"), + product.get("organization_id"), + subscription.get("organization_id"), + ): + if candidate: + return str(candidate).strip() + return "" + + +def _organization_mismatch(event: dict, data: dict) -> bool: + """True only when ``POLAR_ORGANIZATION_ID`` is configured AND the event carries a + DIFFERENT organization id. The check is documented (webhooks.py env docs) but was + never enforced. Fails closed on a concrete mismatch; when the env var is unset, or + the payload carries no org id to compare, it does not block fulfillment (so a + payload-shape assumption can never strand a real purchase).""" + expected = os.environ.get("POLAR_ORGANIZATION_ID", "").strip() + if not expected: + return False + found = _event_organization_id(event, data) + if not found: + logger.warning("polar webhook: POLAR_ORGANIZATION_ID set but event carries no " + "organization id to verify — proceeding") + return False + return not hmac.compare_digest(found, expected) + + +# Events that END entitlement IMMEDIATELY -> revoke every key for the subscription (keys +# are cloud-enforced, so revocation takes effect at the next lease renewal, ~24h). +# +# A plain cancel-at-period-end (``subscription.canceled``) is deliberately NOT here: the +# customer paid for the current period and keeps their plan until it ends — their +# period-bounded key simply expires (Polar later fires ``subscription.revoked`` when access +# actually ends). Only a REFUND (or an explicit access revocation) removes access now: +# order.refunded -> the money was returned, so pull the license immediately. +# subscription.revoked -> Polar has revoked access (refund-driven, admin action, or the +# end of a cancel-at-period-end period). +_REVOKING_EVENTS = frozenset({ + "order.refunded", + "subscription.revoked", +}) + + @router.post("/webhooks/polar") async def polar_webhook(request: Request): """Receive Polar ``order.paid`` events, issue a signed license key, and email @@ -356,6 +475,33 @@ async def polar_webhook(request: Request): event_type = (event.get("type") or "").strip() data = event.get("data") or {} + # Reject a signed event from a DIFFERENT Polar organization when POLAR_ORGANIZATION_ID + # is configured (the documented-but-previously-unenforced control). No-op when unset. + if _organization_mismatch(event, data): + logger.warning("polar webhook: organization id mismatch — rejecting") + return JSONResponse({"error": "organization mismatch"}, status_code=403) + + # Negative lifecycle: refund / chargeback / hard cancellation revokes every key for + # the subscription. Keys are cloud-enforced, so revocation bites at the next lease + # renewal (~24h). revoke_by_subscription is idempotent, so a redelivery is harmless. + if event_type in _REVOKING_EVENTS: + sub_id = _subscription_id(data) + if not sub_id and event_type.startswith("subscription."): + sub_id = str(data.get("id") or "").strip()[:128] + if not sub_id: + return JSONResponse({"status": "ignored", "reason": "no subscription id", + "type": event_type}, status_code=202) + try: + from engraphis.inspector import license_registry as _reg + revoked = await asyncio.to_thread(_reg.revoke_by_subscription, sub_id) + except Exception: # noqa: BLE001 — retryable: let Polar redeliver the revoke + logger.exception("polar webhook: revocation failed for %s", sub_id) + return JSONResponse({"error": "revocation failed"}, status_code=503) + logger.info("polar webhook: %s revoked %d key(s) for subscription %s", + event_type, revoked, sub_id) + return JSONResponse({"status": "revoked", "keys_revoked": revoked, + "type": event_type}, status_code=202) + # Route by event type and derive a stable per-fulfillment key so we issue exactly # ONE key per order and ONE per trial, no matter which/how many events fire: # order.paid -> paid activation, trial conversion, and each renewal @@ -370,14 +516,17 @@ async def polar_webhook(request: Request): # and unrelated updates cannot spam replacement keys. # A non-trial subscription.created is a no-op: its paid key comes from order.paid, so # a canceled trial can never keep Pro — the short trial key just expires. - pending_seat_baseline = None # (sub_id, seats), persisted only after key issuance + pending_seat_baseline = None # (sub_id, seats, event_ts); persisted after key issuance if event_type == "order.paid": from engraphis.inspector.webhooks import ( _extract_seats, handle_order_paid as _fulfill) fulfillment_key = "order:" + str(data.get("id") or webhook_id) sub_id = _subscription_id(data) if sub_id: - pending_seat_baseline = (sub_id, _extract_seats(data)) + # event_ts stays None: an Order object's modified_at is a different clock from + # the Subscription's, so it must not seed the seat-ordering anchor (which is + # compared only against subscription.updated modified_at values). + pending_seat_baseline = (sub_id, _extract_seats(data), None) elif event_type == "subscription.created": if str(data.get("status", "")).strip().lower() != "trialing": return JSONResponse({"status": "ignored", "reason": "not a trial", @@ -386,7 +535,7 @@ async def polar_webhook(request: Request): _extract_seats, handle_subscription_created as _fulfill) sub_id = str(data.get("id") or webhook_id) fulfillment_key = "trial:" + sub_id - pending_seat_baseline = (sub_id, _extract_seats(data)) + pending_seat_baseline = (sub_id, _extract_seats(data), _event_modified_at(data)) elif event_type == "subscription.updated": status = str(data.get("status", "")).strip().lower() sub_id = str(data.get("id") or "").strip()[:128] @@ -395,25 +544,43 @@ async def polar_webhook(request: Request): "subscription", "type": event_type}, status_code=202) from engraphis.inspector.webhooks import _extract_seats new_seats = _extract_seats(data) + event_ts = _event_modified_at(data) try: - prior_seats = get_known_seats(sub_id) + prior = get_seat_baseline(sub_id) except WebhookStateError: logger.exception("polar webhook: could not read seat baseline") return JSONResponse({"error": "webhook state unavailable"}, status_code=503) - if prior_seats is None: + if prior is None: # First sighting seeds the baseline; the initial paid key came from order.paid. try: - persisted = record_known_seats(sub_id, new_seats) + persisted = record_known_seats(sub_id, new_seats, event_ts) except WebhookStateError: logger.exception("polar webhook: could not seed seat baseline") return JSONResponse({"error": "webhook state unavailable"}, status_code=503) reason = "baseline recorded" if persisted else "durable baseline unavailable" return JSONResponse({"status": "ignored", "reason": reason, "type": event_type}, status_code=202) + prior_seats, prior_ts = prior + # Out-of-order guard: if this delivery is OLDER than the last one we acted on for + # this subscription, ignore it — a delayed redelivery of a stale seat count must + # not regress a newer one (and revoke the correct key). Only applies when both + # timestamps are known; without them we fall back to seat-count comparison. + if event_ts is not None and prior_ts is not None and event_ts <= prior_ts: + return JSONResponse({"status": "ignored", "reason": "out-of-order update", + "type": event_type}, status_code=202) if prior_seats == new_seats: + # No seat change. Keep the ordering anchor current (so a later, genuinely + # older delivery is still recognized as stale) but do not re-issue. + if event_ts is not None and (prior_ts is None or event_ts > prior_ts): + try: + record_known_seats(sub_id, new_seats, event_ts) + except WebhookStateError: + logger.exception("polar webhook: could not advance seat anchor") + return JSONResponse({"error": "webhook state unavailable"}, + status_code=503) return JSONResponse({"status": "ignored", "reason": "no seat-count change", "type": event_type}, status_code=202) - pending_seat_baseline = (sub_id, new_seats) + pending_seat_baseline = (sub_id, new_seats, event_ts) from engraphis.inspector.webhooks import handle_subscription_updated as _fulfill # webhook-id is covered by the signature and stable across retries of one # delivery. Versioning by it permits legitimate A -> B -> A seat cycles while @@ -423,21 +590,40 @@ async def polar_webhook(request: Request): return JSONResponse({"status": "ignored", "type": event_type}, status_code=202) # Two-layer dedup: delivery-level (a retry of this exact webhook) and - # fulfillment-level (one key per order/trial/update version). + # fulfillment-level (one key per order/trial/update version). Each claim is + # tri-state so an in-flight attempt is answered RETRYABLE (503) rather than a 2xx + # "duplicate": a 2xx cancels Polar's retries, and if the in-flight attempt crashed + # before minting the key the purchase would be lost with no future delivery to + # reclaim the slot at the TTL. Only a genuinely COMPLETED claim is a duplicate. delivery_claim = "dlv:" + webhook_id fulfillment_claim = "ful:" + fulfillment_key delivery_reserved = False try: - if not reserve_webhook(delivery_claim): + delivery_state = claim_webhook(delivery_claim) + if delivery_state == "fulfilled": logger.info("polar webhook: duplicate delivery %s ignored", webhook_id) return JSONResponse( {"status": "duplicate", "key_issued": False}, status_code=202) + if delivery_state == "in_flight": + logger.info("polar webhook: delivery %s already in flight — retry later", + webhook_id) + return JSONResponse({"status": "processing", "key_issued": False}, + status_code=503) delivery_reserved = True - if not reserve_webhook(fulfillment_claim): + fulfillment_state = claim_webhook(fulfillment_claim) + if fulfillment_state == "fulfilled": complete_webhook(delivery_claim) logger.info("polar webhook: %s already fulfilled — no second key", fulfillment_key) return JSONResponse({"status": "already_fulfilled", "key_issued": False}, status_code=202) + if fulfillment_state == "in_flight": + # A concurrent delivery for the same order/trial/version is minting the key. + # Release our delivery claim and have Polar retry — by then it's fulfilled. + _release_claims(delivery_claim) + logger.info("polar webhook: %s fulfillment in flight — retry later", + fulfillment_key) + return JSONResponse({"status": "processing", "key_issued": False}, + status_code=503) except WebhookStateError: if delivery_reserved: _release_claims(delivery_claim) diff --git a/engraphis/cloud_license.py b/engraphis/cloud_license.py index e4151a69..2b2e14cd 100644 --- a/engraphis/cloud_license.py +++ b/engraphis/cloud_license.py @@ -28,6 +28,7 @@ import json import logging import os +import threading import urllib.error import urllib.request import uuid @@ -106,6 +107,8 @@ def validate_cloud_base_url(value: str) -> str: #: call within a run returns the SAME id — so the trial HMAC verifies and leases #: bind consistently instead of churning a fresh uuid on every call. _machine_id_cache: dict = {} +# Serializes first-run id generation so concurrent threads don't each mint a device id. +_machine_id_lock = threading.Lock() def cloud_url() -> str: @@ -124,29 +127,47 @@ def machine_id() -> str: cached = _machine_id_cache.get(key) if cached: return cached - try: - mid = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip() - if mid: - _machine_id_cache[key] = mid - return mid - except OSError: - pass - mid = uuid.uuid4().hex - try: - _MACHINE_ID_FILE.parent.mkdir(parents=True, exist_ok=True) - _MACHINE_ID_FILE.write_text(mid, encoding="utf-8") + with _machine_id_lock: + # Re-check under the lock: another thread may have generated + cached the id while + # we waited, so we don't mint a second one (which would burn an extra Team seat). + cached = _machine_id_cache.get(key) + if cached: + return cached try: - os.chmod(_MACHINE_ID_FILE, 0o600) + mid = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip() + if mid: + _machine_id_cache[key] = mid + return mid except OSError: pass - except OSError as exc: - logger.warning( - "machine_id: could not persist device id to %s (%s); using an in-process id " - "for this run. Trial and cloud-lease binding need a writable home directory " - "— mount a persistent volume for %s (or set HOME to a writable path).", - _MACHINE_ID_FILE, exc, _MACHINE_ID_FILE.parent) - _machine_id_cache[key] = mid # stable for the rest of the process regardless - return mid + mid = uuid.uuid4().hex + try: + _MACHINE_ID_FILE.parent.mkdir(parents=True, exist_ok=True) + # Atomic create-if-absent so two PROCESSES racing a first run don't each persist + # a DIFFERENT id (registering the same device twice and burning a seat). The + # exclusive create fails if another process already wrote one; adopt that value. + try: + fd = os.open(str(_MACHINE_ID_FILE), + os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(mid) + except FileExistsError: + existing = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip() + if existing: + mid = existing + try: + os.chmod(_MACHINE_ID_FILE, 0o600) + except OSError: + pass + except OSError as exc: + logger.warning( + "machine_id: could not persist device id to %s (%s); using an in-process " + "id for this run. Trial and cloud-lease binding need a writable home " + "directory — mount a persistent volume for %s (or set HOME to a writable " + "path).", + _MACHINE_ID_FILE, exc, _MACHINE_ID_FILE.parent) + _machine_id_cache[key] = mid # stable for the rest of the process regardless + return mid # ── lease token (same ENGR1-style envelope, distinct prefix) ───────────────────────── diff --git a/engraphis/config.py b/engraphis/config.py index 8916743c..7fb6ceb6 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -17,13 +17,13 @@ #: Vendor-hosted managed service for sync, leases, trials, and invite delivery. This is #: the default target when ``ENGRAPHIS_RELAY_URL`` is not overridden. -DEFAULT_RELAY_URL = "https://engraphis-production.up.railway.app" +DEFAULT_RELAY_URL = "https://team.engraphis.com" -# Keys issued while the Cloudflare custom domain was the default carry that URL inside -# their signed payload. Preserve the signature, but route that unavailable host to the -# live Railway service. Arbitrary signed URLs remain authoritative. +# Keys issued before the custom domain migration carry this URL inside their signed +# payload. Preserve the signature, but route that one retired vendor host to the current +# managed service. Arbitrary signed URLs remain authoritative. RETIRED_RELAY_URLS = frozenset({ - "https://team.engraphis.com", + "https://engraphis-production.up.railway.app", }) @@ -55,7 +55,8 @@ class Settings: api_token: str = field(default_factory=lambda: _env("ENGRAPHIS_API_TOKEN", "")) # Comma-separated CORS allow-list. Defaults to loopback only (local-first). cors_origins: list = field( - default_factory=lambda: _parse_origins(_env("ENGRAPHIS_CORS_ORIGINS", "")) + default_factory=lambda: _parse_origins(_env("ENGRAPHIS_CORS_ORIGINS", ""), + _env_int("ENGRAPHIS_PORT", 8700)) ) # Optional server-side workspace binding — the hard multi-tenant isolation boundary. # When non-empty, MemoryService refuses any read or write whose @@ -151,10 +152,13 @@ def _parse_headers(raw: str) -> dict: return {} -def _parse_origins(raw: str) -> list: - """CORS allow-list. Empty -> loopback only (safe local-first default).""" +def _parse_origins(raw: str, port: int = 8700) -> list: + """CORS allow-list. Empty -> loopback on the CONFIGURED port (safe local-first default). + + Deriving the default from ``port`` means running the dashboard on a non-default + ENGRAPHIS_PORT doesn't lock its own origin out of the CORS allow-list.""" if not raw.strip(): - return ["http://127.0.0.1:8700", "http://localhost:8700"] + return ["http://127.0.0.1:%d" % port, "http://localhost:%d" % port] return [o.strip() for o in raw.split(",") if o.strip()] diff --git a/engraphis/core/store.py b/engraphis/core/store.py index 580b4e43..0a4c22e1 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -166,6 +166,115 @@ def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *, return True +class _SerializedConnection: + """Serializes access to one sqlite3 connection shared across threads. + + The Store opens a SINGLE connection with ``check_same_thread=False`` and shares it + across the threadpool FastAPI runs sync handlers on. A bare sqlite3 connection is not + safe for concurrent multi-thread use: interleaved statements corrupt cursors, and — + because a connection has ONE transaction — one thread's ``commit()``/``rollback()`` + lands on another thread's uncommitted writes, so a rollback can silently discard them. + (Per-thread connections are not an option: the sqlite-vec extension and FTS state are + loaded into THIS connection, and a ``:memory:`` DB can't be shared across connections + at all.) + + This wrapper holds a reentrant lock for the DURATION of each write transaction — + pinned on the first statement that opens one (detected via ``in_transaction``) and + released on commit/rollback — so transactions never interleave. Read-only statements + lock only for the individual call. Two safety nets keep a stuck transaction from + deadlocking the process: a statement that raises while a transaction is open rolls it + back and frees the pin, and lock acquisition times out (raising, not blocking forever). + Non-statement attributes/methods (``in_transaction``, ``enable_load_extension`` at + setup, ...) pass straight through. + """ + + _ACQUIRE_TIMEOUT = 60.0 + + def __init__(self, raw) -> None: + object.__setattr__(self, "_raw", raw) + object.__setattr__(self, "_lock", threading.RLock()) + object.__setattr__(self, "_pin", threading.local()) + + def __getattr__(self, name): + return getattr(self._raw, name) + + def __setattr__(self, name, value): + setattr(self._raw, name, value) + + def _pinned(self) -> bool: + return getattr(self._pin, "held", False) + + def _acquire(self) -> None: + if not self._lock.acquire(timeout=self._ACQUIRE_TIMEOUT): + raise sqlite3.OperationalError( + "store write lock timeout — a transaction appears stuck") + + def _run(self, fn, *a, **k): + self._acquire() + try: + result = fn(*a, **k) + except BaseException: + # Do NOT roll back here: a failed statement in sqlite leaves any open + # transaction intact, and callers depend on that — e.g. probing an optional + # table inside a larger write transaction, catching the error, and continuing. + # Only manage the lock (identically to the success path); the pin is released + # by the eventual commit/rollback, or by the acquire timeout as a backstop. + self._settle() + raise + self._settle() + return result + + def _settle(self) -> None: + """After a statement, hold exactly one pinned lock acquire for this thread while a + write transaction is open (released on commit/rollback); otherwise release this + call's acquire so read-only statements don't hold the lock.""" + if self._raw.in_transaction: + if self._pinned(): + self._lock.release() # already pinned; drop this call's acquire + else: + self._pin.held = True # keep this acquire as the transaction pin + else: + self._lock.release() # no open transaction; release now + + def _finish(self, fn): + self._acquire() + try: + fn() + finally: + if self._pinned(): + self._pin.held = False + self._lock.release() # release the transaction pin + self._lock.release() # release this call's acquire + + def execute(self, *a, **k): + return self._run(self._raw.execute, *a, **k) + + def executemany(self, *a, **k): + return self._run(self._raw.executemany, *a, **k) + + def executescript(self, *a, **k): + return self._run(self._raw.executescript, *a, **k) + + def commit(self): + self._finish(self._raw.commit) + + def rollback(self): + self._finish(self._raw.rollback) + + def close(self): + self._raw.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + if exc_type is None: + self.commit() + else: + self.rollback() + return False + + class Store: """A connection to one Engraphis v2 database (one file, or ``:memory:``).""" @@ -178,10 +287,14 @@ def __init__(self, path: str = ":memory:", *, if connect is not None: # Injected connection factory (e.g. the SQLCipher encrypted backend). It owns # opening + keying + row_factory; the core never imports the concrete driver. - self.conn = connect(path) + raw_conn = connect(path) else: - self.conn = sqlite3.connect(path, timeout=30, check_same_thread=False) - self.conn.row_factory = sqlite3.Row + raw_conn = sqlite3.connect(path, timeout=30, check_same_thread=False) + raw_conn.row_factory = sqlite3.Row + # Serialize the shared connection so concurrent threadpool handlers can't interleave + # transactions on it (see _SerializedConnection). All Store/service/backend access + # goes through self.conn, so wrapping here covers every writer. + self.conn = _SerializedConnection(raw_conn) self.conn.execute("PRAGMA journal_mode=WAL") self.conn.execute("PRAGMA foreign_keys=ON") self.conn.execute("PRAGMA synchronous=NORMAL") @@ -297,6 +410,11 @@ def create_workspace(self, name: str, *, settings: Optional[dict] = None) -> str return wid def get_or_create_workspace(self, name: str) -> str: + # Authorize on the RETRIEVE path too, not just create — otherwise a workspace + # outside ENGRAPHIS_WORKSPACES that already exists in the DB (e.g. predating the + # allow-list, or arriving via sync) could be handed back, silently bypassing the + # isolation boundary _authorize_workspace is meant to enforce ("create or retrieve"). + self._authorize_workspace(name) row = self.conn.execute("SELECT id FROM workspaces WHERE name=?", (name,)).fetchone() if row: return row["id"] diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 9202d6a9..03c3dd36 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -232,6 +232,24 @@ def _clamp_ts(v: Any, now: float) -> Optional[float]: return max(0.0, min(f, now + TS_FUTURE_SKEW)) +# World-time validity ceiling (year ~2100). ``valid_from``/``valid_to`` are WORLD time — +# a fact may legitimately be true until a future date — and, unlike the system timestamps, +# do NOT feed the LWW version key (see _version_key), so a future value can't pin content. +# Bound only to a sane far-future ceiling to reject absurd/overflow values. +_WORLD_TS_MAX = 4_102_444_800.0 + + +def _clamp_world_ts(v: Any) -> Optional[float]: + """Coerce a world-time validity timestamp, allowing legitimate FUTURE values (bounded + to a far-future ceiling). Clamping these to ``now + skew`` like the system timestamps + truncated real future validity, and the earliest-wins merge then spread the truncation + to every device.""" + f = _as_float(v, None) + if f is None: + return None + return max(0.0, min(f, _WORLD_TS_MAX)) + + def _clamp_str(v: Any, n: int) -> str: s = v if isinstance(v, str) else ("" if v is None else str(v)) return _CONTROL_RE.sub("", s)[:n] @@ -345,8 +363,10 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]: stability=_clamp_num(d.get("stability"), 0.0, MAX_STABILITY, 1.0), access_count=min(MAX_ACCESS_COUNT, max(0, _as_int(d.get("access_count"), 0))), last_access=_clamp_ts(d.get("last_access"), now), - valid_from=_clamp_ts(d.get("valid_from"), now), - valid_to=_clamp_ts(d.get("valid_to"), now), + # World-time validity may be in the future; system timestamps may not (they feed + # the version key / anti-poison defense). + valid_from=_clamp_world_ts(d.get("valid_from")), + valid_to=_clamp_world_ts(d.get("valid_to")), ingested_at=_clamp_ts(d.get("ingested_at"), now), expired_at=_clamp_ts(d.get("expired_at"), now), pinned=bool(d.get("pinned")), sensitivity=sens, diff --git a/engraphis/engines/recall.py b/engraphis/engines/recall.py index a456cd05..3739835b 100644 --- a/engraphis/engines/recall.py +++ b/engraphis/engines/recall.py @@ -74,8 +74,10 @@ def recall( } -def recall_master(*, namespace: str, max_chunks: int = 10) -> dict[str, Any]: - """Recall the highest-retention memories in a namespace (no prompt needed).""" +def recall_master(*, namespace: Optional[str] = None, max_chunks: int = 10) -> dict[str, Any]: + """Recall the highest-retention memories in a namespace (no prompt needed). A ``None`` + namespace recalls across ALL namespaces — used by the consciousness loop, which has no + single namespace to anchor to.""" candidates = mem_store.all_vectors(namespace=namespace) if not candidates: return {"context": [], "chunks": [], "count": 0, "llmContextMessage": ""} diff --git a/engraphis/engines/reweight.py b/engraphis/engines/reweight.py index e364e79b..c3da3e86 100644 --- a/engraphis/engines/reweight.py +++ b/engraphis/engines/reweight.py @@ -76,6 +76,30 @@ def apply_interaction_boost(mem_id: int, interaction_level: str) -> None: conn.commit() +def boost_entity_memories(namespace: str, entity_name: str, + interaction_level: str) -> int: + """Apply an interaction boost to memories that mention *entity_name*. + + This is what makes a recorded interaction actually reinforce memory (interaction-aware + reinforcement); previously ``apply_interaction_boost`` had no caller, so interactions + were logged but never affected retention. Matching is a bounded name-substring lookup + (the entity name is a BOUND parameter — no SQL injection). Returns how many memories + were reinforced.""" + name = (entity_name or "").strip() + if not name: + return 0 + conn = get_conn() + like = "%" + name + "%" + rows = conn.execute( + "SELECT id FROM memories WHERE namespace=? AND (title LIKE ? OR content LIKE ?) " + "LIMIT 100", + (namespace, like, like), + ).fetchall() + for r in rows: + apply_interaction_boost(r["id"], interaction_level) + return len(rows) + + def decay_pass(namespace: Optional[str] = None) -> int: """Background decay: reduce stability for stale memories. Returns rows touched.""" return mem_store.apply_decay_to_all(namespace, settings.decay_halflife_days) diff --git a/engraphis/engines/thoughts.py b/engraphis/engines/thoughts.py index ddf6bf5c..463d651a 100644 --- a/engraphis/engines/thoughts.py +++ b/engraphis/engines/thoughts.py @@ -27,7 +27,10 @@ def synthesize_thoughts( thought_prompt: Optional[str] = None, ) -> dict[str, Any]: """Recall recent high-salience memories, synthesize a thought via LLM, optionally persist.""" - ctx = recall_engine.recall_master(namespace=namespace or "_global", max_chunks=max_chunks) + # Pass the namespace through as-is: ``None`` recalls across ALL namespaces (the + # consciousness loop calls this with namespace=None). Coercing to a nonexistent + # "_global" namespace made every global synthesis silently recall nothing and no-op. + ctx = recall_engine.recall_master(namespace=namespace, max_chunks=max_chunks) chunks = ctx.get("chunks", []) if not chunks: return {"thought": None, "source_count": 0, "persisted": False, "reason": "no_memories"} diff --git a/engraphis/inspector/auth.py b/engraphis/inspector/auth.py index 9e217e1f..581af2b9 100644 --- a/engraphis/inspector/auth.py +++ b/engraphis/inspector/auth.py @@ -137,11 +137,12 @@ def min_role(method: str, path: str) -> str: return "viewer" if path in ( "/api/code/index", "/api/workspaces/import-folder", "/api/resources/postgres", - "/api/sync/run", + "/api/sync/run", "/api/workspaces/delete", "/api/workspaces/merge", ): # These operations read server-local files or make a caller-selected outbound - # connection, or mutate every shared workspace through the account-wide relay. - # Only an administrator may choose those sources/actions. + # connection, mutate every shared workspace through the account-wide relay, or + # IRREVERSIBLY destroy/combine a whole shared workspace (delete/merge). Those are + # not ordinary member governance actions — only an administrator may choose them. return "admin" if path.startswith("/api/auth/users") or path.startswith("/api/auth/audit") \ or path == "/api/auth/overview" or path in ( @@ -224,7 +225,8 @@ def _clean_email(email) -> str: @_serialized def create_user(self, email: str, name: str, password: str, role: str, - *, seat_limit: Optional[int] = None) -> dict: + *, seat_limit: Optional[int] = None, + require_empty: bool = False) -> dict: # The very first user (bootstrap admin, called from /api/auth/setup on a # zero-user store) is exempt from the license gate. Every user after that # still requires an active Team license — this only closes the chicken-and-egg @@ -257,6 +259,12 @@ def create_user(self, email: str, name: str, password: str, role: str, conn.execute("BEGIN IMMEDIATE") started = True if int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]) > 0: + # require_empty makes /api/auth/setup atomic: the first-admin bootstrap must + # create EXACTLY ONE user, so if any exists by the time we hold the write + # lock, refuse — this closes the setup TOCTOU where two concurrent requests + # both passed the router's count check and both created an admin. + if require_empty: + raise AuthError("team already set up") from engraphis.licensing import require_feature require_feature("team") if seat_limit is not None and int(conn.execute( @@ -435,11 +443,16 @@ def _locked_until(self, email: str) -> float: return fails[-1] + LOCKOUT_SECONDS return 0.0 - @_serialized def login(self, email: str, password: str, *, ip: Optional[str] = None) -> dict: """Verify credentials → new session. Raises :class:`AuthError` (generic message — never reveals which of email/password was wrong) or the lockout notice. + NOT ``@_serialized`` as a whole: PBKDF2 password verification is CPU-bound (~100ms) + and must NOT hold the AuthStore lock, because the async auth middleware resolves + sessions/tokens under that SAME lock — holding it across the hash blocked the whole + event loop for the hash duration, so a burst of logins stalled every request + (an unauthenticated DoS). The lock is taken only for the fast DB / throttle steps. + Deliberately does NOT gate on a live Team license (see ``licensing.require_feature ("team")``, still enforced by :meth:`create_user`). It used to — 2026-07-12 incident: an expired/lapsed license blocked EVERY login, including the admin's own, with no @@ -453,28 +466,33 @@ def login(self, email: str, password: str, *, ip: Optional[str] = None) -> dict: refused — this only stops a license hiccup from locking out people who already have an account. Existing sessions still cap out at ``SESSION_TTL_SECONDS``.""" email = self._clean_email(email) - until = self._locked_until(email) - if until > time.time(): - self.record_event("login.locked", actor_email=email, ip=ip) - raise AuthError("too many failed attempts — try again in a minute") - row = self.conn.execute( - "SELECT id, pw_hash, disabled FROM users WHERE email=?", (email,)).fetchone() - # Always run one PBKDF2 even for unknown emails (no user-enumeration timing). + # Phase 1 (locked): throttle gate + fetch the row. + with self._lock: + until = self._locked_until(email) + if until > time.time(): + self.record_event("login.locked", actor_email=email, ip=ip) + raise AuthError("too many failed attempts — try again in a minute") + row = self.conn.execute( + "SELECT id, pw_hash, disabled FROM users WHERE email=?", (email,)).fetchone() + # Phase 2 (UNLOCKED): the expensive PBKDF2. Always hash once, even for unknown + # emails, to keep timing constant (no user enumeration). encoded = row["pw_hash"] if row else _hash_password("x", iterations=self.iterations) ok = _verify_password(password or "", encoded) - if not ok or row is None or row["disabled"]: - fails = self._failures.pop(email, []) - fails.append(time.time()) - self._failures[email] = fails - self._prune_throttle_maps(time.time()) - self.record_event("login.failed", actor_email=email, ip=ip, - detail=("account_disabled" if row and row["disabled"] - else "bad_credentials")) - raise AuthError("invalid email or password") - self._failures.pop(email, None) - token = self.create_session(row["id"]) - user = self.get_user(row["id"]) - self.record_event("login.success", actor_id=row["id"], actor_email=email, ip=ip) + # Phase 3 (locked): record the outcome and mint the session. + with self._lock: + if not ok or row is None or row["disabled"]: + fails = self._failures.pop(email, []) + fails.append(time.time()) + self._failures[email] = fails + self._prune_throttle_maps(time.time()) + self.record_event("login.failed", actor_email=email, ip=ip, + detail=("account_disabled" if row and row["disabled"] + else "bad_credentials")) + raise AuthError("invalid email or password") + self._failures.pop(email, None) + token = self.create_session(row["id"]) + user = self.get_user(row["id"]) + self.record_event("login.success", actor_id=row["id"], actor_email=email, ip=ip) user["token"] = token return user diff --git a/engraphis/inspector/license_cloud.py b/engraphis/inspector/license_cloud.py index ed9a5ae1..1823b5ab 100644 --- a/engraphis/inspector/license_cloud.py +++ b/engraphis/inspector/license_cloud.py @@ -80,13 +80,20 @@ async def register(request: Request): raise LicenseError("this license has been revoked") now = time.time() - conn = _conn() - try: - # Claim (or refresh) this device's seat. Reclaims seats whose lease has lapsed - # first, then enforces the per-license cap; raises LicenseError (→ 402) if full. - reg.claim_seat(conn, lic, machine_id, now=now) - finally: - conn.close() + # Team is the only device-capped tier (it is seat-priced). Pro is intentionally NOT + # device-capped here: its value is one person syncing their own machines, and + # ``account_id`` isolation already separates customers. Seat-capping every plan would + # make a Pro customer's second device fail registration → the client maps that 402 to + # "revoked" and drops to the free tier, breaking the flagship multi-device Pro feature. + # Mirrors sync_relay._authorize so the two enforcement points can't drift. + if lic.plan == "team": + conn = _conn() + try: + # Claim (or refresh) this device's seat. Reclaims seats whose lease has lapsed + # first, then enforces the per-license cap; raises LicenseError (→ 402) if full. + reg.claim_seat(conn, lic, machine_id, now=now) + finally: + conn.close() try: # ensure it's in the issued registry reg.record_issued(key) @@ -447,8 +454,13 @@ def _client_ip(request: Request) -> str: if "*" not in allowed and direct not in allowed: return direct fwd = request.headers.get("x-forwarded-for", "") - first = fwd.split(",", 1)[0].strip() - return first[:64] if first else direct + # A trusted proxy APPENDS the address it observed to the right of X-Forwarded-For, so + # the rightmost entry is the hop our proxy actually saw. Everything to its left is + # client-supplied and therefore spoofable — taking the leftmost token would let an + # attacker mint a fresh rate-limit bucket per request by pre-seeding the header. We + # trust exactly one hop (the direct peer), so use the last entry. + parts = [p.strip() for p in fwd.split(",") if p.strip()] + return parts[-1][:64] if parts else direct def _bump_trial_rate(ip: str) -> bool: diff --git a/engraphis/inspector/license_registry.py b/engraphis/inspector/license_registry.py index 18d07b98..caee411f 100644 --- a/engraphis/inspector/license_registry.py +++ b/engraphis/inspector/license_registry.py @@ -198,6 +198,32 @@ def revoke_superseded(subscription_id: str, keep_key_id: str, *, conn.close() +def revoke_by_subscription(subscription_id: str, *, + db_path: Optional[str] = None) -> int: + """Revoke EVERY active key issued for *subscription_id*. Returns the number changed. + + Used by the negative half of the billing lifecycle (refund / chargeback / hard + cancellation): unlike :func:`revoke_superseded`, this keeps no key — the customer is + no longer entitled to any. Keys are cloud-enforced (``enforce=cloud``), so the + revocation takes effect at the next lease renewal (within one lease TTL, ~24h). + Idempotent: a second call simply changes nothing. + """ + subscription_id = (subscription_id or "").strip()[:128] + if not subscription_id: + return 0 + conn = connect(db_path) + try: + with conn: + cur = conn.execute( + "UPDATE issued_licenses SET status='revoked', revoked_at=? " + "WHERE subscription_id=? AND status!='revoked'", + (time.time(), subscription_id), + ) + return cur.rowcount + finally: + conn.close() + + def is_revoked(key_id: str, *, db_path: Optional[str] = None) -> bool: """True only if the key is present AND explicitly revoked. diff --git a/engraphis/inspector/webhooks.py b/engraphis/inspector/webhooks.py index 3b3501b4..45b06130 100644 --- a/engraphis/inspector/webhooks.py +++ b/engraphis/inspector/webhooks.py @@ -93,16 +93,50 @@ def _load_signing_secret() -> bytes: return _read_seed_file(Path(_DEFAULT_KEY_PATH)) +def _product_plan_overrides() -> dict: + """Operator-configured exact product-name → plan map (``ENGRAPHIS_POLAR_PRODUCT_MAP``). + + JSON object of ``{"": "pro"|"team", ...}`` matched case-insensitively + and exactly, letting an operator pin tiering precisely instead of relying on the + substring heuristic — e.g. a product literally named "Engraphis Enterprise" that + should map to ``team``. Consulted before the built-in substring rules. Malformed + JSON or unknown plan values are ignored (the substring fallback still applies), so a + bad env var can never stiff a paying customer. + """ + raw = os.environ.get("ENGRAPHIS_POLAR_PRODUCT_MAP", "").strip() + if not raw: + return {} + try: + import json + parsed = json.loads(raw) + except (ValueError, TypeError): + logger.warning("ENGRAPHIS_POLAR_PRODUCT_MAP is not valid JSON — ignoring") + return {} + if not isinstance(parsed, dict): + return {} + out = {} + for name, plan in parsed.items(): + plan = str(plan or "").strip().lower() + if plan in PLAN_FEATURES: + out[str(name).strip().lower()] = plan + return out + + def _map_polar_product_to_plan(product_name: str) -> str: """Map Polar product name to an Engraphis plan tier. - Match on substrings so "Engraphis Pro Monthly" and "Engraphis Pro Annual" both - resolve to ``pro``. A paid order with an *unrecognized* product name still - resolves to ``pro`` (never free) and logs loudly: a customer who paid must never - be silently stiffed with a useless free-tier key. Correct Pro-vs-Team routing - depends on the Polar product name containing "pro"/"team" — keep them named so. + An operator-configured exact-name override (``ENGRAPHIS_POLAR_PRODUCT_MAP``) wins so + tiering can be pinned to real business data rather than inferred. Otherwise match on + substrings so "Engraphis Pro Monthly" and "Engraphis Pro Annual" both resolve to + ``pro``. A paid order with an *unrecognized* product name still resolves to ``pro`` + (never free) and logs loudly: a customer who paid must never be silently stiffed with + a useless free-tier key. Correct Pro-vs-Team routing depends on the Polar product name + containing "pro"/"team" (or an explicit override) — keep them named so. """ name = (product_name or "").lower() + override = _product_plan_overrides().get(name.strip()) + if override: + return override if "team" in name: return "team" if "pro" in name: @@ -132,6 +166,32 @@ def _key_days(product_name: str, metadata: dict) -> int: return 35 +# Grace added over a subscription's current_period_end when re-issuing a key mid-cycle, +# so a slightly-late renewal webhook never briefly locks out a paying customer. Kept +# small (matches the 5-day grace baked into the 35-day monthly _key_days window) — the +# whole point of bounding to the period end is that a mid-cycle change must NOT hand out +# a fresh full 35/395-day window that outlives the paid period. +_KEY_PERIOD_GRACE_DAYS = 5 + + +def _subscription_key_days(payload: dict, product_name: str, metadata: dict, + *, now: Optional[float] = None) -> int: + """Validity (in days) for a key re-issued from a Subscription object mid-cycle. + + Bounds the key to the subscription's ``current_period_end`` (+ a small fixed grace) + rather than a fresh full ``_key_days`` window measured from now. Without this, a + late-cycle seat change would mint a key valid a whole extra billing cycle past the + paid period (≈12 months for annual) — and since cancellation is enforced by letting + the period-bounded key expire, that overrun is unpaid access. Falls back to + :func:`_key_days` only when ``current_period_end`` is absent from the payload. + """ + now = now if now is not None else time.time() + end = _parse_ts(payload.get("current_period_end")) + if end and end > now: + return max(1, math.ceil((end - now) / 86400) + _KEY_PERIOD_GRACE_DAYS) + return _key_days(product_name, metadata) + + def _plan_label(product_name: str) -> str: """Clean tier label for customer-facing email copy ("Pro"/"Team").""" return _map_polar_product_to_plan(product_name).title() @@ -745,7 +805,10 @@ def handle_subscription_updated(payload: dict) -> Optional[str]: product = payload.get("product") or {} product_name = _extract_product_name(payload) seats = _extract_seats(payload) - days = _key_days(product_name, product.get("metadata") or {}) + # Bound the replacement key to the subscription's current paid period, NOT a fresh + # full window from now — a mid-cycle seat change must not extend entitlement past the + # period the customer has actually paid through. + days = _subscription_key_days(payload, product_name, product.get("metadata") or {}) logger.info("seat count changed for %s (%s) -> %d seats, re-issuing key", email_addr, product_name, seats) return _issue_and_email( diff --git a/engraphis/licensing.py b/engraphis/licensing.py index 9b9e8aac..5f1b6048 100644 --- a/engraphis/licensing.py +++ b/engraphis/licensing.py @@ -21,6 +21,7 @@ import json import os import re +import threading import time from dataclasses import dataclass, field from pathlib import Path @@ -515,6 +516,11 @@ def parse_key(key: str, *, now: Optional[float] = None) -> License: _cached: Optional[License] = None _cache_error: str = "" _cache_recheck_at: float = float("inf") # wall-clock time after which the cache is stale +# The cache globals above are read + mutated from FastAPI's threadpool AND invalidated +# by cloud_license on denial, so all access is serialized. Reentrant so current_license +# can be called from a context that already holds it. Without this, a concurrent +# invalidate_cache() between the "is not None" check and the return could return None. +_cache_lock = threading.RLock() #: Cloud-mode cache lifetime. Each refresh contacts the license server; an authoritative #: denial fails closed immediately, while a transient network failure may continue using @@ -584,8 +590,9 @@ def invalidate_cache() -> None: immediately instead of lingering until the lease TTL — without forcing the test suite to reach into private module state.""" global _cached, _cache_recheck_at - _cached = None - _cache_recheck_at = float("inf") + with _cache_lock: + _cached = None + _cache_recheck_at = float("inf") def current_license(*, refresh: bool = False) -> License: @@ -595,36 +602,42 @@ def current_license(*, refresh: bool = False) -> License: free tier and the reason is kept in :func:`license_error`. """ global _cached, _cache_error, _cache_recheck_at - if _cached is not None and not refresh and time.time() < _cache_recheck_at: - return _cached + # Fast path under the lock: a valid, unexpired cache entry is returned atomically so a + # concurrent invalidate_cache() can't null it out between the check and the return. + with _cache_lock: + if _cached is not None and not refresh and time.time() < _cache_recheck_at: + return _cached + # The cloud gate does network I/O, so run it OUTSIDE the lock (two threads racing a + # cache miss just do redundant, idempotent work — last store wins). Online-only: + # entitlement comes ONLY from a signature-valid key that ALSO passes the server-side + # cloud gate. No key, or a server-denied key ⇒ the free tier. material = _read_key_material() + lic: Optional[License] = None + reason = "" if material: try: lic = parse_key(material) except LicenseError as exc: - _cache_error = str(exc) # bad key → fall through to trial/free + lic, reason = None, str(exc) # bad key → free else: - allowed, reason = _cloud_gate(lic, material) - if allowed: - _cached, _cache_error = lic, "" - _cache_recheck_at = _license_recheck_at(lic) - return _cached - _cache_error = reason # cloud denied (revoked/unregistered) → trial/free - else: - _cache_error = "" - # Online-only: entitlement comes ONLY from a signature-valid key that ALSO passes the - # server-side cloud gate above. There is no local/offline trial grant anymore — the - # free trial is a real, short-lived, server-issued key (see start_trial) that flows - # through the exact same gate. No key, or a server-denied key ⇒ the free tier. - _cached = License.free() - # A configured key that temporarily failed its cloud gate must retry automatically. - # Caching the free fallback forever forced users to restart or paste the same key - # again after an outage. No-key free installs remain permanently cached. - _cache_recheck_at = ( - time.time() + _CLOUD_RECHECK_SECONDS - if material else _license_recheck_at(_cached) - ) - return _cached + allowed, gate_reason = _cloud_gate(lic, material) + if not allowed: + lic, reason = None, gate_reason # cloud denied (revoked/unregistered) → free + with _cache_lock: + if lic is not None: + _cached, _cache_error = lic, "" + _cache_recheck_at = _license_recheck_at(lic) + else: + _cache_error = reason + _cached = License.free() + # A configured key that temporarily failed its cloud gate must retry + # automatically. Caching the free fallback forever forced users to restart or + # paste the same key again after an outage. No-key free installs stay cached. + _cache_recheck_at = ( + time.time() + _CLOUD_RECHECK_SECONDS + if material else _license_recheck_at(_cached) + ) + return _cached # ── one-time local free trial (grants Pro features, no key, no phone-home) ──────────── diff --git a/engraphis/redirector.py b/engraphis/redirector.py index 9bc8d432..cfb70ce9 100644 --- a/engraphis/redirector.py +++ b/engraphis/redirector.py @@ -11,18 +11,31 @@ import uvicorn +def _redirect_base() -> str: + """Canonical dashboard base URL to redirect to, taken from CONFIGURATION rather than + the request's Host / X-Forwarded-Host headers. Reflecting those untrusted headers into + the redirect target made this an open redirect (a spoofed header could send a victim to + an attacker-controlled origin). ``ENGRAPHIS_DASHBOARD_URL`` wins; otherwise fall back to + the local dashboard on :8700.""" + base = os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip().rstrip("/") + if base: + return base + host = os.environ.get("ENGRAPHIS_HOST", "127.0.0.1") + return "http://%s:8700" % host + + def create_app() -> FastAPI: app = FastAPI(title="Engraphis Redirect", docs_url=None, redoc_url=None) + base = _redirect_base() @app.get("/{path:path}", include_in_schema=False) async def redirect(request: Request, path: str): - scheme = request.headers.get("x-forwarded-proto", request.url.scheme) - host = request.headers.get("x-forwarded-host", request.headers.get("host", "")) - host = host.partition(":")[0] + # Only the PATH and query from the request are preserved; the destination origin is + # fixed by config, so this can't be turned into an open redirect. + target = base + "/" + path qs = request.url.query - target = f"{scheme}://{host}:8700/{path}" if qs: - target += f"?{qs}" + target += "?" + qs return RedirectResponse(target, status_code=301) return app diff --git a/engraphis/routes/memory.py b/engraphis/routes/memory.py index a78f7b5b..4c4226e6 100644 --- a/engraphis/routes/memory.py +++ b/engraphis/routes/memory.py @@ -171,8 +171,10 @@ async def list_documents(namespace: Optional[str] = None, limit: Optional[int] = @router.get("/documents/{document_id}") async def get_document(document_id: str, namespace: Optional[str] = None): - """GET /memory/documents/{documentId} — get a single document.""" - doc = mem_store.get_memory(namespace or "_global", document_id) + """GET /memory/documents/{documentId} — get a single document. Without ``namespace``, + look it up across all namespaces instead of a nonexistent ``_global`` one (which made + the query always 404).""" + doc = mem_store.find_document(document_id, namespace) if not doc: raise HTTPException(404, f"Document {document_id} not found") return _ok(doc) @@ -218,11 +220,14 @@ async def chat_memory_context(req: ChatRequest): user_msg = next((m for m in reversed(req.messages) if m.get("role") == "user"), None) if not user_msg: raise HTTPException(400, "At least one user message is required") - ctx = recall_engine.recall(namespace=None, prompt=user_msg["content"], num_chunks=10) + user_content = user_msg.get("content") + if not user_content or not str(user_content).strip(): + raise HTTPException(400, "The latest user message must have non-empty 'content'") + ctx = recall_engine.recall(namespace=None, prompt=user_content, num_chunks=10) try: with LLMClient() as llm: answer = llm.chat_with_context( - user_prompt=user_msg["content"], + user_prompt=user_content, context=ctx.get("llmContextMessage", ""), temperature=req.temperature, max_tokens=req.maxTokens or req.max_tokens, @@ -243,6 +248,7 @@ async def record_interactions(req: InteractionRequest): raise HTTPException(400, "entityNames is required") levels = req.interactionLevels or req.interaction_levels level = req.interactionLevel or req.interaction_level or (levels[0] if levels else "view") + reinforced = 0 for name in names: ledger_store.record_interaction( namespace=req.namespace, @@ -251,7 +257,11 @@ async def record_interactions(req: InteractionRequest): description=req.description, timestamp=req.timestamp, ) - return _ok({"recorded": len(names), "namespace": req.namespace, "level": level}) + # Actually reinforce memories mentioning the entity — otherwise the signal is only + # logged and never affects retention. + reinforced += reweight.boost_entity_memories(req.namespace, name, level) + return _ok({"recorded": len(names), "namespace": req.namespace, "level": level, + "memories_reinforced": reinforced}) @router.post("/interact") @@ -287,7 +297,15 @@ async def prune_memory(req: PruneRequest): """ from engraphis.engines.reweight import retention_score - threshold = req.min_retention if req.min_retention is not None else (req.minRetention or 0.05) + # Prefer snake_case, then camelCase, then the default — but honor an explicit 0.0 + # (``req.minRetention or 0.05`` wrongly treated 0.0 as unset and deleted memories the + # caller asked to keep by requesting a zero threshold). + if req.min_retention is not None: + threshold = req.min_retention + elif req.minRetention is not None: + threshold = req.minRetention + else: + threshold = 0.05 dry_run = req.dry_run if req.dry_run is not None else bool(req.dryRun) keep_pinned = req.keepPinned if req.keepPinned is not None else True max_delete = max(1, min(req.maxDelete or 500, 10000)) @@ -352,8 +370,9 @@ async def recall_memories(req: RecallMemoriesRequest): @router.post("/memories/context") async def memories_context(namespace: Optional[str] = None, maxChunks: Optional[int] = 10): - """POST /memory/memories/context — recall context.""" - result = recall_engine.recall_master(namespace=namespace or "_global", max_chunks=maxChunks or 10) + """POST /memory/memories/context — recall context. Without a namespace, recall across + all of them (not a nonexistent '_global', which always returned nothing).""" + result = recall_engine.recall_master(namespace=namespace, max_chunks=maxChunks or 10) return _ok(result) diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index 37488362..32f32b8f 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -714,6 +714,9 @@ class _IntentRememberReq(BaseModel): @router.post("/intent/remember") def intent_remember(req: _IntentRememberReq): + # Team-gated exactly like /api/remember: this is the intent-native agent WRITE path + # onto this cloud instance's store, so a free/lapsed instance must not host it (402). + _paid("team") return _run( service().intent_remember, req.text, workspace=req.workspace, repo=req.repo, title=req.title, mtype=req.mtype, scope=req.scope, importance=req.importance, @@ -734,6 +737,8 @@ class _IntentLinkReq(BaseModel): @router.post("/intent/link") def intent_link(req: _IntentLinkReq): + # Team-gated like /api/remember and /api/intent/remember — same cloud write surface. + _paid("team") return _run( service().intent_link, req.source_id, req.target_id, workspace=req.workspace, repo=req.repo, relation=req.relation, layer=req.layer, reason=req.reason, diff --git a/engraphis/routes/v2_team.py b/engraphis/routes/v2_team.py index 95a487c8..ebb290bd 100644 --- a/engraphis/routes/v2_team.py +++ b/engraphis/routes/v2_team.py @@ -168,9 +168,24 @@ def _auth_iterations() -> int: return PBKDF2_ITERATIONS +def _cookie_secure(request: Request) -> bool: + """Whether the session cookie should carry the Secure flag. + + True when the EXTERNAL connection is HTTPS — including behind a TLS-terminating proxy + (Railway et al.) that forwards plain HTTP internally but sets ``X-Forwarded-Proto: + https``. Trusting ``request.url.scheme`` alone dropped Secure for every proxied + deployment, letting the session cookie ride over cleartext HTTP. (Honouring the + forwarded proto can only ADD Secure — making the cookie more restrictive — so a forged + header cannot downgrade cookie security.)""" + if request.url.scheme == "https": + return True + xfp = request.headers.get("x-forwarded-proto", "").split(",")[0].strip().lower() + return xfp == "https" + + def _set_cookie(response: Response, token: str, *, secure: bool = False) -> None: - """Set the dashboard session cookie. Secure flag mirrors the Inspector pattern: - True when the request arrived over HTTPS, so the cookie is never sent cleartext. + """Set the dashboard session cookie. Secure flag comes from :func:`_cookie_secure` + (HTTPS, including via a TLS-terminating proxy), so the cookie is never sent cleartext. TTL matches the server-side session expiry (SESSION_TTL_SECONDS) — the browser drops the cookie exactly when the server stops honouring it.""" response.set_cookie(_COOKIE, token, httponly=True, samesite="strict", @@ -236,14 +251,18 @@ def setup(body: SetupReq, request: Request, response: Response): if store.count_users() > 0: raise HTTPException(status_code=400, detail={"error": "team already set up"}) try: - admin = store.create_user(body.email, body.name, body.password, "admin") + # require_empty closes the TOCTOU: the zero-user check and the INSERT are atomic + # inside create_user's write transaction, so concurrent setups can't both create + # an admin (the router check above is just a fast-path). + admin = store.create_user(body.email, body.name, body.password, "admin", + require_empty=True) store.record_event("team.setup", actor_id=admin["id"], actor_email=admin["email"], detail="initial admin created") u = store.login(body.email, body.password, ip=request.client.host if request.client else None) except AuthError as exc: raise HTTPException(status_code=400, detail={"error": str(exc)}) - _set_cookie(response, u.pop("token"), secure=request.url.scheme == "https") + _set_cookie(response, u.pop("token"), secure=_cookie_secure(request)) return {"user": u} @router.post("/login") @@ -253,7 +272,7 @@ def login(body: LoginReq, request: Request, response: Response): u = store.login(body.email, body.password, ip=ip) except AuthError as exc: raise HTTPException(status_code=401, detail={"error": str(exc)}) - _set_cookie(response, u.pop("token"), secure=request.url.scheme == "https") + _set_cookie(response, u.pop("token"), secure=_cookie_secure(request)) return {"user": u} @router.post("/forgot") @@ -291,7 +310,7 @@ def reset(body: ResetReq, request: Request, response: Response): u = store.reset_password(body.token, body.password) except AuthError as exc: raise HTTPException(status_code=400, detail={"error": str(exc)}) - _set_cookie(response, u.pop("token"), secure=request.url.scheme == "https") + _set_cookie(response, u.pop("token"), secure=_cookie_secure(request)) return {"user": u} @router.post("/logout") diff --git a/engraphis/stores/__init__.py b/engraphis/stores/__init__.py index e52d4f8e..5da3b8d9 100644 --- a/engraphis/stores/__init__.py +++ b/engraphis/stores/__init__.py @@ -157,6 +157,13 @@ def init_db() -> None: Path(settings.db_path).parent.mkdir(parents=True, exist_ok=True) conn = get_conn() conn.executescript(_SCHEMA) + # Additive column for DBs created before it existed (SQLite has no "ADD COLUMN IF NOT + # EXISTS"). ``last_decay`` is the anchor the background decay pass advances each run so + # elapsed time is counted exactly once — without it, decay compounded on every tick. + try: + conn.execute("ALTER TABLE memories ADD COLUMN last_decay REAL") + except sqlite3.OperationalError: + pass # column already exists conn.commit() # Ensure a default vault exists from engraphis.stores.vaults import ensure_default_vault diff --git a/engraphis/stores/vectors.py b/engraphis/stores/vectors.py index 40525b84..a06eada2 100644 --- a/engraphis/stores/vectors.py +++ b/engraphis/stores/vectors.py @@ -91,12 +91,32 @@ def list_documents( sql += " LIMIT ?" params.append(limit) if offset: + # SQLite requires LIMIT before OFFSET; without an explicit limit, LIMIT -1 means + # "no limit" so an offset alone stays valid SQL instead of a syntax error (500). + if not limit: + sql += " LIMIT -1" sql += " OFFSET ?" params.append(offset) rows = conn.execute(sql, params).fetchall() return [_row_to_mem(r) for r in rows] +def find_document(document_id: str, namespace: Optional[str] = None) -> Optional[dict[str, Any]]: + """Fetch a memory by ``document_id``. With a namespace, scope to it; without one, return + the most recently updated match across all namespaces (document_id is only unique + per-namespace, so a bare lookup picks the newest rather than always missing).""" + conn = get_conn() + if namespace: + row = conn.execute( + "SELECT * FROM memories WHERE namespace=? AND document_id=?", + (namespace, document_id)).fetchone() + else: + row = conn.execute( + "SELECT * FROM memories WHERE document_id=? ORDER BY updated_at DESC LIMIT 1", + (document_id,)).fetchone() + return _row_to_mem(row) if row else None + + def delete_memory_document(document_id: str, namespace: str) -> int: conn = get_conn() cur = conn.execute( @@ -235,29 +255,47 @@ def set_retention(mem_id: int, stability: float, surprise: float) -> None: def apply_decay_to_all(namespace: Optional[str], halflife_days: float) -> int: """Ebbinghaus decay pass: reduce stability for memories not recently accessed. - Returns the number of rows touched.""" + Returns the number of memories whose stability was reduced. + + Decay is anchored on ``last_decay`` (advanced every pass) rather than recomputed from + ``last_access`` each run, so a given interval of not-being-accessed is decayed exactly + ONCE. This makes the pass idempotent and FREQUENCY-INDEPENDENT: the per-interval + factors multiply to ``0.5 ** (total_elapsed / halflife)``, so running it every 60s or + once a day converges to the same stability. The old formula reapplied a fixed + days-since-access factor to the already-decayed value on every tick, which — on the + ~60s consciousness loop — collapsed every memory's stability to the floor within + minutes. Memories reinforced since the last pass keep their boosted stability and just + have their anchor moved forward (subconscious forgetting targets the un-recalled).""" conn = get_conn() now = now_ts() + halflife = max(halflife_days, 0.1) rows = conn.execute( - "SELECT id, stability, last_access, access_count FROM memories" + "SELECT id, stability, last_access, last_decay FROM memories" + (" WHERE namespace=?" if namespace else ""), ([namespace] if namespace else []), ).fetchall() touched = 0 for r in rows: - days_since = (now - r["last_access"]) / 86400.0 - if days_since < 1e-6: + anchor = r["last_decay"] if r["last_decay"] is not None else r["last_access"] + # Reinforced since the last decay: keep the boosted stability, reset the anchor. + if r["last_access"] > anchor: + conn.execute("UPDATE memories SET last_decay=? WHERE id=?", (now, r["id"])) continue - decay_factor = 0.5 ** (days_since / max(halflife_days, 0.1)) - new_stab = max(r["stability"] * (0.5 + 0.5 * decay_factor), 0.01) - if abs(new_stab - r["stability"]) > 1e-6: + delta_days = (now - anchor) / 86400.0 + if delta_days <= 1e-6: + continue + new_stab = max(r["stability"] * (0.5 ** (delta_days / halflife)), 0.01) + if abs(new_stab - r["stability"]) > 1e-9: conn.execute( - "UPDATE memories SET stability=? WHERE id=?", - (new_stab, r["id"]), + "UPDATE memories SET stability=?, last_decay=? WHERE id=?", + (new_stab, now, r["id"]), ) touched += 1 - if touched: - conn.commit() + else: + # Already at the floor (or no measurable change): still advance the anchor so + # the elapsed interval isn't recounted next pass. + conn.execute("UPDATE memories SET last_decay=? WHERE id=?", (now, r["id"])) + conn.commit() return touched diff --git a/tests/test_autosync.py b/tests/test_autosync.py index 939c02fc..07b308bd 100644 --- a/tests/test_autosync.py +++ b/tests/test_autosync.py @@ -79,6 +79,11 @@ def test_sync_auto_toggle_is_admin_only_but_members_still_write(): assert min_role("POST", "/api/resources/postgres") == "admin" assert min_role("POST", "/api/sync/run") == "admin" assert min_role("POST", "/api/code/path") == "viewer" + # Irreversibly destroying/combining a whole shared workspace is admin-only, not an + # ordinary member action (a plain member must not be able to delete/merge everyone's + # workspace). + assert min_role("POST", "/api/workspaces/delete") == "admin" + assert min_role("POST", "/api/workspaces/merge") == "admin" # Members keep "store + view": they can write memories; viewers only read. assert min_role("POST", "/api/correct") == "member" assert min_role("GET", "/api/memories") == "viewer" diff --git a/tests/test_billing.py b/tests/test_billing.py index 89bc6b77..7659dc23 100644 --- a/tests/test_billing.py +++ b/tests/test_billing.py @@ -133,7 +133,7 @@ def test_issued_key_migrates_retired_relay_url(monkeypatch): monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) monkeypatch.setenv( "ENGRAPHIS_KEY_CLOUD_URL", - "https://team.engraphis.com/", + "https://engraphis-production.up.railway.app/", ) key = issue_key("buyer@example.com", product_name="Engraphis Pro", days=30) assert parse_key(key).cloud_url == DEFAULT_RELAY_URL @@ -921,3 +921,165 @@ def fail_record(_key): assert revoke_calls == [] assert LR.is_revoked(old_id) is False + + +# ── #3: in-flight vs completed must not be conflated ─────────────────────────── +def test_claim_webhook_is_tristate(monkeypatch, tmp_path): + monkeypatch.setenv("ENGRAPHIS_WEBHOOK_STATE", str(tmp_path / "wh.db")) + assert B.claim_webhook("wid") == "claimed" # fresh slot + assert B.claim_webhook("wid") == "in_flight" # held, younger than the TTL + B.complete_webhook("wid") + assert B.claim_webhook("wid") == "fulfilled" # completed → a true duplicate + + +def test_in_flight_delivery_is_retryable_not_duplicate(monkeypatch): + # The crash-window regression: a delivery whose earlier attempt is still in flight + # (or crashed mid-fulfillment, within the TTL) must get a RETRYABLE 503 so Polar + # keeps retrying — NOT a 2xx "duplicate", which would cancel retries and lose the key. + client = _inspector_client(monkeypatch) + assert B.claim_webhook("dlv:evt_inflight") == "claimed" # simulate an in-flight attempt + body = (b'{"type":"order.paid","data":{"customer":{"email":"buyer@example.com"},' + b'"product":{"name":"Engraphis Pro"}}}') + r = _post(client, WHSEC, "evt_inflight", body) + assert r.status_code == 503 + assert r.json()["status"] == "processing" + + +# ── #2: refund / cancellation / revocation revokes issued keys ───────────────── +def test_order_refunded_revokes_subscription_keys(monkeypatch): + from engraphis.inspector import license_registry as LR + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) + key = WH.issue_key("buyer@example.com", product_name="Engraphis Team", seats=3, + days=395, subscription_id="sub_refund") + kid = parse_key(key).key_id + assert LR.is_revoked(kid) is False + client = _inspector_client(monkeypatch) + body = _body({"type": "order.refunded", "data": { + "subscription_id": "sub_refund", "customer": {"email": "buyer@example.com"}}}) + r = _post(client, WHSEC, "evt_refund", body) + assert r.status_code == 202 + assert r.json()["status"] == "revoked" and r.json()["keys_revoked"] == 1 + assert LR.is_revoked(kid) is True + + +def test_subscription_revoked_removes_access_by_top_level_id(monkeypatch): + # subscription.* payloads ARE a Subscription object, so the id is top-level (not + # subscription_id). An explicit access revocation must still find and revoke the key. + from engraphis.inspector import license_registry as LR + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) + key = WH.issue_key("buyer@example.com", product_name="Engraphis Pro", seats=1, + days=395, subscription_id="sub_revoked") + kid = parse_key(key).key_id + client = _inspector_client(monkeypatch) + body = _body({"type": "subscription.revoked", "data": { + "id": "sub_revoked", "status": "canceled", + "customer": {"email": "buyer@example.com"}}}) + r = _post(client, WHSEC, "evt_revoked", body) + assert r.status_code == 202 and r.json()["status"] == "revoked" + assert LR.is_revoked(kid) is True + + +def test_subscription_canceled_keeps_plan_until_period_end(monkeypatch): + # Cancel-at-period-end must NOT revoke: the buyer paid for the period and keeps their + # plan until their period-bounded key naturally expires. Only refund / revoked pulls + # access immediately. + from engraphis.inspector import license_registry as LR + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) + key = WH.issue_key("buyer@example.com", product_name="Engraphis Team", seats=3, + days=395, subscription_id="sub_keep") + kid = parse_key(key).key_id + client = _inspector_client(monkeypatch) + body = _body({"type": "subscription.canceled", "data": { + "id": "sub_keep", "status": "active", + "customer": {"email": "buyer@example.com"}}}) + r = _post(client, WHSEC, "evt_keep", body) + assert r.status_code == 202 and r.json()["status"] == "ignored" + assert LR.is_revoked(kid) is False # key stays valid until it expires + + +# ── #4: a mid-cycle reissue is bounded to the paid period, not a full new window ─ +def test_seat_change_key_is_bounded_to_current_period_end(monkeypatch): + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) + key = WH.handle_subscription_updated({ + "id": "sub_pe", "status": "active", "seats": 5, + "customer": {"email": "lead@example.com"}, + "product": {"name": "Engraphis Team Annual"}, + "current_period_end": _iso_in_days(20)}) + lic = parse_key(key) + days_left = (lic.expires - time.time()) / 86400 + # bounded to ~20d period end (+5d grace), NOT a fresh 395-day annual window from now + assert 20 <= days_left <= 30, f"expected period-bounded key, got {days_left:.1f}d" + + +def test_seat_change_without_period_end_falls_back_to_key_days(monkeypatch): + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_VENDOR_SIGNING_KEY", VENDOR_SEED) + key = WH.handle_subscription_updated({ + "id": "sub_nope", "status": "active", "seats": 5, + "customer": {"email": "lead@example.com"}, + "product": {"name": "Engraphis Team"}}) # monthly, no period end + days_left = (parse_key(key).expires - time.time()) / 86400 + assert 30 < days_left <= 36, f"expected ~35d monthly fallback, got {days_left:.1f}d" + + +# ── #5: an out-of-order (older) subscription.updated must not regress a newer count ─ +def _sub_updated_body_ts(sub_id, seats, modified_at, status="active", + product="Engraphis Team"): + return _body({"type": "subscription.updated", "data": { + "id": sub_id, "status": status, "seats": seats, "modified_at": modified_at, + "customer": {"email": "lead@example.com"}, "product": {"name": product}}}) + + +def test_out_of_order_subscription_update_is_ignored(monkeypatch): + client = _inspector_client(monkeypatch) + base = datetime.now(timezone.utc) + t0 = base.isoformat() + t1 = (base + timedelta(days=1)).isoformat() + t2 = (base + timedelta(days=2)).isoformat() + # first sighting seeds baseline (seats=3 @ t0) + r0 = _post(client, WHSEC, "evt_oo_seed", _sub_updated_body_ts("sub_oo", 3, t0)) + assert r0.json()["reason"] == "baseline recorded" + # newer delivery: 3 -> 7 @ t2 -> fulfilled + r2 = _post(client, WHSEC, "evt_oo_new", _sub_updated_body_ts("sub_oo", 7, t2)) + assert r2.json()["key_issued"] is True + # OLDER delivery arrives late: seats=5 @ t1 (< t2) -> ignored as out-of-order + r1 = _post(client, WHSEC, "evt_oo_old", _sub_updated_body_ts("sub_oo", 5, t1)) + assert r1.json()["status"] == "ignored" + assert r1.json()["reason"] == "out-of-order update" + assert B.get_known_seats("sub_oo") == 7 # NOT regressed to 5 + + +# ── #7: organization enforcement + explicit product-tier override ────────────── +def test_organization_id_mismatch_is_rejected(monkeypatch): + monkeypatch.setenv("POLAR_ORGANIZATION_ID", "org_expected") + client = _inspector_client(monkeypatch) + body = _body({"type": "order.paid", "data": { + "organization_id": "org_other", + "customer": {"email": "buyer@example.com"}, + "product": {"name": "Engraphis Pro"}}}) + r = _post(client, WHSEC, "evt_org_bad", body) + assert r.status_code == 403 and r.json()["error"] == "organization mismatch" + + +def test_organization_id_match_is_allowed(monkeypatch): + monkeypatch.setenv("POLAR_ORGANIZATION_ID", "org_ok") + client = _inspector_client(monkeypatch) + body = _body({"type": "order.paid", "data": { + "organization_id": "org_ok", + "customer": {"email": "buyer@example.com"}, + "product": {"name": "Engraphis Pro"}}}) + r = _post(client, WHSEC, "evt_org_ok", body) + assert r.status_code == 202 and r.json()["key_issued"] is True + + +def test_product_map_override_pins_tier(monkeypatch): + from engraphis.inspector import webhooks as WH + monkeypatch.setenv("ENGRAPHIS_POLAR_PRODUCT_MAP", '{"Engraphis Enterprise": "team"}') + assert WH._map_polar_product_to_plan("Engraphis Enterprise") == "team" + monkeypatch.delenv("ENGRAPHIS_POLAR_PRODUCT_MAP", raising=False) + # without the override an unrecognized product still defaults to Pro (never free) + assert WH._map_polar_product_to_plan("Engraphis Enterprise") == "pro" diff --git a/tests/test_cloud_license.py b/tests/test_cloud_license.py index 68adc705..ebf847f8 100644 --- a/tests/test_cloud_license.py +++ b/tests/test_cloud_license.py @@ -93,8 +93,11 @@ def test_register_rejects_expired_key(): def test_seat_cap_enforced(): + # Seat caps apply to TEAM (the seat-priced tier). Pro is the individual multi-device + # tier and is intentionally NOT device-capped (see test_pro_register_is_not_device_capped + # and sync_relay.py) — so this exercises the cap with a team key. c = _app() - key = _key(seats=1) + key = _key(plan="team", seats=1) assert c.post("/license/v1/register", json={"key": key, "machine_id": "A"}).status_code == 200 # a second distinct machine exceeds the 1-seat cap over = c.post("/license/v1/register", json={"key": key, "machine_id": "B"}) @@ -103,6 +106,20 @@ def test_seat_cap_enforced(): assert c.post("/license/v1/register", json={"key": key, "machine_id": "A"}).status_code == 200 +def test_pro_register_is_not_device_capped(): + """Pro is the individual multi-device tier: one person's many devices all register + under the same key. The register endpoint issues the online-enforcement lease that + gates EVERY paid feature, so seat-capping Pro here would lock a paying customer's + second device out of all Pro features — including the multi-device sync they bought. + Mirrors test_pro_relay_is_not_device_capped for the register/lease path.""" + c = _app() + key = _key(plan="pro", seats=1) # Pro keys are minted seats=1 + for m in ("p1", "p2", "p3"): + r = c.post("/license/v1/register", json={"key": key, "machine_id": m}) + assert r.status_code == 200, r.text + assert r.json()["plan"] == "pro" + + def test_verify_endpoint_reflects_status(): c = _app() key = _key() @@ -680,8 +697,8 @@ def fake_register(base, k, mid, **kw): def test_retired_baked_in_url_migrates_to_current_relay(monkeypatch): - """Existing signed keys must survive the vendor's domain-to-Railway migration.""" - key = _enforced_key(cloud_url="https://team.engraphis.com") + """Existing signed keys must survive the vendor's Railway-to-domain migration.""" + key = _enforced_key(cloud_url="https://engraphis-production.up.railway.app") lic_parsed = parse_key(key) calls = {} @@ -695,7 +712,7 @@ def fake_register(base, k, mid, **kw): monkeypatch.setattr(cloud_license, "register", fake_register) monkeypatch.setenv("ENGRAPHIS_LICENSE_KEY", key) got = licensing.current_license(refresh=True) - assert calls["base"] == DEFAULT_RELAY_URL == "https://engraphis-production.up.railway.app" + assert calls["base"] == DEFAULT_RELAY_URL == "https://team.engraphis.com" assert got.plan == "pro" and got.has("sync") @@ -1132,6 +1149,27 @@ def test_start_team_trial_rate_limits_by_trusted_forwarded_source(monkeypatch, t assert other.status_code == 200 +def test_start_team_trial_ignores_client_prepended_forwarded_prefix(monkeypatch): + """A client cannot mint fresh rate-limit buckets by prepending its own value to + X-Forwarded-For: the trusted proxy appends the real client IP to the RIGHT, so only + the rightmost entry is authoritative. Simulate a Railway-style single trusted hop + (``*``) that saw one real client (203.0.113.9) but a client that keeps rotating a + spoofed left prefix — the cap must still bite on the real (rightmost) address.""" + monkeypatch.setenv("ENGRAPHIS_FORWARDED_ALLOW_IPS", "*") + monkeypatch.setattr(license_cloud, "_trial_rate_limit_per_hour", lambda: 2) + c = _app() + _capture_verify_url(monkeypatch) + for i in range(2): + r = c.post("/license/v1/start-trial", + json={"machine_id": "dev-%d" % i, "email": "dev%d@example.com" % i}, + headers={"X-Forwarded-For": "10.0.0.%d, 203.0.113.9" % i}) + assert r.status_code == 200, r.text + over = c.post("/license/v1/start-trial", + json={"machine_id": "dev-over", "email": "over@example.com"}, + headers={"X-Forwarded-For": "10.0.0.250, 203.0.113.9"}) + assert over.status_code == 429 + + def test_start_team_trial_ignores_forwarded_source_from_untrusted_peer(monkeypatch): """Spoofing X-Forwarded-For cannot evade a direct-peer rate limit.""" monkeypatch.setattr(license_cloud, "_trial_rate_limit_per_hour", lambda: 2) diff --git a/tests/test_config.py b/tests/test_config.py index fadafc6e..25b14545 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,7 +9,7 @@ from engraphis.config import Settings -RETIRED_RELAY_URL = "https://team.engraphis.com" +RETIRED_RELAY_URL = "https://engraphis-production.up.railway.app" def test_rerank_model_defaults_to_empty(monkeypatch): @@ -17,6 +17,23 @@ def test_rerank_model_defaults_to_empty(monkeypatch): assert Settings().rerank_model == "" +def test_cors_default_origins_follow_configured_port(): + # The empty-CORS default derives loopback origins from the port, so running on a + # non-default ENGRAPHIS_PORT doesn't lock the dashboard's own origin out. + assert config._parse_origins("", 9000) == [ + "http://127.0.0.1:9000", "http://localhost:9000"] + # Explicit origins pass through unchanged. + assert config._parse_origins("https://app.example.com", 9000) == [ + "https://app.example.com"] + + +def test_cors_origins_use_engraphis_port_env(monkeypatch): + monkeypatch.delenv("ENGRAPHIS_CORS_ORIGINS", raising=False) + monkeypatch.setenv("ENGRAPHIS_PORT", "9100") + assert Settings().cors_origins == [ + "http://127.0.0.1:9100", "http://localhost:9100"] + + def test_rerank_model_read_from_env(monkeypatch): monkeypatch.setenv("ENGRAPHIS_RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2") assert Settings().rerank_model == "cross-encoder/ms-marco-MiniLM-L-6-v2" @@ -63,7 +80,7 @@ def test_license_server_url_migrates_retired_signed_host(monkeypatch): monkeypatch.setattr(config.settings, "relay_url", config.DEFAULT_RELAY_URL) monkeypatch.delenv("ENGRAPHIS_CLOUD_URL", raising=False) assert config.resolve_license_server_url( - "https://team.engraphis.com/", + "https://engraphis-production.up.railway.app/", ) == config.DEFAULT_RELAY_URL diff --git a/tests/test_core_store.py b/tests/test_core_store.py index 05eee76d..9fd069cb 100644 --- a/tests/test_core_store.py +++ b/tests/test_core_store.py @@ -23,6 +23,46 @@ def test_schema_version(store): assert store.schema_version == 3 +def test_concurrent_writes_do_not_corrupt_or_lose_data(tmp_path): + # The shared connection is serialized (_SerializedConnection): concurrent threadpool + # writers must not interleave transactions on it. Every write from every thread must + # land, with no "database is locked"/cursor-corruption errors. + import threading + + store = Store(str(tmp_path / "concurrent.db")) + errors: list = [] + n_threads, per = 8, 25 + + def worker(t: int) -> None: + try: + for i in range(per): + store.create_workspace("ws-%d-%d" % (t, i)) + except Exception as exc: # noqa: BLE001 — surface for the assertion + errors.append(exc) + + threads = [threading.Thread(target=worker, args=(t,)) for t in range(n_threads)] + for th in threads: + th.start() + for th in threads: + th.join() + + count = store.conn.execute("SELECT COUNT(*) AS n FROM workspaces").fetchone()["n"] + store.close() + assert not errors, errors + assert count == n_threads * per + + +def test_wrapper_releases_lock_after_a_failing_statement(tmp_path): + # A statement that raises mid-transaction must roll back and free the write lock, or + # the next writer would deadlock on the shared connection. + store = Store(str(tmp_path / "recover.db")) + with pytest.raises(Exception): + store.conn.execute("INSERT INTO does_not_exist(x) VALUES (1)") + # The lock is free again: a normal write still succeeds. + assert store.create_workspace("after-error") + store.close() + + def test_v3_migration_classifies_existing_graph_layers_once(tmp_path): db = tmp_path / "v2.db" original = Store(str(db)) diff --git a/tests/test_cron_write_tools_workspace_default.py b/tests/test_cron_write_tools_workspace_default.py new file mode 100644 index 00000000..310d03a1 --- /dev/null +++ b/tests/test_cron_write_tools_workspace_default.py @@ -0,0 +1,93 @@ +"""Regression: every cron-auto-fired WRITE memory tool must default workspace to 'default'. + +The 2026-07 fleet-wide outage ("workspace Field required", 200+ occurrences in +gateway.log) happened because ``engraphis_start_session`` hard-required ``workspace`` +while the scheduled fleet (hourly-dev-engineer, ops-heartbeat, dashboard-refresh, +vault-cron-health, ...) calls these tools WITHOUT one. That single tool was fixed and +locked by ``test_start_session_workspace_default.py``. + +But the *same class* of outage hits every auto-fired WRITE tool the fleet calls without a +workspace. ``engraphis_remember`` and ``engraphis_record_event`` are the actual memory +*write* path: if either regresses to a required ``workspace`` (e.g. someone drops the +``= "default"`` while adding a field, or re-tightens ``min_length=1`` into a required +positional), the fleet silently loses ALL memory writes again — a strictly worse failure +than the session tool, because the data never lands at all. + +This test pins the (workspace defaults to 'default') contract for all three auto-fired +write/session tools at two levels: + * signature level — asserts the ``workspace`` parameter still carries the ``"default"`` + default (this is the *exact* thing that broke: a missing default), and + * behavioral level — omitting ``workspace`` succeeds and lands the write in 'default'. + +NOTE: ``engraphis_recall`` is deliberately excluded. It is a READ tool whose ``workspace`` +defaults to ``None`` (search-broadly semantics), so omitting it returns an empty/again +result set rather than crashing with "workspace Field required". Do not "helpfully" add +recall here — its None default is intentional and different from the write-path contract. +""" +import inspect +import json + +import pytest + +pytest.importorskip("mcp", reason="optional 'mcp' extra not installed") + +# Tools the scheduled fleet fires WITHOUT a workspace arg on the write path. +CRON_WRITE_TOOLS = ( + "engraphis_start_session", + "engraphis_remember", + "engraphis_record_event", +) + + +def _module_with_memory_db(monkeypatch): + import engraphis.mcp_server as srv + from engraphis.service import MemoryService + + monkeypatch.setattr(srv, "_service", MemoryService.create(":memory:")) + return srv + + +@pytest.mark.parametrize("tool_name", CRON_WRITE_TOOLS) +def test_cron_write_tool_workspace_defaults_to_default(monkeypatch, tool_name): + """Signature guard: the exact regression (a removed/renamed default) fails right here.""" + srv = _module_with_memory_db(monkeypatch) + param = inspect.signature(getattr(srv, tool_name)).parameters.get("workspace") + assert param is not None, f"{tool_name} lost its 'workspace' parameter" + assert param.default == "default", ( + f"{tool_name}.workspace default is {param.default!r}, not 'default' — cron calls " + "that omit workspace will fail with 'workspace Field required' (fleet-wide " + "memory-write outage)." + ) + + +def test_remember_omitted_workspace_stores_in_default(monkeypatch): + srv = _module_with_memory_db(monkeypatch) + out = json.loads(srv.engraphis_remember( + content="cron write path must not require an explicit workspace")) + assert out["stored"] is True + assert out["workspace"] == "default" + + +def test_record_event_omitted_workspace_succeeds_in_default(monkeypatch): + srv = _module_with_memory_db(monkeypatch) + # The fleet always opens a session first, which provisions the 'default' workspace. + started = json.loads(srv.engraphis_start_session()) + assert started["workspace"] == "default" + + omitted = json.loads(srv.engraphis_record_event( + kind="ops_probe", content="hourly-dev-engineer regression event (omitted ws)")) + # record_event returns {"id","kind"}; a required-workspace regression would instead + # surface as an "Error: ... workspace Field required" string that fails json.loads or + # carries no id. + assert isinstance(omitted, dict) and omitted.get("id"), ( + f"record_event with omitted workspace did not return an id: {omitted!r}") + assert omitted["kind"] == "ops_probe" + + # The omitted-workspace path must behave like passing workspace="default" explicitly + # (parity check; combined with the signature guard above this pins "omitted -> default" + # without depending on retrieval-ranking heuristics). + explicit = json.loads(srv.engraphis_record_event( + kind="ops_probe", content="hourly-dev-engineer regression event (explicit default)", + workspace="default")) + assert isinstance(explicit, dict) and explicit.get("id"), ( + f"record_event with workspace='default' did not return an id: {explicit!r}") diff --git a/tests/test_decay_idempotent.py b/tests/test_decay_idempotent.py new file mode 100644 index 00000000..ac76feb0 --- /dev/null +++ b/tests/test_decay_idempotent.py @@ -0,0 +1,76 @@ +"""Regression: the v1 background decay pass must be idempotent / frequency-independent. + +The old ``apply_decay_to_all`` reapplied a fixed days-since-access factor to the +already-decayed stored stability on EVERY call, so the ~60s consciousness loop compounded +decay and collapsed every memory's stability to the floor within minutes. The fix anchors +decay on ``last_decay`` (advanced each pass), so a given interval is decayed exactly once. +""" +import threading + +from engraphis.config import settings +from engraphis.stores import get_conn, init_db, now_ts +from engraphis.stores import vectors as mem_store + + +def _setup(monkeypatch, tmp_path): + monkeypatch.setattr(settings, "db_path", str(tmp_path / "decay.db")) + # The v1 store keeps a process-global thread-local connection; reset it so this test + # binds to its own DB rather than a stale conn left by an earlier test file. + monkeypatch.setattr("engraphis.stores._local", threading.local()) + init_db() + + +def _insert(namespace, doc_id, stability, last_access): + conn = get_conn() + conn.execute( + "INSERT INTO memories (namespace, document_id, title, content, metadata, " + "created_at, updated_at, last_access, access_count, stability, surprise, " + "memory_type) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (namespace, doc_id, "", "content", "{}", last_access, last_access, last_access, + 0, stability, 1.0, "semantic")) + conn.commit() + + +def _stability(namespace, doc_id): + return get_conn().execute( + "SELECT stability FROM memories WHERE namespace=? AND document_id=?", + (namespace, doc_id)).fetchone()["stability"] + + +def test_decay_is_frequency_independent(monkeypatch, tmp_path): + _setup(monkeypatch, tmp_path) + now = now_ts() + # Two identical memories last accessed 10 days ago, in separate namespaces so each can + # be decayed a different number of times. + _insert("freq", "m", 5.0, now - 10 * 86400) + _insert("once", "m", 5.0, now - 10 * 86400) + + for _ in range(50): # the compounding scenario (loop every tick) + mem_store.apply_decay_to_all("freq", 7.0) + mem_store.apply_decay_to_all("once", 7.0) # the same elapsed time, applied once + + s_freq = _stability("freq", "m") + s_once = _stability("once", "m") + # 50 rapid passes must land at the SAME stability as a single pass over the same + # elapsed interval — not 50x compounded down to the floor. + assert abs(s_freq - s_once) < 0.05, (s_freq, s_once) + # ~10 days at a 7-day half-life leaves 5.0 * 0.5**(10/7) ≈ 1.86, nowhere near the floor. + assert s_freq > 1.0 + + +def test_reinforced_memory_is_not_decayed_that_interval(monkeypatch, tmp_path): + _setup(monkeypatch, tmp_path) + now = now_ts() + _insert("ns", "hot", 5.0, now - 10 * 86400) + # First pass anchors + decays the stale memory once. + mem_store.apply_decay_to_all("ns", 7.0) + decayed = _stability("ns", "hot") + assert decayed < 5.0 + # Simulate reinforcement: the memory is accessed now (last_access moves past the anchor). + conn = get_conn() + conn.execute("UPDATE memories SET last_access=? WHERE namespace=? AND document_id=?", + (now_ts(), "ns", "hot")) + conn.commit() + # A subsequent pass must NOT decay it further — it was just accessed. + mem_store.apply_decay_to_all("ns", 7.0) + assert abs(_stability("ns", "hot") - decayed) < 1e-9 diff --git a/tests/test_memory_routes_fixes.py b/tests/test_memory_routes_fixes.py new file mode 100644 index 00000000..a0506cb7 --- /dev/null +++ b/tests/test_memory_routes_fixes.py @@ -0,0 +1,112 @@ +"""Regressions for v1 memory-route / store correctness bugs: + +- list_documents(offset=..) with no limit generated invalid SQL (OFFSET without LIMIT). +- GET /memory/documents/{id} without ?namespace looked up a nonexistent '_global' ns. +- POST /memory/prune coerced an explicit minRetention=0.0 to 0.05 and over-pruned. +- POST /memory/conversations crashed (500) on a user message missing 'content'. +- POST /memory/interactions recorded signals that never reinforced any memory. +""" +import threading + +import pytest + +from engraphis.config import settings +from engraphis.stores import get_conn, init_db, now_ts +from engraphis.stores import vectors as mem_store +from engraphis.engines import reweight + + +def _setup_store(monkeypatch, tmp_path): + monkeypatch.setattr(settings, "db_path", str(tmp_path / "mem.db")) + monkeypatch.setattr("engraphis.stores._local", threading.local()) + init_db() + + +def test_list_documents_offset_without_limit_is_valid(monkeypatch, tmp_path): + _setup_store(monkeypatch, tmp_path) + for i in range(3): + mem_store.upsert_memory(namespace="ns", document_id="d%d" % i, title="t", + content="c%d" % i) + # offset with no limit must not raise "OFFSET without LIMIT" (previously a 500). + rest = mem_store.list_documents(namespace="ns", offset=1) + assert len(rest) == 2 + + +def test_find_document_without_namespace(monkeypatch, tmp_path): + _setup_store(monkeypatch, tmp_path) + mem_store.upsert_memory(namespace="vault", document_id="doc1", title="t", content="hi") + # No namespace: still found (old code queried a nonexistent '_global' ns and 404'd). + assert mem_store.find_document("doc1") is not None + assert mem_store.find_document("doc1", "vault") is not None + assert mem_store.find_document("nope") is None + + +def test_recall_master_none_namespace_recalls_across_all(monkeypatch, tmp_path): + _setup_store(monkeypatch, tmp_path) + import numpy as np + from engraphis.engines import recall as recall_engine + vec = np.ones(8, dtype=np.float32) + mem_store.upsert_memory(namespace="ns1", document_id="a", title="t", content="alpha", + vector=vec) + mem_store.upsert_memory(namespace="ns2", document_id="b", title="t", content="beta", + vector=vec) + # namespace=None must recall across ALL namespaces, not a nonexistent '_global' (which + # made the consciousness loop's thought synthesis silently no-op). + out = recall_engine.recall_master(namespace=None, max_chunks=10) + assert out["count"] >= 2 + + +def test_interactions_reinforce_matching_memories(monkeypatch, tmp_path): + _setup_store(monkeypatch, tmp_path) + mem_store.upsert_memory(namespace="ns", document_id="d1", title="About Alice", + content="Alice ships the release") + mem_store.upsert_memory(namespace="ns", document_id="d2", title="About Bob", + content="Bob reviews code") + before = get_conn().execute( + "SELECT stability FROM memories WHERE document_id='d1'").fetchone()["stability"] + n = reweight.boost_entity_memories("ns", "Alice", "engage") + assert n == 1 # only the Alice memory matched + after = get_conn().execute( + "SELECT stability FROM memories WHERE document_id='d1'").fetchone()["stability"] + assert after > before # it was actually reinforced + # Bob's memory is untouched. + bob = get_conn().execute( + "SELECT stability FROM memories WHERE document_id='d2'").fetchone()["stability"] + assert bob == 1.0 + + +# ── app-level route regressions ──────────────────────────────────────────────── +pytest.importorskip("fastapi", reason="full-stack extra not installed") +from fastapi.testclient import TestClient # noqa: E402 + + +def _client(monkeypatch, tmp_path): + _setup_store(monkeypatch, tmp_path) + monkeypatch.setattr(settings, "loop_interval", 0) + monkeypatch.setattr(settings, "embed_model", "") + from engraphis.app import create_app + return TestClient(create_app()) + + +def test_prune_honors_explicit_zero_threshold(monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as c: + # A memory last accessed long ago has near-zero retention. + mem_store.upsert_memory(namespace="ns", document_id="old", title="t", + content="stale", created_at=now_ts() - 100 * 86400) + get_conn().execute( + "UPDATE memories SET last_access=?, stability=0.5 WHERE document_id='old'", + (now_ts() - 100 * 86400,)) + get_conn().commit() + # threshold 0.0 => delete only retention < 0 => nothing (old code coerced to 0.05). + r = c.post("/memory/prune", + json={"namespace": "ns", "minRetention": 0.0, "dryRun": True}) + assert r.status_code == 200, r.text + data = r.json()["data"] + assert data.get("candidates", data.get("wouldDelete", 0)) == 0 or \ + data.get("count", 0) == 0 + + +def test_conversations_missing_content_is_400_not_500(monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as c: + r = c.post("/memory/conversations", json={"messages": [{"role": "user"}]}) + assert r.status_code == 400 diff --git a/tests/test_round17_fixes.py b/tests/test_round17_fixes.py new file mode 100644 index 00000000..583200fa --- /dev/null +++ b/tests/test_round17_fixes.py @@ -0,0 +1,54 @@ +"""Regressions for sync/store integrity fixes: +- get_or_create_workspace bypassed the workspace allow-list on the retrieve path. +- sync clamped future world-time validity (valid_to/valid_from) to now+skew. +- autosync _record reset the policy to the disabled default on a transient read failure. +""" +import time + +import pytest + + +def test_get_or_create_workspace_enforces_allowlist(tmp_path): + from engraphis.core import ids + from engraphis.core.store import Store + s = Store(str(tmp_path / "w.db"), allowed_workspaces={"allowed"}) + # A workspace outside the allow-list already exists in the DB (predates the allow-list, + # or arrived via sync). The RETRIEVE path must refuse it, not hand it back. + s.conn.execute("INSERT INTO workspaces(id, name, created_at, settings) VALUES (?,?,?,?)", + (ids.new_id("workspace"), "secret", 0.0, "{}")) + s.conn.commit() + with pytest.raises(ValueError): + s.get_or_create_workspace("secret") + assert s.get_or_create_workspace("allowed") # allowed workspace still works + s.close() + + +def test_sync_apply_preserves_future_world_validity(): + from engraphis.core.sync import dict_to_record + future = time.time() + 5 * 365 * 86400 # ~5 years out (a fact valid until then) + rec = dict_to_record({"id": "mem_x", "content": "c", "valid_to": future, + "valid_from": future - 86400}) + assert rec is not None + assert rec.valid_to > time.time() + 365 * 86400 # NOT truncated to now+skew + # System timestamps are still clamped near now (they feed the version key / anti-poison). + poisoned = dict_to_record({"id": "mem_y", "content": "c", "ingested_at": future, + "last_access": future}) + assert poisoned.ingested_at <= time.time() + 10 * 86400 + + +def test_autosync_record_preserves_policy_on_unreadable_file(monkeypatch, tmp_path): + from engraphis import autosync + target = tmp_path / "autosync.json" + target.mkdir() # exists but read_text raises OSError + monkeypatch.setattr(autosync, "policy_path", lambda: target) + wrote = [] + monkeypatch.setattr(autosync, "_write", lambda doc: wrote.append(doc)) + autosync._record({"workspaces": 1}) + assert wrote == [] # never clobber the policy on a read hiccup + + +def test_autosync_record_creates_on_fresh_install(monkeypatch, tmp_path): + from engraphis import autosync + monkeypatch.setattr(autosync, "policy_path", lambda: tmp_path / "fresh.json") + autosync._record({"workspaces": 1}, now=123.0) + assert autosync.load_policy()["last_run"] == 123.0 diff --git a/tests/test_security_fixes.py b/tests/test_security_fixes.py new file mode 100644 index 00000000..af28ca1b --- /dev/null +++ b/tests/test_security_fixes.py @@ -0,0 +1,53 @@ +"""Regressions for auth/security fixes: atomic setup, cookie Secure behind a proxy, and +the redirector open-redirect.""" +import pytest + +pytest.importorskip("fastapi", reason="full-stack extra not installed") + + +def test_create_user_require_empty_is_atomic_bootstrap(tmp_path): + from engraphis.inspector.auth import AuthError, AuthStore + store = AuthStore(str(tmp_path / "users.db"), iterations=1000) + store.create_user("admin@x.co", "Admin", "Sup3rSecret!", "admin", require_empty=True) + # A second require_empty bootstrap must be refused — /api/auth/setup creates only the + # first admin, atomically (closes the multi-admin TOCTOU). + with pytest.raises(AuthError): + store.create_user("eve@x.co", "Eve", "Sup3rSecret!", "admin", require_empty=True) + assert store.count_users() == 1 + + +def test_cookie_secure_honors_forwarded_proto(): + from engraphis.routes.v2_team import _cookie_secure + + class _Req: + def __init__(self, scheme, headers): + self.url = type("U", (), {"scheme": scheme})() + self.headers = headers + + assert _cookie_secure(_Req("https", {})) is True + # Behind a TLS-terminating proxy the internal scheme is http but XFP says https. + assert _cookie_secure(_Req("http", {"x-forwarded-proto": "https"})) is True + assert _cookie_secure(_Req("http", {})) is False + + +def test_redirector_ignores_spoofed_host(monkeypatch): + from fastapi.testclient import TestClient + monkeypatch.delenv("ENGRAPHIS_DASHBOARD_URL", raising=False) + monkeypatch.setenv("ENGRAPHIS_HOST", "127.0.0.1") + from engraphis.redirector import create_app + c = TestClient(create_app()) + r = c.get("/memories?q=1", headers={"X-Forwarded-Host": "evil.com", + "Host": "evil.com"}, follow_redirects=False) + assert r.status_code == 301 + loc = r.headers["location"] + assert "evil.com" not in loc # spoofed host is not reflected + assert loc.startswith("http://127.0.0.1:8700/memories") + + +def test_redirector_uses_configured_dashboard_url(monkeypatch): + from fastapi.testclient import TestClient + monkeypatch.setenv("ENGRAPHIS_DASHBOARD_URL", "https://dash.example.com") + from engraphis.redirector import create_app + c = TestClient(create_app()) + r = c.get("/x", headers={"X-Forwarded-Host": "evil.com"}, follow_redirects=False) + assert r.headers["location"] == "https://dash.example.com/x" diff --git a/tests/test_sync_dashboard.py b/tests/test_sync_dashboard.py index 7114dc72..7e839120 100644 --- a/tests/test_sync_dashboard.py +++ b/tests/test_sync_dashboard.py @@ -78,7 +78,7 @@ def test_sync_status_locked_without_key(monkeypatch, tmp_path): def test_sync_status_migrates_retired_relay_url(monkeypatch, tmp_path): monkeypatch.setattr( - settings, "relay_url", "https://team.engraphis.com/") + settings, "relay_url", "https://engraphis-production.up.railway.app/") with _client(monkeypatch, tmp_path) as c: assert c.get("/api/sync/status").json()["relay_url"] == DEFAULT_RELAY_URL