diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9a9397..dbd2269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install (core + server/mcp/code extras; no torch — the offline gate) @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.9" - name: Install (numpy-only core — the minimum supported runtime) @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" - name: Build sdist + wheel and verify a clean install diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..137c371 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,48 @@ +name: release + +# Publish Engraphis to PyPI when a GitHub Release is published. +# Uses PyPI Trusted Publishing (OIDC) — NO API token is stored in the repo. +# One-time setup before your first tagged release: see docs/RELEASE.md. + +on: + release: + types: [published] + workflow_dispatch: {} # also allow a manual run from the Actions tab + +permissions: + contents: read + +jobs: + build: + name: Build sdist + wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Build distributions + run: | + python -m pip install --upgrade pip build twine + python -m build + - name: Validate metadata (twine check) + run: twine check dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: pypi # configure this env name on the PyPI trusted publisher + permissions: + id-token: write # REQUIRED for trusted publishing (OIDC) + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/docs/LAUNCH-POST.md b/docs/LAUNCH-POST.md new file mode 100644 index 0000000..8ffe13d --- /dev/null +++ b/docs/LAUNCH-POST.md @@ -0,0 +1,121 @@ +# Engraphis — Launch copy (paste-ready) + +Built from your own landing page + GTM positioning. Post the code publicly and get +`pip install engraphis` working **first** — every version below assumes a stranger can try it +in one command. Fill the two brackets (`REPO_URL`, `DEMO_URL`) and go. + +--- + +## 1. Show HN (Hacker News) + +**Title:** +`Show HN: Engraphis – local-first memory for AI coding agents (MCP, no cloud)` + +**Body:** +``` +Every new session, my coding agent started from zero — re-asking what package manager I use, +re-learning the codebase, forgetting past decisions. The hosted memory services fix this by +making you ship your context to their cloud and bill per memory. I wanted the opposite, so I +built Engraphis. + +It's a local-first memory engine for agents. The core is 100% on your machine (SQLite + local +embeddings, numpy-only), Apache-2.0, no API key for memory, no per-token cost, no data leaving +your box. You bring your own LLM only for optional chat/synthesis. + +The headline is a native MCP server — `claude mcp add engraphis -- engraphis-mcp` — so Claude +Code, Cursor, Cline, Zed, and Windsurf plug in and stop forgetting across sessions and repos. + +What's actually different under the hood: +- Bi-temporal facts: contradictions get versioned (valid_from/valid_to), not clobbered — you + can ask "why do we know this?" and "what did we believe as of last week?" +- Hybrid recall: vector + keyword + a knowledge graph (Personalized PageRank), fused and reranked. +- An Ebbinghaus forgetting curve so stale facts decay and reinforced ones stick. +- Code-aware: it indexes a repo into a function/class/call graph so "what calls this?" is cheap. + +Try it in ~2 min: + pip install engraphis + claude mcp add engraphis -- engraphis-mcp + +Repo: REPO_URL +60-sec demo: DEMO_URL + +It's early (v0.1, beta) and I'd love feedback — especially on recall quality vs. mem0/Zep/Letta +and on what you'd need before trusting it with a real project's memory. +``` +*Post Tue–Thu, ~8–10am ET. Reply to every comment for the first few hours — HN rewards presence.* + +--- + +## 2. Reddit (r/LocalLLaMA, r/ClaudeAI, r/cursor) + +**Title:** +`I built a local-first memory engine for coding agents — MCP server, runs on your machine, no per-token cost` + +**Body:** +``` +My agents kept forgetting everything between sessions, and I didn't want to send my private +codebase context to a hosted memory cloud that bills per memory. So I made Engraphis — a +self-hosted memory engine with a native MCP server. + +- Local + private: SQLite + local embeddings, offline by default, Apache-2.0. +- Plugs into Claude Code / Cursor / Cline / Zed / Windsurf via MCP (`remember` / `recall`, + scoped to workspace → repo → session). +- Actually remembers the right things: hybrid recall (vector + keyword + graph), an Ebbinghaus + decay curve, and bi-temporal truth so contradictions are versioned instead of overwritten. +- There's a Memory Inspector UI where you can see exactly when/why a fact changed. + +Install: + pip install engraphis + claude mcp add engraphis -- engraphis-mcp + +Repo + 60s demo in the comments. It's v0.1 and I'm looking for honest feedback — what would you +need before you'd let it hold your project's memory? +``` +*Put the links in the first comment, not the post body — several of these subs down-rank link posts.* + +--- + +## 3. X / Twitter thread + +``` +1/ Your AI coding agent forgets everything between sessions. + +Hosted memory tools "fix" this by making you ship your context to their cloud and billing you +per memory. + +I built the opposite: Engraphis — local-first memory that runs on your machine. Free core, open +source. 🧵 + +2/ One command to give any MCP agent long-term memory: + + pip install engraphis + claude mcp add engraphis -- engraphis-mcp + +Works with Claude Code, Cursor, Cline, Zed, Windsurf. Your data never leaves your box. + +3/ It's not just a vector store: +• bi-temporal facts (contradictions are versioned, not clobbered — ask "why do we know this?") +• hybrid recall: vector + keyword + knowledge graph +• an Ebbinghaus forgetting curve so stale facts fade, reinforced ones stick +• indexes your repo's call graph + +4/ There's also a Memory Inspector UI: search, timeline, and the exact word-level diff of when a +fact changed and why. + +60-second demo: DEMO_URL + +5/ It's early (v0.1) and open source (Apache-2.0). Repo + docs: REPO_URL + +If your agent keeps re-learning your codebase, I'd love for you to try it and tell me where recall +falls short. +``` + +--- + +## Notes +- Everything above is claims your README already backs up. Don't add benchmarks you haven't run + honestly — "v0.1, looking for feedback" is a stronger and safer launch posture than big numbers. +- Pick **one** primary channel for launch day (Show HN is the highest-leverage for a dev tool). + Post the others the same week, not the same hour. +- The single biggest predictor of a good launch here is the **60-second demo video**. Make that + before you write another word. diff --git a/docs/MONETIZATION-FAST.md b/docs/MONETIZATION-FAST.md new file mode 100644 index 0000000..3af2561 --- /dev/null +++ b/docs/MONETIZATION-FAST.md @@ -0,0 +1,67 @@ +# Engraphis — Making money faster (freemium, done right) + +Your instinct is correct: **free trial → subscribe** is the right model. One correction saves you +from killing your own product — **do not put the usage limit on the local core.** Here's why, then +exactly where the paywall goes and how to get cash fastest. + +## Why not limit the local engine +1. **It's unenforceable.** The core is Apache-2.0 and runs on the *user's* machine. Anyone can read + the code and delete the limit, or fork it. A cap there is theater. +2. **There's no cost to recoup.** mem0 and Zep meter you because *they* pay for the servers your + memories live on. Engraphis's free core runs on the user's own hardware — you pay nothing per + memory, so there's nothing to bill back. A limit there is pure friction that kills adoption and + throws away your one winning line — *"no per-token cost, no metering"* — the exact thing that + beats mem0 and fills the gap Zep left when it dropped self-hosting. + +The limit belongs where you **actually bear cost and can enforce it**: hosted sync, team memory, +and closed-source Pro add-ons. + +## The model — same "try free → subscribe," placed correctly + +| Tier | Price (anchored to competitors) | What's in it | Where the limit lives | +|---|---|---|---| +| **Free** (local, OSS, unlimited) | $0 | Engine + MCP server + CLI + single-user local Inspector. Yours forever, on your machine. | None — this is the trial that hooks them | +| **Pro** (individual) | ~$15–19/mo or **$99/yr founding** | Hosted memory **sync across your devices**, cloud backup, Inspector Pro dashboard, priority support | Free = 1 device / small sync quota → Pro = unlimited | +| **Team** | ~$25–40/seat/mo | **Shared team memory**, RBAC/SSO, audit exports, encryption-at-rest | Per seat | +| **Cloud / Enterprise** (later) | custom | Managed hosting, SLA, on-prem help | — | + +The "try it, then subscribe" path stays intact: they run the free local tool, love it, and the +moment they want their memory **on their laptop *and* desktop, backed up, or shared with a +teammate** — that's the wall, and it's one worth paying to cross. Price it flat and undercut mem0 +($19/$79/$249); "no per-memory metering" *is* your ad. + +> Prices anchored to your own GTM table (mem0 $19/$79/$249, Letta $20, Zep $125+). Re-check before +> publishing — they move. + +## What to build first for money +**Multi-device / team memory sync.** It's the natural upgrade (persistent memory that people +immediately want *everywhere*), it's server-side so the limit is real, and your GTM already named +it as the Pro wedge. Your Memory Inspector already exists — hosting it as the Pro dashboard is a +fast paid surface. Roughly 2–4 focused weeks *after* launch. Don't start it until the free tool is +live and getting users. + +## Fastest actual cash — in order (the "faster" you asked for) +Subscriptions need volume to matter (2% of 50 users = 1 customer), so the fast money at launch +isn't the subscription — it's capturing *committed demand*: + +1. **Founding-member prepay (launch day).** Stand up a Stripe/Gumroad link: + *"Engraphis Pro — Founding: $99/yr, locked forever, hosted sync the day it ships."* You collect + real cash and validate willingness-to-pay **before** building the cloud. Single fastest honest + dollar — and the prepay count tells you whether Pro is even worth building. +2. **GitHub Sponsors** — already configured in `FUNDING.yml`. Turn on tiers; announce it in the + launch post. Small but immediate. +3. **Paid setup/tuning for teams** — *"I'll deploy + tune Engraphis for your team, $X."* Fastest + dollars-per-hour for a technical founder; doesn't scale, but it funds runway and puts you in the + room with your first paying teams. + +## Sequencing +Launch free → capture founding prepay + waitlist on day one → build team/multi-device sync as the +first paid feature → convert the waitlist. **Distribution first; monetize the demand.** Trying to +charge before anyone uses it is exactly why it feels stuck. + +--- +*Not legal or financial advice. You can keep the core Apache-2.0 and ship Pro features as a +separate, commercially-licensed package (`engraphis-pro`) — Apache is permissive, so new code can +carry your terms while the open core stays open. If you ever accept outside code contributions, +use a contributor agreement so you retain the right to do this — looks solo today, so low risk. +Validate the "Engraphis" trademark and your license terms with a professional before charging.* diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..4ad2b90 --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,85 @@ +# Releasing Engraphis to PyPI + +Goal: make `pip install engraphis` work for a stranger. The name is **free** on PyPI as of +2026-07-03. Do this once and Week 2 of the ship plan is done. + +There are two paths. **First release → do Path A (manual, 10 min).** Every release after → +Path B is automatic (`release.yml` is already committed; you just publish a GitHub Release). + +--- + +## 0. Pre-flight fixes (5 min, do these first) + +- [ ] **Fix the project URLs.** `pyproject.toml [project.urls]` points at + `github.com/engraphis/engraphis`, but the repo is at `github.com/Coding-Dev-Tools/engraphis`. + Either update the three URLs to the real remote, **or** create an `engraphis` GitHub org and + move the repo there (nicer branding, and the URLs already assume it). Broken links on the + PyPI page look abandoned — fix before publishing. +- [ ] **Confirm version.** `version = "0.1.0"` is fine for the first upload. Bump it for every + later release (PyPI refuses to overwrite an existing version). +- [ ] **Confirm `LICENSE` and `README.md` exist at repo root** (they do) — PyPI renders the README + as the project page. +- [ ] Merge your working branch to `main` and push, so you're releasing the real code. + +--- + +## Path A — First release (manual, with an API token) + +The first upload has to create the project, so do it by hand once. + +```bash +cd C:\GitHub\engraphis +python -m pip install --upgrade build twine +python -m build # writes dist/engraphis-0.1.0.tar.gz + .whl +twine check dist/* # metadata sanity check — must say PASSED +``` + +Get a token: log in at https://pypi.org → Account settings → **API tokens** → *Add token* +(scope: "Entire account" for the first upload; you can scope it to the project afterward). + +```bash +twine upload dist/* +# username: __token__ +# password: pypi-AgEI... (paste the token) +``` + +Verify in a clean environment: + +```bash +python -m venv /tmp/e && /tmp/e/Scripts/activate # (Windows: \tmp\e\Scripts\activate) +pip install engraphis +engraphis-mcp --help # the headline command exists → success +``` + +Then delete the token if it was account-scoped, and switch to Path B for everything after. + +--- + +## Path B — Every release after (automatic, no tokens) + +`release.yml` publishes via **PyPI Trusted Publishing (OIDC)** — no secrets stored. One-time setup: + +1. On PyPI → your project → **Manage → Publishing → Add a trusted publisher → GitHub**. Enter: + - Owner: `Coding-Dev-Tools` (or your `engraphis` org) + - Repository: `engraphis` + - Workflow filename: `release.yml` + - Environment name: `pypi` ← must match the `environment:` in the workflow +2. In the GitHub repo → **Settings → Environments → New environment → `pypi`**. (Optionally add a + required-reviewer rule so a release waits for your one-click approval — a nice safety gate.) + +To cut a release from then on: + +```bash +# bump version in pyproject.toml first, commit, then: +git tag v0.1.1 && git push origin v0.1.1 +``` + +Then on GitHub → **Releases → Draft a new release** → pick the tag → **Publish**. The workflow +builds, runs `twine check`, and publishes to PyPI automatically. + +--- + +## After it's live +- [ ] Update the README install line from `pip install -e .` to `pip install engraphis` + (and `pip install "engraphis[mcp]"` for the MCP server). +- [ ] Tweet/post the one command. "`pip install engraphis`" *is* the launch — see `LAUNCH-POST.md`. diff --git a/docs/SHIP-IT-PLAN.md b/docs/SHIP-IT-PLAN.md new file mode 100644 index 0000000..b5da702 --- /dev/null +++ b/docs/SHIP-IT-PLAN.md @@ -0,0 +1,52 @@ +# Engraphis — Ship-It Plan (one page, on purpose) + +You don't have a marketing problem or a mindset problem. **Engraphis is built.** It's just +never been *shipped*: it's not on PyPI, your best code isn't merged to `main`, and no human +has been told it exists. Your landing copy and go-to-market doc are already written and sharp. +This plan just executes the last, smallest, scariest step. + +**Positioning (straight from your own GTM — don't reopen it):** +> Local-first memory for coding agents. MCP server, runs on your machine, no cloud, no per-token cost. + +**The one rule for the next month:** no new features, no new audits, no new handoff docs, no +touching the other three products or Hermes. Only the checkboxes below. Everything else is parked. + +--- + +## ~7 hrs/week × 4 weeks + +### Week 1 — Publish the code (≈3–4 hrs) +- [ ] Commit the working tree; merge `improve/engraphis-20260702` → `main`; push. +- [ ] Confirm the GitHub repo is **public** and the README renders correctly. +- [ ] Drop one screenshot or GIF of the Memory Inspector near the top of the README. + +### Week 2 — Let a stranger install it in 2 minutes (≈3 hrs) +- [ ] Publish to PyPI so `pip install engraphis` works. **The name is free.** Add a release workflow (you already have `ci.yml` to copy from). +- [ ] Record a 60–90s demo: add engraphis to Claude Code → it recalls a fact across a restart → open the Inspector. **This one asset is 80% of your marketing.** + +### Week 3 — Tell the warm audience (≈2 hrs) +- [ ] Post the launch write-up (already drafted → `LAUNCH-POST.md`) to **Show HN**, **r/LocalLLaMA / r/ClaudeAI**, and as an **X thread**. +- [ ] Spend that day replying to comments. That is the entire "promotion." + +### Week 4 — Turn attention into a money signal (≈2 hrs) +- [ ] Announce **GitHub Sponsors** (already configured in `FUNDING.yml`). +- [ ] Add a **"Pro / Team tier — join the waitlist"** email-capture link to the README + landing page. Waitlist size tells you whether the paid tier is worth building. + +--- + +## The money question, straight + +An Apache-2.0 core won't produce revenue on day one — and that's fine. What you're building +*this* month is **users + GitHub stars + a waitlist** in a hot category (mem0/Zep/Letta are +funded; Zep abandoned self-hosting — that gap is yours). Money comes next, from the open-core +**Pro/Team tier your GTM already defines**, or sponsorship, once there's adoption. +Distribution first; monetize the demand. Trying to charge before anyone uses it is the reason +it feels stuck. + +## The mental-state question, straight + +The motivation you're waiting for arrives *after* the first person says "oh, nice" — not +before. And promoting feels awful right now partly because promoting something nobody can +install feels like lying. Once `pip install engraphis` works and the repo is public, +"promotion" stops being a performance and becomes "showing people a thing that works." That's +a completely different, much lighter feeling — and it's two weeks away, not a personality change away. diff --git a/engraphis/backends/graph_extractor.py b/engraphis/backends/graph_extractor.py new file mode 100644 index 0000000..a7b5bd7 --- /dev/null +++ b/engraphis/backends/graph_extractor.py @@ -0,0 +1,201 @@ +"""Deterministic free-text -> entity/relation graph feeder. + +The general-purpose text->graph path v2 lacked: v2 had graph CRUD (``core.store``) +and a *code*-symbol indexer (``backends.codegraph``), and a *fact* extractor +(``backends.extractor``, text -> memory notes), but nothing that turned a free-text +memory into knowledge-graph **entities and relations**. This module adds it. + +It ports v1's proven, dependency-free regex NER (``engraphis/engines/ingest.py``) +behind the same factory shape as ``get_extractor`` / ``get_embedder`` / +``get_reranker`` -- so it runs in the numpy-only offline gate and is opt-in by +configuration only (``ENGRAPHIS_GRAPH_EXTRACTOR = none | regex``). The default +``none`` -> ``NullGraphExtractor`` writes nothing, so the write path is byte-for-byte +unchanged unless explicitly enabled. + +Security: extracted names are untrusted input (indirect prompt injection can steer +what lands in text; memory poisoning is an explicit threat model -- SECURITY.md). +Every extracted string is defanged (control/escape chars stripped, length-capped) +before it reaches the graph, the same rule ``service.py`` and the fact ``extractor`` +apply to direct writes, and every node/edge is written scoped to the caller's +``(workspace_id, repo_id)`` -- it cannot cross the isolation boundary. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Optional + +from engraphis.core.interfaces import Edge, Node + +# ── Regex NER (ported from engraphis/engines/ingest.py — the v1 heuristic path) ── +# Capitalized multi-word sequences, emails, URLs/hashtags, quoted names/mentions. +_ENTITY_RE = re.compile( + r"\b([A-Z][a-z]+(?:-[A-Za-z]+)*(?:\s+[A-Z][a-z]+(?:-[A-Za-z]+)*){0,3})\b" + r"|([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})" + r"|(#[a-zA-Z][a-zA-Z0-9_-]+)" + r"|(@[a-zA-Z][a-zA-Z0-9_-]+)" +) +_RELATION_RE = re.compile( + r"\b(?:is|are|was|were|has|have|had|owns|works at|lives in|prefers|likes|" + r"dislikes|uses|manages|created|founded|located in|part of|member of)\b", + re.IGNORECASE, +) +_STOPWORDS = { + "The", "This", "That", "These", "Those", "A", "An", "And", "But", "Or", + "If", "Then", "When", "Where", "What", "Who", "How", "Why", "It", "Is", + "Was", "Are", "Were", "Has", "Have", "Had", "Will", "Would", "Could", + "Should", "May", "Might", "Can", "Did", "Do", "Does", "Not", "No", "Yes", + "User", "We", "They", "He", "She", "His", "Her", "Their", "Our", "My", + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "January", "February", "March", "April", "May", "June", "July", "August", + "September", "October", "November", "December", +} + +# Extracted text is untrusted: strip the same control chars service.py strips from +# direct writes so an entity name can't smuggle a hidden-instruction / escape payload +# into the graph, and cap length so a pathological match can't bloat a node. +_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") +_MAX_NAME = 200 +_MAX_RELATIONS = 20 + + +def _defang(value: str) -> str: + return _CONTROL_RE.sub("", value or "").strip()[:_MAX_NAME] + + +@dataclass +class GraphExtraction: + """Result of one extraction pass: named entities and (src, relation, dst) triples.""" + entities: list[tuple[str, str]] = field(default_factory=list) # (name, etype) + relations: list[tuple[str, str, str]] = field(default_factory=list) # (src, rel, dst) + + +def _extract_entities(text: str) -> list[tuple[str, str]]: + seen: set[str] = set() + out: list[tuple[str, str]] = [] + for m in _ENTITY_RE.finditer(text or ""): + raw = (m.group(0) or "").strip() + if not raw or raw in _STOPWORDS or raw.lower() in ("user", "the user"): + continue + if raw.startswith("#"): + ent, etype = raw, "hashtag" + elif raw.startswith("@"): + ent, etype = raw, "mention" + elif "@" in raw and "." in raw: + ent, etype = raw, "email" + else: + ent, etype = raw, "person_or_concept" + key = ent.lower() + if key not in seen: + seen.add(key) + out.append((ent, etype)) + return out + + +def _extract_entities_from_doc(title: str, content: str) -> list[tuple[str, str]]: + """Extract from ``title`` and ``content`` as SEPARATE passes, then merge by name. + + Never concatenate first: the capitalized-word pattern has no notion of a + title/content boundary, so matching ``f"{title}\\n\\n{content}"`` lets one match + bridge the boundary (title "Meeting Notes" + content "Alice Johnson ..." -> one + garbled node "Meeting Notes\\n\\nAlice Johnson"), fragmenting a real entity. This + regression is covered by tests/test_graph_extractor.py. + """ + seen: set[str] = set() + out: list[tuple[str, str]] = [] + for text in (content, title): # content first — the stronger signal + for ent, etype in _extract_entities(text): + key = ent.lower() + if key not in seen: + seen.add(key) + out.append((ent, etype)) + return out + + +def _extract_relations(text: str, entities: list[tuple[str, str]]) -> list[tuple[str, str, str]]: + """Simple subject-relation-object triples via regex proximity (±60 chars).""" + if len(entities) < 2: + return [] + ent_names = [e[0] for e in entities] + relations: list[tuple[str, str, str]] = [] + for m in _RELATION_RE.finditer(text or ""): + rel = (m.group(0) or "").lower() + window = text[max(0, m.start() - 60): m.end() + 60] + nearby = [name for name in ent_names if name in window] + if len(nearby) >= 2: + relations.append((nearby[0], rel, nearby[1])) + return relations[:_MAX_RELATIONS] + + +class NullGraphExtractor: + """Default backend: extracts nothing, so the write path is unchanged.""" + + def extract(self, content: str, *, title: str = "") -> GraphExtraction: + return GraphExtraction() + + +class RegexGraphExtractor: + """Dependency-free heuristic NER: capitalized names, emails, #tags, @mentions, + plus proximity relations. No network, no model — safe in the offline gate.""" + + def extract(self, content: str, *, title: str = "") -> GraphExtraction: + entities = _extract_entities_from_doc(title, content) + full_text = f"{title}\n\n{content}" if title else content + relations = _extract_relations(full_text, entities) + return GraphExtraction(entities=entities, relations=relations) + + +def get_graph_extractor(kind: str = "none"): + """Factory mirroring ``get_extractor``: config in, backend out. ``kind='regex'`` + -> heuristic NER; anything else (incl. ``'none'``) -> the no-op passthrough.""" + if (kind or "none").lower() == "regex": + return RegexGraphExtractor() + return NullGraphExtractor() + + +def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] = None, + title: str = "", extractor: Any = None, + provenance: Optional[dict] = None) -> dict: + """Extract entities/relations from free text and write them into the knowledge + graph, scoped to ``(workspace_id, repo_id)``. + + * Entities are de-duplicated by the store (``upsert_entity`` returns the existing + id for a repeat name/etype in the same scope). + * Relations connect two extracted entities by their node ids, and are skipped if + an equivalent live edge already exists — so re-feeding the same text is + idempotent. + * A ``None`` / ``NullGraphExtractor`` writes nothing and returns zero counts, so + the default path is a genuine no-op. + + Returns ``{"entities": , "relations": }``. + """ + extractor = extractor or NullGraphExtractor() + result = extractor.extract(content, title=title) + + name_to_id: dict[str, str] = {} + for name, etype in result.entities: + clean = _defang(name) + if not clean or clean in name_to_id: + continue + name_to_id[clean] = store.upsert_entity( + Node(id="", name=clean, ntype=_defang(etype), + workspace_id=workspace_id, repo_id=repo_id) + ) + + prov = dict(provenance or {}) + written_relations = 0 + for src, rel, dst in result.relations: + s, d = _defang(src), _defang(dst) + sid, did = name_to_id.get(s), name_to_id.get(d) + if not sid or not did or sid == did: + continue + existing = {(e.src, e.dst, e.relation) for e in store.neighbors([sid])} + relation = _defang(rel) or "related" + if (sid, did, relation) in existing: + continue + store.upsert_edge(Edge(id="", src=sid, dst=did, relation=relation, + workspace_id=workspace_id, repo_id=repo_id, + provenance=prov)) + written_relations += 1 + + return {"entities": len(name_to_id), "relations": written_relations} diff --git a/engraphis/config.py b/engraphis/config.py index ff71fe0..ed5112e 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -84,6 +84,12 @@ class Settings: default_factory=lambda: _parse_headers(_env("ENGRAPHIS_LLM_EXTRA_HEADERS", "")) ) + # Optional cross-encoder reranker model. Empty (default) -> IdentityReranker (offline). + rerank_model: str = field(default_factory=lambda: _env("ENGRAPHIS_RERANK_MODEL", "")) + + # Optional graph extractor: "none" (default) or "regex" for heuristic NER. + graph_extractor: str = field(default_factory=lambda: _env("ENGRAPHIS_GRAPH_EXTRACTOR", "none").lower()) + loop_interval: int = field(default_factory=lambda: _env_int("ENGRAPHIS_LOOP_INTERVAL", 60)) loop_top_k: int = field(default_factory=lambda: _env_int("ENGRAPHIS_LOOP_TOP_K", 20)) decay_halflife_days: float = field( diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index 4ab270e..304ca9f 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -32,7 +32,8 @@ class MemoryEngine: def __init__(self, store: Store, embedder, vector_index, reranker=None, - *, auto_evolve: bool = True, extractor=None) -> None: + *, auto_evolve: bool = True, extractor=None, + graph_extractor=None) -> None: self.store = store self.embedder = embedder self.index = vector_index @@ -43,13 +44,17 @@ def __init__(self, store: Store, embedder, vector_index, reranker=None, self.auto_evolve = auto_evolve # Optional fact extractor (core.interfaces.Extractor). None = raw passthrough. self.extractor = extractor + # Optional graph extractor (backends.graph_extractor). None = no graph population. + self.graph_extractor = graph_extractor @classmethod def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, embed_dim: int = 256, vector_backend: str = "auto", rerank_model: Optional[str] = None, extractor: str = "none", + graph_extractor: str = "none", auto_evolve: bool = True) -> "MemoryEngine": from engraphis.backends.extractor import PassthroughExtractor, get_extractor + from engraphis.backends.graph_extractor import get_graph_extractor as _get_ge store = Store(db_path) embedder = get_embedder(embed_model, embed_dim) index = get_vector_index(store, dim=embedder.dim, prefer=vector_backend) @@ -57,8 +62,9 @@ def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, ext = get_extractor(extractor) if isinstance(ext, PassthroughExtractor): ext = None # ingest() treats None as passthrough + ge = _get_ge(graph_extractor) if graph_extractor and graph_extractor != "none" else None return cls(store, embedder, index, reranker, auto_evolve=auto_evolve, - extractor=ext) + extractor=ext, graph_extractor=ge) # ── write ───────────────────────────────────────────────────────────────── def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = None, @@ -128,6 +134,15 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, except Exception: pass + # Optional graph population (backends.graph_extractor) + if self.graph_extractor is not None: + try: + from engraphis.backends.graph_extractor import feed as _graph_feed + _graph_feed(self.store, content, workspace_id=workspace_id, + repo_id=repo_id, title=title, extractor=self.graph_extractor) + except Exception: + pass + if decision is not None and decision.op == ResolutionOp.INVALIDATE: self.store.close_validity(decision.target_id, reason=decision.reason) try: diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..6d3e798 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,36 @@ +"""Config wiring tests — env vars must reach Settings, and the offline defaults must hold. + +Covers the ENGRAPHIS_RERANK_MODEL knob added so the cross-encoder reranker (the biggest +precision win on top of hybrid retrieval, MASTER_PLAN.md §7) can be turned on by config +instead of only in code. The default must stay empty so the offline/numpy-only CI path is +unchanged (empty -> None -> IdentityReranker, no torch). +""" +from engraphis.config import Settings + + +def test_rerank_model_defaults_to_empty(monkeypatch): + monkeypatch.delenv("ENGRAPHIS_RERANK_MODEL", raising=False) + assert Settings().rerank_model == "" + + +def test_rerank_model_read_from_env(monkeypatch): + monkeypatch.setenv("ENGRAPHIS_RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2") + assert Settings().rerank_model == "cross-encoder/ms-marco-MiniLM-L-6-v2" + + +def test_empty_rerank_model_normalizes_to_none(monkeypatch): + # This is the exact expression the service builders pass: + # MemoryService.create(..., rerank_model=settings.rerank_model or None) + # Empty must become None so get_reranker returns the offline IdentityReranker. + monkeypatch.delenv("ENGRAPHIS_RERANK_MODEL", raising=False) + assert (Settings().rerank_model or None) is None + + +def test_service_builds_offline_with_default_rerank_model(monkeypatch): + # End-to-end: with no rerank model configured, a MemoryService builds on numpy alone + # (DeterministicEmbedder + IdentityReranker) and serves a round-trip — the CI path. + monkeypatch.delenv("ENGRAPHIS_RERANK_MODEL", raising=False) + from engraphis.service import MemoryService + svc = MemoryService.create(":memory:", rerank_model=(Settings().rerank_model or None)) + assert svc.remember("a durable fact", workspace="w", repo="r")["stored"] is True + assert svc.recall("a durable fact", workspace="w", repo="r")["count"] >= 1 diff --git a/tests/test_graph_extractor.py b/tests/test_graph_extractor.py new file mode 100644 index 0000000..1a96f08 --- /dev/null +++ b/tests/test_graph_extractor.py @@ -0,0 +1,126 @@ +"""Offline tests for the free-text -> entity/relation graph feeder +(engraphis.backends.graph_extractor). numpy-only: no model download, no network. + +Covers the regex NER (entity types, single-word stopwords, the title/content +boundary regression), the scoped feed() writer (entities+relations land in the +graph, scoped to one workspace and isolated from another), idempotent re-feed, +defanging of untrusted extractor output, the default 'none' no-op, and the opt-in +wiring on MemoryEngine.remember (regex on -> graph populated; default -> empty). +""" +from __future__ import annotations + + +from engraphis.backends.graph_extractor import ( + GraphExtraction, NullGraphExtractor, RegexGraphExtractor, feed, get_graph_extractor, +) +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import SearchFilter +from engraphis.core.store import Store + + +def _entity_names(store, workspace_id): + return {r["name"] for r in store.conn.execute( + "SELECT name FROM entities WHERE workspace_id=?", (workspace_id,)).fetchall()} + + +# ── the extractor (pure) ────────────────────────────────────────────────────── + +def test_regex_entities_types_and_stopwords(): + ex = RegexGraphExtractor().extract( + "Alice Johnson emailed bob@acme.com about #launch with @carol. It is done." + ) + names = {n: t for n, t in ex.entities} + assert names.get("Alice Johnson") == "person_or_concept" + assert names.get("bob@acme.com") == "email" + assert names.get("#launch") == "hashtag" + assert names.get("@carol") == "mention" + assert "It" not in names # single-word stopword dropped + + +def test_title_content_boundary_no_bleed(): + # title + content must be NER'd as separate passes; concatenating first lets a + # match bridge the boundary and fragment a real entity. + ex = RegexGraphExtractor().extract("Alice Johnson met Bob Smith.", title="Meeting Notes") + names = [n for n, _ in ex.entities] + assert "Alice Johnson" in names and "Bob Smith" in names + assert not any("\n" in n for n in names) + assert not any("Meeting" in n and "Alice" in n for n in names) + + +def test_relations_by_proximity(): + ex = RegexGraphExtractor().extract("Alice Johnson works at Acme Corporation.") + assert any(r == "works at" and {s, d} == {"Alice Johnson", "Acme Corporation"} + for s, r, d in ex.relations) + + +def test_factory_and_null_default(): + assert isinstance(get_graph_extractor("regex"), RegexGraphExtractor) + assert isinstance(get_graph_extractor("none"), NullGraphExtractor) + assert isinstance(get_graph_extractor(""), NullGraphExtractor) + assert NullGraphExtractor().extract("Alice Johnson works at Acme.").entities == [] + + +# ── the feed() writer (scoped graph) ────────────────────────────────────────── + +def test_feed_writes_scoped_entities_and_edges(): + store = Store(":memory:") + n = feed(store, "Alice Johnson works at Acme Corporation.", + workspace_id="w1", repo_id="r1", extractor=RegexGraphExtractor()) + assert n["entities"] >= 2 and n["relations"] >= 1 + assert {"Alice Johnson", "Acme Corporation"} <= _entity_names(store, "w1") + edges = store.edges_in_scope(SearchFilter(workspace_id="w1")) + assert any(e.relation == "works at" for e in edges) + + +def test_feed_isolation_between_workspaces(): + store = Store(":memory:") + feed(store, "Alice Johnson works at Acme Corporation.", + workspace_id="w1", extractor=RegexGraphExtractor()) + assert _entity_names(store, "w2") == set() + assert store.edges_in_scope(SearchFilter(workspace_id="w2")) == [] + + +def test_feed_idempotent_refeed(): + store, ex = Store(":memory:"), RegexGraphExtractor() + feed(store, "Alice Johnson works at Acme Corporation.", workspace_id="w1", extractor=ex) + before = len(store.edges_in_scope(SearchFilter(workspace_id="w1"))) + again = feed(store, "Alice Johnson works at Acme Corporation.", workspace_id="w1", extractor=ex) + after = len(store.edges_in_scope(SearchFilter(workspace_id="w1"))) + assert after == before and again["relations"] == 0 # no duplicate edges + + +def test_feed_defangs_untrusted_names(): + """Extractor output is untrusted (memory-poisoning threat model): control/escape + chars must be stripped before a name reaches the graph.""" + class _Evil: + def extract(self, content, *, title=""): + return GraphExtraction( + entities=[("Ali\x00ce\x07", "person_or_concept"), ("Bob\x1bSmith", "person_or_concept")], + relations=[("Ali\x00ce\x07", "is", "Bob\x1bSmith")], + ) + store = Store(":memory:") + feed(store, "irrelevant", workspace_id="w1", extractor=_Evil()) + assert _entity_names(store, "w1") == {"Alice", "BobSmith"} + assert len(store.edges_in_scope(SearchFilter(workspace_id="w1"))) == 1 + + +def test_feed_default_none_writes_nothing(): + store = Store(":memory:") + n = feed(store, "Alice Johnson works at Acme Corporation.", workspace_id="w1") # no extractor + assert n == {"entities": 0, "relations": 0} + assert _entity_names(store, "w1") == set() + + +# ── opt-in wiring on the write path ─────────────────────────────────────────── + +def test_remember_default_off_leaves_graph_empty(): + eng = MemoryEngine.create(":memory:") # graph_extractor defaults to "none" + eng.remember("Alice Johnson works at Acme Corporation.", workspace_id="w1", repo_id="r1") + assert eng.store.conn.execute("SELECT COUNT(*) c FROM entities").fetchone()["c"] == 0 + + +def test_remember_regex_on_populates_graph(): + eng = MemoryEngine.create(":memory:", graph_extractor="regex") + eng.remember("Alice Johnson works at Acme Corporation.", workspace_id="w1", repo_id="r1") + assert {"Alice Johnson", "Acme Corporation"} <= _entity_names(eng.store, "w1") + assert len(eng.store.edges_in_scope(SearchFilter(workspace_id="w1"))) >= 1 diff --git a/tests/test_service_isolation.py b/tests/test_service_isolation.py new file mode 100644 index 0000000..2bc9991 --- /dev/null +++ b/tests/test_service_isolation.py @@ -0,0 +1,81 @@ +"""Cross-tenant isolation tests for MemoryService — the server-side workspace binding. + +Locks in the guarantee that a MemoryService bound to a set of workspaces +(``ENGRAPHIS_WORKSPACES``) refuses every read AND write outside that set — knowing the +name of another tenant's workspace is not enough (``service.py:_authorize_workspace``, +MASTER_PLAN.md §16: scope is "enforced server-side on every read/write — never trust +client-supplied scope alone"). + +Writes were always guarded; the read paths (recall / why / timeline / stats / +list_workspaces) are the ones that historically were not, so they get the coverage here. +This is the first test over ``service.py`` at all — the isolation boundary was previously +unpinned, so a refactor could have silently reopened it. + +Offline only: DeterministicEmbedder + NumpyVectorIndex, no torch (AGENTS.md §3). +""" +import pytest + +from engraphis.service import MemoryService, ValidationError + + +def _bound(*workspaces): + """A MemoryService bound to ``workspaces`` — the multi-tenant hard boundary.""" + return MemoryService.create(":memory:", allowed_workspaces=list(workspaces)) + + +# ── writes are refused outside the binding (regression guard; was already enforced) ── +def test_remember_rejects_unpermitted_workspace(): + with pytest.raises(ValidationError): + _bound("team-a").remember("secret", workspace="team-b", repo="r1") + + +# ── reads are refused outside the binding (the historically weak paths) ────────────── +def test_recall_rejects_unpermitted_workspace(): + with pytest.raises(ValidationError): + _bound("team-a").recall("q", workspace="team-b") + + +def test_recall_without_workspace_is_refused_on_a_bound_instance(): + # A workspace-less recall on a bound instance would read across every tenant — the + # exact boundary the binding exists to enforce. + with pytest.raises(ValidationError): + _bound("team-a").recall("q") + + +def test_why_rejects_unpermitted_workspace(): + with pytest.raises(ValidationError): + _bound("team-a").why("q", workspace="team-b") + + +def test_timeline_rejects_unpermitted_workspace(): + with pytest.raises(ValidationError): + _bound("team-a").timeline("q", workspace="team-b") + + +def test_stats_without_workspace_is_refused_on_a_bound_instance(): + with pytest.raises(ValidationError): + _bound("team-a").stats() + + +# ── the binding permits its own workspace, and reads its own data back ─────────────── +def test_bound_instance_allows_and_reads_its_own_workspace(): + svc = _bound("team-a") + stored = svc.remember("alpha fact", workspace="team-a", repo="r1") + assert stored["stored"] is True + hit = svc.recall("alpha fact", workspace="team-a", repo="r1") + assert hit["count"] >= 1 + + +def test_bound_list_workspaces_shows_only_permitted(): + svc = _bound("team-a") + svc.remember("alpha fact", workspace="team-a", repo="r1") + names = {w["name"] for w in svc.list_workspaces()["workspaces"]} + assert names == {"team-a"} # never leaks a workspace outside the binding (service.py:486) + + +# ── the unbound single-tenant default stays global (no accidental lockout) ─────────── +def test_unbound_instance_permits_global_recall(): + svc = MemoryService.create(":memory:") # no binding + svc.remember("alpha fact", workspace="team-a", repo="r1") + out = svc.recall("alpha fact") # workspace-less recall is allowed when unbound + assert "memories" in out