From 2b7c03e76d6c110f1055bd44405cd031035d9e60 Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Thu, 4 Jun 2026 21:26:33 -0400 Subject: [PATCH 1/9] Update docs and bump Obsidian plugin version Format migration list in CLAUDE.md for readability Remove references to deleted design documentation files Document --limit flag for embeddings backfill command --- CLAUDE.md | 29 ++++++++++++++++++++++------- README.md | 3 ++- plugins/obsidian/package-lock.json | 4 ++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 009ed68..b2e03f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,7 +153,7 @@ Every mutation (create / update / archive / unarchive / trash / recover / purge) Lineage is a separate `trace_lineage` join table populated from the `derived_from` frontmatter field. `trace_lineage` supports both directions: `cortex.Get()` reads `derived_from`, and `cortex.DerivedBy()` reads the reverse edge. -**Cortex identity.** Every Cortex has a stable ULID stored in `cortex.md` as `id` and surfaced via the `cortex_identity` MCP tool. The display name (`name`) can be renamed without affecting federation; the ID is the federation key. Vector clocks are keyed on cortex IDs, not names, so two peers that share a display name can no longer silently collapse into one bucket. The federation syncer pins a peer's ID on first contact and refuses to talk to an endpoint whose advertised ID has changed (peer reset, replaced, or restored from another cortex's backup). The same-name guardrail is still enforced at config time as a UX safety net even though identity is now ID-based. See `docs/design/cortex-uuid-plan.md` for the full design. +**Cortex identity.** Every Cortex has a stable ULID stored in `cortex.md` as `id` and surfaced via the `cortex_identity` MCP tool. The display name (`name`) can be renamed without affecting federation; the ID is the federation key. Vector clocks are keyed on cortex IDs, not names, so two peers that share a display name can no longer silently collapse into one bucket. The federation syncer pins a peer's ID on first contact and refuses to talk to an endpoint whose advertised ID has changed (peer reset, replaced, or restored from another cortex's backup). The same-name guardrail is still enforced at config time as a UX safety net even though identity is now ID-based. Federation is opt-in via a `federation:` block in `cortex.md` (peers + interval). When `noema serve --transport http` starts on a Cortex with peers configured, a background syncer polls each peer's `sync_events` MCP tool over Streamable HTTP (the `/mcp` endpoint, MCP 2025-03-26), replays new events into the local Cortex, and merges the remote vector clock into the local one. Replayed events are stored with their original ID, cortex_id, and origin so the event log becomes the union of all peers' events (no event amplification). When two peers update the same trace concurrently — vector clocks neither dominate nor are dominated — Noema creates a `type: divergence` Trace preserving every conflicting version under deterministically-sorted `### Version from ()` headers, rather than overwriting. Resolve via `noema resolve --accept ` or `--custom ` (or the `resolve_divergence` MCP tool). @@ -169,8 +169,6 @@ In publish mode, mutating MCP tools (`create_trace`, `update_trace`, `append_tra Each peer can also declare `mode: paused` to temporarily skip syncing without losing cursor/identity state. CLI commands: `noema federation set-mode `, `noema federation pause-peer `, `noema federation resume-peer `. -The full design rationale, edge cases, and phase breakdown live in `docs/design/federation-plan.md`. - **Content hashing and source-locking.** Every trace mutation (`Add`, `Update`, `Append`) computes a SHA-256 hash of the body (`sha256:`) and stores it in the `content_hash` frontmatter field and DB column. The hash travels through federation events so peers receive it. Three frontmatter fields support integrity: - `content_hash` — current body hash, recomputed on every write @@ -181,7 +179,7 @@ The full design rationale, edge cases, and phase breakdown live in `docs/design/ CLI: `noema verify` is a subcommand group for integrity checks. `noema verify traces` checks all trace file hashes against their frontmatter `content_hash` (with `--backfill` for old traces); `noema verify cortex` validates the manifest, user config, DB, access posture, and federation config; `noema verify drift` checks federated traces against their `source_hash`. Bare `noema verify` runs `verify traces` for back-compat. The legacy top-level `noema drift` still works as a hidden alias for `noema verify drift` and will be removed in a future release. -**MCP access posture.** The HTTP MCP endpoint runs either in **open mode** (no auth, the default — suitable only for loopback) or in **keyed mode** (every request must carry `Authorization: Bearer `). Keyed mode is mandatory for federation rings and any non-loopback deployment, and it also requires TLS — the server refuses to start with a bearer key over plaintext HTTP. The key is supplied via `NOEMA_MCP_KEY` (env var, wins if both are set) or via an optional `access.shared_key_file` block in `cortex.md` pointing at a 0600 sidecar file. The server logs the active posture on startup as `access=keyed source=env|file fingerprint=SHA256:...`, and `noema federation key fingerprint` reproduces the same non-secret fingerprint for out-of-band verification across ring members. The federation syncer automatically injects the local host's bearer key into every outbound `sync_events` call; there is no per-peer key config, so every host in a ring must share one key. Mixed-mode rings (some keyed, some open) are not supported — they isolate by design. See `docs/design/mcp-auth-plan.md` for the full threat model and decision log, and `internal/mcp/middleware.go` for the CORS + auth middleware chain (CORS outermost so browser preflights bypass the auth gate as the spec requires). +**MCP access posture.** The HTTP MCP endpoint runs either in **open mode** (no auth, the default — suitable only for loopback) or in **keyed mode** (every request must carry `Authorization: Bearer `). Keyed mode is mandatory for federation rings and any non-loopback deployment, and it also requires TLS — the server refuses to start with a bearer key over plaintext HTTP. The key is supplied via `NOEMA_MCP_KEY` (env var, wins if both are set) or via an optional `access.shared_key_file` block in `cortex.md` pointing at a 0600 sidecar file. The server logs the active posture on startup as `access=keyed source=env|file fingerprint=SHA256:...`, and `noema federation key fingerprint` reproduces the same non-secret fingerprint for out-of-band verification across ring members. The federation syncer automatically injects the local host's bearer key into every outbound `sync_events` call; there is no per-peer key config, so every host in a ring must share one key. Mixed-mode rings (some keyed, some open) are not supported — they isolate by design. See `internal/mcp/middleware.go` for the CORS + auth middleware chain (CORS outermost so browser preflights bypass the auth gate as the spec requires). **TLS cert lifecycle.** When `noema serve --transport http` is started with TLS (via `--tls-cert`/`--tls-key` or the manifest fallback `access.tls_cert_path` / `access.tls_key_path`), the leaf cert is parsed at startup and classified against its `NotAfter`. An already-expired or not-yet-valid cert is a hard startup error — the server refuses to come up rather than presenting a cert clients will reject anyway. The `--insecure-allow-expired` flag exists as an escape hatch for in-place rotation but logs a loud warning line. Certs within 7 days of expiry produce a warning at startup; the server still starts. A background `cert-monitor` goroutine (mirroring the federation syncer's lifecycle) re-checks the cert every hour while serving and logs a band-transition line as the cert crosses the 90 / 30 / 7 / expired thresholds. `noema verify cortex` reads the same manifest fields and surfaces upcoming expiry passively (`[warn]` ≤ 7 days, `[fail]` once expired or not-yet-valid). See `internal/tlsutil/` for the parser/classifier, `internal/mcp/certmonitor.go` for the loop, and `cmd_serve.go:gateTLSExpiry` for the startup gate. @@ -237,9 +235,26 @@ Schema changes must always be transparent and non-destructive. Rules: - If a migration would require removing or restructuring data, provide a separate explicit `noema migrate` command with a clear description of what it does, requiring user confirmation - Migration failures abort startup with a clear error; they never partially apply -The runner is a thin hand-rolled loop over embedded `*.sql` files in `internal/db/migrations/`, sorted by leading version number. Currently applied: `001_initial.sql`, `002_trash.sql`, `003_events_and_lineage.sql` (event log + `trace_lineage` + `traces.origin` column), `004_federation_state.sql` (key-value store for vector clocks and per-peer cursors), `005_cortex_identity.sql` (`cortex_id` column on `traces` and `events`), `006_content_hash.sql` (`content_hash` column on `traces`), and `007_source_locking.sql` (`source_locked` + `source_hash` columns on `traces`). - -`cortex.md` carries a `version` field. The current `ManifestVersion` is **2** — version 2 carries a stable cortex `id` (a ULID) and re-keys the local event log + vector clock from cortex-name to cortex-id. Cortexes written by older binaries must run `noema migrate cortex-id` (an explicit, interactive, backed-up migration in `internal/cli/cmd_migrate.go`) before they can be opened by this binary; the `--reset` flag handles the case where the directory is a copy of another cortex (Gotcha #3 in the design doc). +The runner is a thin hand-rolled loop over embedded `*.sql` files in `internal/db/migrations/`, sorted by leading version number. Currently applied: + +- `001_initial.sql` — base trace metadata, tags, and FTS tables +- `002_trash.sql` — soft-delete/trash lifecycle +- `003_events_and_lineage.sql` — event log, `trace_lineage`, and `traces.origin` +- `004_federation_state.sql` — key-value store for vector clocks and peer cursors +- `005_cortex_identity.sql` — `cortex_id` on `traces` and `events` +- `006_content_hash.sql` — `content_hash` on `traces` +- `007_source_locking.sql` — `source_locked` and `source_hash` on `traces` +- `008_fts5_include_tags.sql` — include tags in the FTS5 search document +- `009_memory_tiering.sql` — short/mid/long tier metadata and tier votes +- `010_long_term_trigger_refine.sql` — long-term immutability trigger refinements +- `011_purge_columns.sql` — purge audit metadata +- `012_lineage_count_view.sql` — derived-from count view for consolidation signals +- `013_long_term_trigger_visibility.sql` — long-tier visibility trigger behavior +- `014_trace_usage.sql` — per-peer read/modify usage signals +- `015_search_hit_count.sql` — search hit usage signal for top-N results +- `016_trace_embeddings.sql` — local semantic-search embedding index + +`cortex.md` carries a `version` field. The current `ManifestVersion` is **2** — version 2 carries a stable cortex `id` (a ULID) and re-keys the local event log + vector clock from cortex-name to cortex-id. Cortexes written by older binaries must run `noema migrate cortex-id` (an explicit, interactive, backed-up migration in `internal/cli/cmd_migrate.go`) before they can be opened by this binary; the `--reset` flag handles the case where the directory is a copy of another cortex and needs a fresh identity. ### Cortex Creation diff --git a/README.md b/README.md index 557de7c..c8880a2 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,8 @@ noema search [flags] Full-text search (FTS5). --semantic / rank by embedding similarity (needs a search: block + backfill) noema similar [--limit N] Find traces related to (BM25; --semantic / --hybrid for embeddings) noema embeddings status Show semantic-search embedding coverage (embedded / stale / missing) -noema embeddings backfill [--force] Embed traces that are missing or stale (for semantic search) +noema embeddings backfill [--force] [--limit N] + Embed traces that are missing or stale (for semantic search) noema archive Archive a Trace noema unarchive Restore an archived Trace diff --git a/plugins/obsidian/package-lock.json b/plugins/obsidian/package-lock.json index 09be21c..5d9338a 100644 --- a/plugins/obsidian/package-lock.json +++ b/plugins/obsidian/package-lock.json @@ -1,12 +1,12 @@ { "name": "noema-obsidian", - "version": "0.1.0", + "version": "0.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "noema-obsidian", - "version": "0.1.0", + "version": "0.2.1", "license": "MIT", "devDependencies": { "@types/node": "^20", From 98eb74751f9ea9b30ccdaee794b05ee4e7d5afbf Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:00 -0400 Subject: [PATCH 2/9] test: migration 017 backfill + cross-cortex signing wire interop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add regression coverage for the federation event-signing feature: - migration017_test.go: assert the signature/pubkey columns exist as NOT NULL DEFAULT '' and that a row inserted without them backfills to the empty (unsigned) sentinel rather than NULL, the property a mixed-version ring relies on to treat pre-signing events as unsigned. - signing_interop_test.go: exercise the full cross-cortex wire path the in-process unit tests skip — emit on cortex A, JSON-serialize exactly as sync_events transmits, deserialize, and ReplayEvent on cortex B. Covers signed round-trip under enforce (with TOFU pin), in-transit tamper rejection, the mixed-version unsigned-upstream matrix (enforce rejects / off accepts), and no-silent-downgrade-after-pin. Co-Authored-By: Claude Opus 4.8 --- internal/cortex/signing_interop_test.go | 226 ++++++++++++++++++++++++ internal/db/migration017_test.go | 94 ++++++++++ 2 files changed, 320 insertions(+) create mode 100644 internal/cortex/signing_interop_test.go create mode 100644 internal/db/migration017_test.go diff --git a/internal/cortex/signing_interop_test.go b/internal/cortex/signing_interop_test.go new file mode 100644 index 0000000..b371261 --- /dev/null +++ b/internal/cortex/signing_interop_test.go @@ -0,0 +1,226 @@ +package cortex + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Fail-Safe/Noema/internal/event" + "github.com/Fail-Safe/Noema/internal/eventsig" + "github.com/Fail-Safe/Noema/internal/federation" + "github.com/Fail-Safe/Noema/internal/trace" +) + +// These tests exercise the cross-cortex wire path that the in-process unit +// tests in signing_test.go deliberately skip: a real cortex A emits a signed +// event, the event is serialized to JSON exactly as sync_events transmits it, +// deserialized on the far side, and replayed into a real cortex B. This is the +// only place a JSON-canonicalization drift in the signed preimage (Data, +// vclock, or timestamp bytes) would surface — struct-to-struct replay can't +// catch it because it never round-trips through the wire encoding. + +// openSignedCortexWithID opens a signed cortex with a caller-chosen id so two +// distinct peers can coexist in one test (openVerifyCortex hardcodes one id). +func openSignedCortexWithID(t *testing.T, id, mode string) *Cortex { + t.Helper() + dir := filepath.Join(t.TempDir(), "c") + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + _, pub, seed, err := eventsig.Generate() + if err != nil { + t.Fatal(err) + } + writeSeedFile(t, dir, "noema-signing.key", seed, 0o600) + m := Manifest{ + Name: "c", + Created: "2026-01-01T00:00:00Z", + Version: ManifestVersion, + ID: id, + Signing: &SigningConfig{PublicKey: pub, PrivateKeyFile: "noema-signing.key"}, + Federation: &FederationConfig{Verify: mode}, + } + if err := WriteManifest(dir, m); err != nil { + t.Fatal(err) + } + cx, err := Open("c", dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { cx.Close() }) + return cx +} + +// openUnsignedCortexWithMode opens an unsigned cortex (no keygen) that still +// runs in the given verify mode — the "pre-signing peer that just upgraded its +// binary" shape used by the mixed-version cases. +func openUnsignedCortexWithMode(t *testing.T, id, mode string) *Cortex { + t.Helper() + dir := filepath.Join(t.TempDir(), "u") + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + m := Manifest{ + Name: "u", + Created: "2026-01-01T00:00:00Z", + Version: ManifestVersion, + ID: id, + Federation: &FederationConfig{Verify: mode}, + } + if err := WriteManifest(dir, m); err != nil { + t.Fatal(err) + } + cx, err := Open("u", dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { cx.Close() }) + return cx +} + +// emitAndWire adds a trace to src, then returns its create event after a full +// JSON marshal/unmarshal round trip — the exact bytes a peer receives from +// sync_events. The wire form, not the in-memory struct, is what gets replayed. +func emitAndWire(t *testing.T, src *Cortex, title, body string) event.Event { + t.Helper() + tr := trace.New(title, "fact", "agent-1", []string{"x"}, body) + if err := src.Add(tr); err != nil { + t.Fatalf("Add: %v", err) + } + evs, err := src.Events(tr.ID) + if err != nil { + t.Fatalf("Events: %v", err) + } + var create *event.Event + for i := range evs { + if evs[i].Action == event.ActionCreate { + create = &evs[i] + } + } + if create == nil { + t.Fatal("no create event emitted") + } + raw, err := json.Marshal(create) + if err != nil { + t.Fatalf("marshal event: %v", err) + } + var wire event.Event + if err := json.Unmarshal(raw, &wire); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + return wire +} + +const ( + idPeerA = "01HVPEERA00000000000000000" + idPeerB = "01HVPEERB00000000000000000" +) + +// TestInterop_SignedRoundTripEnforce is the core gap-closer: A signs a create, +// it travels over the JSON wire, and an enforce-mode B accepts it (TOFU-pinning +// A's key) and materializes the trace — proving the signature survives the wire +// encoding and the pinned key persists for the next event. +func TestInterop_SignedRoundTripEnforce(t *testing.T) { + a := openSignedCortexWithID(t, idPeerA, VerifyOff) + b := openSignedCortexWithID(t, idPeerB, VerifyEnforce) + + wire := emitAndWire(t, a, "Shared fact", "body that crosses the wire") + if wire.Signature == "" || wire.PubKey == "" { + t.Fatal("emitted wire event lacks signature/pubkey") + } + + if err := b.ReplayEvent(wire); err != nil { + t.Fatalf("enforce-mode B should accept A's validly signed event over the wire: %v", err) + } + + // B TOFU-pinned A's key under A's cortex_id. + pinned, err := federation.NewState(b.DB.DB).GetCortexPubKey(idPeerA) + if err != nil { + t.Fatal(err) + } + if pinned != wire.PubKey { + t.Fatalf("B pinned %q, want A's key %q", pinned, wire.PubKey) + } + + // B materialized the trace and stored the event with signature intact, so + // it can re-serve it transitively. + if _, err := b.Get(wire.TraceID); err != nil { + t.Fatalf("trace not materialized on B: %v", err) + } + stored, err := b.Events(wire.TraceID) + if err != nil { + t.Fatal(err) + } + if len(stored) == 0 || stored[0].Signature != wire.Signature { + t.Fatal("B did not persist the originating signature for transitive relay") + } +} + +// TestInterop_WireTamperRejected mutates a field after the event is on the wire +// (as a malicious relay would) and confirms enforce-mode B rejects it. The +// pinned-key path is exercised by pre-pinning A's key first. +func TestInterop_WireTamperRejected(t *testing.T) { + a := openSignedCortexWithID(t, idPeerA, VerifyOff) + b := openSignedCortexWithID(t, idPeerB, VerifyEnforce) + + wire := emitAndWire(t, a, "Authentic", "untampered body") + if err := federation.NewState(b.DB.DB).SetCortexPubKey(idPeerA, wire.PubKey); err != nil { + t.Fatal(err) + } + + tampered := wire + tampered.Origin = "relay-in-the-middle" // Origin is part of the signed preimage + if err := b.ReplayEvent(tampered); err == nil { + t.Fatal("enforce-mode B must reject an event tampered in transit") + } + + // The untampered original still verifies under the same pinned key. + if err := b.ReplayEvent(wire); err != nil { + t.Fatalf("the authentic event should still verify: %v", err) + } +} + +// TestInterop_MixedVersionUnsignedUpstream is the backward/forward-compat +// matrix: an unsigned (pre-signing) peer's event is rejected by an enforce peer +// but accepted by an off peer, so upgrading the binary without enabling verify +// never breaks an existing ring. +func TestInterop_MixedVersionUnsignedUpstream(t *testing.T) { + upstream := openUnsignedCortexWithMode(t, idPeerA, VerifyOff) + wire := emitAndWire(t, upstream, "Legacy", "from an unsigned cortex") + if wire.Signature != "" { + t.Fatalf("unsigned cortex should emit no signature, got %q", wire.Signature) + } + + enforce := openUnsignedCortexWithMode(t, idPeerB, VerifyEnforce) + if err := enforce.ReplayEvent(wire); err == nil { + t.Fatal("enforce mode must reject an unsigned upstream event") + } + + off := openUnsignedCortexWithMode(t, idPeerB, VerifyOff) + if err := off.ReplayEvent(wire); err != nil { + t.Fatalf("off mode must accept an unsigned upstream event (backward compat): %v", err) + } +} + +// TestInterop_NoSilentDowngradeAfterPin closes the downgrade hole at the replay +// layer: once B has pinned A's key, an event from A's cortex_id that arrives +// stripped of its signature (an attacker or a buggy relay trying to dodge +// verification) is rejected under enforce rather than silently accepted. +func TestInterop_NoSilentDowngradeAfterPin(t *testing.T) { + a := openSignedCortexWithID(t, idPeerA, VerifyOff) + b := openSignedCortexWithID(t, idPeerB, VerifyEnforce) + + wire := emitAndWire(t, a, "Pinned", "establishes the pin") + if err := b.ReplayEvent(wire); err != nil { + t.Fatalf("first signed event should pin and verify: %v", err) + } + + downgraded := wire + downgraded.ID = "01HVEVENTDOWNGRADE00000000" + downgraded.Signature = "" + downgraded.PubKey = "" + if err := b.ReplayEvent(downgraded); err == nil { + t.Fatal("a pinned cortex cannot silently downgrade to unsigned under enforce") + } +} diff --git a/internal/db/migration017_test.go b/internal/db/migration017_test.go new file mode 100644 index 0000000..c89069c --- /dev/null +++ b/internal/db/migration017_test.go @@ -0,0 +1,94 @@ +package db_test + +import ( + "testing" + + "github.com/Fail-Safe/Noema/internal/db" +) + +// ---- Migration 017: event signature + pubkey ---- + +// TestMigration017_ColumnsAndSchema locks in the wire-format guarantee from +// 017_event_signature.sql: both columns exist on events, are NOT NULL, and +// default to the empty (= unsigned) sentinel. The defaults are load-bearing — +// scanEvents never special-cases NULL, and an empty signature is the agreed +// "produced before signing was configured" marker. +func TestMigration017_ColumnsAndSchema(t *testing.T) { + conn, err := db.Open(t.TempDir()) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer conn.Close() + + rows, err := conn.Query(`PRAGMA table_info(events)`) + if err != nil { + t.Fatalf("PRAGMA table_info: %v", err) + } + defer rows.Close() + type colInfo struct { + notnull int + dflt string + } + cols := map[string]colInfo{} + for rows.Next() { + var cid, notnull, pk int + var name, ctype string + var dflt *string + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + t.Fatalf("Scan: %v", err) + } + var d string + if dflt != nil { + d = *dflt + } + cols[name] = colInfo{notnull: notnull, dflt: d} + } + for _, c := range []string{"signature", "pubkey"} { + info, ok := cols[c] + if !ok { + t.Errorf("events.%s missing after migration 017", c) + continue + } + if info.notnull != 1 { + t.Errorf("events.%s should be NOT NULL", c) + } + // SQLite renders a string default as the quoted literal ''. + if info.dflt != "''" { + t.Errorf("events.%s default = %q, want \"''\"", c, info.dflt) + } + } +} + +// TestMigration017_BackfillsUnsignedSentinel is the backward-compat regression: +// a row inserted without the signing columns (the shape every pre-017 event +// has) must read back as the empty sentinel rather than NULL. This is exactly +// what the NOT NULL DEFAULT '' columns guarantee for rows that predate signing, +// and what lets a mixed-version ring treat old events as cleanly unsigned. +func TestMigration017_BackfillsUnsignedSentinel(t *testing.T) { + conn, err := db.Open(t.TempDir()) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer conn.Close() + + // Insert using only the pre-signing columns, omitting signature/pubkey. + if _, err := conn.Exec( + `INSERT INTO events (id, action, trace_id, cortex_id, origin, timestamp) VALUES (?, ?, ?, ?, ?, ?)`, + "01EVENT017", "create", "20260610-t", "01CORTEX", "peer-a", "2026-06-10T00:00:00Z", + ); err != nil { + t.Fatalf("insert pre-signing event: %v", err) + } + + var sig, pub string + if err := conn.QueryRow( + `SELECT signature, pubkey FROM events WHERE id = ?`, "01EVENT017", + ).Scan(&sig, &pub); err != nil { + t.Fatalf("read back event: %v", err) + } + if sig != "" { + t.Errorf("signature = %q, want empty sentinel", sig) + } + if pub != "" { + t.Errorf("pubkey = %q, want empty sentinel", pub) + } +} From 2a6dcbafe527fb3caf451069a00fc7bdf1816a73 Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:01 -0400 Subject: [PATCH 3/9] feat(federation): authenticate events with Ed25519 signatures Sign every emitted event with a per-cortex Ed25519 key and verify replayed events against the originating cortex's pinned key, moving the federation trust anchor from the shared transport key to the cortex that authored each event. Once cortex_id is authenticated, source-locking is enforced on the replay path: a source-locked trace may only be mutated by its owning cortex. - eventsig: canonical, length-prefixed, domain-separated preimage with sign/verify and ed25519: key+signature encoding. - noema keygen generates/rotates the key into a 0600 sidecar and records the public key in cortex.md; the cortex_identity handshake advertises it. - Key distribution is trust-on-first-use keyed on cortex_id, with conflict detection and downgrade resistance. A high-assurance peer's key can be hard-pinned via pubkey: on its cortex.md entry, which overrides TOFU and refuses a mismatching handshake regardless of verify mode. - Verification is staged via federation.verify (off default | warn | enforce) so a mixed-version ring never hard-partitions; migration 017 adds the additive, NOT NULL DEFAULT '' signature/pubkey columns. Co-Authored-By: Claude Opus 4.8 --- README.md | 29 ++ SECURITY.md | 21 + internal/cli/cmd_federation.go | 7 + internal/cli/cmd_keygen.go | 153 +++++++ internal/cli/cmd_keygen_test.go | 132 ++++++ internal/cli/cmd_serve.go | 2 +- internal/cli/root.go | 2 +- internal/cortex/access.go | 35 +- internal/cortex/cortex.go | 117 ++++- internal/cortex/signing.go | 202 +++++++++ internal/cortex/signing_test.go | 408 ++++++++++++++++++ .../db/migrations/017_event_signature.sql | 28 ++ internal/event/event.go | 41 +- internal/event/store.go | 13 +- internal/eventsig/eventsig.go | 224 ++++++++++ internal/eventsig/eventsig_test.go | 229 ++++++++++ internal/federation/federation.go | 7 +- internal/federation/signing_pin_test.go | 127 ++++++ internal/federation/state.go | 23 + internal/federation/syncer.go | 104 ++++- internal/mcp/server.go | 20 + 21 files changed, 1889 insertions(+), 35 deletions(-) create mode 100644 internal/cli/cmd_keygen.go create mode 100644 internal/cli/cmd_keygen_test.go create mode 100644 internal/cortex/signing.go create mode 100644 internal/cortex/signing_test.go create mode 100644 internal/db/migrations/017_event_signature.sql create mode 100644 internal/eventsig/eventsig.go create mode 100644 internal/eventsig/eventsig_test.go create mode 100644 internal/federation/signing_pin_test.go diff --git a/README.md b/README.md index c8880a2..532a0b4 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ Noema gives AI agents — and the humans working alongside them — a persistent | **Trace** | A single memory — one markdown file + its database row | | **Cortex** | A named collection of Traces, stored in a directory you control | +Contributor and architecture notes: + +- [Repository Guidelines](AGENTS.md) +- [Architecture](docs/architecture.md) +- [Development Guide](docs/development.md) + A Trace has a **type** that describes its intent: | Type | Meaning | @@ -661,6 +667,29 @@ noema verify drift # check federated traces against source hashes noema edit --force # override source-lock ``` +Locally, source-locking is enforced by refusing foreign-origin mutations. Across a federation, that guarantee is only as strong as the events that carry it — which is what **event signing** (below) protects. + +### Federation event signing + +Content hashing proves a body matches its hash; it does not prove *who* wrote the event. On a shared-key federation, any peer holding the key could otherwise forge events under another cortex's identity, overwrite source-locked traces, or rewrite `source_hash` so drift checks still pass. Event signing closes that gap by authenticating the originating cortex with an Ed25519 signature. + +```bash +noema keygen # generate this cortex's Ed25519 signing key +noema keygen --force # rotate the key (peers must re-pin) +``` + +Once a cortex has a key, it signs every event it emits. Peers learn its public key through the `cortex_identity` handshake and pin it per cortex id (trust-on-first-use); a later key change is refused until you run `noema federation reset-peer`. Verification of *incoming* events is staged so a mixed-version ring never hard-partitions, controlled by `federation.verify` in `cortex.md`: + +| `federation.verify` | Behavior on a bad/unsigned event | +|---|---| +| `off` (default) | accept — no change for cortexes that haven't enabled signing | +| `warn` | accept, but log every unsigned/forged/unverifiable event | +| `enforce` | reject — only correctly-signed events from their owning cortex are replayed | + +Under `enforce`, source-lock enforcement extends to replay: a locked trace can only be mutated by the cortex that owns it. + +Trust-on-first-use has a first-contact window. For a high-assurance peer you can skip it by hard-pinning the peer's key out-of-band — add `pubkey: ed25519:` to that peer's entry under `federation.peers` in `cortex.md`. The peer must then advertise exactly that key at the handshake or the sync is refused (overriding TOFU); to rotate, edit the pinned value. + ### Authentication Federation peers share a single bearer key — the same `NOEMA_MCP_KEY` / `access.shared_key_file` described in [Shared-key authentication](#shared-key-authentication) above. When the syncer polls a peer's `sync_events` tool, it automatically attaches `Authorization: Bearer ` from the local host's active key; nothing peer-specific lives in `cortex.md`. This means: diff --git a/SECURITY.md b/SECURITY.md index 46e2109..cae46c2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,6 +28,27 @@ Out of scope: - Vulnerabilities that require local filesystem access to the cortex directory (Noema trusts the local operator) - Issues in dependencies — please report those upstream, but feel free to flag them here if Noema's usage makes the issue exploitable +## Federation Trust Model + +Federation authenticity rests on two layers. The transport layer (the shared +bearer key, optionally over TLS) controls *who may connect*. The event layer +(per-cortex Ed25519 signatures) authenticates *which cortex authored each +event*, independent of which peer relayed it. A cortex that has run `noema keygen` signs every event it emits; +peers pin its public key per cortex id on first contact and, when +`federation.verify` is set to `enforce`, reject any event that is not correctly +signed by its owning cortex — which also extends source-lock enforcement to the +replay path. + +Known residual: key distribution defaults to trust-on-first-use, so an attacker +who intercepts the very first handshake or event from a cortex id can pin their +own key for it. This is mitigated by running the handshake over TLS and, for a +high-assurance peer, by hard-pinning its key out-of-band: set `pubkey:` on that +peer's entry in `cortex.md` and the peer must advertise exactly that key or the +sync is refused, bypassing first-use entirely. Even so, signing makes federation +*authenticated-trust*, not zero-trust. Reports that defeat the intended +guarantees **after** a key is correctly pinned — signature forgery, verification +bypass, source-lock bypass under `enforce`, or downgrade attacks — are in scope. + ## Supported Versions Only the latest release is actively supported with security patches. Upgrade to the latest version before reporting. diff --git a/internal/cli/cmd_federation.go b/internal/cli/cmd_federation.go index e37f5fa..c101b6a 100644 --- a/internal/cli/cmd_federation.go +++ b/internal/cli/cmd_federation.go @@ -365,6 +365,13 @@ func runFederationResetPeer(out io.Writer, in io.Reader, cx *cortex.Cortex, name delete(vc, s.pinnedID) bucketsDropped++ } + // Drop the pinned federation signing key for this cortex_id too, + // so a peer that rotated its key (`noema keygen --force`) gets + // re-pinned cleanly on the next handshake instead of failing the + // signing-key mismatch check forever. + if err := state.Delete(federation.CortexPubKeyKey(s.pinnedID)); err != nil { + return fmt.Errorf("clearing signing-key pin for peer %q: %w", s.name, err) + } } } diff --git a/internal/cli/cmd_keygen.go b/internal/cli/cmd_keygen.go new file mode 100644 index 0000000..2ee7a55 --- /dev/null +++ b/internal/cli/cmd_keygen.go @@ -0,0 +1,153 @@ +package cli + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/Fail-Safe/Noema/internal/config" + "github.com/Fail-Safe/Noema/internal/cortex" + "github.com/Fail-Safe/Noema/internal/eventsig" +) + +// defaultSigningKeyFile is the sidecar filename written into the cortex +// directory when keygen is not told otherwise. It mirrors the convention of +// keeping per-cortex secrets beside cortex.md. +const defaultSigningKeyFile = "noema-signing.key" + +func keygenCmd() *cobra.Command { + var force bool + + cmd := &cobra.Command{ + Use: "keygen", + Short: "Generate this Cortex's Ed25519 federation signing key", + Long: `Generate the Ed25519 keypair this Cortex uses to sign federated events. + +Signing authenticates the cortex that produced an event, so peers can refuse +forged or tampered events instead of trusting any holder of the shared +federation key. + +What it does: + 1. Reads cortex.md and requires a stable cortex id (run + 'noema migrate cortex-id' first if missing). + 2. Generates a fresh Ed25519 keypair. + 3. Writes the private seed to a 0600 sidecar file (default + ` + defaultSigningKeyFile + `) beside cortex.md. The seed is never printed. + 4. Records the public key and the sidecar path in cortex.md under 'signing'. + +Peers learn the public key automatically on their next sync via the +cortex_identity handshake and pin it on first contact (trust-on-first-use). + +Use --force to rotate: it overwrites the existing key. Rotation invalidates +every signature this cortex emitted under the old key, so peers that pinned the +old key will reject this cortex until they re-pin — coordinate a rotation the +same way you would a shared-key change.`, + Example: " noema keygen\n noema keygen --cortex research\n noema keygen --force", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("loading config: %w", err) + } + name := cortexFlag + if name == "" { + name = os.Getenv("NOEMA_CORTEX") + } + if name == "" { + name = cfg.Default + } + if name == "" { + return fmt.Errorf("no cortex specified: use --cortex, set NOEMA_CORTEX, or run `noema use `") + } + entry, ok := cfg.Cortexes[name] + if !ok { + return fmt.Errorf("unknown cortex %q", name) + } + return runKeygen(cmd.OutOrStdout(), name, entry.Path, force) + }, + } + cmd.Flags().BoolVar(&force, "force", false, "overwrite an existing signing key (rotation); peers must re-pin") + _ = cmd.RegisterFlagCompletionFunc("cortex", cortexNameCompletions) + return cmd +} + +// runKeygen is the testable core: it operates on a cortex directory, generates +// and persists the signing key, and updates cortex.md. It deliberately never +// writes the private seed to out — only the public key and file paths. +func runKeygen(out io.Writer, name, dir string, force bool) error { + m, err := cortex.ReadManifest(dir) + if err != nil { + return fmt.Errorf("reading cortex.md: %w", err) + } + if m.ID == "" { + return fmt.Errorf( + "cortex %q has no stable id yet; run `noema migrate cortex-id --cortex %s` before generating a signing key", + name, name, + ) + } + + // Idempotency: a cortex that already has a working signing key is left + // alone unless --force is given, so re-running keygen is safe. + if !force && m.Signing != nil && m.Signing.PublicKey != "" { + if _, err := cortex.LoadSigningKey(dir, m.Signing); err == nil { + fmt.Fprintf(out, "Cortex %q already has a signing key:\n public key: %s\nUse --force to rotate.\n", name, m.Signing.PublicKey) + return nil + } + // Configured but unreadable (missing/corrupt sidecar) — fall through + // and regenerate so the operator isn't stuck, but say why. + fmt.Fprintf(out, "Cortex %q has a configured signing key that could not be loaded; regenerating.\n", name) + } + + keyFile := defaultSigningKeyFile + if m.Signing != nil && m.Signing.PrivateKeyFile != "" { + keyFile = m.Signing.PrivateKeyFile + } + keyPath := keyFile + if !filepath.IsAbs(keyPath) { + keyPath = filepath.Join(dir, keyFile) + } + + _, pub, seed, err := eventsig.Generate() + if err != nil { + return fmt.Errorf("generating signing key: %w", err) + } + + // Write the seed with owner-only permissions. O_TRUNC so a rotation + // cleanly replaces the old seed; an explicit Chmod guards the case where + // the file already existed with looser bits. + if err := writeSecretFile(keyPath, seed+"\n"); err != nil { + return fmt.Errorf("writing signing key file: %w", err) + } + + m.Signing = &cortex.SigningConfig{PublicKey: pub, PrivateKeyFile: keyFile} + if err := cortex.WriteManifest(dir, m); err != nil { + return fmt.Errorf("writing cortex.md: %w", err) + } + + fmt.Fprintf(out, "Generated signing key for cortex %q.\n", name) + fmt.Fprintf(out, " public key: %s\n", pub) + fmt.Fprintf(out, " private key: %s (mode 0600 — keep it secret, never commit it)\n", keyPath) + fmt.Fprintf(out, " cortex.md: signing block updated\n") + fmt.Fprintln(out, "\nPeers pin this public key on their next sync (trust-on-first-use).") + return nil +} + +// writeSecretFile writes content to path with 0600 permissions, truncating any +// existing file and forcing the mode even if the file pre-existed with looser +// bits. Used for the signing-key seed sidecar. +func writeSecretFile(path, content string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return err + } + if _, err := f.WriteString(content); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Chmod(path, 0o600) +} diff --git a/internal/cli/cmd_keygen_test.go b/internal/cli/cmd_keygen_test.go new file mode 100644 index 0000000..2091615 --- /dev/null +++ b/internal/cli/cmd_keygen_test.go @@ -0,0 +1,132 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/Fail-Safe/Noema/internal/cortex" +) + +// writeTestManifest drops a minimal cortex.md with the given id into dir. +func writeTestManifest(t *testing.T, dir, id string) { + t.Helper() + md := "---\nname: test\ncreated: 2026-01-01T00:00:00Z\nversion: 2\n" + if id != "" { + md += "id: " + id + "\n" + } + md += "---\n" + if err := os.WriteFile(filepath.Join(dir, "cortex.md"), []byte(md), 0o640); err != nil { + t.Fatal(err) + } +} + +func TestRunKeygenGeneratesAndPersists(t *testing.T) { + dir := t.TempDir() + writeTestManifest(t, dir, "01HV0000000000000000000CTX") + + var out bytes.Buffer + if err := runKeygen(&out, "test", dir, false); err != nil { + t.Fatalf("runKeygen: %v", err) + } + + // The seed must never appear in user-facing output. + m, err := cortex.ReadManifest(dir) + if err != nil { + t.Fatal(err) + } + if m.Signing == nil || m.Signing.PublicKey == "" { + t.Fatal("manifest signing block was not written") + } + if m.Signing.PrivateKeyFile != defaultSigningKeyFile { + t.Fatalf("unexpected key file: %s", m.Signing.PrivateKeyFile) + } + + // Sidecar exists with owner-only perms (Unix). + keyPath := filepath.Join(dir, defaultSigningKeyFile) + info, err := os.Stat(keyPath) + if err != nil { + t.Fatalf("sidecar missing: %v", err) + } + if runtime.GOOS != "windows" { + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("sidecar mode %#o, want 0600", perm) + } + } + + // The configured key must load and round-trip. + k, err := cortex.LoadSigningKey(dir, m.Signing) + if err != nil { + t.Fatalf("LoadSigningKey after keygen: %v", err) + } + if k.Public != m.Signing.PublicKey { + t.Fatal("loaded public key does not match manifest") + } + + // Sanity: the secret seed is not echoed to the user. + seed, _ := os.ReadFile(keyPath) + seedLine := strings.TrimSpace(string(seed)) + if seedLine != "" && strings.Contains(out.String(), seedLine) { + t.Fatal("keygen output leaked the private seed") + } +} + +func TestRunKeygenIdempotentWithoutForce(t *testing.T) { + dir := t.TempDir() + writeTestManifest(t, dir, "01HV0000000000000000000CTX") + + var out bytes.Buffer + if err := runKeygen(&out, "test", dir, false); err != nil { + t.Fatal(err) + } + m1, _ := cortex.ReadManifest(dir) + + out.Reset() + if err := runKeygen(&out, "test", dir, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "already has a signing key") { + t.Fatalf("second run should be a no-op, got: %q", out.String()) + } + m2, _ := cortex.ReadManifest(dir) + if m1.Signing.PublicKey != m2.Signing.PublicKey { + t.Fatal("idempotent run must not rotate the key") + } +} + +func TestRunKeygenForceRotates(t *testing.T) { + dir := t.TempDir() + writeTestManifest(t, dir, "01HV0000000000000000000CTX") + + var out bytes.Buffer + if err := runKeygen(&out, "test", dir, false); err != nil { + t.Fatal(err) + } + m1, _ := cortex.ReadManifest(dir) + + out.Reset() + if err := runKeygen(&out, "test", dir, true); err != nil { + t.Fatal(err) + } + m2, _ := cortex.ReadManifest(dir) + if m1.Signing.PublicKey == m2.Signing.PublicKey { + t.Fatal("--force must rotate to a new key") + } +} + +func TestRunKeygenRequiresCortexID(t *testing.T) { + dir := t.TempDir() + writeTestManifest(t, dir, "") // no id + + var out bytes.Buffer + err := runKeygen(&out, "test", dir, false) + if err == nil { + t.Fatal("keygen without a cortex id should error") + } + if !strings.Contains(err.Error(), "migrate cortex-id") { + t.Fatalf("error should point to the fix, got: %v", err) + } +} diff --git a/internal/cli/cmd_serve.go b/internal/cli/cmd_serve.go index 61fa040..8e3eb5a 100644 --- a/internal/cli/cmd_serve.go +++ b/internal/cli/cmd_serve.go @@ -761,7 +761,7 @@ func startSyncer(cx *cortex.Cortex, sharedKey string, fc *cortex.FederationConfi var peers []federation.PeerConfig for _, p := range fc.Peers { peers = append(peers, federation.PeerConfig{ - Name: p.Name, Endpoint: p.Endpoint, CA: p.CA, Mode: p.Mode, + Name: p.Name, Endpoint: p.Endpoint, CA: p.CA, Mode: p.Mode, PubKey: p.PubKey, }) } diff --git a/internal/cli/root.go b/internal/cli/root.go index dd9cdc5..d2baa0e 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -119,7 +119,7 @@ func init() { eventsCmd(), resolveCmd(), memoryCmd(), consolidateCmd(), embeddingsCmd(), ) addGrouped(groupCortex, - initCmd(), useCmd(), cortexCmd(), federationCmd(), migrateCmd(), + initCmd(), useCmd(), cortexCmd(), federationCmd(), migrateCmd(), keygenCmd(), ) addGrouped(groupIntegrity, verifyCmd(), diff --git a/internal/cortex/access.go b/internal/cortex/access.go index 9175064..460018d 100644 --- a/internal/cortex/access.go +++ b/internal/cortex/access.go @@ -107,37 +107,46 @@ func LoadAccessKey(cortexDir string, cfg *AccessConfig) (AccessKey, error) { }, nil } -// loadKeyFile reads a sidecar key file, enforcing permissions and -// format rules. Returns the parsed key with surrounding whitespace +// loadKeyFile reads the MCP shared-key sidecar file, enforcing permissions +// and format rules. Returns the parsed key with surrounding whitespace // stripped. func loadKeyFile(path string) (string, error) { + return loadSidecarLine(path, "access key file") +} + +// loadSidecarLine reads a single-secret sidecar file (the MCP shared key, or +// the Ed25519 signing-key seed), enforcing the same permission, size, and +// single-line format rules for both. The label is woven into every error so +// the message names the right file kind. Returns the first non-empty line +// with surrounding whitespace stripped. +func loadSidecarLine(path, label string) (string, error) { info, err := os.Stat(path) if err != nil { - return "", fmt.Errorf("reading access key file %s: %w", path, err) + return "", fmt.Errorf("reading %s %s: %w", label, path, err) } if info.IsDir() { - return "", fmt.Errorf("access key file %s is a directory", path) + return "", fmt.Errorf("%s %s is a directory", label, path) } if info.Size() > accessKeyMaxBytes { - return "", fmt.Errorf("access key file %s is %d bytes; maximum is %d", path, info.Size(), accessKeyMaxBytes) + return "", fmt.Errorf("%s %s is %d bytes; maximum is %d", label, path, info.Size(), accessKeyMaxBytes) } // SSH-style permissions check: group/other bits must be zero. This // is a Unix-only guardrail — on Windows info.Mode().Perm() does not // carry POSIX bits, so the check is a no-op there (v1 is Unix-first). if mode := info.Mode().Perm(); mode&0o077 != 0 { return "", fmt.Errorf( - "refusing to read access key file %s: mode %#o is too permissive; chmod 600 %s", - path, mode, path, + "refusing to read %s %s: mode %#o is too permissive; chmod 600 %s", + label, path, mode, path, ) } data, err := os.ReadFile(path) if err != nil { - return "", fmt.Errorf("reading access key file %s: %w", path, err) + return "", fmt.Errorf("reading %s %s: %w", label, path, err) } - // Parse: first non-empty trimmed line is the key; two or more - // non-empty lines is an error (v1 does not support multi-key + // Parse: first non-empty trimmed line is the secret; two or more + // non-empty lines is an error (v1 does not support multi-secret // files). CRLF line endings are handled naturally because the // per-line TrimSpace strips the trailing \r. var key string @@ -154,11 +163,11 @@ func loadKeyFile(path string) (string, error) { } switch { case nonEmpty == 0: - return "", fmt.Errorf("access key file %s is empty or whitespace-only", path) + return "", fmt.Errorf("%s %s is empty or whitespace-only", label, path) case nonEmpty > 1: return "", fmt.Errorf( - "access key file %s contains %d non-empty lines; v1 supports a single key per file", - path, nonEmpty, + "%s %s contains %d non-empty lines; v1 supports a single key per file", + label, path, nonEmpty, ) } return key, nil diff --git a/internal/cortex/cortex.go b/internal/cortex/cortex.go index 5342dd6..1d577d7 100644 --- a/internal/cortex/cortex.go +++ b/internal/cortex/cortex.go @@ -20,6 +20,7 @@ import ( "github.com/Fail-Safe/Noema/internal/config" "github.com/Fail-Safe/Noema/internal/db" "github.com/Fail-Safe/Noema/internal/event" + "github.com/Fail-Safe/Noema/internal/eventsig" "github.com/Fail-Safe/Noema/internal/federation" "github.com/Fail-Safe/Noema/internal/trace" ) @@ -39,6 +40,7 @@ type Manifest struct { Version int `yaml:"version"` Access *AccessConfig `yaml:"access,omitempty"` Federation *FederationConfig `yaml:"federation,omitempty"` + Signing *SigningConfig `yaml:"signing,omitempty"` Watch *WatchConfig `yaml:"watch,omitempty"` Consolidation *ConsolidationConfig `yaml:"consolidation,omitempty"` Search *SearchConfig `yaml:"search,omitempty"` @@ -71,6 +73,25 @@ type AccessConfig struct { TLSKeyPath string `yaml:"tls_key_path,omitempty"` } +// SigningConfig holds the cortex's Ed25519 federation signing material. +// PublicKey is safe to keep in the manifest and is what gets advertised to +// peers via cortex_identity; +// PrivateKeyFile points to a 0600 sidecar holding the base64 seed, never the +// secret itself — the same "manifest holds a pointer, not the secret" rule as +// AccessConfig.SharedKeyFile. A cortex with no SigningConfig emits unsigned +// events and federates exactly as before. +type SigningConfig struct { + // PublicKey is the "ed25519:" public key. Advertised in the + // cortex_identity handshake so peers can pin it and verify this + // cortex's events. + PublicKey string `yaml:"public_key,omitempty"` + + // PrivateKeyFile is a path to the sidecar file whose first non-empty + // line is the base64-encoded 32-byte Ed25519 seed. Relative paths are + // resolved against the cortex directory. + PrivateKeyFile string `yaml:"private_key_file,omitempty"` +} + // Federation mode constants. const ( FederationModeSync = "sync" // bidirectional: pull from peers + serve events @@ -84,11 +105,20 @@ const ( PeerModePaused = "paused" // configured but skipped by the syncer ) +// Federation signature-verification mode constants. They control how replayed +// events are checked against their signatures. +const ( + VerifyOff = "off" // do not verify event signatures + VerifyWarn = "warn" // verify and log problems, but accept the event + VerifyEnforce = "enforce" // reject events that fail verification +) + // FederationConfig holds peer declarations for cortex.md. type FederationConfig struct { Mode string `yaml:"mode,omitempty"` // sync | publish | subscribe Peers []PeerEntry `yaml:"peers,omitempty"` Interval string `yaml:"interval,omitempty"` // e.g. "30s", "1m" + Verify string `yaml:"verify,omitempty"` // off | warn | enforce } // EffectiveMode returns the configured federation mode, defaulting to "sync". @@ -99,12 +129,29 @@ func (fc *FederationConfig) EffectiveMode() string { return fc.Mode } +// EffectiveVerify returns the configured signature-verification mode, +// defaulting to "off" so upgrading a cortex never starts rejecting a peer +// that hasn't enabled signing yet. Operators opt into warn/enforce. +func (fc *FederationConfig) EffectiveVerify() string { + if fc == nil || fc.Verify == "" { + return VerifyOff + } + return fc.Verify +} + // PeerEntry is a peer declared in cortex.md. type PeerEntry struct { Name string `yaml:"name"` Endpoint string `yaml:"endpoint"` CA string `yaml:"ca,omitempty"` // path to CA cert for TLS verification Mode string `yaml:"mode,omitempty"` // sync | paused + + // PubKey optionally hard-pins this peer's "ed25519:" federation + // signing key. When set, the peer MUST advertise exactly this key at the + // identity handshake or the sync is refused — out-of-band operator intent + // that takes precedence over trust-on-first-use and closes the TOFU + // first-contact window for high-assurance peers. Empty means TOFU. + PubKey string `yaml:"pubkey,omitempty"` } // EffectiveMode returns the configured peer mode, defaulting to "sync". @@ -470,7 +517,9 @@ type Cortex struct { Name string // human-readable display label Dir string DB *db.DB - forceSourceLock bool // when true, checkSourceLock is a no-op + forceSourceLock bool // when true, checkSourceLock is a no-op + signKey SigningKey // Ed25519 signing material; zero value = unsigned mode + verifyMode string // off | warn | enforce — how replayed signatures are checked } // SetForceSourceLock enables or disables the source-lock override. When @@ -543,12 +592,25 @@ func (m Manifest) ValidateFederation() error { default: return fmt.Errorf("federation.mode %q is not valid; use sync, publish, or subscribe", m.Federation.Mode) } + switch m.Federation.EffectiveVerify() { + case VerifyOff, VerifyWarn, VerifyEnforce: + default: + return fmt.Errorf("federation.verify %q is not valid; use off, warn, or enforce", m.Federation.Verify) + } for _, p := range m.Federation.Peers { switch p.EffectiveMode() { case PeerModeSync, PeerModePaused: default: return fmt.Errorf("federation.peers[%s].mode %q is not valid; use sync or paused", p.Name, p.Mode) } + // A hard-pinned key must be a well-formed "ed25519:" public + // key. Catching a typo here fails fast at startup rather than turning + // every handshake with this peer into a confusing mid-sync rejection. + if p.PubKey != "" { + if _, err := eventsig.ParsePublic(p.PubKey); err != nil { + return fmt.Errorf("federation.peers[%s].pubkey is not a valid signing key: %w", p.Name, err) + } + } } return nil } @@ -1110,6 +1172,19 @@ noema resolve --custom "" # apply a custom merge Either choice updates the original trace, federates the resolution, and trashes the divergence trace. The MCP equivalent is the ` + "`resolve_divergence`" + ` tool. + +## Federation Event Signing + +If this Cortex has run ` + "`noema keygen`" + `, it signs every event it emits with an +Ed25519 key, and peers verify those signatures. The ` + "`federation.verify`" + ` setting in +` + "`cortex.md`" + ` controls how incoming events are checked: ` + "`off`" + ` (default), ` + "`warn`" + ` +(log problems but accept), or ` + "`enforce`" + ` (reject anything not correctly signed by its +owning cortex). Under ` + "`enforce`" + `, source-locking is also enforced on replay — a +source-locked trace can only be changed by the cortex that owns it. + +As an agent you do not handle keys; ` + "`noema keygen`" + ` is an operator step. Just be +aware that under ` + "`enforce`" + ` a federated mutation you make to another cortex's +source-locked trace will be rejected by that cortex. ` } @@ -1152,6 +1227,19 @@ func Open(name, dir string) (*Cortex, error) { return nil, err } } + + // Load federation signing material. A configured-but-broken key is + // a hard error rather than a silent downgrade to unsigned: an + // operator who ran `noema keygen` expects their events to be signed, + // and emitting unsigned events that enforce-mode peers will reject is + // a worse, quieter failure than refusing to start. + sk, skErr := LoadSigningKey(dir, m.Signing) + if skErr != nil { + conn.Close() + return nil, fmt.Errorf("loading signing key for cortex %q: %w", name, skErr) + } + cx.signKey = sk + cx.verifyMode = m.Federation.EffectiveVerify() } // Auto-purge expired trash. Best-effort: errors are silently ignored. @@ -2869,6 +2957,19 @@ func (c *Cortex) emitEvent(tx *sql.Tx, action event.Action, traceID, timestamp s Data: data, VClock: vc, } + // Sign the finalized event (all authenticated fields are set above) so + // peers can verify it came from this cortex, and attach our public key so + // peers that have never handshaked us directly (transitive relay) can + // still verify it. Only local emissions are signed here; replayed remote + // events keep their origin's signature and key. + if c.signKey.Signing() { + sig, err := eventsig.Sign(c.signKey.Private, *e) + if err != nil { + return fmt.Errorf("signing event %s: %w", e.ID, err) + } + e.Signature = sig + e.PubKey = c.signKey.Public + } if err := event.Append(tx, e); err != nil { return err } @@ -2905,6 +3006,20 @@ func setClockTx(tx *sql.Tx, vc federation.VClock) error { // ReplayEvent materializes a remote event locally without emitting a new event. // The remote event is stored in the local log with its original ID and origin. func (c *Cortex) ReplayEvent(e event.Event) error { + // Authenticate the event against its signature before anything else, so a + // forged or tampered event is rejected (enforce) or flagged (warn) before + // it can touch the event log, the trace table, or the filesystem. A no-op + // in the default "off" mode and for cortexes without signing configured. + if err := c.verifyReplayEvent(e); err != nil { + return err + } + // With cortex_id now authenticated, enforce source-locking on replay: a + // locked trace may only be mutated by its owning cortex. This is the + // federation half of CheckSourceLock and the reason signing exists. + if err := c.checkReplaySourceLock(e); err != nil { + return err + } + // Coordination events (consolidation Claim/Success/Fail) carry a // synthetic window ID as trace_id, not a real trace ID. They only // need to land in the event log — no trace-table side effect, no diff --git a/internal/cortex/signing.go b/internal/cortex/signing.go new file mode 100644 index 0000000..265fdb8 --- /dev/null +++ b/internal/cortex/signing.go @@ -0,0 +1,202 @@ +package cortex + +import ( + "crypto/ed25519" + "database/sql" + "errors" + "fmt" + "log" + "path/filepath" + + "github.com/Fail-Safe/Noema/internal/event" + "github.com/Fail-Safe/Noema/internal/eventsig" + "github.com/Fail-Safe/Noema/internal/federation" +) + +// SigningKey is a cortex's resolved Ed25519 signing material. The zero value +// represents unsigned mode: a cortex that has not run `noema keygen` emits +// events without a signature and federates exactly as it did before signing +// existed. +type SigningKey struct { + // Private is the expanded private key used to sign emitted events. Nil + // in unsigned mode. Like the MCP shared key, it must never be logged, + // written to the event log, or echoed in errors. + Private ed25519.PrivateKey + + // Public is the "ed25519:" public key advertised to peers. Safe + // to log and display. Empty in unsigned mode. + Public string +} + +// Signing reports whether usable signing material was loaded. +func (k SigningKey) Signing() bool { return k.Private != nil } + +// PublicKey returns this cortex's "ed25519:" federation signing public +// key, or "" if the cortex is unsigned. It is advertised to peers through the +// cortex_identity handshake so they can pin it and verify this cortex's events. +func (c *Cortex) PublicKey() string { return c.signKey.Public } + +// verifyReplayEvent enforces the federation verify mode against an incoming +// remote event. It is called at the top of ReplayEvent, before any state +// change. In "off" mode it is a no-op. In "warn" it logs problems but accepts; +// in "enforce" it returns an error, which pins the syncer cursor and stops the +// batch (the same path any replay failure takes). +func (c *Cortex) verifyReplayEvent(e event.Event) error { + if c.verifyMode == "" || c.verifyMode == VerifyOff { + return nil + } + + // An event claiming OUR cortex_id must verify under OUR key, never a key + // carried in the event — otherwise a forger could impersonate us by + // emitting under our id with their own pubkey. + if e.CortexID == c.ID { + if c.signKey.Public == "" { + return c.failVerify(e, fmt.Errorf("event claims this cortex's id %s but no local signing key is configured to check it", c.ID)) + } + if e.Signature == "" { + return c.failVerify(e, fmt.Errorf("event %s claims this cortex's id but carries no signature", e.ID)) + } + if err := eventsig.Verify(c.signKey.Public, e, e.Signature); err != nil { + return c.failVerify(e, fmt.Errorf("event %s claims our identity but does not verify under our key: %w", e.ID, err)) + } + return nil + } + + state := federation.NewState(c.DB.DB) + pinned, err := state.GetCortexPubKey(e.CortexID) + if err != nil { + return fmt.Errorf("loading pinned signing key for cortex %s: %w", e.CortexID, err) + } + + keyToUse := pinned + switch { + case pinned != "": + // A known cortex must verify under its pinned key. An event that also + // carries a different key is an impersonation attempt or an + // uncoordinated rotation — refuse it rather than silently re-pinning. + if e.PubKey != "" && !eventsig.PublicKeysEqual(e.PubKey, pinned) { + return c.failVerify(e, fmt.Errorf("event %s pubkey conflicts with the pinned key for cortex %s", e.ID, e.CortexID)) + } + case e.PubKey != "": + // No key pinned yet: trust-on-first-use the key carried in the event, + // but only after the signature actually verifies under it (below). + keyToUse = e.PubKey + default: + return c.failVerify(e, fmt.Errorf("no signing key available for cortex %s (event carries none and none is pinned)", e.CortexID)) + } + + if e.Signature == "" { + return c.failVerify(e, fmt.Errorf("unsigned event %s from cortex %s", e.ID, e.CortexID)) + } + if err := eventsig.Verify(keyToUse, e, e.Signature); err != nil { + return c.failVerify(e, fmt.Errorf("signature verification failed for event %s from cortex %s: %w", e.ID, e.CortexID, err)) + } + + // Verified. If we learned the key from the event, pin it now (TOFU) so a + // later event that swaps the key for this cortex_id is caught as a conflict. + if pinned == "" { + if err := state.SetCortexPubKey(e.CortexID, keyToUse); err != nil { + return fmt.Errorf("pinning signing key for cortex %s: %w", e.CortexID, err) + } + log.Printf("[federation] TOFU-pinned signing key for cortex %s from event %s", e.CortexID, e.ID) + } + return nil +} + +// failVerify applies the verify mode to a verification failure: enforce returns +// an error (rejecting the event); warn logs and accepts. +func (c *Cortex) failVerify(e event.Event, reason error) error { + if c.verifyMode == VerifyEnforce { + return fmt.Errorf("rejecting event %s (verify=enforce): %w", e.ID, reason) + } + log.Printf("[federation] signature warning (action=%s trace=%s): %v — accepted (verify=warn)", e.Action, e.TraceID, reason) + return nil +} + +// checkReplaySourceLock enforces, on the replay path, that a source-locked +// trace may only be mutated by the cortex that owns it. This is the federation +// counterpart to CheckSourceLock (which guards local mutations): without it, +// any peer could overwrite, trash, or purge a source-locked trace by emitting +// an event for it. +// +// The rule is only meaningful once cortex_id is authenticated, so it is gated +// on the verify mode exactly like signature verification: off skips it (a +// spoofed cortex_id would defeat it anyway), enforce rejects a violation, warn +// logs it. By the time this runs in enforce mode, verifyReplayEvent has already +// proven the event's cortex_id, so an attacker cannot pass the owner check by +// claiming the owner's identity — that forgery was already rejected at the +// signature step. This catches the remaining case: an authenticated-but- +// unauthorized cortex trying to mutate another cortex's locked trace. +func (c *Cortex) checkReplaySourceLock(e event.Event) error { + if c.verifyMode == "" || c.verifyMode == VerifyOff { + return nil + } + switch e.Action { + case event.ActionUpdate, event.ActionTrash, event.ActionPurge: + default: + return nil // create/archive/vote/tier/etc. don't overwrite locked content + } + + var sourceLocked int + var owner string + err := c.DB.QueryRow(`SELECT source_locked, cortex_id FROM traces WHERE id = ?`, e.TraceID).Scan(&sourceLocked, &owner) + if errors.Is(err, sql.ErrNoRows) { + return nil // we hold no local copy to protect + } + if err != nil { + return fmt.Errorf("checking source-lock for replayed event %s: %w", e.ID, err) + } + if sourceLocked == 0 || e.CortexID == owner { + return nil + } + + reason := fmt.Errorf("event %s (%s) targets source-locked trace %s owned by cortex %s but originates from cortex %s", + e.ID, e.Action, e.TraceID, owner, e.CortexID) + if c.verifyMode == VerifyEnforce { + return fmt.Errorf("rejecting source-lock violation: %w", reason) + } + log.Printf("[federation] source-lock warning: %v — accepted (verify=warn)", reason) + return nil +} + +// LoadSigningKey resolves a cortex's signing key from its manifest config. +// It returns the zero value (unsigned mode) when no signing config or no +// private-key file is present. The sidecar file is held to the same 0600 / +// size / single-line rules as the MCP shared key. +// +// When the manifest also records a public_key, it is checked against the key +// derived from the private seed: a mismatch is a hard error, because a +// manifest public_key that disagrees with the private key would make every +// signature this cortex emits fail verification on its peers — a silent, +// fleet-wide federation outage that is far better caught at startup. +func LoadSigningKey(cortexDir string, cfg *SigningConfig) (SigningKey, error) { + if cfg == nil || cfg.PrivateKeyFile == "" { + return SigningKey{}, nil + } + + path := cfg.PrivateKeyFile + if !filepath.IsAbs(path) { + path = filepath.Join(cortexDir, path) + } + + seed, err := loadSidecarLine(path, "signing key file") + if err != nil { + return SigningKey{}, err + } + + priv, err := eventsig.PrivateFromSeed(seed) + if err != nil { + return SigningKey{}, fmt.Errorf("parsing signing key %s: %w", path, err) + } + + pub := eventsig.EncodePublic(priv.Public().(ed25519.PublicKey)) + if cfg.PublicKey != "" && !eventsig.PublicKeysEqual(cfg.PublicKey, pub) { + return SigningKey{}, fmt.Errorf( + "signing key mismatch: cortex.md public_key does not match the key derived from %s "+ + "(re-run `noema keygen --force` to regenerate, or fix public_key)", + path, + ) + } + + return SigningKey{Private: priv, Public: pub}, nil +} diff --git a/internal/cortex/signing_test.go b/internal/cortex/signing_test.go new file mode 100644 index 0000000..af0eebb --- /dev/null +++ b/internal/cortex/signing_test.go @@ -0,0 +1,408 @@ +package cortex + +import ( + "crypto/ed25519" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Fail-Safe/Noema/internal/event" + "github.com/Fail-Safe/Noema/internal/eventsig" + "github.com/Fail-Safe/Noema/internal/federation" + "github.com/Fail-Safe/Noema/internal/trace" +) + +// makeSignedTraceEvent builds a create/update event with a valid content hash +// and signs it, so it survives both signature verification and replayCreate/ +// replayUpdate's own content-hash check. +func makeSignedTraceEvent(t *testing.T, priv ed25519.PrivateKey, pub, cortexID, origin, id, traceID string, action event.Action, body string, sourceLocked bool) event.Event { + t.Helper() + payload := map[string]any{ + "title": "Locked", + "type": "fact", + "author": "author-1", + "origin": origin, + "body": body, + "content_hash": trace.ContentHash(body), + "source_locked": sourceLocked, + } + data, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + e := event.Event{ + ID: id, + Action: action, + TraceID: traceID, + CortexID: cortexID, + Origin: origin, + Timestamp: "2026-01-01T00:00:00Z", + Data: data, + } + sig, err := eventsig.Sign(priv, e) + if err != nil { + t.Fatal(err) + } + e.Signature = sig + e.PubKey = pub + return e +} + +const selfCortexID = "01HVSELF000000000000000000" + +// openVerifyCortex opens a signed cortex with the given federation verify mode +// and returns it along with its own keypair (for self-origin event tests). +func openVerifyCortex(t *testing.T, mode string) (*Cortex, ed25519.PrivateKey, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "v") + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + selfPriv, selfPub, selfSeed, err := eventsig.Generate() + if err != nil { + t.Fatal(err) + } + writeSeedFile(t, dir, "noema-signing.key", selfSeed, 0o600) + m := Manifest{ + Name: "v", + Created: "2026-01-01T00:00:00Z", + Version: ManifestVersion, + ID: selfCortexID, + Signing: &SigningConfig{PublicKey: selfPub, PrivateKeyFile: "noema-signing.key"}, + Federation: &FederationConfig{Verify: mode}, + } + if err := WriteManifest(dir, m); err != nil { + t.Fatal(err) + } + cx, err := Open("v", dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { cx.Close() }) + return cx, selfPriv, selfPub +} + +// makeEvent builds a remote event and signs it with priv, attaching pub. +func makeEvent(t *testing.T, priv ed25519.PrivateKey, pub, cortexID, id string) event.Event { + t.Helper() + e := event.Event{ + ID: id, + Action: event.ActionCreate, + TraceID: "20260101-remote-trace", + CortexID: cortexID, + Origin: "peer", + Timestamp: "2026-01-01T00:00:00Z", + Data: event.NormalizeData(nil), + } + sig, err := eventsig.Sign(priv, e) + if err != nil { + t.Fatal(err) + } + e.Signature = sig + e.PubKey = pub + return e +} + +func TestVerifyReplay_OffIsNoop(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyOff) + // A totally bogus event (no signature, unknown cortex) is accepted in off mode. + e := event.Event{ID: "01HVEVENT00000000000000001", Action: event.ActionCreate, TraceID: "20260101-x", CortexID: "01HVPEER000000000000000000"} + if err := cx.verifyReplayEvent(e); err != nil { + t.Fatalf("off mode should accept anything, got: %v", err) + } +} + +func TestVerifyReplay_EnforceAcceptsValidPinned(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + peerPriv, peerPub, _, _ := eventsig.Generate() + const peerID = "01HVPEER000000000000000000" + if err := federation.NewState(cx.DB.DB).SetCortexPubKey(peerID, peerPub); err != nil { + t.Fatal(err) + } + e := makeEvent(t, peerPriv, peerPub, peerID, "01HVEVENT00000000000000002") + if err := cx.verifyReplayEvent(e); err != nil { + t.Fatalf("valid signed event from pinned cortex should verify: %v", err) + } +} + +func TestVerifyReplay_EnforceRejectsUnsigned(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + e := event.Event{ID: "01HVEVENT00000000000000003", Action: event.ActionCreate, TraceID: "20260101-x", CortexID: "01HVPEER000000000000000000"} + if err := cx.verifyReplayEvent(e); err == nil { + t.Fatal("enforce mode should reject an unsigned event from an unpinned cortex") + } +} + +func TestVerifyReplay_EnforceRejectsBadSignature(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + peerPriv, peerPub, _, _ := eventsig.Generate() + const peerID = "01HVPEER000000000000000000" + e := makeEvent(t, peerPriv, peerPub, peerID, "01HVEVENT00000000000000004") + e.Origin = "tampered" // invalidates the signature; pubkey still present + if err := cx.verifyReplayEvent(e); err == nil { + t.Fatal("enforce mode should reject an event whose signature no longer matches") + } +} + +func TestVerifyReplay_EnforceTOFUThenConflict(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + const peerID = "01HVPEER000000000000000000" + privA, pubA, _, _ := eventsig.Generate() + privB, pubB, _, _ := eventsig.Generate() + + // First sighting: no pin yet → TOFU-pin pubA after verifying. + first := makeEvent(t, privA, pubA, peerID, "01HVEVENT00000000000000005") + if err := cx.verifyReplayEvent(first); err != nil { + t.Fatalf("first event should TOFU-pin and verify: %v", err) + } + pinned, _ := federation.NewState(cx.DB.DB).GetCortexPubKey(peerID) + if !eventsig.PublicKeysEqual(pinned, pubA) { + t.Fatalf("expected pubA pinned, got %q", pinned) + } + + // A later event for the same cortex_id carrying a different key is a + // conflict — even though it is validly signed under its own key. + second := makeEvent(t, privB, pubB, peerID, "01HVEVENT00000000000000006") + if err := cx.verifyReplayEvent(second); err == nil { + t.Fatal("a key swap for a pinned cortex_id must be rejected") + } +} + +func TestVerifyReplay_EnforceRejectsSelfImpersonation(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + // An attacker forges an event under OUR cortex_id with their own key. + attackerPriv, attackerPub, _, _ := eventsig.Generate() + e := makeEvent(t, attackerPriv, attackerPub, selfCortexID, "01HVEVENT00000000000000007") + if err := cx.verifyReplayEvent(e); err == nil { + t.Fatal("an event claiming our identity signed by a foreign key must be rejected") + } +} + +func TestVerifyReplay_EnforceAcceptsValidSelf(t *testing.T) { + cx, selfPriv, selfPub := openVerifyCortex(t, VerifyEnforce) + // A genuine loopback of our own event verifies under our own key. + e := makeEvent(t, selfPriv, selfPub, selfCortexID, "01HVEVENT00000000000000008") + if err := cx.verifyReplayEvent(e); err != nil { + t.Fatalf("our own signed event should verify on loopback: %v", err) + } +} + +func TestVerifyReplay_WarnAcceptsButLogs(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyWarn) + // Unsigned event from an unpinned cortex: warn logs but accepts (returns nil). + e := event.Event{ID: "01HVEVENT00000000000000009", Action: event.ActionCreate, TraceID: "20260101-x", CortexID: "01HVPEER000000000000000000"} + if err := cx.verifyReplayEvent(e); err != nil { + t.Fatalf("warn mode should accept (log only), got: %v", err) + } +} + +// TestReplaySourceLock_RejectsForeignMutation is the Phase 6 payoff: with +// cortex_id authenticated, a source-locked trace can only be mutated by its +// owning cortex — the original Finding #1 hole, now closed. +func TestReplaySourceLock_RejectsForeignMutation(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyEnforce) + const tid = "20260101-locked-trace" + + // Publisher P creates a source-locked trace; replay accepts and records P + // as the owning cortex. + privP, pubP, _, _ := eventsig.Generate() + const peerP = "01HVPEERP00000000000000000" + create := makeSignedTraceEvent(t, privP, pubP, peerP, "peer-p", "01HVEV0000000000000000CRE", tid, event.ActionCreate, "original body", true) + if err := cx.ReplayEvent(create); err != nil { + t.Fatalf("replay of P's signed create should succeed: %v", err) + } + + // Attacker X — validly signed as itself — tries to overwrite P's locked + // trace. Identity forgery is impossible (it would fail signature checks), + // so X emits honestly under its own id; the source-lock owner check is + // what stops it. + privX, pubX, _, _ := eventsig.Generate() + const peerX = "01HVPEERX00000000000000000" + attack := makeSignedTraceEvent(t, privX, pubX, peerX, "peer-x", "01HVEV0000000000000000ATK", tid, event.ActionUpdate, "attacker body", true) + if err := cx.ReplayEvent(attack); err == nil { + t.Fatal("a foreign cortex mutating a source-locked trace must be rejected") + } + + // The owner P may update its own locked trace. + legit := makeSignedTraceEvent(t, privP, pubP, peerP, "peer-p", "01HVEV0000000000000000UPD", tid, event.ActionUpdate, "P's revised body", true) + if err := cx.ReplayEvent(legit); err != nil { + t.Fatalf("owner P updating its own locked trace should succeed: %v", err) + } +} + +// TestReplaySourceLock_OffModeSkips documents that without authenticated +// cortex_id (off mode) the owner check is intentionally not applied, since a +// spoofed cortex_id would defeat it anyway. +func TestReplaySourceLock_OffModeSkips(t *testing.T) { + cx, _, _ := openVerifyCortex(t, VerifyOff) + const tid = "20260101-locked-trace" + + privP, pubP, _, _ := eventsig.Generate() + create := makeSignedTraceEvent(t, privP, pubP, "01HVPEERP00000000000000000", "peer-p", "01HVEV0000000000000000CRE", tid, event.ActionCreate, "original body", true) + if err := cx.ReplayEvent(create); err != nil { + t.Fatalf("create replay: %v", err) + } + privX, pubX, _, _ := eventsig.Generate() + attack := makeSignedTraceEvent(t, privX, pubX, "01HVPEERX00000000000000000", "peer-x", "01HVEV0000000000000000ATK", tid, event.ActionUpdate, "other body", true) + if err := cx.ReplayEvent(attack); err != nil { + t.Fatalf("off mode does not enforce source-lock on replay, expected accept: %v", err) + } +} + +func writeSeedFile(t *testing.T, dir, name, seed string, mode os.FileMode) { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(seed+"\n"), mode); err != nil { + t.Fatal(err) + } + // os.WriteFile honors umask, so force the exact mode the test wants. + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } +} + +func TestLoadSigningKeyUnsigned(t *testing.T) { + for _, cfg := range []*SigningConfig{nil, {}, {PublicKey: "ed25519:x"}} { + k, err := LoadSigningKey(t.TempDir(), cfg) + if err != nil { + t.Fatalf("cfg %+v: unexpected error %v", cfg, err) + } + if k.Signing() { + t.Fatalf("cfg %+v: expected unsigned mode", cfg) + } + } +} + +func TestLoadSigningKeyValid(t *testing.T) { + dir := t.TempDir() + _, pub, seed, err := eventsig.Generate() + if err != nil { + t.Fatal(err) + } + writeSeedFile(t, dir, "noema-signing.key", seed, 0o600) + + k, err := LoadSigningKey(dir, &SigningConfig{PublicKey: pub, PrivateKeyFile: "noema-signing.key"}) + if err != nil { + t.Fatalf("LoadSigningKey: %v", err) + } + if !k.Signing() { + t.Fatal("expected signing mode") + } + if k.Public != pub { + t.Fatalf("public key mismatch: got %s want %s", k.Public, pub) + } +} + +func TestLoadSigningKeyRejectsLoosePerms(t *testing.T) { + dir := t.TempDir() + _, pub, seed, _ := eventsig.Generate() + writeSeedFile(t, dir, "noema-signing.key", seed, 0o644) + + _, err := LoadSigningKey(dir, &SigningConfig{PublicKey: pub, PrivateKeyFile: "noema-signing.key"}) + if err == nil { + t.Fatal("expected error for world-readable signing key file") + } +} + +func TestLoadSigningKeyPublicMismatch(t *testing.T) { + dir := t.TempDir() + _, _, seed, _ := eventsig.Generate() + _, otherPub, _, _ := eventsig.Generate() + writeSeedFile(t, dir, "noema-signing.key", seed, 0o600) + + _, err := LoadSigningKey(dir, &SigningConfig{PublicKey: otherPub, PrivateKeyFile: "noema-signing.key"}) + if err == nil { + t.Fatal("expected mismatch error when manifest public_key disagrees with the private seed") + } +} + +// TestEmitSignsEvents is the Phase 3 end-to-end check: a cortex configured +// with a signing key must produce events that verify under its public key, +// and any tampering with a stored event must break that verification. +func TestEmitSignsEvents(t *testing.T) { + dir := filepath.Join(t.TempDir(), "signed") + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + _, pub, seed, err := eventsig.Generate() + if err != nil { + t.Fatal(err) + } + writeSeedFile(t, dir, "noema-signing.key", seed, 0o600) + + m := Manifest{ + Name: "signed", + Created: "2026-01-01T00:00:00Z", + Version: ManifestVersion, + ID: "01HV0000000000000000000CTX", + Signing: &SigningConfig{PublicKey: pub, PrivateKeyFile: "noema-signing.key"}, + } + if err := WriteManifest(dir, m); err != nil { + t.Fatal(err) + } + + cx, err := Open("signed", dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer cx.Close() + + tr := trace.New("My fact", "fact", "agent-1", []string{"a"}, "Some body content.") + if err := cx.Add(tr); err != nil { + t.Fatalf("Add: %v", err) + } + + evs, err := cx.Events(tr.ID) + if err != nil { + t.Fatalf("Events: %v", err) + } + var create *event.Event + for i := range evs { + if evs[i].Action == event.ActionCreate { + create = &evs[i] + } + } + if create == nil { + t.Fatal("no create event recorded") + } + if create.Signature == "" { + t.Fatal("emitted event was not signed") + } + if err := eventsig.Verify(pub, *create, create.Signature); err != nil { + t.Fatalf("emitted signature does not verify: %v", err) + } + + // Tampering with any authenticated field must invalidate the signature. + tampered := *create + tampered.Origin = "attacker" + if err := eventsig.Verify(pub, tampered, create.Signature); err == nil { + t.Fatal("tampered event still verified — signature does not cover Origin") + } +} + +// TestUnsignedCortexEmitsUnsigned confirms the backward-compatible path: a +// cortex with no signing config emits events with an empty signature and does +// not error. +func TestUnsignedCortexEmitsUnsigned(t *testing.T) { + dir := filepath.Join(t.TempDir(), "plain") + cx, err := Open("plain", dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer cx.Close() + + tr := trace.New("Plain", "note", "agent-1", nil, "body") + if err := cx.Add(tr); err != nil { + t.Fatalf("Add: %v", err) + } + evs, err := cx.Events(tr.ID) + if err != nil { + t.Fatal(err) + } + for _, e := range evs { + if e.Signature != "" { + t.Fatalf("unsigned cortex produced a signature: %q", e.Signature) + } + } +} diff --git a/internal/db/migrations/017_event_signature.sql b/internal/db/migrations/017_event_signature.sql new file mode 100644 index 0000000..f5f6ccc --- /dev/null +++ b/internal/db/migrations/017_event_signature.sql @@ -0,0 +1,28 @@ +-- Adds the per-event Ed25519 signature that authenticates the originating +-- cortex per the federation signing wire spec (algorithm, canonical preimage, +-- "ed25519:" encoding, verification modes). +-- +-- Before signing, an event's authenticity rested entirely on the shared +-- bearer key at the transport hop, so any key-holding peer in a federation +-- could forge events as any cortex and overwrite or delete source-locked +-- traces. The signature moves the trust anchor to the originating cortex: +-- replay verifies each event under that cortex's pinned public key. +-- +-- The columns are NOT NULL DEFAULT '' rather than nullable so existing rows +-- backfill cleanly to the empty (= unsigned) sentinel and scans never have +-- to special-case NULL. An empty signature means "produced before signing +-- was configured"; how that is treated on replay is governed by the +-- federation verify mode (off | warn | enforce), not by this schema. These +-- are pure index/audit columns on the local event log and are never the +-- basis for federation identity decisions on their own -- they authenticate +-- the event body that is already synced, so a mixed-version ring degrades +-- gracefully: older peers omit the fields and are treated as unsigned. +-- +-- pubkey carries the signer's "ed25519:" public key with the event. +-- Federation gossips events transitively (peer A relays peer B's events to +-- peer C, which never handshakes B directly), so the signing key has to +-- travel with the event for a verifier to check a cortex_id it has not +-- directly pinned. It is stored so relayed events keep the key when this +-- cortex re-serves them via sync_events. +ALTER TABLE events ADD COLUMN signature TEXT NOT NULL DEFAULT ''; +ALTER TABLE events ADD COLUMN pubkey TEXT NOT NULL DEFAULT ''; diff --git a/internal/event/event.go b/internal/event/event.go index 0d4ca97..20e8bc0 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -44,12 +44,41 @@ const ( // on replay. Federation, vector clocks, and divergence detection all key on // CortexID; Origin is purely for audit-trail rendering. type Event struct { - ID string `json:"id"` // ULID + ID string `json:"id"` // ULID Action Action `json:"action"` TraceID string `json:"trace_id"` - CortexID string `json:"cortex_id"` // stable ULID identity (federation key) - Origin string `json:"origin"` // display name at write time - Timestamp string `json:"timestamp"` // RFC3339 UTC - Data json.RawMessage `json:"data,omitempty"` // action-specific payload - VClock map[string]uint64 `json:"vclock,omitempty"` // vector clock keyed on cortex IDs + CortexID string `json:"cortex_id"` // stable ULID identity (federation key) + Origin string `json:"origin"` // display name at write time + Timestamp string `json:"timestamp"` // RFC3339 UTC + Data json.RawMessage `json:"data,omitempty"` // action-specific payload + VClock map[string]uint64 `json:"vclock,omitempty"` // vector clock keyed on cortex IDs + + // Signature authenticates the originating cortex. It is the + // "ed25519:" form defined by the federation signing wire spec, + // computed over the canonical preimage of every other field. Empty + // for events produced before signing was configured; how empty or + // invalid signatures are treated is governed by the federation verify + // mode (off | warn | enforce). It is never part of its own preimage. + Signature string `json:"signature,omitempty"` + + // PubKey is the signer's "ed25519:" public key, carried so that + // a verifier can check an event whose CortexID it has never directly + // handshaked — federation gossips events transitively, so the signing + // key has to travel with the event. Like Signature, it is identifying + // metadata, NOT part of the signed preimage: an event claiming a + // CortexID that is already pinned must verify under the *pinned* key, so + // a forged PubKey can't impersonate a known cortex. Empty for unsigned + // events. + PubKey string `json:"pubkey,omitempty"` +} + +// NormalizeData returns the canonical on-the-wire form of an event payload. +// A nil or empty payload becomes the two bytes `{}`. Both the event store +// (before persistence) and the signing preimage builder MUST use this so the +// bytes that get stored, transmitted, and signed are byte-identical. +func NormalizeData(data json.RawMessage) json.RawMessage { + if len(data) == 0 { + return json.RawMessage("{}") + } + return data } diff --git a/internal/event/store.go b/internal/event/store.go index 0c06585..a7d2286 100644 --- a/internal/event/store.go +++ b/internal/event/store.go @@ -5,7 +5,7 @@ import ( "encoding/json" ) -const eventColumns = `id, action, trace_id, cortex_id, origin, timestamp, data, vclock` +const eventColumns = `id, action, trace_id, cortex_id, origin, timestamp, data, vclock, signature, pubkey` // Append inserts an event into the event log within an existing transaction. func Append(tx *sql.Tx, e *Event) error { @@ -13,13 +13,10 @@ func Append(tx *sql.Tx, e *Event) error { if err != nil { return err } - data := e.Data - if data == nil { - data = json.RawMessage("{}") - } + data := NormalizeData(e.Data) _, err = tx.Exec( - `INSERT INTO events (`+eventColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - e.ID, string(e.Action), e.TraceID, e.CortexID, e.Origin, e.Timestamp, string(data), string(vclock), + `INSERT INTO events (`+eventColumns+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + e.ID, string(e.Action), e.TraceID, e.CortexID, e.Origin, e.Timestamp, string(data), string(vclock), e.Signature, e.PubKey, ) return err } @@ -62,7 +59,7 @@ func scanEvents(rows *sql.Rows) ([]Event, error) { for rows.Next() { var e Event var data, vclock string - if err := rows.Scan(&e.ID, &e.Action, &e.TraceID, &e.CortexID, &e.Origin, &e.Timestamp, &data, &vclock); err != nil { + if err := rows.Scan(&e.ID, &e.Action, &e.TraceID, &e.CortexID, &e.Origin, &e.Timestamp, &data, &vclock, &e.Signature, &e.PubKey); err != nil { return nil, err } e.Data = json.RawMessage(data) diff --git a/internal/eventsig/eventsig.go b/internal/eventsig/eventsig.go new file mode 100644 index 0000000..6055751 --- /dev/null +++ b/internal/eventsig/eventsig.go @@ -0,0 +1,224 @@ +// Package eventsig implements the federation event-signing wire spec: +// an Ed25519 signature over a canonical, +// length-prefixed, domain-separated preimage of an event's authenticated +// fields. It is the single source of truth for how signatures are produced +// and verified; the emit path (cortex.emitEvent) and the replay path +// (cortex.ReplayEvent) both go through here so the two can never drift. +// +// This package is pure: it depends only on the event shape and the standard +// library, never on the database or filesystem. Keeping it side-effect-free +// is what makes the canonical preimage exhaustively testable. +package eventsig + +import ( + "bytes" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "sort" + "strings" + + "github.com/Fail-Safe/Noema/internal/event" +) + +// domainTag is the version-pinned domain-separation prefix that opens every +// preimage. It guarantees a Noema event signature can never be valid for any +// other protocol's message, or for a future signing-format version, even when +// an attacker can influence the bytes that follow. The trailing newline is +// part of the tag. +const domainTag = "noema-event-sig-v1\n" + +// keyAlg is the only signature scheme this version understands. Encoded keys +// and signatures carry it as a prefix ("ed25519:") so a verifier +// rejects an unrecognised scheme rather than guessing. The base64 is standard +// (padded) encoding of the raw bytes. +const keyAlg = "ed25519" + +var keyPrefix = keyAlg + ":" + +var ( + // ErrBadPublicKey is returned when a public key string is missing the + // scheme prefix, is not valid base64, or is not the right length. + ErrBadPublicKey = errors.New("eventsig: malformed public key") + // ErrBadSignature is returned when a signature string is malformed + // (wrong prefix, bad base64, wrong length) — distinct from a + // well-formed signature that simply does not verify. + ErrBadSignature = errors.New("eventsig: malformed signature") + // ErrBadSeed is returned when a private-key seed string is malformed. + ErrBadSeed = errors.New("eventsig: malformed seed") + // ErrVerify is returned when a well-formed signature does not verify + // against the event under the given public key. + ErrVerify = errors.New("eventsig: signature verification failed") +) + +// Preimage builds the canonical byte string that gets signed. The layout is +// fixed by the federation signing wire spec; any change here is a +// wire-format break and must bump domainTag. The event's own Signature field +// is deliberately excluded. +func Preimage(e event.Event) []byte { + var b bytes.Buffer + b.WriteString(domainTag) + writeField(&b, []byte(e.ID)) + writeField(&b, []byte(string(e.Action))) + writeField(&b, []byte(e.TraceID)) + writeField(&b, []byte(e.CortexID)) + writeField(&b, []byte(e.Origin)) + writeField(&b, []byte(e.Timestamp)) + writeField(&b, vclockBytes(e.VClock)) + + // The signature covers a hash of the payload rather than the payload + // itself: this binds content_hash, source_hash, source_locked, and body + // (all carried inside Data) without an unbounded preimage. NormalizeData + // matches exactly what the event store persists and what sync_events + // transmits, so the verifier re-hashes identical bytes. + sum := sha256.Sum256(event.NormalizeData(e.Data)) + writeField(&b, sum[:]) + + return b.Bytes() +} + +// vclockBytes serializes a vector clock deterministically: entries sorted by +// cortex-id key in byte order, each emitted as field(key) || u64be(value). +// A nil/empty clock yields an empty slice (the caller still wraps it in an +// outer field()). +func vclockBytes(vc map[string]uint64) []byte { + if len(vc) == 0 { + return nil + } + keys := make([]string, 0, len(vc)) + for k := range vc { + keys = append(keys, k) + } + sort.Strings(keys) + + var b bytes.Buffer + for _, k := range keys { + writeField(&b, []byte(k)) + var v [8]byte + binary.BigEndian.PutUint64(v[:], vc[k]) + b.Write(v[:]) + } + return b.Bytes() +} + +// writeField appends a 4-byte big-endian length prefix followed by the bytes. +// Length-prefixing makes the concatenation unambiguous: no choice of field +// values can be re-partitioned into a different sequence of fields. +func writeField(b *bytes.Buffer, v []byte) { + var n [4]byte + binary.BigEndian.PutUint32(n[:], uint32(len(v))) + b.Write(n[:]) + b.Write(v) +} + +// Sign returns the "ed25519:" signature of the event under priv. +func Sign(priv ed25519.PrivateKey, e event.Event) (string, error) { + if len(priv) != ed25519.PrivateKeySize { + return "", fmt.Errorf("%w: private key is %d bytes, want %d", ErrBadSeed, len(priv), ed25519.PrivateKeySize) + } + sig := ed25519.Sign(priv, Preimage(e)) + return keyPrefix + base64.StdEncoding.EncodeToString(sig), nil +} + +// Verify checks that sig is a valid signature for e under the encoded public +// key. It returns nil on success, ErrBadPublicKey/ErrBadSignature for +// malformed inputs, or ErrVerify when a well-formed signature does not match. +// Callers distinguish these because the verify mode treats a malformed or +// failed signature the same way but logs them differently. +func Verify(pub string, e event.Event, sig string) error { + pk, err := ParsePublic(pub) + if err != nil { + return err + } + raw, err := decodeScheme(sig, ed25519.SignatureSize) + if err != nil { + return fmt.Errorf("%w: %v", ErrBadSignature, err) + } + if !ed25519.Verify(pk, Preimage(e), raw) { + return ErrVerify + } + return nil +} + +// EncodePublic renders a public key as "ed25519:". +func EncodePublic(pub ed25519.PublicKey) string { + return keyPrefix + base64.StdEncoding.EncodeToString(pub) +} + +// ParsePublic decodes an "ed25519:" public key. +func ParsePublic(s string) (ed25519.PublicKey, error) { + raw, err := decodeScheme(s, ed25519.PublicKeySize) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadPublicKey, err) + } + return ed25519.PublicKey(raw), nil +} + +// EncodeSeed renders a 32-byte private-key seed as base64 for the sidecar +// file. The seed, not the expanded private key, is what gets stored. +func EncodeSeed(seed []byte) string { + return base64.StdEncoding.EncodeToString(seed) +} + +// PrivateFromSeed expands a base64-encoded 32-byte seed into a usable private +// key via ed25519.NewKeyFromSeed. +func PrivateFromSeed(s string) (ed25519.PrivateKey, error) { + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(s)) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadSeed, err) + } + if len(raw) != ed25519.SeedSize { + return nil, fmt.Errorf("%w: seed is %d bytes, want %d", ErrBadSeed, len(raw), ed25519.SeedSize) + } + return ed25519.NewKeyFromSeed(raw), nil +} + +// Generate creates a fresh keypair and returns the private key, its encoded +// public key ("ed25519:"), and the base64 seed for the sidecar file. +func Generate() (priv ed25519.PrivateKey, pub string, seed string, err error) { + s := make([]byte, ed25519.SeedSize) + if _, err = rand.Read(s); err != nil { + return nil, "", "", fmt.Errorf("eventsig: generating seed: %w", err) + } + priv = ed25519.NewKeyFromSeed(s) + return priv, EncodePublic(priv.Public().(ed25519.PublicKey)), EncodeSeed(s), nil +} + +// PublicKeysEqual reports whether two encoded public keys denote the same key, +// tolerating base64/whitespace differences by comparing the decoded bytes in +// constant time. Used by the TOFU pinning layer to decide whether an +// advertised key matches the one already pinned for a cortex. +func PublicKeysEqual(a, b string) bool { + pa, err := ParsePublic(a) + if err != nil { + return false + } + pb, err := ParsePublic(b) + if err != nil { + return false + } + return subtle.ConstantTimeCompare(pa, pb) == 1 +} + +// decodeScheme strips the mandatory "ed25519:" prefix, base64-decodes the +// remainder, and checks it is exactly wantLen bytes. +func decodeScheme(s string, wantLen int) ([]byte, error) { + s = strings.TrimSpace(s) + rest, ok := strings.CutPrefix(s, keyPrefix) + if !ok { + return nil, fmt.Errorf("missing %q scheme prefix", keyAlg) + } + raw, err := base64.StdEncoding.DecodeString(rest) + if err != nil { + return nil, fmt.Errorf("base64: %v", err) + } + if len(raw) != wantLen { + return nil, fmt.Errorf("decoded %d bytes, want %d", len(raw), wantLen) + } + return raw, nil +} diff --git a/internal/eventsig/eventsig_test.go b/internal/eventsig/eventsig_test.go new file mode 100644 index 0000000..e8b7de4 --- /dev/null +++ b/internal/eventsig/eventsig_test.go @@ -0,0 +1,229 @@ +package eventsig + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/Fail-Safe/Noema/internal/event" +) + +func sampleEvent() event.Event { + return event.Event{ + ID: "01HV000000000000000000000A", + Action: event.ActionUpdate, + TraceID: "20260609-why-we-chose-go", + CortexID: "01HV0000000000000000000CTX", + Origin: "research-cortex", + Timestamp: "2026-06-09T14:23:00Z", + Data: json.RawMessage(`{"title":"Why Go","content_hash":"abc","source_locked":true}`), + VClock: map[string]uint64{"01HV0000000000000000000CTX": 7, "01HV00000000000000000PEER": 3}, + } +} + +func mustKeypair(t *testing.T) ed25519.PrivateKey { + t.Helper() + priv, _, _, err := Generate() + if err != nil { + t.Fatalf("Generate: %v", err) + } + return priv +} + +func pub(priv ed25519.PrivateKey) string { + return EncodePublic(priv.Public().(ed25519.PublicKey)) +} + +func TestSignVerifyRoundTrip(t *testing.T) { + priv := mustKeypair(t) + e := sampleEvent() + + sig, err := Sign(priv, e) + if err != nil { + t.Fatalf("Sign: %v", err) + } + if !strings.HasPrefix(sig, keyPrefix) { + t.Fatalf("signature missing scheme prefix: %q", sig) + } + if err := Verify(pub(priv), e, sig); err != nil { + t.Fatalf("Verify of own signature failed: %v", err) + } +} + +func TestSignatureIsDeterministic(t *testing.T) { + priv := mustKeypair(t) + e := sampleEvent() + a, _ := Sign(priv, e) + b, _ := Sign(priv, e) + if a != b { + t.Fatalf("Ed25519 signatures should be deterministic: %q != %q", a, b) + } +} + +// TestTamperEveryField is the core security property: mutating any signed +// field must break verification. A gap here is a forgeable field. +func TestTamperEveryField(t *testing.T) { + priv := mustKeypair(t) + e := sampleEvent() + sig, _ := Sign(priv, e) + pk := pub(priv) + + tampers := map[string]func(*event.Event){ + "ID": func(x *event.Event) { x.ID = "01HV000000000000000000000B" }, + "Action": func(x *event.Event) { x.Action = event.ActionTrash }, + "TraceID": func(x *event.Event) { x.TraceID = "20260609-something-else" }, + "CortexID": func(x *event.Event) { x.CortexID = "01HV0000000000000000000EVL" }, + "Origin": func(x *event.Event) { x.Origin = "attacker" }, + "Timestamp": func(x *event.Event) { x.Timestamp = "2026-06-09T14:23:01Z" }, + "Data": func(x *event.Event) { + x.Data = json.RawMessage(`{"title":"Why Go","content_hash":"abc","source_locked":false}`) + }, + "VClock value": func(x *event.Event) { + x.VClock = map[string]uint64{"01HV0000000000000000000CTX": 8, "01HV00000000000000000PEER": 3} + }, + "VClock add key": func(x *event.Event) { + x.VClock = map[string]uint64{"01HV0000000000000000000CTX": 7, "01HV00000000000000000PEER": 3, "01HV0000000000000000000NEW": 1} + }, + "VClock drop key": func(x *event.Event) { x.VClock = map[string]uint64{"01HV0000000000000000000CTX": 7} }, + } + + for name, mutate := range tampers { + t.Run(name, func(t *testing.T) { + tampered := sampleEvent() + mutate(&tampered) + err := Verify(pk, tampered, sig) + if !errors.Is(err, ErrVerify) { + t.Fatalf("tampering %s did not invalidate the signature (err=%v)", name, err) + } + }) + } +} + +// TestSignatureFieldExcludedFromPreimage: the Signature field must not feed its +// own preimage, or signing would be impossible to reproduce on the verifier. +func TestSignatureFieldExcludedFromPreimage(t *testing.T) { + priv := mustKeypair(t) + e := sampleEvent() + sig, _ := Sign(priv, e) + + withSig := sampleEvent() + withSig.Signature = sig // verifier receives the event with the signature populated + if err := Verify(pub(priv), withSig, sig); err != nil { + t.Fatalf("populating Signature changed the preimage: %v", err) + } +} + +// TestDataNormalizationConsistency: nil, empty, and explicit "{}" payloads must +// all hash to the same preimage, matching what the event store persists. +func TestDataNormalizationConsistency(t *testing.T) { + base := sampleEvent() + base.VClock = nil + + variants := []json.RawMessage{nil, {}, json.RawMessage("{}")} + var want []byte + for i, d := range variants { + e := base + e.Data = d + got := Preimage(e) + if i == 0 { + want = got + continue + } + if string(got) != string(want) { + t.Fatalf("variant %d (%q) preimage differs from nil payload", i, string(d)) + } + } +} + +// TestVClockOrderIndependence: the canonical vclock must not depend on Go map +// iteration order. Building the same logical clock twice must yield one preimage. +func TestVClockOrderIndependence(t *testing.T) { + a := sampleEvent() + a.VClock = map[string]uint64{"aaa": 1, "bbb": 2, "ccc": 3} + b := sampleEvent() + b.VClock = map[string]uint64{"ccc": 3, "bbb": 2, "aaa": 1} + if string(Preimage(a)) != string(Preimage(b)) { + t.Fatal("preimage depends on vclock map order") + } +} + +func TestVerifyWrongKey(t *testing.T) { + signer := mustKeypair(t) + other := mustKeypair(t) + e := sampleEvent() + sig, _ := Sign(signer, e) + if err := Verify(pub(other), e, sig); !errors.Is(err, ErrVerify) { + t.Fatalf("verification under a different key should fail with ErrVerify, got %v", err) + } +} + +func TestMalformedInputs(t *testing.T) { + priv := mustKeypair(t) + e := sampleEvent() + goodSig, _ := Sign(priv, e) + goodPub := pub(priv) + + t.Run("missing sig prefix", func(t *testing.T) { + bare := strings.TrimPrefix(goodSig, keyPrefix) + if err := Verify(goodPub, e, bare); !errors.Is(err, ErrBadSignature) { + t.Fatalf("want ErrBadSignature, got %v", err) + } + }) + t.Run("bad sig base64", func(t *testing.T) { + if err := Verify(goodPub, e, keyPrefix+"!!!not base64!!!"); !errors.Is(err, ErrBadSignature) { + t.Fatalf("want ErrBadSignature, got %v", err) + } + }) + t.Run("wrong sig length", func(t *testing.T) { + if err := Verify(goodPub, e, keyPrefix+"YWJj"); !errors.Is(err, ErrBadSignature) { + t.Fatalf("want ErrBadSignature, got %v", err) + } + }) + t.Run("missing pub prefix", func(t *testing.T) { + bare := strings.TrimPrefix(goodPub, keyPrefix) + if err := Verify(bare, e, goodSig); !errors.Is(err, ErrBadPublicKey) { + t.Fatalf("want ErrBadPublicKey, got %v", err) + } + }) + t.Run("unknown scheme", func(t *testing.T) { + if _, err := ParsePublic("rsa:YWJj"); !errors.Is(err, ErrBadPublicKey) { + t.Fatalf("want ErrBadPublicKey, got %v", err) + } + }) +} + +func TestPrivateFromSeedRoundTrip(t *testing.T) { + priv, pubEnc, seed, err := Generate() + if err != nil { + t.Fatalf("Generate: %v", err) + } + restored, err := PrivateFromSeed(seed) + if err != nil { + t.Fatalf("PrivateFromSeed: %v", err) + } + e := sampleEvent() + sig, _ := Sign(restored, e) + if err := Verify(pubEnc, e, sig); err != nil { + t.Fatalf("signature from seed-restored key failed to verify: %v", err) + } + // And the restored key must match the original. + if string(restored) != string(priv) { + t.Fatal("seed round-trip produced a different private key") + } +} + +func TestPublicKeysEqual(t *testing.T) { + priv := mustKeypair(t) + a := pub(priv) + if !PublicKeysEqual(a, " "+a+"\n") { + t.Fatal("PublicKeysEqual should tolerate surrounding whitespace") + } + if PublicKeysEqual(a, pub(mustKeypair(t))) { + t.Fatal("distinct keys reported equal") + } + if PublicKeysEqual(a, "garbage") { + t.Fatal("malformed key reported equal") + } +} diff --git a/internal/federation/federation.go b/internal/federation/federation.go index 68dcb6f..5449097 100644 --- a/internal/federation/federation.go +++ b/internal/federation/federation.go @@ -14,8 +14,13 @@ const ( type PeerConfig struct { Name string `yaml:"name"` Endpoint string `yaml:"endpoint"` - CA string `yaml:"ca,omitempty"` // path to CA certificate for TLS verification + CA string `yaml:"ca,omitempty"` // path to CA certificate for TLS verification Mode string `yaml:"mode,omitempty"` // sync | paused + + // PubKey, when non-empty, hard-pins this peer's "ed25519:" + // signing key: the peer must advertise exactly it at the handshake or the + // sync is refused, overriding trust-on-first-use. + PubKey string `yaml:"pubkey,omitempty"` } // Config holds federation settings from cortex.md. diff --git a/internal/federation/signing_pin_test.go b/internal/federation/signing_pin_test.go new file mode 100644 index 0000000..cca7f45 --- /dev/null +++ b/internal/federation/signing_pin_test.go @@ -0,0 +1,127 @@ +package federation + +import ( + "testing" + + "github.com/Fail-Safe/Noema/internal/eventsig" +) + +func genPub(t *testing.T) string { + t.Helper() + _, pub, _, err := eventsig.Generate() + if err != nil { + t.Fatal(err) + } + return pub +} + +// TestPinSigningKey exercises the trust-on-first-use policy for federation +// signing keys: first contact pins, a matching re-advertisement is accepted, a +// changed key is refused, and an empty advertisement neither pins nor clears. +func TestPinSigningKey(t *testing.T) { + s := newSyncerForTest(t, &fakeReplayer{}) + const ctxID = "01HV0000000000000000000CTX" + pubA := genPub(t) + pubB := genPub(t) + + // First contact: pin. + if err := s.pinSigningKey("peer-a", ctxID, pubA, ""); err != nil { + t.Fatalf("first pin should succeed: %v", err) + } + got, _ := s.state.GetCortexPubKey(ctxID) + if !eventsig.PublicKeysEqual(got, pubA) { + t.Fatalf("key not pinned: got %q", got) + } + + // Re-advertising the same key is fine. + if err := s.pinSigningKey("peer-a", ctxID, pubA, ""); err != nil { + t.Fatalf("re-advertising same key should succeed: %v", err) + } + + // A different key for the same cortex_id is refused. + if err := s.pinSigningKey("peer-a", ctxID, pubB, ""); err == nil { + t.Fatal("changed signing key should be refused") + } + // ...and the original pin is untouched. + got, _ = s.state.GetCortexPubKey(ctxID) + if !eventsig.PublicKeysEqual(got, pubA) { + t.Fatal("refused mismatch must not overwrite the pinned key") + } + + // An empty advertisement is a no-op that preserves the existing pin + // (downgrade protection): a peer can't drop its key to dodge verification. + if err := s.pinSigningKey("peer-a", ctxID, "", ""); err != nil { + t.Fatalf("empty advertisement should be a no-op: %v", err) + } + got, _ = s.state.GetCortexPubKey(ctxID) + if !eventsig.PublicKeysEqual(got, pubA) { + t.Fatal("empty advertisement must not clear the pin") + } + + // A different cortex_id pins independently. + const otherCtx = "01HV00000000000000000OTHER" + if err := s.pinSigningKey("peer-b", otherCtx, pubB, ""); err != nil { + t.Fatalf("independent cortex pin should succeed: %v", err) + } + got, _ = s.state.GetCortexPubKey(otherCtx) + if !eventsig.PublicKeysEqual(got, pubB) { + t.Fatal("second cortex_id should pin its own key") + } +} + +// TestHardPinSigningKey covers the operator-configured hard-pin path, which +// overrides trust-on-first-use: a matching advertisement is accepted and +// pinned, a differing or absent advertisement fails the handshake, and a +// hard-pin overwrites a stale TOFU pin (explicit config beats first-use). +func TestHardPinSigningKey(t *testing.T) { + const ctxID = "01HV0000000000000000000CTX" + hardPin := genPub(t) + other := genPub(t) + + // Match on first contact: accept and pin the hard-pinned key. + t.Run("match pins", func(t *testing.T) { + s := newSyncerForTest(t, &fakeReplayer{}) + if err := s.pinSigningKey("peer-a", ctxID, hardPin, hardPin); err != nil { + t.Fatalf("advertised key matching the hard-pin should be accepted: %v", err) + } + got, _ := s.state.GetCortexPubKey(ctxID) + if !eventsig.PublicKeysEqual(got, hardPin) { + t.Fatalf("hard-pin not stored: got %q", got) + } + }) + + // A differing advertisement fails the handshake regardless of TOFU state. + t.Run("mismatch rejected", func(t *testing.T) { + s := newSyncerForTest(t, &fakeReplayer{}) + if err := s.pinSigningKey("peer-a", ctxID, other, hardPin); err == nil { + t.Fatal("an advertised key that differs from the hard-pin must be refused") + } + if got, _ := s.state.GetCortexPubKey(ctxID); got != "" { + t.Fatalf("a rejected hard-pin handshake must not store anything, got %q", got) + } + }) + + // A peer that advertises no key cannot satisfy a hard-pin (no silent + // downgrade to unsigned for a high-assurance peer). + t.Run("empty advertisement rejected", func(t *testing.T) { + s := newSyncerForTest(t, &fakeReplayer{}) + if err := s.pinSigningKey("peer-a", ctxID, "", hardPin); err == nil { + t.Fatal("a hard-pinned peer that advertises no key must be refused") + } + }) + + // A hard-pin overwrites a pre-existing, differing TOFU pin. + t.Run("overrides stale TOFU pin", func(t *testing.T) { + s := newSyncerForTest(t, &fakeReplayer{}) + if err := s.state.SetCortexPubKey(ctxID, other); err != nil { + t.Fatal(err) + } + if err := s.pinSigningKey("peer-a", ctxID, hardPin, hardPin); err != nil { + t.Fatalf("hard-pin should override a stale TOFU pin: %v", err) + } + got, _ := s.state.GetCortexPubKey(ctxID) + if !eventsig.PublicKeysEqual(got, hardPin) { + t.Fatalf("hard-pin did not override the stale pin: got %q", got) + } + }) +} diff --git a/internal/federation/state.go b/internal/federation/state.go index abeeeb2..4bd6397 100644 --- a/internal/federation/state.go +++ b/internal/federation/state.go @@ -168,3 +168,26 @@ func (s *State) SetPeerSeen(name, timestamp string) error { func (s *State) SetPeerCortexID(name, cortexID string) error { return s.Set(PeerCortexIDKey(name), cortexID) } + +// CortexPubKeyKey returns the federation_state key under which a cortex's +// pinned Ed25519 public key is stored. It is keyed on cortex_id rather than +// peer name on purpose: events gossip transitively, so the verifier must be +// able to look up the signing key for any originating cortex_id it sees, +// including third parties it has no direct peer entry for. +func CortexPubKeyKey(cortexID string) string { + return "cortexkey:" + cortexID +} + +// GetCortexPubKey returns the pinned public key for a cortex_id, or "" if none +// is pinned yet. +func (s *State) GetCortexPubKey(cortexID string) (string, error) { + return s.Get(CortexPubKeyKey(cortexID)) +} + +// SetCortexPubKey pins (or overwrites) the public key for a cortex_id. Callers +// own the trust policy — whether an overwrite is allowed (rotation), refused +// (handshake identity-change), or flagged (replay conflict) — and call this +// only once they have decided to (re-)pin. +func (s *State) SetCortexPubKey(cortexID, pubkey string) error { + return s.Set(CortexPubKeyKey(cortexID), pubkey) +} diff --git a/internal/federation/syncer.go b/internal/federation/syncer.go index 27ab32b..623918e 100644 --- a/internal/federation/syncer.go +++ b/internal/federation/syncer.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Fail-Safe/Noema/internal/event" + "github.com/Fail-Safe/Noema/internal/eventsig" "github.com/mark3labs/mcp-go/client" "github.com/mark3labs/mcp-go/client/transport" "github.com/mark3labs/mcp-go/mcp" @@ -616,6 +617,7 @@ func (s *Syncer) verifyPeerIdentity(mcpClient *client.Client, peer PeerConfig) e ID string `json:"id"` Name string `json:"name"` Version int `json:"version"` + PubKey string `json:"pubkey,omitempty"` Rank *RankEntry `json:"rank,omitempty"` } if err := json.Unmarshal([]byte(text), &identity); err != nil { @@ -655,15 +657,14 @@ func (s *Syncer) verifyPeerIdentity(mcpClient *client.Client, peer PeerConfig) e if err != nil { return fmt.Errorf("loading pinned cortex id for peer %q: %w", peer.Name, err) } - if pinned == "" { + switch { + case pinned == "": // First successful handshake — pin the ID. if err := s.state.SetPeerCortexID(peer.Name, identity.ID); err != nil { return fmt.Errorf("pinning cortex id for peer %q: %w", peer.Name, err) } log.Printf("[federation] peer %q identity pinned: %s (%s)", peer.Name, identity.Name, identity.ID) - return nil - } - if pinned != identity.ID { + case pinned != identity.ID: return fmt.Errorf( "peer %q identity mismatch: pinned id is %s but the endpoint now reports %s. "+ "This can happen if the peer was reset, replaced, or restored from a different cortex's backup. "+ @@ -671,6 +672,101 @@ func (s *Syncer) verifyPeerIdentity(mcpClient *client.Client, peer PeerConfig) e peer.Name, pinned, identity.ID, peer.Name, ) } + + // The cortex_id is trusted now (freshly pinned or matched its pin). Pin or + // verify the peer's federation signing key under that cortex_id. + return s.pinSigningKey(peer.Name, identity.ID, identity.PubKey, peer.PubKey) +} + +// pinSigningKey records a peer's advertised federation signing public key, +// keyed on its cortex_id rather than its peer name so that transitively-relayed +// events from the same cortex verify under the same key. It mirrors the +// cortex_id pin: first contact establishes trust-on-first-use, and a later +// change is refused with a reset-peer escape hatch, because a peer that swaps +// its signing key is indistinguishable from an impostor without an +// out-of-band check. +// +// An empty advertised key (an older peer, or an unsigned cortex) is a no-op: +// it neither pins nor clears. Preserving any existing pin means a peer cannot +// silently downgrade to unsigned to dodge enforce-mode replay verification. +// +// hardPin is the operator-asserted "ed25519:" key from this peer's +// cortex.md entry, or "" for trust-on-first-use. When set it overrides TOFU +// entirely (see hardPinSigningKey): the peer must advertise exactly it, and a +// mismatch fails the handshake regardless of verify mode. +func (s *Syncer) pinSigningKey(peerName, cortexID, advertised, hardPin string) error { + if hardPin != "" { + return s.hardPinSigningKey(peerName, cortexID, advertised, hardPin) + } + if advertised == "" { + return nil + } + pinned, err := s.state.GetCortexPubKey(cortexID) + if err != nil { + return fmt.Errorf("loading pinned signing key for peer %q: %w", peerName, err) + } + if pinned == "" { + if err := s.state.SetCortexPubKey(cortexID, advertised); err != nil { + return fmt.Errorf("pinning signing key for peer %q: %w", peerName, err) + } + log.Printf("[federation] peer %q signing key pinned (%s)", peerName, cortexID) + return nil + } + if !eventsig.PublicKeysEqual(pinned, advertised) { + return fmt.Errorf( + "peer %q signing-key mismatch: the pinned key for cortex %s differs from the one now advertised. "+ + "If the peer intentionally rotated its key (`noema keygen --force`), run `noema federation reset-peer %s` "+ + "to clear the pin — the next sync re-pins the new key", + peerName, cortexID, peerName, + ) + } + return nil +} + +// hardPinSigningKey enforces an operator-configured hard-pin: the key declared +// on the peer's cortex.md entry is authoritative, so trust-on-first-use does +// not apply. The peer must advertise exactly the hard-pinned key — an empty or +// differing advertisement fails the handshake regardless of verify mode, +// mirroring how a changed cortex_id always refuses. This is what closes the +// TOFU first-contact window for high-assurance peers. +// +// On a match, the cortex_id → pubkey store is set to the hard-pin so the replay +// path (including transitively-relayed events from this cortex_id) verifies +// against the operator-asserted key. A pre-existing TOFU pin that differs is +// overwritten — explicit out-of-band config beats first-use — and the override +// is logged so the change is auditable. To rotate a hard-pinned peer the +// operator edits cortex.md; no reset-peer is needed. +func (s *Syncer) hardPinSigningKey(peerName, cortexID, advertised, hardPin string) error { + if advertised == "" { + return fmt.Errorf( + "peer %q is hard-pinned to a signing key in cortex.md but advertised none; "+ + "the peer must run `noema keygen`, or remove the pubkey pin to fall back to trust-on-first-use", + peerName, + ) + } + if !eventsig.PublicKeysEqual(advertised, hardPin) { + return fmt.Errorf( + "peer %q signing-key mismatch: the key advertised by cortex %s does not match the pubkey hard-pinned "+ + "for this peer in cortex.md. If the peer intentionally rotated its key, update the pinned pubkey; "+ + "otherwise this may be an impersonation attempt", + peerName, cortexID, + ) + } + pinned, err := s.state.GetCortexPubKey(cortexID) + if err != nil { + return fmt.Errorf("loading pinned signing key for peer %q: %w", peerName, err) + } + if eventsig.PublicKeysEqual(pinned, hardPin) { + return nil + } + if err := s.state.SetCortexPubKey(cortexID, hardPin); err != nil { + return fmt.Errorf("applying hard-pin for peer %q: %w", peerName, err) + } + if pinned == "" { + log.Printf("[federation] peer %q signing key hard-pinned from cortex.md (%s)", peerName, cortexID) + } else { + log.Printf("[federation] peer %q hard-pin in cortex.md overrode the prior trust-on-first-use key (%s)", peerName, cortexID) + } return nil } diff --git a/internal/mcp/server.go b/internal/mcp/server.go index acc1b3e..4c6f0a3 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -766,6 +766,13 @@ func NewServer(cx *cortex.Cortex, noemaVersion string, federationMode string) *s "version": m.Version, "mode": m.Federation.EffectiveMode(), } + // Advertise the federation signing public key so peers can pin it + // (trust-on-first-use) and verify this cortex's events. Omitted when + // the cortex is unsigned; older peers ignore the field. The public + // key is not a secret. + if pub := cx.PublicKey(); pub != "" { + payload["pubkey"] = pub + } // Piggyback the current consolidation rank on the identity // response so federated peers can observe eligibility without a // separate round-trip. Missing state (feature off, eligibility @@ -1282,6 +1289,19 @@ Both list_traces and search_traces prefix each row with a tier glyph (s=short, m=mid, L=long) so you can see the tier without a second lookup. get_trace surfaces the full tier name on a dedicated "Tier:" line in the metadata block. +## Federation Event Signing +content_hash proves a body matches its hash but not which cortex authored an event. +Run noema keygen (CLI) to give a cortex an Ed25519 signing key; it then signs every +event it emits and advertises its public key through the cortex_identity handshake. +Peers pin that key per cortex id on first contact and refuse a later change until +noema federation reset-peer clears it. + +Incoming events are checked according to federation.verify in cortex.md: off (default, +no change), warn (log unsigned/forged events but accept), or enforce (reject anything +that is not correctly signed by its owning cortex). Under enforce, source-locking is +also enforced on replay: a locked trace can only be mutated by the cortex that owns it, +so a peer cannot overwrite, trash, or purge another cortex's locked trace. + ## External Filesystem Edits Whenever noema serve is running (stdio OR http), a background watcher observes the traces/, archive/traces/, and trash/traces/ directories. From 1599bbcd57c18f29c6f1428847fbf100eda20db6 Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:02 -0400 Subject: [PATCH 4/9] docs: restructure contributor docs into AGENTS.md + docs/ Replace the root CLAUDE.md with AGENTS.md and move reference material into docs/architecture.md and docs/development.md, linked from the README. Update the Makefile and a migration comment that pointed at CLAUDE.md, and ignore the generated docs/layout.md. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + AGENTS.md | 70 +++++ CLAUDE.md | 302 --------------------- Makefile | 2 +- docs/architecture.md | 102 +++++++ docs/development.md | 98 +++++++ internal/db/migrations/014_trace_usage.sql | 4 +- 7 files changed, 273 insertions(+), 306 deletions(-) create mode 100644 AGENTS.md delete mode 100644 CLAUDE.md create mode 100644 docs/architecture.md create mode 100644 docs/development.md diff --git a/.gitignore b/.gitignore index 1afbdad..807f381 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,6 @@ # Local dev .claude/ +docs/layout.md install.sh push-binary.sh diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a4eeda0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,70 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +Noema is a Go 1.26 CLI/MCP server. The executable entry point is +`cmd/noema/main.go`; implementation lives under `internal/`, grouped by domain: +`cli`, `cortex`, `db`, `mcp`, `federation`, `watch`, `consolidation`, `embed`, +`trace`, and related helpers. SQLite migrations are in `internal/db/migrations/`. +Tests generally sit next to the code they cover as `*_test.go`. See +[docs/architecture.md](docs/architecture.md) for the system model and +[docs/development.md](docs/development.md) for expanded workflow notes. + +Plugins are separate: `plugins/obsidian/` is a TypeScript Obsidian plugin, and +`plugins/hermes/` is a Python Hermes memory provider with pytest tests. Shared +scripts live in `scripts/`; security notes and playbooks live in `SECURITY.md` +and `tests/`. + +## Build, Test, and Development Commands + +- `make build`: builds the local development binary at `./noema` with version + metadata. +- `go run ./cmd/noema`: runs the CLI without keeping a binary. +- `go build ./...`: builds all Go packages; this is part of CI. +- `make test` or `go test ./...`: runs the Go test suite. +- `go test -race ./...`: runs tests with race detection, matching CI’s stronger + check. +- `make vet` or `go vet ./...`: runs Go vet. +- `cd plugins/obsidian && npm ci && npm run build && npx tsc --noEmit`: validates + the Obsidian plugin. +- `cd plugins/hermes && pytest`: runs Hermes plugin tests when Python changes. + +## Coding Style & Naming Conventions + +Use `gofmt`/`go fmt ./...` for Go formatting and keep packages small and +domain-named. Exported Go identifiers need clear names and documentation when +they form public package surface. Prefer existing Cobra command patterns in +`internal/cli/` and existing storage/mutation APIs in `internal/cortex/` over new +cross-cutting abstractions. SQL migrations use zero-padded numeric prefixes such +as `016_trace_embeddings.sql`. + +For TypeScript, keep sources in `plugins/obsidian/src/` and let the existing +esbuild and TypeScript configs define output and type checks. + +## Testing Guidelines + +Add focused `*_test.go` files beside changed Go code. Cover migrations, +watcher behavior, federation/vector-clock logic, MCP command surfaces, and +concurrency-sensitive paths when touched. Use race tests for background workers, +syncers, and server code. Plugin changes should include plugin-local tests or +build/type-check verification. + +## Commit & Pull Request Guidelines + +Recent history uses concise subjects with conventional prefixes where helpful: +`feat(tls): ...`, `release: v0.14.0 ...`, `chore: ...`. Keep commit subjects +specific and public-safe. Active feature work should target `next`; `main` is +for release and hotfix flow. + +PRs should describe behavior changes, link issues when applicable, and list the +checks run. Include screenshots only for Obsidian UI changes. Before opening a +PR, run Go build/vet/tests plus plugin checks for any plugin touched. + +## Agent-Specific Instructions + +At session start, read Noema preference traces with `tag: "user-preference"` +before assuming defaults. Treat trace bodies as binding unless the current user +message overrides them. Do not print secrets or secret-bearing HTTP bodies; +verify reachability with status codes, hashes, or length checks. Keep +public-facing commits, PRs, fixtures, and docs free of private hostnames, +personal identifiers, and internal agent/cortex names unless explicitly approved. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index b2e03f0..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,302 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -Also read .claude/layout.md when it exists. - -## Project - -**Noema** — "The intentional memory layer for your AI agents." - -Noema → The project -Cortex → The collection / instance -Trace → A memory or piece of knowledge - -## Etymology - -From Husserl's phenomenology: Noema is the intentional content of a mental act — the "what" that a thought is about, as it appears to consciousness. - -The paired concept is Noesis — the act of thinking itself. Noema is what is thought; Noesis is the thinking of it. Traces are the collective units [memory entry] comprised of a markdown doc + corresponding DB rows. The Cortex is the collection of Traces. - -## Commands - -```bash -go build ./cmd/noema # build the binary -go run ./cmd/noema # run without building -go build ./... # build all packages (CI check) -go test ./... # run all tests -go vet ./... # static analysis -``` - -## Branch Strategy - -- `main` is the stable/release branch -- Use a `feature` or `bug` branch for active work; PRs go feature/bug-branch → `next` -- Use a `next` branch for staging/testing upcoming (beta) work; PRs go `next` → `main` - -## Tech Stack - -Super simple, super basic. Intentionally lightweight. Transparently open. - -**Language:** Go (v1). Revisit Rust if MCP server concurrency demands it. -- SQLite: `modernc.org/sqlite` (pure-Go, no CGo required) -- TUI: Charm / Bubble Tea -- Search: SQLite FTS5 (lexical, always on) plus an **opt-in semantic / hybrid** - layer (embeddings + pure-Go cosine; see Semantic Search below). No CGo, no - vector extension, no external vector DB. - -Markdown files are the content holders of data. The SQLite DB is a basic schema to hold pieces of metadata about the markdown files (location, author, type, tag(s), ...) in order to provide mapping, related contexts, search performance. - -The collective unit [memory entry] comprised of markdown doc + corresponding DB rows will be known as a 'Trace'. - -A Trace shall have a single markdown doc, though the doc may be as small or large as needed. Traces may be categorized as one of several types based on intent. They may be tagged in such a way as to allow for distinct Traces to be formed into logical groupings or patterns as the data grows. - -A collection or instance containing Traces will be known as a 'Cortex'. - -### Trace Structure - -Every Trace is a markdown file with YAML frontmatter followed by free-form content: - -```markdown ---- -id: 20260329-why-we-chose-go -title: Why we chose Go -type: decision -author: research-agent-1 -tags: [go, language, architecture] -derived_from: [20260328-language-candidates] -origin: research-cortex -created: 2026-03-29T14:23:00Z -updated: 2026-03-29T14:23:00Z ---- - -Body content here. -``` - -All frontmatter fields are indexed in the DB. `author` is a free-form string — it can be a human username, an agent name, or omitted. Multi-agent systems use it to identify which peer wrote a given Trace. - -`derived_from` is an optional list of trace IDs this Trace was derived from — it builds a lineage graph queryable via `trace_lineage`. `origin` is the name of the Cortex that produced the Trace; it is set automatically on creation and used by federation to attribute remote Traces. Both fields use `omitempty`, so older Traces without them parse unchanged. - -### Trace Types - -Every Trace has exactly one type: - -| Type | Meaning | -|---|---| -| `fact` | A discrete thing that is true | -| `decision` | A choice made and why | -| `preference` | A behavioral or stylistic lean | -| `context` | Situational background | -| `skill` | A learned capability or procedure | -| `intent` | Something that needs to happen (pickup is autonomous) | -| `observation` | Something witnessed but not yet verified | -| `note` | Anything else | -| `divergence` | A concurrent edit conflict, auto-created by federation sync | - -### Cortex Storage Layout - -Each Cortex is a named directory the user manages. Layout: - -``` -/ - cortex.md ← Cortex manifest (see below) - archive/ - traces/ ← Archived traces, same format as active traces - db/ - noema.db ← SQLite DB: metadata, tags, FTS5 index - traces/ - 20260329-why-we-chose-go.md - 20260329-another-trace.md - trash/ - traces/ ← Removed traces, kept for 30 days [user-configurable] -``` - -Trace filenames follow the pattern `YYYYMMDD-slugified-title.md` (ISO 8601). The markdown files are the source of truth for content; the DB is the index. - -**`cortex.md` manifest** — a markdown file with YAML frontmatter. The -YAML block holds the manifest (minimal — not a config file); any prose -below the closing fence is preserved verbatim on write, so users can -keep free-form notes about the cortex alongside its metadata. - -```markdown ---- -id: 01JQ3Z4XKP8VY6H7B9W2R5T8MN -name: my-cortex -purpose: "Primary memory for the research agent cluster" -owner: mark -created: 2026-03-29 -version: 2 ---- - -Optional free-form notes about this cortex. -``` - -For back-compat, `ReadManifest` also accepts legacy bare-YAML files -(no `---` fences) written by older binaries; the next `WriteManifest` -silently upgrades them to the framed form. - -### Archiving - -Archiving is non-destructive and fully reversible: - -- `noema archive ` — moves the markdown file to `archive/traces/`, sets `archived_at` timestamp in DB -- `noema unarchive ` — moves it back to `traces/`, clears `archived_at` -- Archived Traces are **hidden by default** in all list/search output -- Surface them with `--archived` (archived only) or `--all` (active + archived) -- DB query pattern: active = `WHERE archived_at IS NULL`; archived = `WHERE archived_at IS NOT NULL` - -### Event Log, Lineage, and Federation - -Every mutation (create / update / archive / unarchive / trash / recover / purge) is recorded as an immutable event in the `events` table inside the same SQLite transaction as the mutation itself. Events carry a ULID, an action, the trace ID, the **stable cortex ID** (`cortex_id`, a ULID), the originating Cortex display name (`origin`, mutable), an RFC3339 timestamp, a JSON snapshot of the trace state (for create/update), and a vector clock snapshot keyed on cortex IDs. `Remove()` (hard delete) and `Sync()` (filesystem reconciliation) intentionally emit no events — hard deletes are local-only escape hatches and Sync is reconciliation, not a semantic mutation. - -**Filesystem watcher.** Under `noema serve` (stdio OR http), a background watcher in `internal/watch/` observes `traces/`, `archive/traces/`, and `trash/traces/` with fsnotify. External edits (Obsidian, VS Code, Finder drag, `rm`, iCloud sync from another device, a second noema process on the same cortex) are dispatched through the same `Cortex` mutation methods as MCP tools — `Add`, `Update`, and the `MarkArchivedNoMove` / `MarkTrashedNoMove` / `IngestExternalDelete` / `ApplyExternalPurge` helpers that update DB state and emit events without attempting a file move the OS already did. Loopback is prevented by content-hash comparison: a write whose body hash matches the DB's `content_hash` is Noema's own and gets skipped. Source-locked foreign traces are refused on external edit. The watcher is on by default; opt out with `watch: { enabled: false }` in `cortex.md`. Debounce window defaults to 300ms to collapse editor save bursts. **Atomic-save guard:** when reconcile sees a missing file with a known DB row, it waits one debounce window and re-stats before treating the absence as a delete. Editors like Obsidian save by removing then rewriting the file at the same path, briefly leaving it missing on disk; without this guard the watcher misclassifies the gap as `IngestExternalDelete` and trashes the live trace. **Frontmatter heal:** if a tracked active trace's file is rewritten without its frontmatter delimiter (Obsidian saving plain text, a script stripping YAML), the watcher reconstructs the frontmatter from the DB row + the file's raw content as body and emits an `Update` instead of silently skipping the edit. Source-locked foreign traces are excluded from heal so authoritative state stays untouched. See `internal/watch/reconcile.go` for the action dispatch table. Federation propagation is still HTTP-only (peers need a network endpoint), but external-edit events land in the local log during stdio sessions and flow outward the next time an HTTP serve runs. - -Lineage is a separate `trace_lineage` join table populated from the `derived_from` frontmatter field. `trace_lineage` supports both directions: `cortex.Get()` reads `derived_from`, and `cortex.DerivedBy()` reads the reverse edge. - -**Cortex identity.** Every Cortex has a stable ULID stored in `cortex.md` as `id` and surfaced via the `cortex_identity` MCP tool. The display name (`name`) can be renamed without affecting federation; the ID is the federation key. Vector clocks are keyed on cortex IDs, not names, so two peers that share a display name can no longer silently collapse into one bucket. The federation syncer pins a peer's ID on first contact and refuses to talk to an endpoint whose advertised ID has changed (peer reset, replaced, or restored from another cortex's backup). The same-name guardrail is still enforced at config time as a UX safety net even though identity is now ID-based. - -Federation is opt-in via a `federation:` block in `cortex.md` (peers + interval). When `noema serve --transport http` starts on a Cortex with peers configured, a background syncer polls each peer's `sync_events` MCP tool over Streamable HTTP (the `/mcp` endpoint, MCP 2025-03-26), replays new events into the local Cortex, and merges the remote vector clock into the local one. Replayed events are stored with their original ID, cortex_id, and origin so the event log becomes the union of all peers' events (no event amplification). When two peers update the same trace concurrently — vector clocks neither dominate nor are dominated — Noema creates a `type: divergence` Trace preserving every conflicting version under deterministically-sorted `### Version from ()` headers, rather than overwriting. Resolve via `noema resolve --accept ` or `--custom ` (or the `resolve_divergence` MCP tool). - -**Federation modes.** The `federation.mode` field in `cortex.md` controls sync directionality: - -| Mode | Syncer runs? | `sync_events` serves? | HTTP write tools? | -|------|-------------|----------------------|-------------------| -| `sync` (default) | Yes | Yes | Yes | -| `publish` | No | Yes | Blocked (read-only for remote peers) | -| `subscribe` | Yes | Blocked | Yes | - -In publish mode, mutating MCP tools (`create_trace`, `update_trace`, `append_trace`, `delete_trace`, `recover_trace`, `archive_trace`, `unarchive_trace`, `resolve_divergence`) return a clear error on the HTTP transport. The local stdio transport retains full write access so operators can manage content locally. In subscribe mode, `sync_events` returns an error so remote peers cannot pull events from this cortex. - -Each peer can also declare `mode: paused` to temporarily skip syncing without losing cursor/identity state. CLI commands: `noema federation set-mode `, `noema federation pause-peer `, `noema federation resume-peer `. - -**Content hashing and source-locking.** Every trace mutation (`Add`, `Update`, `Append`) computes a SHA-256 hash of the body (`sha256:`) and stores it in the `content_hash` frontmatter field and DB column. The hash travels through federation events so peers receive it. Three frontmatter fields support integrity: - -- `content_hash` — current body hash, recomputed on every write -- `source_hash` — the origin's `content_hash` at publish time, carried through federation -- `source_locked` — when `true`, consumer-side tooling refuses `Update`, `Trash`, and `Remove` on this trace (only enforced when `origin != local cortex name`; the publisher can always edit its own traces) - -`Archive` and `Unarchive` remain allowed on source-locked traces (non-destructive). CLI commands accept `--force` to bypass the lock. MCP tools surface `ErrSourceLocked` as a tool error. The TUI blocks the `e` key with a status message. - -CLI: `noema verify` is a subcommand group for integrity checks. `noema verify traces` checks all trace file hashes against their frontmatter `content_hash` (with `--backfill` for old traces); `noema verify cortex` validates the manifest, user config, DB, access posture, and federation config; `noema verify drift` checks federated traces against their `source_hash`. Bare `noema verify` runs `verify traces` for back-compat. The legacy top-level `noema drift` still works as a hidden alias for `noema verify drift` and will be removed in a future release. - -**MCP access posture.** The HTTP MCP endpoint runs either in **open mode** (no auth, the default — suitable only for loopback) or in **keyed mode** (every request must carry `Authorization: Bearer `). Keyed mode is mandatory for federation rings and any non-loopback deployment, and it also requires TLS — the server refuses to start with a bearer key over plaintext HTTP. The key is supplied via `NOEMA_MCP_KEY` (env var, wins if both are set) or via an optional `access.shared_key_file` block in `cortex.md` pointing at a 0600 sidecar file. The server logs the active posture on startup as `access=keyed source=env|file fingerprint=SHA256:...`, and `noema federation key fingerprint` reproduces the same non-secret fingerprint for out-of-band verification across ring members. The federation syncer automatically injects the local host's bearer key into every outbound `sync_events` call; there is no per-peer key config, so every host in a ring must share one key. Mixed-mode rings (some keyed, some open) are not supported — they isolate by design. See `internal/mcp/middleware.go` for the CORS + auth middleware chain (CORS outermost so browser preflights bypass the auth gate as the spec requires). - -**TLS cert lifecycle.** When `noema serve --transport http` is started with TLS (via `--tls-cert`/`--tls-key` or the manifest fallback `access.tls_cert_path` / `access.tls_key_path`), the leaf cert is parsed at startup and classified against its `NotAfter`. An already-expired or not-yet-valid cert is a hard startup error — the server refuses to come up rather than presenting a cert clients will reject anyway. The `--insecure-allow-expired` flag exists as an escape hatch for in-place rotation but logs a loud warning line. Certs within 7 days of expiry produce a warning at startup; the server still starts. A background `cert-monitor` goroutine (mirroring the federation syncer's lifecycle) re-checks the cert every hour while serving and logs a band-transition line as the cert crosses the 90 / 30 / 7 / expired thresholds. `noema verify cortex` reads the same manifest fields and surfaces upcoming expiry passively (`[warn]` ≤ 7 days, `[fail]` once expired or not-yet-valid). See `internal/tlsutil/` for the parser/classifier, `internal/mcp/certmonitor.go` for the loop, and `cmd_serve.go:gateTLSExpiry` for the startup gate. - -### MCP Tools - -The MCP server (`internal/mcp/server.go`) exposes the full Cortex surface: `get_instructions`, `list_traces`, `get_trace`, `create_trace`, `update_trace`, `append_trace`, `delete_trace`, `recover_trace`, `archive_trace`, `unarchive_trace`, `search_traces`, `trace_history`, `trace_lineage`, `resolve_divergence`, `cortex_identity`, `sync_events`, `federation_status`, and `announce_peer`. The `get_instructions` tool returns a live reference guide for the active Cortex; agents should call it first in any new session. `cortex_identity` returns the stable ULID + display name + manifest version and is called by federation peers on every sync to verify the remote endpoint still belongs to the cortex they paired with. `cortex_identity` also piggybacks the local consolidation rank (see below) so remote peers can observe eligibility without a separate round-trip. - -### Mid → long graduation (Phase 15) - -The three-tier model completes with an automatic mid→long promoter. A graduation pass runs on the same scheduler cadence as the heuristic short→mid pass (`ChainPasses` composes them into one PassFn) and evaluates every mid-tier trace older than `consolidation.graduation.min_age_days` (default 14) against a simple AND-gate: `(read_count + search_hit_count) >= min_read_count` (default 3) AND (modify_count == 0 if `require_unmodified`) AND `tier_votes >= 0`. The AND-gate is deliberate — the long tier is meant to be auditable and slow-moving, not probabilistic. Explicit curation is available via `noema memory promote [--to mid|long]`; `noema memory demote ` handles mid→short only (long-term demotion goes through `noema memory purge` to preserve the ceremony). The underlying `internal/cortex/candidates.go:GraduationCandidates` query flips `PromotionCandidates`' `created_at >= cutoff` to `<= cutoff` so it narrows to traces that have had time to prove durability. Every tier transition emits `ActionPromote` / `ActionDemote` which replicate through federation via the new replay handlers in `internal/cortex/cortex.go:replayTierChange`. - -### Search-hit instrumentation (Phase 15.5) - -`search_traces` and `find_similar_traces` bump a per-peer `search_hit_count` column in `trace_usage` for the top-N results (default N=3 via `cortex.DefaultSearchHitTopN`) when the caller is `ActorAgent`. This complements `read_count` (which only bumps on `get_trace` / `noema_recall`): auto-injection providers like the Hermes plugin consume search output without ever drilling into individual traces, so without this signal their cortexes generate no usage data and traces never accumulate enough to graduate. The graduation gate and the heuristic short→mid scorer both fold `search_hit_count` into the read bucket at 1:1, which preserves the option to reweight later (the columns stay separate on the wire and in the DB) without a schema change. Long-tier traces are skipped on bump — see `internal/cortex/actor.go` `SearchAs` / `FindSimilarAs` for the rationale and the `bumpSearchHitsForIDs` guard. Migration `015_search_hit_count.sql` adds the column with default 0; older peers in a federation ring stay wire-compatible because `federation.TraceUsage.SearchHitCount` uses `omitempty`. - -### Semantic Search (opt-in) - -Lexical FTS5 search is always on. An **opt-in semantic layer** adds embedding-based ranking, kept pure-Go: `modernc.org/sqlite` can't load the `sqlite-vec` C extension, so embeddings are stored as a `BLOB` (migration `016_trace_embeddings.sql`: `trace_id, embedding_model, dim, embedding, source_hash, updated_at`) and cosine similarity is computed in Go (brute-force; interactive to ~50k traces — a feasibility spike measured ~7ms/query at 10k×768-dim). Embeddings are a **local derived index, never federated** (vectors from different models aren't comparable, and trace bodies + `content_hash` already sync) — so there is no `sync_events`/wire-format impact, like FTS5. - -Enable via a `search:` block in `cortex.md` (off by default): `semantic_enabled: true`, `embedding_model` (required — no baked-in default), `embedding_endpoint` (OpenAI-compatible `/embeddings`; inherits `consolidation.local_llm_endpoint` when empty), optional `api_key_env`, `default_mode` (lexical|semantic|hybrid), `hybrid_weight` (0..1, default 0.5), `max_chars` (per-trace embed-text budget, default 32000 — lower it for small-context embed models), `embed_interval_seconds` (serve maintainer cadence, default 300). `Manifest.ValidateSearch` enforces the cross-field rules. The embedder reuses `consolidation.HTTPLLMClient` via an `Embed()` method; the `cortex.Embedder` interface is defined in `internal/cortex` to avoid an import cycle and injected by the CLI/serve layer. - -- **Storage / lifecycle** (`internal/cortex/embedding.go`, `embedding_store.go`): the BLOB codec (1-byte version + LE float32, L2-normalized so cosine == dot), `EmbedBackfill` (idempotent, resumable, `--force`/`--limit`, model-change re-embed; staleness keyed on `source_hash` vs `content_hash`), and `EmbeddingStatus`. Body text = `title + "\n\n" + body`, rune-truncated to `max_chars`. -- **Query** (`semantic.go`): `SemanticSearch` (embeds the query), `SemanticSimilar` (reuses the source trace's stored vector), `HybridSearch`/`HybridSimilar` (Reciprocal Rank Fusion, K=60, of BM25-ranked `lexicalRanked` + cosine, weighted by `hybrid_weight`). Non-finite or dimension-mismatched stored vectors are skipped at the consumer so a corrupted BLOB can't poison ranking. Actor wrappers (`*As`) bump `search_hit_count` for `ActorAgent`, matching the lexical path. -- **CLI**: `noema embeddings status|backfill [--force] [--limit]`; `noema search --semantic|--hybrid`; `noema similar --semantic|--hybrid`. -- **MCP**: `search_traces` and `find_similar_traces` take a `mode` arg (`lexical`|`semantic`|`hybrid`); the server builds the embedder from the manifest and **degrades to lexical with a bracketed note** (never an error) when semantic is unconfigured, the endpoint is unreachable, or the source isn't embedded. -- **Serve-time freshness** (`internal/embed/Maintainer`): under `noema serve`, a background loop runs a periodic idempotent backfill (gated on the background-work lock) so new/edited traces get embedded without a manual backfill. It is NOT a per-mutation hook — embedding never blocks or fails a write; freshness is eventual, bounded by `embed_interval_seconds`. - -### Automatic LLM distillation (auto_distillation_enabled) - -`consolidation.llm_enabled: true` alone only wires up `noema consolidate`; the scheduled agent still runs heuristic-only promotion on cron/idle/threshold triggers. Setting `consolidation.auto_distillation_enabled: true` folds the LLM pipeline into every trigger via `consolidation.DistillationPass` (at `internal/consolidation/distill_pass.go`), which `cmd_serve.go:startConsolidator` prepends to the chain so each fire runs distillation → heuristic → graduation. DistillationPass deliberately swallows `NewHTTPLLMClient` and `RunLLMPass` errors (logging instead of propagating) so an unreachable endpoint degrades to heuristic + graduation rather than aborting the pass. Context cancellation is the one exception — it propagates so the election gate and agent unwind cleanly on shutdown. `Manifest.ValidateConsolidation` rejects `auto_distillation_enabled: true` without `llm_enabled: true` + `local_llm_endpoint` + `model_name` at load time. - -### Consolidation coordination in a federation - -When multiple federated peers have `consolidation.enabled` + `consolidation.llm_enabled` with a reachable `local_llm_endpoint`, exactly one peer runs each consolidation cycle. Each peer's `internal/consolidation.EligibilityLoop` advertises a random rank in `[1,99]` via `federation_state` and the `cortex_identity` heartbeat. At trigger time the `Election.Decide` evaluator picks the highest-ranked eligible peer (with lex-max `cortex_id` breaking ties); the `consolidation.WithElection` wrapper emits `consolidation_claim`, waits one quiet period (`2 × federation.interval`) for conflicting claims, runs the pass, and emits `consolidation_success` or `consolidation_fail`. Subscribe-mode cortexes advertise rank 0 and never win; paused peers naturally drop out via staleness. Zero new config knobs — coordination is automatic when `federation.peers` is non-empty. Single-node cortexes skip the gate entirely to avoid emitting noise for degenerate one-peer elections. - -### Hermes Memory Provider Plugin - -A Python plugin at `plugins/hermes/` implements the Hermes `MemoryProvider` ABC, letting any [Hermes](https://hermes-agent.nousresearch.com/) agent use a Noema Cortex as its memory backend. The plugin communicates with Noema via MCP (JSON-RPC 2.0 over stdio or HTTP) — it does not import any Go code. - -**Files:** `__init__.py` (NoemaMemoryProvider + register()), `transport.py` (StdioTransport, HttpTransport), `plugin.yaml` (metadata), `README.md` (setup). - -**Agent-facing tools:** `noema_search`, `noema_remember`, `noema_recall`, `noema_list`, `noema_update`, `noema_lineage` — mapped to `search_traces`, `create_trace`, `get_trace`, `list_traces`, `update_trace`, `trace_lineage` respectively. - -**Internal tools used:** `get_instructions` (system prompt), `cortex_identity` (connectivity check), `append_trace` (session log), `archive_trace` (session end). - -### DB Schema Migrations - -Schema changes must always be transparent and non-destructive. Rules: - -- Migrations are versioned SQL files embedded in the binary (`embed.FS`), run in order on startup if the DB is behind the current version -- The DB tracks the applied version in a `schema_migrations` table (one row per applied version) -- Migrations only use `ALTER TABLE ... ADD COLUMN`, new table creation, or new index creation — never `DROP`, never destructive `ALTER` -- If a migration would require removing or restructuring data, provide a separate explicit `noema migrate` command with a clear description of what it does, requiring user confirmation -- Migration failures abort startup with a clear error; they never partially apply - -The runner is a thin hand-rolled loop over embedded `*.sql` files in `internal/db/migrations/`, sorted by leading version number. Currently applied: - -- `001_initial.sql` — base trace metadata, tags, and FTS tables -- `002_trash.sql` — soft-delete/trash lifecycle -- `003_events_and_lineage.sql` — event log, `trace_lineage`, and `traces.origin` -- `004_federation_state.sql` — key-value store for vector clocks and peer cursors -- `005_cortex_identity.sql` — `cortex_id` on `traces` and `events` -- `006_content_hash.sql` — `content_hash` on `traces` -- `007_source_locking.sql` — `source_locked` and `source_hash` on `traces` -- `008_fts5_include_tags.sql` — include tags in the FTS5 search document -- `009_memory_tiering.sql` — short/mid/long tier metadata and tier votes -- `010_long_term_trigger_refine.sql` — long-term immutability trigger refinements -- `011_purge_columns.sql` — purge audit metadata -- `012_lineage_count_view.sql` — derived-from count view for consolidation signals -- `013_long_term_trigger_visibility.sql` — long-tier visibility trigger behavior -- `014_trace_usage.sql` — per-peer read/modify usage signals -- `015_search_hit_count.sql` — search hit usage signal for top-N results -- `016_trace_embeddings.sql` — local semantic-search embedding index - -`cortex.md` carries a `version` field. The current `ManifestVersion` is **2** — version 2 carries a stable cortex `id` (a ULID) and re-keys the local event log + vector clock from cortex-name to cortex-id. Cortexes written by older binaries must run `noema migrate cortex-id` (an explicit, interactive, backed-up migration in `internal/cli/cmd_migrate.go`) before they can be opened by this binary; the `--reset` flag handles the case where the directory is a copy of another cortex and needs a fresh identity. - -### Cortex Creation - -``` -noema init --name # creates ~/.noema// -noema init --name --path # creates // -``` - -Both forms scaffold the full Cortex layout (see above) and write `cortex.md`. - -### Config File - -`~/.config/noema/config.yaml` (XDG standard). Stores the active default Cortex name and any user-level settings. - -### Cortex Selection - -Priority order (highest first): - -1. `--cortex ` flag on any command -2. `NOEMA_CORTEX` environment variable -3. Default Cortex set via `noema use ` (persisted in config) - -## Usage Intents - -### CLI - -The binary shall provide straightforward capabilities to insert, edit, list, and remove Traces in the Cortex. Entry and modification of Traces should be in a user-friendly way, such as: - -Enter/Modify Trace -Subject/Title: -Tag(s) [comma or semicolon separated]: -Body/Content: - - -Additionally, for other tooling not supporting MCP, we should allow for CLI arguments (positional or named) to be consumed for usage against the Cortex. - -### TUI - -This should provide an intuitive one-stop-shop approach to insertion, modification, listing, and removal of Traces in the Cortex. - -### MCP - -A lightweight MCP server is spawned with `noema serve` (or `noema server`). It exposes Cortex operations — list, create, update, delete, search, audit, lineage, federation — to any MCP consumer. - -**Transports:** both `stdio` and Streamable HTTP must be supported, so the server works with Claude Code agents, GitHub Copilot extensions, Zed, and any other MCP-compatible tooling. The transport is selected by `--transport stdio|http`. The HTTP transport is the **Streamable HTTP** transport from the [MCP 2025-03-26 spec](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports#streamable-http) — a single endpoint at `/mcp` that handles JSON-RPC POSTs and optional SSE streaming on the same path. The legacy two-endpoint SSE transport has been removed; passing `--transport sse` returns a clear migration error. HTTP mode requires an explicit `--host` (binding to `0.0.0.0`/`::` is rejected to prevent accidental network exposure) and supports `--tls-cert`/`--tls-key` for HTTPS. Federation only runs in HTTP mode because peers need a network endpoint to call `sync_events` on; a single Cortex can serve stdio (for a local agent) and HTTP (for federation) from separate processes thanks to SQLite WAL mode. diff --git a/Makefile b/Makefile index e48e3f2..024cf06 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Noema build targets. # # Dev builds live at the repo root as ./noema (matching the convention in -# CLAUDE.md and .gitignore). Release builds land in ./dist/ with an +# AGENTS.md and .gitignore). Release builds land in ./dist/ with an # explicit - suffix so cross-compiled artifacts are self- # identifying when scp'd onto a peer host. # diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..aa92975 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,102 @@ +# Architecture + +Noema is a local-first memory layer for humans and AI agents. It exposes a CLI, +a TUI, and an MCP server over stdio or Streamable HTTP. The core domain terms +are: + +- **Noema**: the project and binary. +- **Cortex**: one named memory collection stored in a directory. +- **Trace**: one memory entry, represented by a markdown file plus indexed + SQLite metadata. + +## Storage Model + +A Cortex is a user-managed directory: + +```text +/ + cortex.md + db/noema.db + traces/ + archive/traces/ + trash/traces/ +``` + +Trace markdown files are the source of truth. Each file contains YAML +frontmatter followed by free-form body content: + +```markdown +--- +id: 20260329-why-we-chose-go +title: Why we chose Go +type: decision +author: research-agent-1 +tags: [go, architecture] +derived_from: [20260328-language-candidates] +origin: research-cortex +created: 2026-03-29T14:23:00Z +updated: 2026-03-29T14:23:00Z +--- + +Body content here. +``` + +Valid trace types are `fact`, `decision`, `preference`, `context`, `skill`, +`intent`, `observation`, `note`, and `divergence`. + +## Database And Migrations + +SQLite stores indexes, tags, lineage, event history, federation state, usage +signals, and optional embeddings. Migrations live in +`internal/db/migrations/`, are embedded into the binary, and run in version +order. Schema changes must be transparent and non-destructive: add columns, +tables, or indexes; do not drop or reshape stored data inside automatic +migrations. Destructive or structural migrations need an explicit CLI command +with clear operator confirmation. + +## Event Log, Lineage, And Federation + +Mutations create immutable events in the SQLite event log within the same +transaction as the content change. Events include a ULID, action, trace ID, +cortex ID, origin, timestamp, vector clock snapshot, and trace state where +needed. Lineage is tracked separately from `derived_from` so Noema can query +both ancestors and descendants. + +Federation is opt-in through the `federation:` block in `cortex.md`. Peers sync +over HTTP by pulling `sync_events`, replaying remote events, and merging vector +clocks. Concurrent updates that neither vector clock dominates create a +`divergence` trace rather than silently overwriting content. + +Events are authenticated with per-cortex Ed25519 signatures. A cortex that has +run `noema keygen` signs every event it emits and advertises its public key +through the `cortex_identity` handshake; peers pin that key per cortex id +(trust-on-first-use) and, under `federation.verify: enforce`, reject events that +are not correctly signed by their owning cortex. This is what makes source-lock +enforceable on the replay path and not just for local mutations. + +## Watcher And External Edits + +`noema serve` starts a filesystem watcher unless disabled in `cortex.md`. It +observes active, archived, and trashed trace directories, then reconciles +external edits through the same mutation paths used by CLI and MCP tools. +Content hashes prevent Noema from reacting to its own writes. The watcher also +guards against atomic-save gaps and can heal missing frontmatter from the DB row +for locally-owned traces. + +## MCP And Access Posture + +The MCP server exposes Cortex operations such as list, get, create, update, +archive, search, lineage, history, federation status, and sync. Stdio mode is +for local agent integration. HTTP mode uses the Streamable HTTP `/mcp` endpoint +and requires an explicit host. Keyed HTTP mode requires TLS and authenticates +requests with `Authorization: Bearer ` from `NOEMA_MCP_KEY` or a configured +sidecar key file. + +## Search And Consolidation + +Lexical FTS5 search is always available. Semantic and hybrid search are opt-in: +embeddings are stored locally as a derived index and are never federated. + +Noema supports short, mid, and long memory tiers. Consolidation can promote +short-term traces heuristically or through an optional local LLM pipeline; a +separate graduation pass promotes durable mid-tier traces to long-term memory. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..123a8d8 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,98 @@ +# Development Guide + +This guide covers local development and release-facing workflow. For the +system model, see [Architecture](architecture.md). + +## Toolchain + +Noema is a Go 1.26 module. The main binary is built from `./cmd/noema`; most +implementation code is under `internal/`. The project intentionally uses a +small dependency set: + +- SQLite through `modernc.org/sqlite`, so normal release builds can be static + with `CGO_ENABLED=0`. +- Cobra for CLI commands in `internal/cli/`. +- Bubble Tea and related Charm libraries for TUI code. +- SQLite FTS5 for default lexical search. + +The Obsidian plugin in `plugins/obsidian/` uses Node 20, TypeScript, and +esbuild. The Hermes plugin in `plugins/hermes/` is Python and talks to Noema +through MCP; it does not import Go code. + +## Common Commands + +```sh +make build +go run ./cmd/noema +go build ./... +go test ./... +go test -race ./... +go vet ./... +``` + +`make build` writes a development binary to `./noema` and injects +`internal/cli.Version` from `git describe --tags --always --dirty`. + +Release-style local builds: + +```sh +make release +make release-linux +``` + +Release targets use `CGO_ENABLED=0`, `-trimpath`, and stripped linker flags, +then write artifacts to `dist/` with OS/architecture suffixes. + +## Plugin Checks + +Validate the Obsidian plugin when changing `plugins/obsidian/`: + +```sh +cd plugins/obsidian +npm ci +npm run build +npx tsc --noEmit +``` + +Validate the Hermes plugin when changing `plugins/hermes/`: + +```sh +cd plugins/hermes +pytest +``` + +## Testing Expectations + +Add focused `*_test.go` files beside changed Go code. Migration, watcher, +federation, MCP, and CLI behavior should be covered at the package level. Use +`go test -race ./...` for changes involving background workers, HTTP serving, +federation sync, SQLite concurrency, or filesystem watching. + +CI runs `go mod tidy` and fails if `go.mod` or `go.sum` would change, then +runs `go build ./...`, `go vet ./...`, Staticcheck, `go test -race ./...`, and +the Obsidian plugin build/typecheck. + +## Branches, Commits, And PRs + +`main` is the stable release branch. Active feature and bug branches should +target `next`; release PRs move `next` to `main`. + +Commit subjects are concise and often use conventional prefixes, for example +`feat(tls): refuse serve on expired certs`, `release: v0.14.0 ...`, or +`chore: normalize fixture names`. Keep subjects and PR text public-safe. + +PRs should describe behavior changes, link issues when relevant, and list the +checks run. Include screenshots only for UI changes, especially Obsidian plugin +work. + +## Release Notes + +Release automation runs from `v*` tags through GoReleaser. For a local dry run: + +```sh +goreleaser release --snapshot --clean --skip=publish +``` + +The release workflow builds cross-platform archives, checksums, and Homebrew +formula/cask updates. Regular pushes to `main` or `next` do not publish +release artifacts. diff --git a/internal/db/migrations/014_trace_usage.sql b/internal/db/migrations/014_trace_usage.sql index 876f46b..23c8333 100644 --- a/internal/db/migrations/014_trace_usage.sql +++ b/internal/db/migrations/014_trace_usage.sql @@ -10,9 +10,7 @@ -- local cortex ID (from cortex.md) to know which peer to credit the -- historical counters to, and migrations don't have access to it. -- --- See feat/trace-usage-storage and the CLAUDE.md "Memory tier" --- description for full context on why federation-wide signal is the --- right input for consolidation-election decisions. +-- See docs/architecture.md for the memory-tier and federation overview. CREATE TABLE trace_usage ( trace_id TEXT NOT NULL, From a763dd0a7793b2be51cae03599c510f247e080e7 Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:03 -0400 Subject: [PATCH 5/9] docs: warn that event signing only protects under verify=enforce Make explicit in the federation signing section that off (default) checks nothing and warn logs but still applies forged events, so the forgery and source-lock-bypass risks stay open until peers are moved to enforce. off/warn are a rollout on-ramp, not protection. Co-Authored-By: Claude Opus 4.8 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 532a0b4..a32d037 100644 --- a/README.md +++ b/README.md @@ -686,6 +686,8 @@ Once a cortex has a key, it signs every event it emits. Peers learn its public k | `warn` | accept, but log every unsigned/forged/unverifiable event | | `enforce` | reject — only correctly-signed events from their owning cortex are replayed | +> **Signing only protects you under `enforce`.** Generating a key and signing emitted events changes nothing about what a cortex *accepts*: in the default `off` mode no signature is checked, and `warn` logs problems but still applies the event. Until every cortex you trust is set to `verify: enforce`, the forgery and source-lock-bypass risks above remain fully open — `off`/`warn` are a rollout on-ramp, not protection. Move to `enforce` once you have confirmed the peers it talks to are signing. + Under `enforce`, source-lock enforcement extends to replay: a locked trace can only be mutated by the cortex that owns it. Trust-on-first-use has a first-contact window. For a high-assurance peer you can skip it by hard-pinning the peer's key out-of-band — add `pubkey: ed25519:` to that peer's entry under `federation.peers` in `cortex.md`. The peer must then advertise exactly that key at the handshake or the sync is refused (overriding TOFU); to rotate, edit the pinned value. From 6030c77a4250bc3f9cdd8759da51085f2983089d Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:04 -0400 Subject: [PATCH 6/9] fix(keygen): write the signing-key seed atomically Replace the O_TRUNC-then-write with a temp-file + fsync + rename so a crash or error mid-write can never truncate or partially overwrite an existing key. A key clobbered halfway through a `keygen --force` rotation would otherwise force every peer to re-pin. The temp file sits beside the target to keep the rename on one filesystem, and the 0600 mode is forced explicitly so it holds even over a looser-permission original. Co-Authored-By: Claude Opus 4.8 --- internal/cli/cmd_keygen.go | 36 +++++++++++++++++++------ internal/cli/cmd_keygen_test.go | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/internal/cli/cmd_keygen.go b/internal/cli/cmd_keygen.go index 2ee7a55..e2d9d19 100644 --- a/internal/cli/cmd_keygen.go +++ b/internal/cli/cmd_keygen.go @@ -134,20 +134,40 @@ func runKeygen(out io.Writer, name, dir string, force bool) error { return nil } -// writeSecretFile writes content to path with 0600 permissions, truncating any -// existing file and forcing the mode even if the file pre-existed with looser -// bits. Used for the signing-key seed sidecar. +// writeSecretFile atomically writes content to path with 0600 permissions. +// It writes a temp file in the same directory, fsyncs it, and renames it into +// place, so a crash or error mid-write can never truncate or partially +// overwrite an existing key: the path always resolves to either the complete +// old contents or the complete new ones. This matters for the signing-key +// seed sidecar — a key clobbered halfway through a `keygen --force` rotation +// would force every peer to re-pin. The temp file lives beside the target so +// the rename stays on one filesystem (cross-device renames are not atomic). func writeSecretFile(path, content string) error { - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + tmp, err := os.CreateTemp(filepath.Dir(path), ".noema-signing-*.tmp") if err != nil { return err } - if _, err := f.WriteString(content); err != nil { - f.Close() + // Best-effort cleanup on any error path; a no-op once the rename succeeds. + defer os.Remove(tmp.Name()) + + // os.CreateTemp already uses 0600, but force it explicitly so the guarantee + // does not depend on that detail. + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.WriteString(content); err != nil { + tmp.Close() + return err + } + // Flush to disk before the rename so the swapped-in file can't be left + // empty if the machine loses power right after the rename lands. + if err := tmp.Sync(); err != nil { + tmp.Close() return err } - if err := f.Close(); err != nil { + if err := tmp.Close(); err != nil { return err } - return os.Chmod(path, 0o600) + return os.Rename(tmp.Name(), path) } diff --git a/internal/cli/cmd_keygen_test.go b/internal/cli/cmd_keygen_test.go index 2091615..10657aa 100644 --- a/internal/cli/cmd_keygen_test.go +++ b/internal/cli/cmd_keygen_test.go @@ -130,3 +130,50 @@ func TestRunKeygenRequiresCortexID(t *testing.T) { t.Fatalf("error should point to the fix, got: %v", err) } } + +// TestWriteSecretFileAtomicReplace locks in the rotation guarantees: the file +// lands at exactly 0600 with the new contents even when it pre-existed with +// looser bits, and the atomic temp-then-rename leaves no stray temp files in +// the directory. +func TestWriteSecretFileAtomicReplace(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits are not enforced on Windows") + } + dir := t.TempDir() + path := filepath.Join(dir, "noema-signing.key") + + // Pre-existing file with deliberately loose permissions and old contents. + if err := os.WriteFile(path, []byte("old-seed\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := writeSecretFile(path, "new-seed\n"); err != nil { + t.Fatalf("writeSecretFile: %v", err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new-seed\n" { + t.Fatalf("content = %q, want the replacement seed", got) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("mode = %#o, want 0600 even over a 0644 original", perm) + } + + // The atomic write must not leave its temp file behind. + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Name() != "noema-signing.key" { + t.Fatalf("unexpected leftover file in dir: %q", e.Name()) + } + } +} From da1dcd2dc38fa9093b18e83747515ab44d7db3f7 Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:05 -0400 Subject: [PATCH 7/9] feat(keygen): gitignore the signing-key seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the seed file to a .gitignore at the cortex root so it can't be accidentally committed if the cortex directory is (or becomes) a git repo. Idempotent, creates .gitignore if absent, and skips a key stored outside the cortex directory (which a .gitignore there couldn't cover). A gitignore failure warns but does not fail keygen — the key is valid either way. Also refresh the now-stale O_TRUNC comment on the write path. Co-Authored-By: Claude Opus 4.8 --- internal/cli/cmd_keygen.go | 64 +++++++++++++++++++++++++++++++-- internal/cli/cmd_keygen_test.go | 38 ++++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/internal/cli/cmd_keygen.go b/internal/cli/cmd_keygen.go index e2d9d19..1510c0b 100644 --- a/internal/cli/cmd_keygen.go +++ b/internal/cli/cmd_keygen.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/spf13/cobra" @@ -114,13 +115,17 @@ func runKeygen(out io.Writer, name, dir string, force bool) error { return fmt.Errorf("generating signing key: %w", err) } - // Write the seed with owner-only permissions. O_TRUNC so a rotation - // cleanly replaces the old seed; an explicit Chmod guards the case where - // the file already existed with looser bits. + // Write the seed atomically with owner-only permissions, so a crash during + // a rotation can't leave the key truncated (see writeSecretFile). if err := writeSecretFile(keyPath, seed+"\n"); err != nil { return fmt.Errorf("writing signing key file: %w", err) } + // Best-effort guard against committing the seed if the cortex directory is + // a git repo: add it to .gitignore. A failure here doesn't invalidate the + // key, so warn rather than abort. + ignored, giErr := ensureSeedGitignored(dir, keyPath) + m.Signing = &cortex.SigningConfig{PublicKey: pub, PrivateKeyFile: keyFile} if err := cortex.WriteManifest(dir, m); err != nil { return fmt.Errorf("writing cortex.md: %w", err) @@ -130,10 +135,63 @@ func runKeygen(out io.Writer, name, dir string, force bool) error { fmt.Fprintf(out, " public key: %s\n", pub) fmt.Fprintf(out, " private key: %s (mode 0600 — keep it secret, never commit it)\n", keyPath) fmt.Fprintf(out, " cortex.md: signing block updated\n") + switch { + case giErr != nil: + fmt.Fprintf(out, " warning: could not update .gitignore (%v); never commit %s\n", giErr, filepath.Base(keyPath)) + case ignored: + fmt.Fprintf(out, " .gitignore: %s ignored\n", filepath.Base(keyPath)) + } fmt.Fprintln(out, "\nPeers pin this public key on their next sync (trust-on-first-use).") return nil } +// ensureSeedGitignored makes sure the signing-key seed can't be accidentally +// committed if the cortex directory is (or later becomes) a git repo, by adding +// it to a .gitignore at the cortex root. It is idempotent (an existing entry is +// left alone) and creates the .gitignore if absent. Returns true when the seed +// is now ignored. A key stored outside the cortex directory is left to the +// operator — a .gitignore here couldn't cover it — and returns false, nil. +func ensureSeedGitignored(dir, keyPath string) (bool, error) { + rel, err := filepath.Rel(dir, keyPath) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return false, nil + } + rel = filepath.ToSlash(rel) + + gitignore := filepath.Join(dir, ".gitignore") + data, err := os.ReadFile(gitignore) + if err != nil && !os.IsNotExist(err) { + return false, err + } + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == rel { + return true, nil // already ignored + } + } + + var entry string + switch { + case len(data) == 0: + entry = "# Noema signing-key seed — never commit this\n" + rel + "\n" + case !strings.HasSuffix(string(data), "\n"): + entry = "\n" + rel + "\n" + default: + entry = rel + "\n" + } + f, err := os.OpenFile(gitignore, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return false, err + } + if _, err := f.WriteString(entry); err != nil { + f.Close() + return false, err + } + if err := f.Close(); err != nil { + return false, err + } + return true, nil +} + // writeSecretFile atomically writes content to path with 0600 permissions. // It writes a temp file in the same directory, fsyncs it, and renames it into // place, so a crash or error mid-write can never truncate or partially diff --git a/internal/cli/cmd_keygen_test.go b/internal/cli/cmd_keygen_test.go index 10657aa..d1742ad 100644 --- a/internal/cli/cmd_keygen_test.go +++ b/internal/cli/cmd_keygen_test.go @@ -131,6 +131,44 @@ func TestRunKeygenRequiresCortexID(t *testing.T) { } } +// TestRunKeygenGitignoresSeed verifies the seed is added to a .gitignore at the +// cortex root and that a rotation doesn't duplicate the entry. +func TestRunKeygenGitignoresSeed(t *testing.T) { + dir := t.TempDir() + writeTestManifest(t, dir, "01HV0000000000000000000CTX") + + var out bytes.Buffer + if err := runKeygen(&out, "test", dir, false); err != nil { + t.Fatal(err) + } + + count := func() int { + data, err := os.ReadFile(filepath.Join(dir, ".gitignore")) + if err != nil { + t.Fatalf("reading .gitignore: %v", err) + } + n := 0 + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == defaultSigningKeyFile { + n++ + } + } + return n + } + if got := count(); got != 1 { + t.Fatalf("expected the seed ignored exactly once, got %d occurrences", got) + } + + // Rotation must not append a duplicate entry. + out.Reset() + if err := runKeygen(&out, "test", dir, true); err != nil { + t.Fatal(err) + } + if got := count(); got != 1 { + t.Fatalf("rotation duplicated the .gitignore entry: %d occurrences", got) + } +} + // TestWriteSecretFileAtomicReplace locks in the rotation guarantees: the file // lands at exactly 0600 with the new contents even when it pre-existed with // looser bits, and the atomic temp-then-rename leaves no stray temp files in From 7fed8a7db965d5c2230552d5713fd09696c77dad Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:06 -0400 Subject: [PATCH 8/9] fix(federation): don't flag a federated receiver as a copied directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit detectCopiedDirectory keyed on "any foreign cortex_id with none of our own" — which is the normal state of a receiver/subscribe cortex after its first federation sync, since peer events are replayed under the originating cortex's id. The guard runs on every Open, so it made such a cortex unopenable: serve restart and every CLI command (including the reset-peer recovery the signing feature points operators to) failed with "appears to be a copy", and the suggested escape hatch (migrate cortex-id --reset) didn't even help — it only re-keys rows this cortex authored. Scope the check to locally-authored history: trip only when events with origin == this cortex's name are recorded under a cortex_id other than the one cortex.md declares — the actual copy/re-id signature, and exactly the rows --reset re-keys. Peer-replayed events no longer count. Pre-existing since the cortex-ULID work; surfaced by end-to-end federation testing of the signing feature. Co-Authored-By: Claude Opus 4.8 --- internal/cortex/cortex.go | 62 ++++++++++++++++---------------- internal/cortex/cortex_test.go | 65 ++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 30 deletions(-) diff --git a/internal/cortex/cortex.go b/internal/cortex/cortex.go index 1d577d7..03c0b93 100644 --- a/internal/cortex/cortex.go +++ b/internal/cortex/cortex.go @@ -1217,10 +1217,11 @@ func Open(name, dir string) (*Cortex, error) { cx.ID = m.ID // Copied-directory detection (Gotcha #3 in the design doc). - // If the manifest claims an ID but the events table has rows under - // a *different* cortex_id (and none under this one), the directory - // was copied from another machine and would silently corrupt vector - // clocks if allowed to run. Refuse with a clear escape hatch. + // If the manifest claims an ID but events this cortex *authored* are + // recorded under a different cortex_id, the directory was copied or + // re-identified and would silently corrupt vector clocks if allowed to + // run. Peer events replayed via federation are excluded — see + // detectCopiedDirectory. Refuse with a clear escape hatch. if cx.ID != "" { if err := cx.detectCopiedDirectory(); err != nil { conn.Close() @@ -1345,41 +1346,42 @@ func (c *Cortex) backfillTraceUsage() error { return nil } -// detectCopiedDirectory refuses to start if the events table contains rows -// from a different cortex ID than the one declared in cortex.md, and zero -// rows under the declared ID. That signature only happens when a Cortex -// directory has been copied wholesale from another machine — the new -// "instance" inherits the original's ULID but has none of its own writes. -// Allowing this would create two physical Cortexes claiming the same -// identity in any federation they joined. +// detectCopiedDirectory refuses to start when events this cortex *authored* +// (origin == its display name) are recorded under a cortex_id other than the +// one cortex.md now declares. That is the signature of a directory copied or +// re-identified from another instance: its own history lives under a stale +// identity, and running it would create two physical Cortexes claiming the same +// id in any federation they joined, silently merging vector clocks. +// +// Crucially, the check keys on origin, NOT on "any foreign cortex_id". Events +// replayed from peers via federation legitimately carry the originating +// cortex's id, so a receiver or subscribe cortex normally holds many +// foreign-id events with none of its own — that is expected, not a copy. The +// earlier id-only heuristic flagged exactly that case, making such a cortex +// unopenable (serve restart and every CLI command, including the reset-peer +// recovery) after its first sync. Scoping to origin matches precisely the rows +// `noema migrate cortex-id --reset` re-keys, so the guard and its remedy agree. func (c *Cortex) detectCopiedDirectory() error { - var distinctIDs, ownRows int + var ownUnderForeignID int if err := c.DB.QueryRow( - `SELECT COUNT(DISTINCT cortex_id) FROM events WHERE cortex_id != ''`, - ).Scan(&distinctIDs); err != nil { - return nil // table missing or empty — fresh cortex, fine - } - if distinctIDs == 0 { - return nil // never written; fresh or pre-migration - } - if err := c.DB.QueryRow( - `SELECT COUNT(*) FROM events WHERE cortex_id = ?`, c.ID, - ).Scan(&ownRows); err != nil { - return nil + `SELECT COUNT(*) FROM events WHERE origin = ? AND cortex_id != '' AND cortex_id != ?`, + c.Name, c.ID, + ).Scan(&ownUnderForeignID); err != nil { + return nil // table missing or unreadable — treat as fresh, don't block } - if ownRows > 0 { - return nil // we have our own writes — legitimate + if ownUnderForeignID == 0 { + return nil // no locally-authored events under a stale id — legitimate } return fmt.Errorf( "cortex %q at %s appears to be a copy of another Cortex.\n"+ - "Its cortex.md declares id=%s but the event log contains no rows under that id\n"+ - "(it has %d distinct other id(s)). Two Cortexes cannot share an identity in a\n"+ - "federation — vector clocks would silently merge and concurrent edits would be\n"+ - "clobbered. To make this directory a distinct Cortex, run:\n"+ + "Its cortex.md declares id=%s but %d event(s) it authored are recorded under a\n"+ + "different cortex_id. Two Cortexes cannot share an identity in a federation —\n"+ + "vector clocks would silently merge and concurrent edits would be clobbered.\n"+ + "To make this directory a distinct Cortex, run:\n"+ " noema migrate cortex-id --cortex %s --reset\n"+ "That will assign a fresh id and re-key the local event log. If this is not a\n"+ "copy and you expected the events to be present, restore from backup instead", - c.Name, c.Dir, c.ID, distinctIDs, c.Name, + c.Name, c.Dir, c.ID, ownUnderForeignID, c.Name, ) } diff --git a/internal/cortex/cortex_test.go b/internal/cortex/cortex_test.go index e186544..05c5db0 100644 --- a/internal/cortex/cortex_test.go +++ b/internal/cortex/cortex_test.go @@ -246,6 +246,71 @@ func TestOpen_PreservesAgentsMDWhenMatching(t *testing.T) { } } +// TestOpen_AllowsFederatedReceiver is the regression for the copied-directory +// guard: a cortex that has only ever received peer events (foreign cortex_id, +// origin = the peer's name) — the normal state of a receiver/subscribe cortex +// after its first sync — must reopen cleanly. The old id-only heuristic wrongly +// flagged this as a copy, making such a cortex unopenable (serve restart and +// every CLI command, including reset-peer) after its first federation sync. +func TestOpen_AllowsFederatedReceiver(t *testing.T) { + dir := t.TempDir() + if _, err := cortex.Create("recv", dir); err != nil { + t.Fatalf("Create: %v", err) + } + root := filepath.Join(dir, "recv") + + cx, err := cortex.Open("recv", root) + if err != nil { + t.Fatalf("first Open: %v", err) + } + // A peer-authored event replayed via federation: foreign cortex_id, origin + // is the PEER's display name, not this cortex's. + if _, err := cx.DB.Exec( + `INSERT INTO events (id, action, trace_id, cortex_id, origin, timestamp) VALUES (?, ?, ?, ?, ?, ?)`, + "01EVPEER0000000000000000A", "create", "20260610-peer", "01PEERCORTEX0000000000000A", "peer-a", "2026-06-10T00:00:00Z", + ); err != nil { + t.Fatalf("seed peer event: %v", err) + } + cx.Close() + + cx2, err := cortex.Open("recv", root) + if err != nil { + t.Fatalf("reopening a receiver with peer-replayed events must succeed, got: %v", err) + } + cx2.Close() +} + +// TestOpen_RejectsReIdentifiedCopy keeps the real copy detection: when events +// THIS cortex authored (origin == its name) are recorded under a cortex_id that +// differs from the one cortex.md now declares, the directory was copied or +// re-identified and must be refused. +func TestOpen_RejectsReIdentifiedCopy(t *testing.T) { + dir := t.TempDir() + if _, err := cortex.Create("copied", dir); err != nil { + t.Fatalf("Create: %v", err) + } + root := filepath.Join(dir, "copied") + + cx, err := cortex.Open("copied", root) + if err != nil { + t.Fatalf("first Open: %v", err) + } + // Locally-authored event (origin == this cortex's name) under a stale id. + if _, err := cx.DB.Exec( + `INSERT INTO events (id, action, trace_id, cortex_id, origin, timestamp) VALUES (?, ?, ?, ?, ?, ?)`, + "01EVOWN00000000000000000A", "create", "20260610-own", "01STALEID00000000000000000", "copied", "2026-06-10T00:00:00Z", + ); err != nil { + t.Fatalf("seed own event under stale id: %v", err) + } + cx.Close() + + if _, err := cortex.Open("copied", root); err == nil { + t.Fatal("a cortex whose own events live under a foreign id must be refused as a copy") + } else if !strings.Contains(err.Error(), "appears to be a copy") { + t.Fatalf("expected copy-detection error, got: %v", err) + } +} + func TestReadManifest(t *testing.T) { dir := t.TempDir() if _, err := cortex.Create("manifested", dir); err != nil { From 2176a83307f22a4492ae78ed59be2a927a33e5ec Mon Sep 17 00:00:00 2001 From: Mark Baker Date: Wed, 10 Jun 2026 00:00:07 -0400 Subject: [PATCH 9/9] feat(federation): rotation-aware reset-peer (--key-rotated) + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A key rotation (noema keygen --force) retires the old key, but a peer's pre-rotation events stay signed with it. A full reset-peer resets the cursor to the start, so the peer re-pulls those old-key events and, under verify=enforce, rejects them (their embedded pubkey conflicts with the newly re-pinned key) — the cursor sticks and recovery stalls. Add reset-peer --key-rotated: clear only the pinned signing key, keeping the cursor, pinned identity, and vclock. An already-caught-up peer then re-pins the new key on its next handshake and pulls only post-rotation events, never re-touching the retired-key history. Document the residual limitation: a from-scratch resync of a rotated cortex (a brand-new peer, or a full reset) still can't replay its pre-rotation events under enforce — there is no key history. Rotate only when forfeiting that to zero-state peers is acceptable. Surfaced by end-to-end federation testing of the signing feature. Co-Authored-By: Claude Opus 4.8 --- README.md | 2 ++ internal/cli/cmd_federation.go | 54 +++++++++++++++++++++++++++-- internal/cli/cmd_federation_test.go | 54 ++++++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a32d037..8352f57 100644 --- a/README.md +++ b/README.md @@ -692,6 +692,8 @@ Under `enforce`, source-lock enforcement extends to replay: a locked trace can o Trust-on-first-use has a first-contact window. For a high-assurance peer you can skip it by hard-pinning the peer's key out-of-band — add `pubkey: ed25519:` to that peer's entry under `federation.peers` in `cortex.md`. The peer must then advertise exactly that key at the handshake or the sync is refused (overriding TOFU); to rotate, edit the pinned value. +**Rotating a key.** `noema keygen --force` retires the old key and signs future events with the new one. Peers that pinned the old key refuse the rotated cortex until they re-pin. For a peer that was already caught up, recover with `noema federation reset-peer --key-rotated`: it drops only the pinned key (re-pinned on the next handshake) while keeping the cursor, so the peer pulls only post-rotation events. **Limitation under `enforce`:** a *from-scratch* resync of a rotated cortex — a brand-new peer, or a full `reset-peer` — cannot replay that cortex's **pre-rotation** events, because they were signed with the now-retired key and there is no key history. The peer pins the new key and rejects the older events. Rotate only when forfeiting verifiable replay of pre-rotation history to peers that resync from zero is acceptable; existing caught-up peers recovered with `--key-rotated` are unaffected. + ### Authentication Federation peers share a single bearer key — the same `NOEMA_MCP_KEY` / `access.shared_key_file` described in [Shared-key authentication](#shared-key-authentication) above. When the syncer polls a peer's `sync_events` tool, it automatically attaches `Authorization: Bearer ` from the local host's active key; nothing peer-specific lives in `cortex.md`. This means: diff --git a/internal/cli/cmd_federation.go b/internal/cli/cmd_federation.go index c101b6a..4548338 100644 --- a/internal/cli/cmd_federation.go +++ b/internal/cli/cmd_federation.go @@ -225,6 +225,7 @@ func federationPeersCmd() *cobra.Command { func federationResetPeerCmd() *cobra.Command { var assumeYes bool + var keyRotated bool cmd := &cobra.Command{ Use: "reset-peer ...", @@ -241,9 +242,18 @@ re-paired) and the local syncer is now refusing to talk to it with a the peer's new cortex_id and the cursor restarts from the beginning of the peer's event log so no events are silently skipped. +For a peer that only rotated its signing key (ran ` + "`noema keygen --force`" + `) +without changing its cortex_id, use --key-rotated instead. That clears just +the pinned signing key — so the new key is re-pinned on the next handshake — +while keeping the cursor and pinned identity, so already-replayed history is +not re-pulled. A full reset would re-pull from the start and reject the +peer's pre-rotation events under verify=enforce, because they were signed +with the now-retired key. (--key-rotated assumes the local cursor was caught +up to the peer before the rotation, which is the normal case.) + This is the supported way to clear stale federation state — never edit the federation_state SQLite table by hand.`, - Example: " noema federation reset-peer peer-b\n noema federation reset-peer peer-b peer-c --yes", + Example: " noema federation reset-peer peer-b\n noema federation reset-peer peer-b --key-rotated\n noema federation reset-peer peer-b peer-c --yes", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { cx, err := resolveCortex() @@ -280,17 +290,18 @@ the federation_state SQLite table by hand.`, } } - return runFederationResetPeer(cmd.OutOrStdout(), cmd.InOrStdin(), cx, args, configured, assumeYes) + return runFederationResetPeer(cmd.OutOrStdout(), cmd.InOrStdin(), cx, args, configured, assumeYes, keyRotated) }, } cmd.Flags().BoolVarP(&assumeYes, "yes", "y", false, "skip the confirmation prompt") + cmd.Flags().BoolVar(&keyRotated, "key-rotated", false, "peer rotated its signing key: clear only the pinned key (keep cursor + identity, no history re-pull)") return cmd } // runFederationResetPeer is split out so tests can drive it without going // through cobra. It assumes args have already been validated against // cortex.md so it never sees an unknown peer name. -func runFederationResetPeer(out io.Writer, in io.Reader, cx *cortex.Cortex, names []string, endpoints map[string]string, assumeYes bool) error { +func runFederationResetPeer(out io.Writer, in io.Reader, cx *cortex.Cortex, names []string, endpoints map[string]string, assumeYes, keyRotatedOnly bool) error { state := federation.NewState(cx.DB.DB) // Snapshot what we're about to delete so the prompt actually shows the @@ -321,6 +332,43 @@ func runFederationResetPeer(out io.Writer, in io.Reader, cx *cortex.Cortex, name }) } + // --key-rotated is a narrow operation: clear only the pinned signing key + // (keyed on the peer's pinned cortex_id), leaving the cursor, identity, and + // vclock intact. The peer re-pins its new key on the next handshake and the + // kept cursor means only post-rotation events (signed with the new key) are + // pulled — so we never re-pull pre-rotation events that would be rejected + // under verify=enforce for carrying the retired key. + if keyRotatedOnly { + fmt.Fprintf(out, "About to clear the pinned signing key for %d peer(s) in cortex %q.\n", len(snaps), cx.Name) + fmt.Fprintln(out, "The cursor and pinned identity are kept, so already-replayed history is not") + fmt.Fprintln(out, "re-pulled; the peer's new key is re-pinned on the next handshake.") + for _, s := range snaps { + fmt.Fprintf(out, " %s (%s) — pinned cortex_id %s (kept)\n", s.name, s.endpoint, emptyDash(s.pinnedID)) + } + if !assumeYes { + fmt.Fprint(out, "Proceed? [y/N]: ") + var resp string + _, _ = fmt.Fscanln(in, &resp) + if resp != "y" && resp != "Y" && resp != "yes" { + return fmt.Errorf("aborted by user") + } + } + cleared := 0 + for _, s := range snaps { + if s.pinnedID == "" { + fmt.Fprintf(out, " %s: no pinned identity yet — nothing to clear\n", s.name) + continue + } + if err := state.Delete(federation.CortexPubKeyKey(s.pinnedID)); err != nil { + return fmt.Errorf("clearing signing-key pin for peer %q: %w", s.name, err) + } + cleared++ + fmt.Fprintf(out, " %s: signing-key pin cleared (cursor and identity kept)\n", s.name) + } + fmt.Fprintf(out, "\nCleared %d signing-key pin(s). Restart `noema serve` (or wait for the next poll) to re-pin the rotated key.\n", cleared) + return nil + } + fmt.Fprintf(out, "About to reset federation state for %d peer(s) in cortex %q:\n", len(snaps), cx.Name) for _, s := range snaps { fmt.Fprintf(out, " %s (%s)\n", s.name, s.endpoint) diff --git a/internal/cli/cmd_federation_test.go b/internal/cli/cmd_federation_test.go index 726940d..cdbfb9d 100644 --- a/internal/cli/cmd_federation_test.go +++ b/internal/cli/cmd_federation_test.go @@ -114,7 +114,7 @@ func TestFederationResetPeer_ClearsAllState(t *testing.T) { } var out bytes.Buffer - if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta"}, endpointMap(cx, t), false); err != nil { + if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta"}, endpointMap(cx, t), false, false); err != nil { t.Fatalf("runFederationResetPeer: %v\noutput:\n%s", err, out.String()) } @@ -158,6 +158,52 @@ func TestFederationResetPeer_ClearsAllState(t *testing.T) { } } +// TestFederationResetPeer_KeyRotated pins the rotation-recovery path: with +// --key-rotated, reset-peer clears ONLY the pinned signing key and leaves the +// cursor, pinned identity, and vclock bucket intact. That lets an already- +// caught-up peer re-pin the rotated key on its next handshake and pull only +// post-rotation events, instead of a full reset re-pulling pre-rotation history +// that would be rejected under verify=enforce for carrying the retired key. +func TestFederationResetPeer_KeyRotated(t *testing.T) { + cx := newCortexWithPeers(t, "alpha", "beta") + + const pinned = "01PEER000000000000000BETA00" + const cursor = "01EVENT00000000000000000001" + seedPeerState(t, cx, "beta", pinned, cursor, "2026-04-07T12:00:00Z") + + st := federation.NewState(cx.DB.DB) + if err := st.SetCortexPubKey(pinned, "ed25519:OLDKEYPLACEHOLDER"); err != nil { + t.Fatalf("seed signing key: %v", err) + } + if err := st.SetClock(federation.VClock{pinned: 5}); err != nil { + t.Fatalf("SetClock: %v", err) + } + + var out bytes.Buffer + if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta"}, endpointMap(cx, t), false, true); err != nil { + t.Fatalf("runFederationResetPeer(--key-rotated): %v\noutput:\n%s", err, out.String()) + } + + // Signing-key pin must be gone so the new key re-pins on next handshake. + if v, _ := st.GetCortexPubKey(pinned); v != "" { + t.Errorf("signing-key pin should be cleared, got %q", v) + } + // Identity, cursor, and vclock bucket must all be KEPT — that's what avoids + // the pre-rotation history re-pull. + if v, _ := st.Get(federation.PeerCortexIDKey("beta")); v != pinned { + t.Errorf("cortex_id pin should be kept, got %q", v) + } + if v, _ := st.Get(federation.PeerCursorKey("beta")); v != cursor { + t.Errorf("cursor should be kept, got %q", v) + } + if vc, _ := st.GetClock(); vc[pinned] != 5 { + t.Errorf("vclock bucket should be kept, got %v", vc) + } + if !strings.Contains(out.String(), "signing-key pin cleared") { + t.Errorf("output missing confirmation:\n%s", out.String()) + } +} + // TestFederationResetPeer_RejectsUnknownPeer pins the typo guard. A reset // against an unknown peer name has to fail loudly with a list of known // peers — silently no-op'ing or matching by prefix would let an operator @@ -215,7 +261,7 @@ func TestFederationResetPeer_NoStateNoOp(t *testing.T) { cx := newCortexWithPeers(t, "alpha", "beta") var out bytes.Buffer - if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta"}, endpointMap(cx, t), false); err != nil { + if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta"}, endpointMap(cx, t), false, false); err != nil { t.Fatalf("runFederationResetPeer: %v\noutput:\n%s", err, out.String()) } @@ -240,7 +286,7 @@ func TestFederationResetPeer_AbortLeavesStateIntact(t *testing.T) { seedPeerState(t, cx, "beta", pinned, "01EVENT00000000000000000001", "2026-04-07T12:00:00Z") var out bytes.Buffer - err := runFederationResetPeer(&out, strings.NewReader("n\n"), cx, []string{"beta"}, endpointMap(cx, t), false) + err := runFederationResetPeer(&out, strings.NewReader("n\n"), cx, []string{"beta"}, endpointMap(cx, t), false, false) if err == nil { t.Fatal("expected error on user abort, got nil") } @@ -276,7 +322,7 @@ func TestFederationResetPeer_MultiplePeers(t *testing.T) { } var out bytes.Buffer - if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta", "gamma"}, endpointMap(cx, t), false); err != nil { + if err := runFederationResetPeer(&out, strings.NewReader("y\n"), cx, []string{"beta", "gamma"}, endpointMap(cx, t), false, false); err != nil { t.Fatalf("runFederationResetPeer: %v\noutput:\n%s", err, out.String()) }