Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Build output
/noema
/dist/
/noema-*-plugin.tar.gz

# SQLite databases
*.db
Expand All @@ -17,6 +18,7 @@
# Caches
**/__pycache__
.pytest_cache
.gocache

# Editor
*.swp
Expand Down
22 changes: 21 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ LDFLAGS_RELEASE := -s -w -X $(VERSION_PKG).Version=$(VERSION)
HOST_OS := $(shell go env GOOS)
HOST_ARCH := $(shell go env GOARCH)

.PHONY: help build release release-linux test vet clean
.PHONY: help build release release-linux test vet obsidian-publish clean

help:
@echo "Noema build targets:"
Expand All @@ -38,6 +38,8 @@ help:
@echo " make release-linux Stripped build for linux/amd64 -> $(DIST_DIR)/$(BIN)-linux-amd64"
@echo " make test go test ./..."
@echo " make vet go vet ./..."
@echo " make obsidian-publish"
@echo " Build and copy Obsidian plugin into the active cortex vault"
@echo " make clean Remove ./$(BIN) and ./$(DIST_DIR)/"
@echo ""
@echo "Version string for the next build: $(VERSION)"
Expand Down Expand Up @@ -69,6 +71,24 @@ test:
vet:
go vet ./...

obsidian-publish:
npm --prefix plugins/obsidian run build
@set -eu; \
cortex_dir="$(OBSIDIAN_CORTEX_DIR)"; \
if [ -z "$$cortex_dir" ]; then \
cortex_dir="$$(noema cortex list | sed -n 's/^[^[:space:]][^[:space:]]*[[:space:]][[:space:]]*\(.*\)[[:space:]][[:space:]]*\*$$/\1/p' | sed 's/[[:space:]]*$$//' | head -n 1)"; \
fi; \
if [ -z "$$cortex_dir" ]; then \
echo "error: no active cortex found; run 'noema use <name>' or pass OBSIDIAN_CORTEX_DIR=/path/to/cortex" >&2; \
exit 1; \
fi; \
dest="$$cortex_dir/.obsidian/plugins/noema"; \
install -d "$$dest"; \
install -m 0644 plugins/obsidian/main.js "$$dest/main.js"; \
install -m 0644 plugins/obsidian/manifest.json "$$dest/manifest.json"; \
install -m 0644 plugins/obsidian/styles.css "$$dest/styles.css"; \
echo "Obsidian plugin published to $$dest"

clean:
rm -f $(BIN)
rm -rf $(DIST_DIR)
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ noema federation resume-peer <name> Resume syncing with a paused peer
noema federation key fingerprint Print the SHA-256 fingerprint of the active MCP shared key (safe to
say aloud over an out-of-band channel to confirm a pairing)

noema keygen [--force] Generate this cortex's Ed25519 federation signing key so it can sign
the events it emits (--force rotates it; peers must re-pin)

noema serve [--transport stdio|http] [--host <addr>] [--tls-cert <file> --tls-key <file>]
Start the MCP server (http requires --host; endpoint is /mcp)
noema serve --print-config Print a ready-to-use .mcp.json snippet and exit
Expand Down
25 changes: 17 additions & 8 deletions internal/consolidation/heuristic.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,12 @@ const MinLineageSourcesForCredit = 2
// Exported for tests; package-private callers don't need it because
// HeuristicPass is the single consumer.
//
// search_hit_count is folded into the reads bucket at the same weight
// because both signals describe passive/weak consumption — the read
// either came from a deliberate get_trace or from being one of the
// top-N hits an agent's search returned. Auto-injection providers like
// Hermes only ever generate search hits, and without this fold-in
// short-tier traces in those cortexes would never accumulate enough
// signal to promote.
// search_hit_count is normally folded into the reads bucket at the same
// weight because both signals describe consumption — the read either came
// from a deliberate get_trace or from being one of the top-N hits an agent's
// search returned. Auto-injection providers like Hermes only ever generate
// search hits, and without this fold-in short-tier traces in those cortexes
// would never accumulate enough signal to promote.
//
// Lineage credit is gated by MinLineageSourcesForCredit: a trace with
// only one derived_from gets zero lineage points, regardless of the
Expand All @@ -145,12 +144,22 @@ const MinLineageSourcesForCredit = 2
// single session trace it was extracted from) from gliding past the
// promotion threshold purely on the lineage head start. Multi-source
// traces still earn the full weight per source.
//
// One-source traces also do not get passive search-hit credit unless they
// have stronger intent (an edit or an explicit tier vote). This keeps
// auto-retrieved Hermes session summaries from leaking into mid-tier just
// because they appeared in search results, while still allowing deliberate
// reads, edits, votes, and real multi-source distillations to promote.
func scoreCandidate(pc cortex.PromotionCandidate, cfg PassConfig) int {
lineageCredit := 0
if pc.DerivedFromCount >= MinLineageSourcesForCredit {
lineageCredit = pc.DerivedFromCount * cfg.WeightLineage
}
return (pc.ReadCount+pc.SearchHitCount)*cfg.WeightReads +
searchHitCredit := pc.SearchHitCount
if pc.DerivedFromCount == 1 && pc.ModifyCount == 0 && pc.TierVotes == 0 {
searchHitCredit = 0
}
return (pc.ReadCount+searchHitCredit)*cfg.WeightReads +
pc.ModifyCount*cfg.WeightModifies +
lineageCredit +
pc.TierVotes*cfg.WeightVotes
Expand Down
18 changes: 13 additions & 5 deletions internal/consolidation/heuristic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ func TestScoreCandidate_BlendedFormula(t *testing.T) {
// Lineage credit is gated at >= 2 sources. A single derived_from
// is provenance, not consolidation — see MinLineageSourcesForCredit.
{"1 lineage ref earns no credit", cortex.PromotionCandidate{DerivedFromCount: 1}, 0},
{"1 lineage ref + search hits earns no passive credit", cortex.PromotionCandidate{SearchHitCount: 5, DerivedFromCount: 1}, 0},
{"1 lineage ref + deliberate reads still counts", cortex.PromotionCandidate{ReadCount: 5, DerivedFromCount: 1}, 5},
{"1 lineage ref + modify unlocks search-hit credit", cortex.PromotionCandidate{SearchHitCount: 3, ModifyCount: 1, DerivedFromCount: 1}, 5},
{"1 lineage ref + vote unlocks search-hit credit", cortex.PromotionCandidate{SearchHitCount: 1, TierVotes: 1, DerivedFromCount: 1}, 6},
{"2 lineage refs earn full credit", cortex.PromotionCandidate{DerivedFromCount: 2}, 6},
{"3 lineage refs scale linearly", cortex.PromotionCandidate{DerivedFromCount: 3}, 9},
{"1 vote", cortex.PromotionCandidate{TierVotes: 1}, 5},
Expand All @@ -75,10 +79,10 @@ func TestScoreCandidate_BlendedFormula(t *testing.T) {
func TestHeuristicPass_PromotesAboveThreshold(t *testing.T) {
cx := &fakeHeuristicCortex{
candidates: []cortex.PromotionCandidate{
{ID: "hot", Tier: trace.TierShort, ReadCount: 10}, // score 10 - promote
{ID: "meh", Tier: trace.TierShort, ReadCount: 2}, // score 2 - skip
{ID: "voted", Tier: trace.TierShort, TierVotes: 1}, // score 5 - promote (at threshold)
{ID: "cold", Tier: trace.TierShort}, // score 0 - skip
{ID: "hot", Tier: trace.TierShort, ReadCount: 10}, // score 10 - promote
{ID: "meh", Tier: trace.TierShort, ReadCount: 2}, // score 2 - skip
{ID: "voted", Tier: trace.TierShort, TierVotes: 1}, // score 5 - promote (at threshold)
{ID: "cold", Tier: trace.TierShort}, // score 0 - skip
{ID: "referenced", Tier: trace.TierShort, DerivedFromCount: 2}, // score 6 - promote (real consolidation)
// Regression guard: a 1-source provenance link with low
// engagement must not glide past the threshold. This is the
Expand All @@ -87,6 +91,10 @@ func TestHeuristicPass_PromotesAboveThreshold(t *testing.T) {
// silently inflating mid-tier before MinLineageSourcesForCredit
// landed. Score: 1*1 (read) + 0 (lineage gated) = 1.
{ID: "session-summary-shaped", Tier: trace.TierShort, ReadCount: 1, DerivedFromCount: 1},
// Regression guard for the 1-source mid leak detector: passive
// search hits alone must not promote a Hermes session-summary
// shaped trace.
{ID: "session-summary-search-hit", Tier: trace.TierShort, SearchHitCount: 10, DerivedFromCount: 1},
},
}
pass := HeuristicPass(cx, PassConfig{}, nil)
Expand All @@ -102,7 +110,7 @@ func TestHeuristicPass_PromotesAboveThreshold(t *testing.T) {
t.Errorf("expected %q to be promoted, promoted=%v", want, cx.promoted)
}
}
for _, shouldSkip := range []string{"meh", "cold", "session-summary-shaped"} {
for _, shouldSkip := range []string{"meh", "cold", "session-summary-shaped", "session-summary-search-hit"} {
if got[shouldSkip] {
t.Errorf("unexpected promotion of %q", shouldSkip)
}
Expand Down
64 changes: 46 additions & 18 deletions internal/cortex/cortex.go
Original file line number Diff line number Diff line change
Expand Up @@ -1311,10 +1311,26 @@ func Open(name, dir string) (*Cortex, error) {
fmt.Fprintf(os.Stderr, "[cortex] trace_usage backfill warning: %v\n", err)
}
}
if err := cx.pruneLegacyVectorClockBuckets(); err != nil {
fmt.Fprintf(os.Stderr, "[cortex] vector-clock cleanup warning: %v\n", err)
}

return cx, nil
}

func (c *Cortex) pruneLegacyVectorClockBuckets() error {
state := federation.NewState(c.DB.DB)
vc, err := state.GetClock()
if err != nil {
return err
}
clean := federation.KeepCortexIDKeys(vc)
if len(clean) == len(vc) {
return nil
}
return state.SetClock(clean)
}

// backfillTraceUsage populates trace_usage from the legacy per-trace
// counter columns, crediting them to the local cortex ID. Idempotent:
// skipped when trace_usage already has rows. Safe on fresh cortexes
Expand Down Expand Up @@ -1346,25 +1362,31 @@ func (c *Cortex) backfillTraceUsage() error {
return nil
}

// detectCopiedDirectory refuses to start when events this cortex *authored*
// (origin == its display name) are recorded under a cortex_id other than the
// one cortex.md now declares. That is the signature of a directory copied or
// re-identified from another instance: its own history lives under a stale
// identity, and running it would create two physical Cortexes claiming the same
// id in any federation they joined, silently merging vector clocks.
// detectCopiedDirectory refuses to start when events that appear locally
// authored are recorded under a cortex_id other than the one cortex.md now
// declares. That is the signature of a directory copied or re-identified from
// another instance: its own history lives under a stale identity, and running
// it would create two physical Cortexes claiming the same id in any federation
// they joined, silently merging vector clocks.
//
// Crucially, the check keys on origin, NOT on "any foreign cortex_id". Events
// replayed from peers via federation legitimately carry the originating
// cortex's id, so a receiver or subscribe cortex normally holds many
// foreign-id events with none of its own — that is expected, not a copy. The
// earlier id-only heuristic flagged exactly that case, making such a cortex
// unopenable (serve restart and every CLI command, including the reset-peer
// recovery) after its first sync. Scoping to origin matches precisely the rows
// `noema migrate cortex-id --reset` re-keys, so the guard and its remedy agree.
// The check cannot rely on origin alone. It is a display label, and real
// federations may contain multiple distinct cortex IDs with the same name. A
// peer event with origin == c.Name is legitimate when the foreign cortex_id is
// one of our pinned peers, so those IDs are excluded from the copy count.
func (c *Cortex) detectCopiedDirectory() error {
var ownUnderForeignID int
if err := c.DB.QueryRow(
`SELECT COUNT(*) FROM events WHERE origin = ? AND cortex_id != '' AND cortex_id != ?`,
`SELECT COUNT(*)
FROM events
WHERE origin = ?
AND cortex_id != ''
AND cortex_id != ?
AND cortex_id NOT IN (
SELECT value
FROM federation_state
WHERE key LIKE 'peer:%:cortex_id'
AND value != ''
)`,
c.Name, c.ID,
).Scan(&ownUnderForeignID); err != nil {
return nil // table missing or unreadable — treat as fresh, don't block
Expand Down Expand Up @@ -2947,6 +2969,7 @@ func (c *Cortex) emitEvent(tx *sql.Tx, action event.Action, traceID, timestamp s
if err != nil {
vc = make(federation.VClock)
}
vc = federation.KeepCortexIDKeys(vc)
vc.Increment(c.ID)

e := &event.Event{
Expand Down Expand Up @@ -3251,10 +3274,12 @@ func (c *Cortex) replayUpdate(e event.Event) error {
if len(e.VClock) > 0 {
localEvents, _ := event.ForTrace(c.DB.DB, e.TraceID)
if lastLocal := lastMutationEvent(localEvents); lastLocal != nil && len(lastLocal.VClock) > 0 {
rel := federation.Compare(lastLocal.VClock, e.VClock)
localClock := federation.KeepCortexIDKeys(lastLocal.VClock)
remoteClock := federation.KeepCortexIDKeys(e.VClock)
rel := federation.Compare(localClock, remoteClock)
if rel == 0 {
// Concurrent — neither clock dominates. Create a divergence trace.
return c.createDivergence(r, e, data.Body, lastLocal.VClock, e.VClock)
return c.createDivergence(r, e, data.Body, localClock, remoteClock)
}
}
}
Expand Down Expand Up @@ -3856,7 +3881,10 @@ func (c *Cortex) MergeClock(remote federation.VClock) error {
if err != nil {
return err
}
merged, err := federation.MergeCapped(local, remote)
merged, err := federation.MergeCapped(
federation.KeepCortexIDKeys(local),
federation.KeepCortexIDKeys(remote),
)
if err != nil {
return err
}
Expand Down
110 changes: 110 additions & 0 deletions internal/cortex/cortex_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,79 @@ func TestOpen_AllowsFederatedReceiver(t *testing.T) {
cx2.Close()
}

func TestOpen_AllowsPinnedPeerWithSameDisplayName(t *testing.T) {
dir := t.TempDir()
if _, err := cortex.Create("agentbrain", dir); err != nil {
t.Fatalf("Create: %v", err)
}
root := filepath.Join(dir, "agentbrain")

cx, err := cortex.Open("agentbrain", root)
if err != nil {
t.Fatalf("first Open: %v", err)
}
peerID := "01PEERCORTEX0000000000000A"
if err := federation.NewState(cx.DB.DB).SetPeerCortexID("peer-a", peerID); err != nil {
t.Fatalf("pin peer cortex id: %v", err)
}
// Multiple machines may intentionally use the same human-readable cortex
// name. The authenticated cortex_id, not origin, is the peer identity.
if _, err := cx.DB.Exec(
`INSERT INTO events (id, action, trace_id, cortex_id, origin, timestamp) VALUES (?, ?, ?, ?, ?, ?)`,
"01EVPEER0000000000000000B", "create", "20260610-peer-same-name", peerID, "agentbrain", "2026-06-10T00:00:00Z",
); err != nil {
t.Fatalf("seed same-name peer event: %v", err)
}
cx.Close()

cx2, err := cortex.Open("agentbrain", root)
if err != nil {
t.Fatalf("reopening with a pinned same-name peer event must succeed, got: %v", err)
}
cx2.Close()
}

func TestOpen_PrunesLegacyVectorClockNameBuckets(t *testing.T) {
dir := t.TempDir()
if _, err := cortex.Create("mycortex", dir); err != nil {
t.Fatalf("Create: %v", err)
}
root := filepath.Join(dir, "mycortex")

cx, err := cortex.Open("mycortex", root)
if err != nil {
t.Fatalf("first Open: %v", err)
}
state := federation.NewState(cx.DB.DB)
if err := state.SetClock(federation.VClock{
cx.ID: 4,
"mycortex": 19,
"legacy-p2": 8,
}); err != nil {
t.Fatalf("SetClock: %v", err)
}
cx.Close()

cx2, err := cortex.Open("mycortex", root)
if err != nil {
t.Fatalf("second Open: %v", err)
}
defer cx2.Close()

vc, err := cx2.GetClock()
if err != nil {
t.Fatalf("GetClock: %v", err)
}
if vc[cx2.ID] != 4 {
t.Errorf("local ULID bucket = %d, want 4 in %v", vc[cx2.ID], vc)
}
for _, legacy := range []string{"mycortex", "legacy-p2"} {
if _, ok := vc[legacy]; ok {
t.Errorf("legacy bucket %q survived Open cleanup: %v", legacy, vc)
}
}
}

// TestOpen_RejectsReIdentifiedCopy keeps the real copy detection: when events
// THIS cortex authored (origin == its name) are recorded under a cortex_id that
// differs from the one cortex.md now declares, the directory was copied or
Expand Down Expand Up @@ -1757,6 +1830,43 @@ func TestVectorClockIncrements(t *testing.T) {
}
}

func TestMergeClock_DropsLegacyNameBuckets(t *testing.T) {
cx := setup(t)
state := federation.NewState(cx.DB.DB)
if err := state.SetClock(federation.VClock{
cx.ID: 4,
"mycortex": 19,
"legacy-p2": 8,
}); err != nil {
t.Fatalf("SetClock: %v", err)
}

remoteID := "01REMOTE000000000000000000"
if err := cx.MergeClock(federation.VClock{
remoteID: 7,
"mycortex": 19,
"legacy-p3": 2,
}); err != nil {
t.Fatalf("MergeClock: %v", err)
}

vc, err := cx.GetClock()
if err != nil {
t.Fatalf("GetClock: %v", err)
}
if vc[cx.ID] != 4 {
t.Errorf("local ULID bucket = %d, want 4 in %v", vc[cx.ID], vc)
}
if vc[remoteID] != 7 {
t.Errorf("remote ULID bucket = %d, want 7 in %v", vc[remoteID], vc)
}
for _, legacy := range []string{"mycortex", "legacy-p2", "legacy-p3"} {
if _, ok := vc[legacy]; ok {
t.Errorf("legacy bucket %q survived merge: %v", legacy, vc)
}
}
}

// ---- Backfill ----

// TestBackfillCreateEvents_SyncedTraceGetsCreate is the headline scenario:
Expand Down
Loading
Loading