feat: implement Phase 4 shared memory — advanced cross-peer features

- Optimize cross-peer queries with peer.representation() instead of peer.chat()
- Extend MemoryRecall handler for glasses cross-peer support
- Add cross-peer context to web chat (chat.ts, chat.baml, bamlActions.ts)
- Add group connections schema and CRUD (connectionGroups, connectionGroupMembers)
- Update synthesis.baml with crossPeerPerspectives in MemoryContext
- Add public query getActiveSharedMemoryConnectionsByMentraId for app layer
- Add getActiveSharedMemoryGroupMembers internal query

Amp-Thread-ID: https://ampcode.com/threads/T-019ca5da-0bdc-71c4-b883-2d206a12db38
Co-authored-by: Amp <amp@ampcode.com>

Author: Ubuntu

apps/application/src/baml_client/inlinedbaml.ts

@@ -96,6 +96,11 @@ export namespace partial_types {
       peerCard: string[]
       recentMessages: string[]
       sessionSummaries: string[]
+      crossPeerPerspectives: CrossPeerPerspective[]
+    }
+    export interface CrossPeerPerspective {
+      label?: string | null
+      perspective?: string | null
     }
     export interface MemoryCore {
       userName?: string | null

apps/application/src/baml_client/partial_types.ts

@@ -45,7 +45,7 @@ export default class TypeBuilder {
 
     LocationLite: ClassViewer<'LocationLite', "lat" | "lon" | "timezone">;
 
-    MemoryContext: ClassViewer<'MemoryContext', "explicitFacts" | "deductiveFacts" | "peerCard" | "recentMessages" | "sessionSummaries">;
+    MemoryContext: ClassViewer<'MemoryContext', "explicitFacts" | "deductiveFacts" | "peerCard" | "recentMessages" | "sessionSummaries" | "crossPeerPerspectives">;
 
     MemoryCore: ClassViewer<'MemoryCore', "userName" | "userFacts" | "deductiveFacts">;
 
@@ -127,7 +127,7 @@ export default class TypeBuilder {
         ]);
 
         this.MemoryContext = this.tb.classViewer("MemoryContext", [
-          "explicitFacts","deductiveFacts","peerCard","recentMessages","sessionSummaries",
+          "explicitFacts","deductiveFacts","peerCard","recentMessages","sessionSummaries","crossPeerPerspectives",
         ]);
 
         this.MemoryCore = this.tb.classViewer("MemoryCore", [

apps/application/src/baml_client/type_builder.ts

@@ -141,6 +141,13 @@ export interface MemoryContext {
   peerCard: string[]
   recentMessages: string[]
   sessionSummaries: string[]
+  crossPeerPerspectives: CrossPeerPerspective[]
+  
+}
+
+export interface CrossPeerPerspective {
+  label: string
+  perspective: string
 
 }
 

apps/application/src/baml_client/types.ts

@@ -1,11 +1,13 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
 import type { Id } from "@convex/_generated/dataModel";
+import { Honcho } from "@honcho-ai/sdk";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
 import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
+import { env } from "../core/env";
 
 const memoryRunCallIds = new WeakMap<AppSession, number>();
 
@@ -198,6 +200,82 @@ export async function MemoryRecall(
 				peerRep = { explicit: [], deductive: [] };
 			}
 
+			// Cross-peer queries: fetch perspectives from connected users
+			let crossPeerPerspectives: Array<{ label: string; perspective: string }> = [];
+			try {
+				const connections = await convexClient.query(
+					api.connections.getActiveSharedMemoryConnectionsByMentraId,
+					{ mentraUserId },
+				);
+
+				const cappedConnections = connections.slice(0, 3);
+				if (cappedConnections.length > 0) {
+					session.logger.info(
+						`[startMemoryRecallFlow] Querying ${cappedConnections.length} cross-peer connections`,
+					);
+
+					const honchoClient = new Honcho({
+						apiKey: env.HONCHO_API_KEY,
+						environment: "production",
+						workspaceId: "with-context",
+					});
+
+					const rawPerspectives = await Promise.all(
+						cappedConnections.map(async (conn) => {
+							try {
+								const connectedPeer = await honchoClient.peer(
+									`${conn.connectedUserId}-diatribe`,
+								);
+								const rep = await connectedPeer.representation({
+									target: `${userId}-diatribe`,
+									searchQuery: textQuery,
+									searchTopK: 5,
+									maxConclusions: 10,
+								});
+								return {
+									label: conn.label ?? "Connected user",
+									perspective: typeof rep.representation === "string" ? rep.representation : "",
+								};
+							} catch (error) {
+								session.logger.warn(
+									`[startMemoryRecallFlow] Cross-peer query failed for ${conn.connectedUserId}: ${error}`,
+								);
+								return null;
+							}
+						}),
+					);
+
+					const validPerspectives = rawPerspectives.filter(
+						(p): p is { label: string; perspective: string } =>
+							p !== null && p.perspective.length > 0,
+					);
+
+					// Sensitivity gate
+					const sensitivityResults = await Promise.all(
+						validPerspectives.map(async (p) => {
+							try {
+								const category = await b.CheckSensitivity(p.perspective);
+								return { ...p, category };
+							} catch {
+								return { ...p, category: "SENSITIVE" as const };
+							}
+						}),
+					);
+
+					crossPeerPerspectives = sensitivityResults
+						.filter((p) => p.category === "SAFE")
+						.map(({ label, perspective }) => ({ label, perspective }));
+
+					session.logger.info(
+						`[startMemoryRecallFlow] Cross-peer: ${validPerspectives.length} valid, ${crossPeerPerspectives.length} safe`,
+					);
+				}
+			} catch (error) {
+				session.logger.warn(
+					`[startMemoryRecallFlow] Cross-peer lookup failed: ${error}`,
+				);
+			}
+
 			// Build memory context
 			const memoryContext = {
 				explicitFacts: peerRep.explicit.map((e) => e.content),
@@ -207,6 +285,7 @@ export async function MemoryRecall(
 				peerCard: contextData.peerCard,
 				recentMessages: contextData.messages.slice(-5).map((m) => m.content),
 				sessionSummaries,
+				crossPeerPerspectives,
 			};
 
 			// Synthesize response with BAML (replaces .chat() call)

apps/application/src/handlers/memory.ts

@@ -29,6 +29,7 @@ class ChatContext {
   userFacts string[] @description("Known facts about the user")
   deductiveFacts string[] @description("Inferred facts about the user")
   conversationHistory ChatConversationMessage[] @description("Previous messages in this chat")
+  crossPeerPerspectives CrossPeerPerspective[] @description("Perspectives from connected users with shared memory enabled")
 }
 
 function InterpretChatMessage(
@@ -50,6 +51,7 @@ Style:
 - Reference session context naturally
 - Keep responses brief - this is chat, not an essay
 - Don't be overly enthusiastic or use excessive exclamation marks
+- If connected user perspectives are available, weave them naturally with attribution (e.g. "Your wife mentioned...", "Alex has been looking into...")
 
 DATE: {{ context.date }}
 
@@ -75,6 +77,14 @@ USER PROFILE:
 - {{ fact }}
 {% endfor %}
 
+{% endif %}
+{% if context.crossPeerPerspectives | length > 0 %}
+CONNECTED USER PERSPECTIVES:
+{% for p in context.crossPeerPerspectives %}
+From "{{ p.label }}":
+{{ p.perspective }}
+{% endfor %}
+
 {% endif %}
 {% if context.conversationHistory | length > 0 %}
 CONVERSATION HISTORY:
@@ -114,6 +124,7 @@ test test_chat_simple_greeting {
         "User is interested in AI technologies"
       ]
       conversationHistory []
+      crossPeerPerspectives []
     }
   }
 }
@@ -142,6 +153,7 @@ test test_chat_with_history {
           createdAt "2024-12-24T16:00:00Z"
         }
       ]
+      crossPeerPerspectives []
     }
   }
 }
@@ -157,6 +169,7 @@ test test_chat_no_sessions {
       userFacts []
       deductiveFacts []
       conversationHistory []
+      crossPeerPerspectives []
     }
   }
 }

baml_src/chat.baml

@@ -4,6 +4,7 @@ class MemoryContext {
   peerCard string[]
   recentMessages string[]
   sessionSummaries string[] @description("Summaries of past sessions, e.g. 'Dec 20: discussed weather and navigation'")
+  crossPeerPerspectives CrossPeerPerspective[] @description("Perspectives from connected users with shared memory enabled")
 }
 
 class MemorySynthesisLines {
@@ -46,6 +47,14 @@ PAST SESSION SUMMARIES:
 - {{ summary }}
 {% endfor %}
 
+{% if context.crossPeerPerspectives | length > 0 %}
+CONNECTED USER PERSPECTIVES:
+{% for p in context.crossPeerPerspectives %}
+From "{{ p.label }}":
+{{ p.perspective }}
+{% endfor %}
+
+{% endif %}
 QUERY: {{ query }}
 
 Instructions:
@@ -56,6 +65,7 @@ Instructions:
 - If you don't have enough information to answer, say so briefly
 - Focus on what's most relevant to the query
 - Ignore any alphanumeric IDs or hashes in the facts (e.g., "j979r44m..." prefixes) — extract only the meaningful content
+- If connected users have relevant perspectives, naturally mention them (e.g. "Sarah thinks...", "Alex mentioned...")
 "#
 }
 
@@ -78,6 +88,7 @@ test test_memory_synthesis_name {
         "What is my name?"
       ]
       sessionSummaries []
+      crossPeerPerspectives []
     }
   }
   @@assert({{ this.lines|length <= 3 }})
@@ -107,6 +118,7 @@ test test_memory_synthesis_children {
         "Tell me about my daughters"
       ]
       sessionSummaries []
+      crossPeerPerspectives []
     }
   }
   @@assert({{ this.lines|length <= 3 }})
@@ -130,6 +142,7 @@ test test_memory_synthesis_no_info {
         "What car do I drive?"
       ]
       sessionSummaries []
+      crossPeerPerspectives []
     }
   }
   @@assert({{ this.lines|length <= 3 }})
@@ -156,6 +169,7 @@ test test_memory_synthesis_age {
         "How old am I?"
       ]
       sessionSummaries []
+      crossPeerPerspectives []
     }
   }
   @@assert({{ this.lines|length <= 3 }})

baml_src/synthesis.baml

@@ -1,6 +1,6 @@
 # Shared Memory Between Users — Implementation Plan
 
-## Status: Architecture Finalized / Ready for Phase 0
+## Status: Phases 0-4 Complete / Ready for Phase 5 (Passive Sharing & Smart Detection)
 
 ## Problem
 
@@ -221,15 +221,16 @@ When a connection is revoked (either side):
 
 ## Implementation Phases
 
-### Phase 0: SDK Evaluation (prerequisite)
+### Phase 0: SDK Evaluation ✅ COMPLETE
 
 **Goal**: Confirm `peer.chat(query, { target })` is available and plan upgrade if needed.
 
-1. Check current `@honcho-ai/sdk` version in `package.json` against latest v3.x
-2. Test if `peer.chat(query, { target })` (dialectic API) works with our version
-3. If upgrade needed, plan a separate PR for SDK migration
-4. Verify backward compatibility with existing `getContext()` / `addMessages()` patterns
-5. **Decision gate**: If dialectic API is unavailable, we need an alternative approach (manual context merging via `getContext()` on both peers)
+**Result**: Our `@honcho-ai/sdk@1.6.0` already supports the full v3 dialectic API including `peer.chat(query, { target })` for cross-peer queries. No SDK upgrade needed.
+
+- `peer.chat(query, { target?: string | Peer, session?: string | Session, reasoningLevel?: string })` — confirmed available
+- `peer.context({ target })` — cross-peer context retrieval confirmed
+- `peer.representation({ target })` — cross-peer representation confirmed
+- Existing `session.getContext()` / `session.addMessages()` patterns remain compatible
 
 ### Phase 1: Connections (Convex only, no Honcho changes)
 
@@ -272,14 +273,19 @@ When a connection is revoked (either side):
 3. Update `InterpretEmailReply` prompt for attributed cross-peer weaving
 4. Test: Ajay replies to an email note about a topic Sarah has discussed, her perspective enriches the reply
 
-### Phase 4: Advanced Cross-Peer Features (future)
+### Phase 4: Advanced Cross-Peer Features ✅ COMPLETE
 
 **Goal**: Deeper cross-peer reasoning and additional surfaces.
 
-1. Use `session.working_rep(peerA, peerB)` for pre-computed cross-peer representations (faster than per-query `peer.chat`)
-2. Extend to real-time glasses responses (MemoryRecall handler) for "What does Sarah think about X?" queries
-3. Add cross-peer context to web chat (`chat.ts` `sendMessage`) for daily recap conversations
-4. Group connections (3+ users, e.g., a team sharing context)
+**Result**: All four sub-tasks implemented.
+
+1. **Optimized with `peer.representation()`**: Replaced `peer.chat()` with `peer.representation({ target, searchQuery, searchTopK: 5, maxConclusions: 10 })` in `followupsChat.ts` and `emailReply.ts`. The `representation()` method returns pre-computed representations without agentic reasoning, which is faster since BAML handles interpretation. Applied across all integration surfaces.
+
+2. **Extended MemoryRecall for glasses**: Added cross-peer support to `apps/application/src/handlers/memory.ts`. Uses public query `connections.getActiveSharedMemoryConnectionsByMentraId` (since the app layer can't access internal queries), queries up to 3 connected peers via `peer.representation()`, gates through `b.CheckSensitivity()` directly, and passes `crossPeerPerspectives` to `b.SynthesizeMemory()`. Updated `baml_src/synthesis.baml` with cross-peer context in `MemoryContext` class and prompt.
+
+3. **Added cross-peer to web chat**: Extended `packages/convex/chat.ts` `sendMessage` with the same cross-peer query pattern (connections → representation → sensitivity gate → BAML). Updated `baml_src/chat.baml` `ChatContext` class with `crossPeerPerspectives` field and prompt section. Updated `bamlActions.ts` `interpretChatMessage` args.
+
+4. **Group connections**: Added `connectionGroups` and `connectionGroupMembers` tables to schema. Added mutations (`createConnectionGroup`, `acceptGroupInvite`, `leaveConnectionGroup`, `toggleGroupSharedMemory`, `updateGroupMemberLabel`), public query (`getGroupsForUser`), and internal query (`getActiveSharedMemoryGroupMembers`) to `connections.ts`.
 
 ### Phase 5: Passive Sharing & Smart Detection (future)