Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions App/backend/local-api-contracts/src/memory-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,6 @@ export const StartTurnOutputSchema = z.object({
turnId: NonEmptyStringSchema,
contextPacketId: NonEmptyStringSchema,
sessionId: NonEmptyStringSchema,
episodeId: NonEmptyStringSchema,
injectedContext: InjectedContextSchema,
searchEventId: NonEmptyStringSchema,
sourceMemoryIds: z.array(NonEmptyStringSchema),
Expand Down Expand Up @@ -333,11 +332,14 @@ export const CompleteTurnOutputSchema = z.object({
sessionId: NonEmptyStringSchema,
episodeId: NonEmptyStringSchema,
rawTurnId: NonEmptyStringSchema,
l1MemoryId: NonEmptyStringSchema,
l1MemoryId: z.string(),
l1MemoryIds: z.array(NonEmptyStringSchema),
closedEpisodeIds: z.array(NonEmptyStringSchema),
scheduledEvolution: z.boolean(),
jobs: z.array(JobRefSchema),
changeSeq: z.number().int().nonnegative(),
serverTime: IsoTimeSchema
serverTime: IsoTimeSchema,
duplicate: z.boolean().optional()
});
export type CompleteTurnOutput = z.infer<typeof CompleteTurnOutputSchema>;

Expand Down Expand Up @@ -423,12 +425,15 @@ export const GetMemoryOutputSchema = z.object({
worldModel: z
.object({
sourceMemoryIds: z.array(NonEmptyStringSchema),
confidence: z.number().optional()
confidence: z.number().optional(),
summary: z.string().optional()
})
.optional(),
skill: z
.object({
invocationGuide: z.string(),
retrievalBlurb: z.string().optional(),
triggerContext: z.string().optional(),
procedure: z.array(z.string()).optional(),
sourcePolicyIds: z.array(NonEmptyStringSchema),
sourceWorldModelIds: z.array(NonEmptyStringSchema),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,6 @@ function startTurnOutput() {
turnId: "turn-1",
contextPacketId: "context-1",
sessionId: "session-1",
episodeId: "episode-1",
injectedContext: { markdown: "", sections: [] },
searchEventId: "search-1",
sourceMemoryIds: [],
Expand All @@ -454,6 +453,8 @@ function completeTurnOutput() {
turnId: "turn-1",
sessionId: "session-1",
l1MemoryId: "memory-1",
l1MemoryIds: ["memory-1"],
closedEpisodeIds: [],
rawTurnId: "raw-1",
episodeId: "episode-1",
scheduledEvolution: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ export function createHttpMemoryClient(
return request("POST", "runWorker", WorkerRunOutputSchema, {
body: {
limit: input.limit,
targetMemoryIds: input.targetMemoryIds
targetMemoryIds: input.targetMemoryIds,
priorityCohortOnly: input.priorityCohortOnly
},
signal: input.signal,
timeoutMs: input.timeoutMs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,6 @@ function startTurnOutput(body: unknown) {
turnId: input.turnId ?? "turn-1",
contextPacketId: "context-1",
sessionId: input.sessionId,
episodeId: "episode-1",
injectedContext: { markdown: "", sections: [] },
searchEventId: "search-1",
sourceMemoryIds: [],
Expand All @@ -455,6 +454,8 @@ function completeTurnOutput() {
turnId: "turn-1",
sessionId: "session-1",
l1MemoryId: "memory-1",
l1MemoryIds: ["memory-1"],
closedEpisodeIds: [],
rawTurnId: "raw-1",
episodeId: "episode-1",
scheduledEvolution: false,
Expand Down
1 change: 1 addition & 0 deletions App/backend/src/adapters/outbound/memory-client/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export interface MemoryClient {
runWorker(input: {
limit: number;
targetMemoryIds?: string[];
priorityCohortOnly?: boolean;
signal?: AbortSignal;
timeoutMs?: number;
}): Promise<WorkerRunOutput>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,6 @@ describe("claude code skill target", () => {
if (url.pathname === "/api/v1/turns/start") {
writeJsonResponse(response, 200, {
turnId: "claude-turn-1",
episodeId: "claude-episode-1",
sourceMemoryIds: ["claude-memory-1"],
injectedContext: { markdown: "Claude historical context" }
});
Expand Down Expand Up @@ -333,7 +332,7 @@ describe("claude code skill target", () => {
answer: "修复已经完成",
sourceMemoryIds: ["claude-memory-1"]
});
expect(requests[3]?.body.episodeId).toBe("claude-episode-1");
expect(requests[3]?.body).not.toHaveProperty("episodeId");
} finally {
await close(server);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,6 @@ describe("codex skill target", () => {
if (request.method === "POST" && url.pathname === "/api/v1/turns/start") {
writeJsonResponse(response, 200, {
turnId: "turn-stop-1",
episodeId: "episode-1",
sourceMemoryIds: ["memory-1"],
injectedContext: { markdown: "Relevant prior context" }
});
Expand Down Expand Up @@ -331,7 +330,7 @@ describe("codex skill target", () => {
source: "codex",
sourceMemoryIds: ["memory-1"]
});
expect(requests[3]?.body.episodeId).toBe("episode-1");
expect(requests[3]?.body).not.toHaveProperty("episodeId");
} finally {
await close(server);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@ describe("cursor skill target", () => {
if (url.pathname === "/api/v1/turns/start") {
writeJsonResponse(response, 200, {
turnId: "cursor-turn-1",
episodeId: "cursor-episode-1",
sourceMemoryIds: ["cursor-memory-1"],
injectedContext: { markdown: "Cursor historical context" }
});
Expand Down Expand Up @@ -296,7 +295,7 @@ describe("cursor skill target", () => {
sourceMemoryIds: ["cursor-memory-1"],
status: "succeeded"
});
expect(requests[3]?.body.episodeId).toBe("cursor-episode-1");
expect(requests[3]?.body).not.toHaveProperty("episodeId");

const cancelledEvent = {
...eventBase,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ describe("opencode skill target", () => {
if (targetUrl.pathname === "/api/v1/turns/start") {
return jsonResponse({
turnId: "memmy-turn-1",
episodeId: "episode-1",
sourceMemoryIds: ["trace-1"],
injectedContext: { markdown: "User prefers concise answers." }
});
Expand Down Expand Up @@ -188,14 +187,14 @@ describe("opencode skill target", () => {
expect(requests.find((request) => request.path.endsWith("/complete"))?.body).toMatchObject({
adapterId: "memmy-opencode-plugin",
sessionId: "memmy-session-1",
episodeId: "episode-1",
query: "请检查 README",
answer: "检查完成",
status: "succeeded",
toolCalls: [{ id: "call-1", name: "read", arguments: { filePath: "README.md" } }],
toolResults: [{ tool_call_id: "call-1", content: "README contents", output: "README contents" }],
sourceMemoryIds: ["trace-1"]
});
expect(requests.find((request) => request.path.endsWith("/complete"))?.body).not.toHaveProperty("episodeId");
} finally {
globalThis.fetch = originalFetch;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export function renderMemmyDefaultContent(source: string): string {
`memmy-memory turn start --source ${source} --session-id "$SESSION_ID" --query "$USER_QUERY"`,
"```",
"",
"Use returned `injectedContext` as historical memory context only. Keep the returned `turnId` for completion; `episodeId` identifies the episode selected at turn start. Keep the current user query separate from recalled memory.",
"Use returned `injectedContext` as historical memory context only. Keep the returned `turnId` for completion; the final `episodeId` is returned by `turn complete`. Keep the current user query separate from recalled memory.",
"",
"At the end of the turn, write the final interaction:",
"",
Expand Down
17 changes: 11 additions & 6 deletions App/backend/src/services/agent-source-scan-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,15 +134,20 @@ export async function runAgentSourceScanJob(
}

callbacks.onResumeChanged({ phase: "summarize", results });
const failures = await agentSources.processImportSummaries(
results.flatMap((result) => result.memoryIds ?? []),
{ ...scanOptions, progressSourceId: job.sourceId }
);
const resultByMemoryId = new Map<string, ScanResult>();
for (const result of results) {
const failures = await agentSources.processImportSummaries(result.memoryIds ?? [], {
...scanOptions,
progressSourceId: result.sourceId
});
result.errors.push(...failures.map((failure) => ({
for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result);
}
for (const failure of failures) {
const result = resultByMemoryId.get(failure.memoryId);
result?.errors.push({
conversationId: failure.memoryId,
reason: failure.reason
})));
});
}
if (job.controller.signal.aborted) {
return;
Expand Down
42 changes: 22 additions & 20 deletions App/backend/src/services/agent-source-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,7 @@ import {
export type { ScanProgress } from "../adapters/outbound/agent-source/types.js";

const SCAN_MESSAGE_YIELD_INTERVAL = 100;
const IMPORT_SUMMARY_PRIORITY_LIMIT = 100;
const IMPORT_SUMMARY_PRIORITY_BATCH_SIZE = 20;
const IMPORT_SUMMARY_STANDARD_BATCH_SIZE = 100;
const IMPORT_WORKER_BATCH_SIZE = 4;
const IMPORT_WORKER_TIMEOUT_MS = 600_000;
const IMPORT_PROGRESS_POLL_INTERVAL_MS = 250;
const INITIAL_GLOBAL_MEMORY_LIMIT = 1_000;
Expand Down Expand Up @@ -122,13 +120,11 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption
async scanAll(scanOptions = {}) {
const collected = await this.collectAll(scanOptions);
const results = await this.ingestCollected(collected, scanOptions);
for (const result of results) {
const failures = await this.processImportSummaries(result.memoryIds ?? [], {
...scanOptions,
progressSourceId: result.sourceId
});
appendProcessingFailures(result, failures);
}
const failures = await this.processImportSummaries(
results.flatMap((result) => result.memoryIds ?? []),
{ ...scanOptions, progressSourceId: "all" }
);
appendProcessingFailuresToResults(results, failures);
return results;
},

Expand Down Expand Up @@ -986,7 +982,6 @@ async function processPendingImportSummaries(
const failures: ProcessingFailure[] = [];
const progressSourceId = scanOptions.progressSourceId ?? "all";
let indexed = 0;
let prioritySummaries = 0;
let lastProgressAt = Date.now();
emitProgress(scanOptions, {
sourceId: progressSourceId,
Expand All @@ -998,20 +993,13 @@ async function processPendingImportSummaries(

while (pendingMemoryIds.size > 0) {
scanOptions.signal?.throwIfAborted();
const limit = prioritySummaries < IMPORT_SUMMARY_PRIORITY_LIMIT
? IMPORT_SUMMARY_PRIORITY_BATCH_SIZE
: IMPORT_SUMMARY_STANDARD_BATCH_SIZE;
const result = await options.memoryClient.runWorker({
limit,
targetMemoryIds: [...pendingMemoryIds],
limit: IMPORT_WORKER_BATCH_SIZE,
priorityCohortOnly: true,
signal: scanOptions.signal,
timeoutMs: IMPORT_WORKER_TIMEOUT_MS
});

prioritySummaries += result.jobs.filter((job) =>
job.jobType === "import_summary" &&
Boolean(job.targetMemoryId && pendingMemoryIds.has(job.targetMemoryId))
).length;
const refreshed = await options.memoryClient.getMemoryProcessingStatus([...pendingMemoryIds]);
const processingByMemoryId = new Map(refreshed.items.map((item) => [item.memoryId, item]));
const activeMemoryIds = new Set(refreshed.items
Expand Down Expand Up @@ -1067,6 +1055,20 @@ function appendProcessingFailures(result: ScanResult, failures: readonly Process
})));
}

function appendProcessingFailuresToResults(
results: readonly ScanResult[],
failures: readonly ProcessingFailure[]
): void {
const resultByMemoryId = new Map<string, ScanResult>();
for (const result of results) {
for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result);
}
for (const failure of failures) {
const result = resultByMemoryId.get(failure.memoryId);
if (result) appendProcessingFailures(result, [failure]);
}
}


async function* toAsyncIterable(messages: readonly ConversationMessage[]): AsyncIterable<ConversationMessage> {
for (const message of messages) {
Expand Down
89 changes: 88 additions & 1 deletion App/backend/src/services/tests/agent-source-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -466,9 +466,92 @@ describe("agent source service", () => {
expect(events).toEqual(["scan:cursor", "scan:custom", "ingest:cursor", "ingest:custom"]);
});

it("enqueues every scanned source into one global priority drain", async () => {
const baseMemoryClient = createMockMemoryClient();
const enqueueCalls: string[][] = [];
const workerCalls: Array<{
targetMemoryIds?: string[];
priorityCohortOnly?: boolean;
}> = [];
const service = createService({
adapters: [
createFakeAdapter("cursor", [createMessage("cursor", 1)]),
createFakeAdapter("custom", [createMessage("custom", 1)])
],
ingestionService: {
async ingest(messages, ctx) {
for await (const _message of messages) {
// Consume the source stream before returning its durable memory id.
}
return {
attempted: 1,
written: 1,
deduped: 0,
failed: 0,
writtenMemories: 1,
dedupedMemories: 0,
failedMemories: 0,
memoryIds: [`memory-${ctx.sourceId}`],
conversations: 1,
completedConversationIds: [],
incompleteConversationIds: [],
failedConversationIds: [],
errors: []
};
}
},
memoryClient: {
...baseMemoryClient,
async enqueueImportSummaries(memoryIds) {
enqueueCalls.push([...(memoryIds ?? [])]);
return {
enqueued: memoryIds?.length ?? 0,
memoryIds: memoryIds ?? [],
serverTime: "2026-05-28T10:00:00.000Z"
};
},
async runWorker(input) {
workerCalls.push(input);
return baseMemoryClient.runWorker(input);
},
async getMemoryProcessingStatus(memoryIds) {
return {
items: memoryIds.map((memoryId) => ({
memoryId,
state: "ready" as const,
stage: null,
activeJobId: null,
attemptCount: 1,
manualRetryCount: 0,
retryAction: "retry" as const,
errorCode: null,
errorMessage: null,
failedAt: null,
updatedAt: "2026-05-28T10:00:00.000Z"
})),
serverTime: "2026-05-28T10:00:00.000Z"
};
}
}
});

await service.scanAll();

expect(enqueueCalls).toEqual([["memory-cursor", "memory-custom"]]);
expect(workerCalls).toEqual([
expect.objectContaining({
limit: 4,
priorityCohortOnly: true
})
]);
expect(workerCalls[0]?.targetMemoryIds).toBeUndefined();
});

it("reconciles summary progress when another worker finishes the scan memories", async () => {
const baseMemoryClient = createMockMemoryClient();
const workerTargets: string[][] = [];
const workerLimits: number[] = [];
const workerPriorityCohorts: Array<boolean | undefined> = [];
let enqueueCalls = 0;
const memoryClient: MemoryClient = {
...baseMemoryClient,
Expand Down Expand Up @@ -500,6 +583,8 @@ describe("agent source service", () => {
},
async runWorker(input) {
workerTargets.push(input.targetMemoryIds ?? []);
workerLimits.push(input.limit);
workerPriorityCohorts.push(input.priorityCohortOnly);
return baseMemoryClient.runWorker(input);
}
};
Expand All @@ -515,7 +600,9 @@ describe("agent source service", () => {
}
})).resolves.toEqual([]);

expect(workerTargets).toEqual([["memory-a", "memory-b"]]);
expect(workerTargets).toEqual([[]]);
expect(workerLimits).toEqual([4]);
expect(workerPriorityCohorts).toEqual([true]);
expect(progress).toEqual([
{ current: 0, total: 2 },
{ current: 2, total: 2 }
Expand Down
Loading