Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 28 additions & 15 deletions Memory/src/storage/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -980,22 +980,11 @@ export class MemoryRepository {
...filter,
status: filter.status ?? ["activated", "resolving"]
});
const clauses = normalized.map(() => {
const columns = [
"lower(memories.id) LIKE ? ESCAPE '\\'",
"lower(COALESCE(memories.memory_key, '')) LIKE ? ESCAPE '\\'",
"lower(memories.memory_value) LIKE ? ESCAPE '\\'",
"lower(memories.properties_json) LIKE ? ESCAPE '\\'",
"lower(memories.info_json) LIKE ? ESCAPE '\\'"
];
if (includeTags) columns.push("lower(memories.tags_json) LIKE ? ESCAPE '\\'");
return `(${columns.join(" OR ")})`;
});
const params = normalized.flatMap((term) => {
const termColumns = normalized.map((term) => likeColumnsForTerm(term, includeTags));
const clauses = termColumns.map((columns) => `(${columns.join(" OR ")})`);
const params = normalized.flatMap((term, index) => {
const pattern = `%${escapeLikePattern(term)}%`;
return includeTags
? [pattern, pattern, pattern, pattern, pattern, pattern]
: [pattern, pattern, pattern, pattern, pattern];
return termColumns[index]!.map(() => pattern);
});
const rows = this.db
.prepare(
Expand Down Expand Up @@ -3855,6 +3844,30 @@ function escapeLikePattern(value: string): string {
return value.replace(/[\\%_]/g, (match) => `\\${match}`);
}

function likeColumnsForTerm(term: string, includeTags: boolean): string[] {
const columns = [
"lower(memories.id) LIKE ? ESCAPE '\\'",
"lower(COALESCE(memories.memory_key, '')) LIKE ? ESCAPE '\\'",
"lower(memories.memory_value) LIKE ? ESCAPE '\\'"
];
// Short ASCII terms ("ts", "id", ...) are substrings of JSON keys present in
// every row's metadata blobs, so matching them there ranks unrelated recent
// memories above real hits. Longer terms and CJK bigrams cannot collide with
// JSON structure and keep their reach into metadata values.
if (!isShortAsciiTerm(term)) {
columns.push(
"lower(memories.properties_json) LIKE ? ESCAPE '\\'",
"lower(memories.info_json) LIKE ? ESCAPE '\\'"
);
}
if (includeTags) columns.push("lower(memories.tags_json) LIKE ? ESCAPE '\\'");
return columns;
}

function isShortAsciiTerm(term: string): boolean {
return /^[\x20-\x7e]{1,2}$/.test(term);
}

function normalizeAgentIdKey(value: string): string {
return value.trim().toLowerCase().replace(/[\s-]+/gu, "_");
}
Expand Down
48 changes: 48 additions & 0 deletions Memory/tests/repository/memory-retrieval-index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,54 @@ describe("memory retrieval indexes", () => {
}
});

it("keeps short ascii pattern terms out of raw JSON metadata columns", () => {
const root = mkdtempSync(join(tmpdir(), "mindock-memory-pattern-json-noise-"));
try {
const db = new MemoryDb({ path: join(root, "memory.sqlite") });
const repos = new Repositories(db.db);
repos.memories.insert({
...traceMemory("trace-birthday-party"),
memoryValue: [
"Summary: planned a birthday party",
"User:",
"help me plan a birthday party",
"Agent:",
"balloons and cake are ready."
].join("\n")
});
repos.memories.insert({
...traceMemory("trace-build-error"),
memoryValue: [
"Summary: ts build error in web",
"User:",
"fix my ts build",
"Agent:",
"done, the build is green."
].join("\n")
});

// Every trace row carries the JSON key "ts" inside properties_json, so a
// two-character query token must only match real content columns.
const hits = repos.memories.searchPatternIds(["ts"], { memoryLayer: "L1" }, 5)
.map((hit) => hit.id);
expect(hits).toContain("trace-build-error");
expect(hits).not.toContain("trace-birthday-party");

// CJK bigrams cannot collide with ASCII JSON keys and keep their reach
// into metadata values.
repos.memories.insert({
...traceMemory("trace-cjk-metadata-only"),
memoryValue: "Summary: metadata-only cjk row"
});
expect(repos.memories.searchPatternIds(["科幻"], { memoryLayer: "L1" }, 5)
.map((hit) => hit.id)).toContain("trace-cjk-metadata-only");

db.close();
} finally {
rmSync(root, { recursive: true, force: true });
}
});

it("filters memories by an inclusive start and exclusive end creation time", () => {
const root = mkdtempSync(join(tmpdir(), "mindock-memory-time-filter-"));
try {
Expand Down