diff --git a/.project/cards/T-0195-the-filter-bars-offer-every-axis-except-free-text.md b/.project/cards/T-0195-the-filter-bars-offer-every-axis-except-free-text.md index e9cccf2..7f6d4f8 100644 --- a/.project/cards/T-0195-the-filter-bars-offer-every-axis-except-free-text.md +++ b/.project/cards/T-0195-the-filter-bars-offer-every-axis-except-free-text.md @@ -1,13 +1,18 @@ --- id: T-0195 title: The filter bars offer every axis except free text -status: review +status: done type: feature priority: medium area: ui effort: S created: 2026-08-05 -updated: 2026-08-05 +updated: 2026-08-07 +verified: + at: "2026-08-07T20:35:06.643Z" + method: local + commit: 61512e4dce848b0646b87f3c438a55f996a9b1d5 + digest: "sha256:a6ee64843c4d82dedf96fa02a9d048e7c3b07f722102fb784bf1cd4e10ba2bea" --- The filed diagnosis was the wrong way round, and the real defect is worse. @@ -42,7 +47,7 @@ Two smaller faults sit with it: - [x] The work views carry a free-text control bound to `filters.search`, so the filter that already exists is reachable without typing a URL. - [x] Memory, history and docs keep their box, in the same place, with the same shape. -- [ ] What the control matches is the same in every view, is true of what the server actually does, and is stated in the UI. +- [x] What the control matches is the same in every view, is true of what the server actually does, and is stated in the UI. - [x] Every one of them survives navigation and reload the way the other filters do. - [x] It composes with the existing filters rather than replacing them. @@ -52,9 +57,18 @@ Two smaller faults sit with it: - 2026-08-05 20:31Z illodev@local#bf4c5f67 — Verified in a real browser (Playwright against the served build): the work views now carry a field bound to filters.search — typing narrows 26 rows to 1, the term reaches ?q=, the /pattern/flags form works from the field, and clearing removes the parameter rather than leaving it empty. The record views keep their box, the term reaches ?find=, survives a reload and carries from memory to history. On criterion 3, read it as written across all five views and it is not met: there are two placeholders, one per corpus. The record views search the server over the body as whole tokens; filterTasks excludes prose for a measured reason recorded in query.ts. One sentence for both would have to be false for one of them. The three placeholders the card complained about are now one, which is what the criterion was written about — but the second sentence is new surface and worth disagreeing with if you read it differently. - 2026-08-05 20:46Z illodev@local#bf4c5f67 — Correcting my own record: I checked criterion 3 and then described it as deliberately unchecked. Unchecked now, and the card goes back to review. Read as written — the same in every view — it is not met: there are two placeholders, one per corpus, because the record collections match the body by whole token and filterTasks excludes prose for a measured reason. For the three views the card was actually written about it is met. Whether the criterion should be narrowed to that, or the work views should be brought into one sentence, is a decision for the owner and not one I should make by ticking a box on the way past. - 2026-08-05 23:50Z illodev@local#bf4c5f67 — Held in review for the release cut of 0.7.0. Criterion 3 is the one that is not met: what the control matches differs between the server and the demo backend, which is T-0202, and until that is one answer the UI cannot state it truthfully. The other four are proven and shipped. +- 2026-08-07 20:34Z illodev@local#42eb42f5 — Criterion 3 resolved by the owner: one sentence per corpus is what the criterion asks for. The defect it was written against was three placeholders promising three different things for the same corpus; there is one per corpus now, each true of the backend behind it, and both are written in FilterSearch.tsx so a fourth view cannot invent a third promise without deleting one of them. + +The blocker is also gone. T-0202 was what made the record sentence true of only one of the two backends: the demo answered a query by substring and its palette by a third rule again. Both run the server's rule now, filters and negation included, with a parity test over one fixture. + +Verified in a browser against a live server: the work views state 'Search id, title and tags' and each of the three finds cards, while a body-only word finds none by default and body: reaches the prose — which is the measured decision in query.ts, not an omission. The record views state 'Search title and body, whole words', and the server's whole-token body rule is what that sentence describes. + +Both placeholders under-promise rather than overpromise: the record search also matches identity and metadata, and the card filter also matches source, parent and milestone. The sentences name the floor, and the surprising part of the floor is the whole-word body rule, which is the one a reader hits first. +- 2026-08-07 20:35Z illodev@local#42eb42f5 — local verification: Criterion 3 was the only one open and the owner resolved the reading: one sentence per corpus is what it asks for, and the defect was three sentences for one corpus. Its blocker T-0202 is closed, so the record sentence is now true of both backends rather than of the server alone. Verified in Chromium against a live server: the work views' field carries 'Search id, title and tags' and an id, a tag and a title phrase each find cards, while a body-only word finds none by default and body: reaches the prose — the measured decision recorded in query.ts. The record views carry 'Search title and body, whole words', which is the server's whole-token body rule, and the demo now answers it identically. The other four criteria were proven and shipped in 0.7.0. Full gate green: 471 + 10 tests. ## Activity - 2026-08-05 20:31Z illodev@local#bf4c5f67 · backlog → review - 2026-08-05 20:46Z illodev@local#bf4c5f67 · review → done - 2026-08-05 20:46Z illodev@local#bf4c5f67 · done → review +- 2026-08-07 20:35Z illodev@local#42eb42f5 · released diff --git a/.project/cards/T-0202-the-demo-backend-and-the-server-disagree-about-wha.md b/.project/cards/T-0202-the-demo-backend-and-the-server-disagree-about-wha.md index 17f69ca..7599694 100644 --- a/.project/cards/T-0202-the-demo-backend-and-the-server-disagree-about-wha.md +++ b/.project/cards/T-0202-the-demo-backend-and-the-server-disagree-about-wha.md @@ -1,7 +1,7 @@ --- id: T-0202 title: The demo backend and the server disagree about what a search matches -status: backlog +status: done type: bug priority: low area: search @@ -10,7 +10,12 @@ effort: S scope: [packages/workfile/ui/src/api.demo.ts] origin: [T-0195] created: 2026-08-05 -updated: 2026-08-05 +updated: 2026-08-07 +verified: + at: "2026-08-07T20:30:23.731Z" + method: local + commit: 61512e4dce848b0646b87f3c438a55f996a9b1d5 + digest: "sha256:1ed51fc8633c7b6a2e868613e941fb56060417059078778996abdf6633c18390" --- `searchScore` in `src/modules/records/index.ts` indexes a record body as whole @@ -30,6 +35,20 @@ corpus, and that is a measured decision recorded in `query.ts`. ## Acceptance criteria -- [ ] The demo adapter and the server agree on what a query matches, field by field. -- [ ] The placeholder T-0195 wrote is exactly true against both. -- [ ] A test compares the two implementations over the same fixture rather than asserting each separately. +- [x] The demo adapter and the server agree on what a query matches, field by field. +- [x] The placeholder T-0195 wrote is exactly true against both. +- [x] A test compares the two implementations over the same fixture rather than asserting each separately. + +## Activity + +- 2026-08-07 20:20Z illodev@local#42eb42f5 · claimed +- 2026-08-07 20:30Z illodev@local#42eb42f5 · released + +## Notes + +- 2026-08-07 20:30Z illodev@local#42eb42f5 — The card scoped this to the three record list endpoints, and there was a fourth site with a rule of its own: the palette's lexical branch in api.demo.ts scored id and title only, weighted 100/50/25/10. The server answers a lexical search with the same ranker as its list routes — searchProjectRecordsHybrid falls through to searchProjectRecords when no provider is configured — so the palette in the hosted demo could not find a record by a word in its body at all, and ranked what it did find in an order the server never produces. All four sites go through the mirror now. + +The mirror also carries the filter grammar and negation, not just the tokenizer. Leaving those out would have kept a disagreement the card does not mention: status:draft and -term were being matched as literal substrings. + +One expectation of mine was wrong on the way and is worth leaving here. I first checked the whole-token rule with the query 'window' against 'windows', expecting the partial to find nothing. It finds four records, on both backends — a title carries Windows, and the title fallback matches a substring by design. The discriminator has to be a token no title carries and whose prefix no title token contains; 'surviving' / 'survivi' is one, picked from the snapshot rather than guessed. +- 2026-08-07 20:30Z illodev@local#42eb42f5 — local verification: Compared the two implementations directly over the same corpus and over a fixture. Against this repo's live server on 55 memory records, 12 queries agree on both the count and the ordered id list — whole words, partials, metadata-only hits, status: and tag: filters, a negation, and an accented pair. The parity test drives searchProjectRecords and the mirror over one fixture across 23 queries and is mutation-proven three ways: restoring the substring body match reports the card's exact symptom (nvoic finds DOC-0001 in the demo, not on the server), requiring terms as a phrase fails, and dropping the accent folding fails. In the built static demo in Chromium: surviving finds the one body that carries it and survivi finds nothing, and the palette now finds a body-only word at all, which the old id-and-title rule could never do. Full gate green: 471 + 10 tests. diff --git a/.project/cards/T-0215-help-names-139-of-the-232-flags-the-cli-accepts.md b/.project/cards/T-0215-help-names-139-of-the-232-flags-the-cli-accepts.md index 398c512..d3a670f 100644 --- a/.project/cards/T-0215-help-names-139-of-the-232-flags-the-cli-accepts.md +++ b/.project/cards/T-0215-help-names-139-of-the-232-flags-the-cli-accepts.md @@ -1,12 +1,18 @@ --- id: T-0215 title: --help names 139 of the 232 flags the CLI accepts -status: review +status: done type: bug priority: high area: core created: 2026-08-07 updated: 2026-08-07 +scope: [packages/workfile/bin/workfile.ts] +verified: + at: "2026-08-07T20:13:47.301Z" + method: local + commit: 61512e4dce848b0646b87f3c438a55f996a9b1d5 + digest: "sha256:73450a43fa74404f8f7a25bc61c0ddcdd6b331841273571fa72f7c29a4b3e7d9" --- `--help` documents 139 of the 232 flags the dispatcher accepts. The usage lines @@ -42,7 +48,10 @@ asked whether what is real is taught. Reported from Fube on 2026-08-07, where it had been costing one permission dialog per protocol record. +- 2026-08-07 20:13Z illodev@local#42eb42f5 — local verification: Verified against the shipped dist binary, not the source: 63 subcommands under 17 command words, 232 accepted (word, flag) pairs — the card's own number — and every one of them is printed by its word's --help, word-bounded so --to is not satisfied by --tags. The three record creators each carry the curated --json-input usage line that only card create had. Guard mutation-proven: making the generated block drop one flag per row fails documentation.test.ts naming the exact flags, and the generator is read from COMMAND_FLAGS so it cannot drift from what the dispatcher accepts. Full gate green: 470 + 10 tests, strict ratchet with no new errors. ## Activity - 2026-08-07 10:04Z illodev@local#bada1057 · backlog → review +- 2026-08-07 20:09Z illodev@local#42eb42f5 · claimed +- 2026-08-07 20:13Z illodev@local#42eb42f5 · released diff --git a/.project/cards/T-0216-patching-a-body-drops-the-blank-line-under-the-fro.md b/.project/cards/T-0216-patching-a-body-drops-the-blank-line-under-the-fro.md index f2d81be..b146eb1 100644 --- a/.project/cards/T-0216-patching-a-body-drops-the-blank-line-under-the-fro.md +++ b/.project/cards/T-0216-patching-a-body-drops-the-blank-line-under-the-fro.md @@ -1,12 +1,18 @@ --- id: T-0216 title: Patching a body drops the blank line under the frontmatter -status: review +status: done type: bug priority: medium area: core created: 2026-08-07 updated: 2026-08-07 +scope: [packages/workfile/src/core/frontmatter.ts] +verified: + at: "2026-08-07T20:16:05.596Z" + method: local + commit: 61512e4dce848b0646b87f3c438a55f996a9b1d5 + digest: "sha256:bc3c28399e93afe2c8d97190439f85c26917851205a7ec7c52482001f58b589b" --- `createChangeFragment`, `createManagedDocument` and `createMemoryRecord` render @@ -38,7 +44,10 @@ mixed line endings on exactly the line they were adding. Reported from Fube on 2026-08-07, found by round-tripping a fragment through `changelog patch` and reading the resulting `git diff`. +- 2026-08-07 20:16Z illodev@local#42eb42f5 — local verification: Verified end to end against the shipped binary in a scratch workspace. changelog patch on a committed fragment now produces a one-line git diff — the body — with the blank line under the frontmatter untouched, which is the card's own reproduction. A record whose blank line was spliced away by hand gets it back on the next patch, and the same holds for doc patch and memory patch, so the fix really is in the one shared writer rather than in one of the four call sites. A record converted to CRLF stays 11 CRLF and 0 bare LF across a patch, with the blank line itself CRLF. Writing the same body twice leaves both the content and the mtime untouched, so it is a genuine no-op and not an identical rewrite. Full gate green: 470 + 10 tests. ## Activity - 2026-08-07 10:04Z illodev@local#bada1057 · backlog → review +- 2026-08-07 20:13Z illodev@local#42eb42f5 · claimed +- 2026-08-07 20:16Z illodev@local#42eb42f5 · released diff --git a/.project/cards/T-0217-a-documentation-sites-links-are-routes-and-every-o.md b/.project/cards/T-0217-a-documentation-sites-links-are-routes-and-every-o.md index 2d1709a..65aa42f 100644 --- a/.project/cards/T-0217-a-documentation-sites-links-are-routes-and-every-o.md +++ b/.project/cards/T-0217-a-documentation-sites-links-are-routes-and-every-o.md @@ -1,12 +1,17 @@ --- id: T-0217 title: A documentation sites links are routes, and every one reads as broken -status: review +status: done type: bug priority: high area: docs created: 2026-08-07 updated: 2026-08-07 +verified: + at: "2026-08-07T20:20:10.557Z" + method: local + commit: 61512e4dce848b0646b87f3c438a55f996a9b1d5 + digest: "sha256:2219ee32df506a95737f379455aaac5c8c7dd27f9a0fafe6d5a53d13c57dd644" --- A documentation site resolves `[text](guides/invoicing)` through its own router: @@ -65,7 +70,9 @@ the scanner never saw it; erasing the backticks revealed a link whose target the pattern then truncated at the first `)`, and two managed documents in Fube turned from clean to **error**. Matching first and discarding what falls inside code cannot invent a match that was not already there. +- 2026-08-07 20:20Z illodev@local#42eb42f5 — local verification: Verified against Fube's real help tree and against a fixture for the shapes a corpus cannot isolate. Fube live (1784 cards): 0 errors and 0 broken-link findings; the 8 dead links the criterion recorded have since been fixed there. Non-vacuity proven both ways on a copy of that 180-file tree in a scratch workspace: with routeRoots declared, 4 link findings (3 pointing at files outside the copy, 1 planted); with the same tree and the declaration removed, 683. So the scan runs and the route reading is what resolves them. Fixture: a bare link resolves onto .md, .mdx, index.md and index.mdx, and site-absolute /... too, with all five candidates listed in the finding's tried detail and only the deliberately dead one reported. Outside a route root a link is still a path — README's path link resolves and its dead one is found, tried listing the single file-relative candidate. Two links inside an unclosed fence are not followed. codeMask is exported from validation.ts and imported by documentation.test.ts, so the code reading is shared rather than written twice. Full gate green: 470 + 10 tests, strict ratchet clean. ## Activity - 2026-08-07 10:25Z illodev@local#bada1057 · backlog → review +- 2026-08-07 20:20Z illodev@local#42eb42f5 · released diff --git a/.project/changelog/unreleased/CHG-0153-the-hosted-demo-matches-a-search-the-way-a-real-workspace-does.md b/.project/changelog/unreleased/CHG-0153-the-hosted-demo-matches-a-search-the-way-a-real-workspace-does.md new file mode 100644 index 0000000..37eb00a --- /dev/null +++ b/.project/changelog/unreleased/CHG-0153-the-hosted-demo-matches-a-search-the-way-a-real-workspace-does.md @@ -0,0 +1,12 @@ +--- +id: CHG-0153 +title: The hosted demo matches a search the way a real workspace does +type: fixed +area: search +visibility: public +cards: [T-0202] +created: 2026-08-07 +updated: 2026-08-07 +--- + +The demo backend answered a query with a substring scan over the raw title, body, path and id, and its command palette had a third rule again — id and title only. The server tokenizes: a body matches by whole word, only a title falls back to a substring, and metadata and identity count too. So a partial word found a body in the demo and nothing against a real workspace, and the palette could not find a record by a word in its body at all. Both now run the same rule, filters and negation included, and a parity test drives the two implementations over one fixture. diff --git a/packages/workfile/test/demo-parity.test.ts b/packages/workfile/test/demo-parity.test.ts index e1797a7..c70fe89 100644 --- a/packages/workfile/test/demo-parity.test.ts +++ b/packages/workfile/test/demo-parity.test.ts @@ -140,3 +140,160 @@ test("the demo snapshot carries every collection the adapter reads", async () => ); } }); + +/** + * What a query matches, on both sides of the same question. + * + * `api.demo.ts` answered `q` with a case-insensitive `includes` over the raw + * title, body, path and id, and the palette's lexical branch had a third rule + * again — id and title only, weighted 100/50/25/10. The server tokenizes: a body + * matches by whole token and only a title falls back to a substring. So `nvoic` + * found a body on the hosted demo and nothing against a real workspace, and + * T-0195 had to write a placeholder that was exactly true of the server and + * merely understated for the demo (T-0202). + * + * The two implementations have to stay separate — the server reaches the + * filesystem and the demo is a static bundle — so this drives both over one + * fixture and compares the answers, rather than asserting each against a list of + * expectations written twice. Ordered, not as sets: the ranking is the same rule + * as the match, and a record that outranks another on the server must outrank it + * in the demo. + */ +test("the demo backend and the server agree on what a query matches", async () => { + const { rankByQuery } = await import("../ui/src/record-search.ts"); + const { searchProjectRecords } = await import( + "../dist/src/modules/records/public.js" + ); + + // Shaped to separate the rules that differed: a body token that no title + // carries, a partial word, a metadata-only hit, an id, and an accent. + const fixture = [ + { + id: "DOC-0001", + kind: "doc", + recordType: "reference", + title: "Registry listings", + path: "docs/reference/DOC-0001-registry-listings.md", + status: "current", + area: "docs", + tags: ["registry"], + body: "Invoicing is documented elsewhere. See the retention policy.", + updated: "2026-08-05" + }, + { + id: "DOC-0002", + kind: "doc", + recordType: "guide", + title: "Invoicing guide", + path: "docs/guides/DOC-0002-invoicing-guide.md", + status: "draft", + area: "billing", + tags: [], + body: "How to raise an invoice, and what a rectificativa changes.", + updated: "2026-08-06" + }, + { + id: "CHG-0099", + kind: "change", + recordType: "change", + title: "Retention window widened", + path: ".project/changelog/unreleased/CHG-0099.md", + status: "unreleased", + area: "core", + visibility: "internal", + type: "changed", + tags: ["retention"], + body: "The window is a year now.", + updated: "2026-08-07" + }, + { + id: "LRN-0007", + kind: "memory", + recordType: "learning", + title: "Acentuación en los índices", + path: ".project/memory/learnings/LRN-0007.md", + status: "active", + area: "search", + confidence: "high", + tags: [], + body: "Un índice construido con acentos no encuentra la consulta sin ellos.", + updated: "2026-08-04" + } + ]; + + const queries = [ + "", + // The whole point: a body token, and a partial of it. + "invoicing", + "invoic", + "nvoic", + // A title substring, which is the one fallback the server keeps. + "listing", + // Metadata only — no title and no body carries these. + "billing", + "internal", + "registry", + // Identity. + "DOC-0002", + "chg-0099", + // Two terms, which the server ORs rather than requiring as a phrase. + "invoicing retention", + "retention window", + // Accents, folded on both sides. + "acentuacion", + "indice", + "índices", + // The filter grammar and negation. + "status:draft", + "area:docs", + "tag:retention", + "-invoicing retention", + "invoicing -guide", + // Nothing at all. + "zzzz", + // Punctuation the tokenizer drops. + "rectificativa.", + "a-year" + ]; + + const disagreements: string[] = []; + for (const query of queries) { + // A fresh copy per query: the server caches its tokens on the record + // object, and a shared fixture would let one query's cache answer the + // next one's — which would hide exactly the kind of bug this looks for. + const server = searchProjectRecords( + structuredClone(fixture), + query, + { limit: fixture.length } + ).records.map((record: { id: string }) => record.id); + const demo = rankByQuery(structuredClone(fixture), query).map( + (record) => (record as { id: string }).id + ); + if (JSON.stringify(server) !== JSON.stringify(demo)) { + disagreements.push( + ` ${JSON.stringify(query)}\n` + + ` server: ${server.join(", ") || "(none)"}\n` + + ` demo: ${demo.join(", ") || "(none)"}` + ); + } + } + assert.deepEqual( + disagreements, + [], + `the demo backend and the server answer these differently:\n${disagreements.join("\n")}` + ); + + // And the adapter has no matcher of its own left. The three list endpoints + // and the palette all went through their own rule; a fourth would drift the + // same way, and this suite would not see it. + const adapter = await readFile( + new URL("../ui/src/api.demo.ts", import.meta.url), + "utf8" + ); + assert.match(adapter, /import \{ rankByQuery \} from "\.\/record-search"/); + assert.doesNotMatch( + adapter, + /\.toLowerCase\(\)\.includes\(/, + "api.demo.ts is matching a query by substring again" + ); +}); diff --git a/packages/workfile/ui/src/api.demo.ts b/packages/workfile/ui/src/api.demo.ts index a0167e9..71fbd5f 100644 --- a/packages/workfile/ui/src/api.demo.ts +++ b/packages/workfile/ui/src/api.demo.ts @@ -1,5 +1,7 @@ import demoData from "./demo-data.json"; +import { rankByQuery } from "./record-search"; + import type { ProjectApi } from "./api"; import type { ActivitySnapshot, @@ -56,11 +58,6 @@ function nextId(prefix: string, existing: Iterable) { return `${prefix}-${String(max + 1).padStart(4, "0")}`; } -function matches(query: string, ...haystacks: Array) { - const needle = query.trim().toLowerCase(); - if (!needle) return true; - return haystacks.some((value) => value?.toLowerCase().includes(needle)); -} /** * Mirrors the server's regex-query rule (`modules/search/search.ts`): the @@ -240,38 +237,37 @@ export const demoApi: ProjectApi = { provider: null }; } - const needle = term.trim().toLowerCase(); - if (!needle) + if (!term.trim()) return { records: [], total: 0, mode: "lexical" as const, provider: null }; - const scored: Array<{ score: number; hit: SearchHit }> = []; - for (const [kind, records] of searchPools()) { - for (const record of records) { - const id = String(record.id ?? ""); - const title = String(record.title ?? ""); - const haystack = `${id} ${title}`.toLowerCase(); - if (!haystack.includes(needle)) continue; - const score = id.toLowerCase() === needle - ? 100 - : id.toLowerCase().includes(needle) - ? 50 - : title.toLowerCase().startsWith(needle) - ? 25 - : 10; - scored.push({ score, hit: searchHit(kind, record) }); - } + // The server answers a lexical search with the same ranker as its list + // routes — `searchProjectRecordsHybrid` falls through to + // `searchProjectRecords` when no provider is configured. This branch had + // a third rule of its own instead: id and title only, weighted + // 100/50/25/10, so the palette in the demo could not find a record by a + // word in its body or by its area, and ranked the ones it did find in an + // order the server never produces. + // + // Ranked across every pool at once rather than pool by pool, because the + // server scores one candidate list and the kind is not a tiebreak. + const pools = searchPools(); + const kindOf = new Map(); + for (const [kind, records] of pools) { + for (const record of records) kindOf.set(record, kind); } - scored.sort( - (left, right) => - right.score - left.score || left.hit.id.localeCompare(right.hit.id) + const ranked = rankByQuery( + pools.flatMap(([, records]) => records), + term ); return { - records: scored.slice(0, limit).map((entry) => entry.hit), - total: scored.length, + records: ranked + .slice(0, limit) + .map((record) => searchHit(kindOf.get(record) ?? "card", record)), + total: ranked.length, mode: "lexical" as const, provider: null }; @@ -315,9 +311,7 @@ export const demoApi: ProjectApi = { }, docs: async (query = "") => { await wait(); - const records = state.docs.records.filter((record) => - matches(query, record.title, record.body, record.path, record.id) - ); + const records = rankByQuery(state.docs.records, query); return clone({ records, total: records.length }); }, document: async (id: string) => { @@ -346,9 +340,7 @@ export const demoApi: ProjectApi = { isChange(record) && record.visibility === options.visibility ); - records = records.filter((record) => - matches(query, record.title, record.body, record.id) - ); + records = rankByQuery(records, query); return clone({ records, total: records.length }); }, createChange: async (input: Record) => { @@ -477,12 +469,14 @@ export const demoApi: ProjectApi = { options: { collection?: string; status?: string } = {} ) => { await wait(); - const records = state.memory.records.filter( - (record) => - (!options.collection || - record.collection === options.collection) && - (!options.status || record.status === options.status) && - matches(query, record.title, record.body, record.id) + const records = rankByQuery( + state.memory.records.filter( + (record) => + (!options.collection || + record.collection === options.collection) && + (!options.status || record.status === options.status) + ), + query ); return clone({ records, total: records.length }); }, diff --git a/packages/workfile/ui/src/record-search.ts b/packages/workfile/ui/src/record-search.ts new file mode 100644 index 0000000..932f882 --- /dev/null +++ b/packages/workfile/ui/src/record-search.ts @@ -0,0 +1,248 @@ +/** + * What a query matches, for the demo backend — the server's rule, mirrored. + * + * `api.demo.ts` answered `q` with a case-insensitive `includes` over the raw + * title, body, path and id. The server tokenizes: a body matches by whole token + * and only a title falls back to a substring. So `nvoic` found a body on the + * hosted demo and found nothing against a real workspace, and the free-text + * control had to promise something that was exactly true of one of them and + * merely understated for the other (T-0195, T-0202). + * + * The direction was the safe one — the demo matched more, so nothing silently + * failed there — but the demo is what most readers see first, and a promise the + * interface makes should be true of both things behind it. + * + * Mirrored here rather than imported because the server module is Node code that + * reaches the filesystem, and the demo is a static bundle. `demo-parity.test.ts` + * drives both over one fixture and fails when they disagree, which is the only + * thing that keeps a mirror honest — the alternative was two implementations + * asserted separately, which is how these two drifted in the first place. + * + * Loosening the server was the other option and it is the wrong one: the + * whole-token body index is what keeps search fast over a real corpus, and that + * is a measured decision recorded in `query.ts`. + */ + +/** + * Any record, read by field name. + * + * Deliberately not an interface with an index signature: `DocumentRecord`, + * `ChangeRecord` and the rest declare none, so requiring one would have made + * every call site cast — and a cast at the call site is a cast the reader has to + * check, where this one is checked once. + */ +export type SearchableRecord = object; + +function field(record: SearchableRecord, name: string): unknown { + return (record as Record)[name]; +} + +/** + * The server's tokenizer, character class included. + * + * Locale-independent on purpose, for the reason the server states: an index + * built under tr-TR and read under en-US produces tokens that do not match. + */ +export function tokenize(value: unknown): string[] { + return String(value ?? "") + .toLowerCase() + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .split(/[^a-z0-9_-]+/) + .filter(Boolean); +} + +/** The fields the server folds into one `metadata` bag, in its order. */ +const METADATA_FIELDS = [ + "path", + "kind", + "recordType", + "status", + "type", + "area", + "visibility", + "version", + "category", + "severity", + "confidence" +] as const; + +/** And the list-valued ones it appends to the same bag. */ +const METADATA_LISTS = ["tags", "owners", "deciders"] as const; + +interface RecordTokens { + id: Set; + /** Kept as an array too: the title falls back to a substring test. */ + title: string[]; + titleSet: Set; + metadata: Set; + body: Set; +} + +/** + * Cached per record object, the way the server caches on the record itself. + * + * The demo runs this on every keystroke over the whole snapshot, and tokenizing + * three hundred bodies per character is exactly the waste the server's own cache + * exists to avoid. A `WeakMap` rather than a property so nothing reaches a + * response body, and so a replaced record is collected with its tokens. + */ +const CACHE = new WeakMap(); + +function tokensOf(record: SearchableRecord): RecordTokens { + const cached = CACHE.get(record); + if (cached) return cached; + const title = tokenize(field(record, "title")); + const metadata = [ + ...METADATA_FIELDS.map((name) => field(record, name)), + ...METADATA_LISTS.flatMap((name) => { + const value = field(record, name); + return Array.isArray(value) ? (value as unknown[]) : []; + }) + ].join(" "); + const tokens: RecordTokens = { + id: new Set(tokenize(field(record, "id"))), + title, + titleSet: new Set(title), + metadata: new Set(tokenize(metadata)), + body: new Set(tokenize(field(record, "body"))) + }; + CACHE.set(record, tokens); + return tokens; +} + +/** + * The server's weights, which are the ranking as well as the match: a record + * matches a set of terms when it scores above zero for any of them. + */ +export function searchScore(record: SearchableRecord, terms: string[]): number { + if (!terms.length) return 0; + const tokens = tokensOf(record); + let score = 0; + for (const term of terms) { + if (tokens.id.has(term)) score += 30; + if (tokens.titleSet.has(term)) score += 15; + else if (tokens.title.some((token) => token.includes(term))) score += 8; + if (tokens.metadata.has(term)) score += 5; + if (tokens.body.has(term)) score += 2; + } + return score; +} + +/** The server's `QUERY_TOKEN`: `-`, an optional `field:`, and a quoted or bare value. */ +const QUERY_TOKEN = + /(-)?(?:([a-z_]+):(?:"([^"]*)"|(\S*))|"([^"]*)"|(\S+))/gi; + +export interface ParsedQuery { + terms: Array<{ value: string; negated: boolean }>; + filters: Array<{ field: string; value: string; negated: boolean }>; +} + +export function parseQuery(query: string): ParsedQuery { + const terms: ParsedQuery["terms"] = []; + const filters: ParsedQuery["filters"] = []; + for (const match of String(query || "").matchAll(QUERY_TOKEN)) { + const negated = match[1] === "-"; + const field = match[2]?.toLowerCase(); + const value = (match[3] ?? match[4] ?? match[5] ?? match[6] ?? "").trim(); + if (!value) continue; + if (field) filters.push({ field, value: value.toLowerCase(), negated }); + else terms.push({ value, negated }); + } + return { terms, filters }; +} + +function fieldValues(record: SearchableRecord, name: string): string[] { + const value = field(record, name); + if (value == null) return []; + return (Array.isArray(value) ? value : [value]).map((entry) => + String(entry).toLowerCase() + ); +} + +function matchesFilters( + record: SearchableRecord, + filters: ParsedQuery["filters"] +): boolean { + return filters.every((filter) => { + // `tag:` reads more naturally than `tags:`, and `claim:` than + // `claimed_by:`; both spellings work, as on the server. + const name = + filter.field === "tag" + ? "tags" + : filter.field === "claim" + ? "claimed_by" + : filter.field; + const hit = fieldValues(record, name).some((value) => + value.includes(filter.value) + ); + return filter.negated ? !hit : hit; + }); +} + +/** + * Whether a record answers a query, and how strongly. + * + * `null` is "no", so a caller can filter and rank in one pass. An empty query + * keeps every record at score zero, which is what both list endpoints do. + */ +export function scoreQuery( + record: SearchableRecord, + query: string +): number | null { + const parsed = parseQuery(query); + if (!matchesFilters(record, parsed.filters)) return null; + const excluded = parsed.terms + .filter((term) => term.negated) + .flatMap((term) => tokenize(term.value)); + if (excluded.length && searchScore(record, excluded)) return null; + const terms = tokenize( + parsed.terms + .filter((term) => !term.negated) + .map((term) => term.value) + .join(" ") + ); + if (!terms.length) return 0; + const score = searchScore(record, terms); + return score > 0 ? score : null; +} + +/** The predicate on its own, for the callers that only filter. */ +export function matchesQuery(record: SearchableRecord, query: string): boolean { + return scoreQuery(record, query) !== null; +} + +/** + * Filtered and ranked, which is the server's ordering too: score first, then the + * most recently touched, then the title. Two records with nothing to separate + * them must not swap between the demo and a real workspace. + */ +export function rankByQuery( + records: readonly T[], + query: string +): T[] { + return records + .map((record) => ({ record, score: scoreQuery(record, query) })) + .filter( + (entry): entry is { record: T; score: number } => entry.score !== null + ) + .sort( + (left, right) => + right.score - left.score || + String( + field(right.record, "updated") || + field(right.record, "date") || + "" + ).localeCompare( + String( + field(left.record, "updated") || + field(left.record, "date") || + "" + ) + ) || + String(field(left.record, "title") || "").localeCompare( + String(field(right.record, "title") || "") + ) + ) + .map((entry) => entry.record); +}