From 939c6dfe2d7c8324c9d10d788dd2221677dcd04d Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 22 Mar 2026 22:40:53 +0000 Subject: [PATCH 1/2] feat: add TAGS() macro and auto-detection for multi-tag AND queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces two complementary solutions for the common mistake of filtering by multiple tags with AND (which always returns zero results): 1. TAGS() macro: explicit opt-in syntax WHERE TAGS('#project', '#active') → path IN (SELECT path FROM tags WHERE tag = '#project' INTERSECT SELECT path FROM tags WHERE tag = '#active') 2. Auto-detection: transparently rewrites the broken pattern WHERE tag = '#a' AND tag = '#b' → path IN (...INTERSECT...) Mixed conditions (AND name = 'test') are preserved correctly. Both use INTERSECT subqueries which are more flexible than an EXISTS approach (no hardcoded table reference required). Adds disableTagAutoDetection setting so advanced users can opt out of the transparent rewrite and handle the SQL themselves. Docs updated in query-vault-content.md and faq/understanding-tags.md. --- docs/faq/understanding-tags.md | 16 ++ docs/query-vault-content.md | 47 ++++- .../codeblockHandler/CodeblockProcessor.ts | 4 +- .../inline/InlineProcessor.ts | 4 +- src/modules/editor/sql/sqlTransformer.ts | 168 ++++++++++++++++- src/modules/editor/sql/transformer.test.ts | 173 ++++++++++++++++-- src/modules/settings/SQLSealSettingsTab.ts | 11 ++ 7 files changed, 401 insertions(+), 22 deletions(-) diff --git a/docs/faq/understanding-tags.md b/docs/faq/understanding-tags.md index bed69d2..0147aa0 100644 --- a/docs/faq/understanding-tags.md +++ b/docs/faq/understanding-tags.md @@ -28,3 +28,19 @@ The tags table is specifically designed to capture all tags found within the vau ## In Summary When using SQLSeal, keep in mind that tags from frontmatter and tags in the tags table serve different purposes and are structured differently. Frontmatter tags are essentially metadata fields of the file and appear as-is in the files table, while the tags table provides a detailed, row-based view of all tags in your vault. + +## Filtering by Multiple Tags + +A common pitfall when querying the `tags` table is writing: + +```sql +WHERE tag = '#project' AND tag = '#active' +``` + +This always returns zero results because no single row can have two different `tag` values at once. SQLSeal automatically detects and corrects this pattern. You can also use the explicit `TAGS()` macro: + +```sqlseal +SELECT * FROM files WHERE TAGS('#project', '#active') +``` + +See [Filter by Multiple Tags](../query-vault-content.md#filter-by-multiple-tags-and-logic) for full details. diff --git a/docs/query-vault-content.md b/docs/query-vault-content.md index ea5c9d4..211c0f9 100644 --- a/docs/query-vault-content.md +++ b/docs/query-vault-content.md @@ -19,11 +19,56 @@ SELECT * FROM files WHERE type = 'resource' The query above will return only files that have property `type` set to value `resource`. ## Filter by Tags -Tags are kept in a separate table `tags`. To select all files that have specific tag, we can perform simple join. +Tags are kept in a separate table `tags`. To select all files that have a specific tag, perform a simple join: ```sqlseal SELECT files.* FROM files JOIN tags ON files.path=tags.path WHERE tag = '#important' ``` +## Filter by Multiple Tags (AND logic) + +Filtering by multiple tags with a plain `AND` clause **always returns zero results**. This is because each row in the `tags` table holds only one tag — a single row can never simultaneously satisfy `tag = '#project' AND tag = '#active'`. + +SQLSeal provides two ways to correctly select files that have *all* of the listed tags: + +### Option 1 — TAGS() macro (recommended) + +Use the built-in `TAGS()` macro in your `WHERE` clause: + +```sqlseal +SELECT * FROM files WHERE TAGS('#project', '#active') +``` + +SQLSeal rewrites this into an efficient INTERSECT query automatically. You can combine it with other conditions too: + +```sqlseal +SELECT * FROM files WHERE TAGS('#project', '#active') AND name LIKE '%meeting%' +``` + +### Option 2 — Auto-detection (transparent) + +If you accidentally write the broken pattern, SQLSeal detects it and fixes it for you: + +```sqlseal +SELECT * FROM files WHERE tag = '#project' AND tag = '#active' +``` + +This is automatically rewritten to the same INTERSECT query as the `TAGS()` macro — no change needed on your part. + +> **Advanced users:** to disable the transparent rewrite and handle the SQL yourself, enable *Disable Tag Auto-Detection* in the SQLSeal settings under *Behavior*. + +### How it works under the hood + +Both approaches produce an INTERSECT subquery: + +```sql +SELECT * FROM files +WHERE path IN ( + SELECT path FROM tags WHERE tag = '#project' + INTERSECT + SELECT path FROM tags WHERE tag = '#active' +) +``` + ## Table Structure See full breakdown in [Data Sources: Vault Data](./data-sources/vault-data.md). diff --git a/src/modules/editor/codeblockHandler/CodeblockProcessor.ts b/src/modules/editor/codeblockHandler/CodeblockProcessor.ts index b5db450..e5a306a 100644 --- a/src/modules/editor/codeblockHandler/CodeblockProcessor.ts +++ b/src/modules/editor/codeblockHandler/CodeblockProcessor.ts @@ -120,7 +120,9 @@ export class CodeblockProcessor extends MarkdownRenderChild { await this.sync.getTablesMappingForContext(this.sourceKey); // Transforming Query - const res = this.tq(this.query, registeredTablesForContext); + const res = this.tq(this.query, registeredTablesForContext, { + disableTagAutoDetection: this.settings.get('disableTagAutoDetection') + }); const transformedQuery = res.sql; if (this.flags.refresh) { diff --git a/src/modules/editor/codeblockHandler/inline/InlineProcessor.ts b/src/modules/editor/codeblockHandler/inline/InlineProcessor.ts index 5fab118..53bbab7 100644 --- a/src/modules/editor/codeblockHandler/inline/InlineProcessor.ts +++ b/src/modules/editor/codeblockHandler/inline/InlineProcessor.ts @@ -38,7 +38,9 @@ export class InlineProcessor extends MarkdownRenderChild { async render() { try { const registeredTablesForContext = await this.sync.getTablesMappingForContext(this.sourcePath); - const transformedQuery = transformQuery(this.query, registeredTablesForContext); + const transformedQuery = transformQuery(this.query, registeredTablesForContext, { + disableTagAutoDetection: this.settings.get('disableTagAutoDetection') + }); // FIXME: settings here instead of plugin if (this.settings.get('enableDynamicUpdates')) { diff --git a/src/modules/editor/sql/sqlTransformer.ts b/src/modules/editor/sql/sqlTransformer.ts index a2d456b..028ed61 100644 --- a/src/modules/editor/sql/sqlTransformer.ts +++ b/src/modules/editor/sql/sqlTransformer.ts @@ -2,13 +2,171 @@ import { uniq } from 'lodash'; import { parse, show, cstVisitor } from 'sql-parser-cst'; /** - * Function transforms SQL query and updates tables into actual names in te database. + * Pre-processes a SQLSeal query by rewriting the TAGS() macro into valid SQL INTERSECT subqueries. + * + * The TAGS() macro solves a common user mistake: filtering by multiple tags with AND + * always returns zero results because a single row in the tags table cannot have two + * different tag values simultaneously. + * + * Usage: + * WHERE TAGS('#project', '#active') + * Becomes: + * WHERE path IN (SELECT path FROM tags WHERE tag = '#project' INTERSECT SELECT path FROM tags WHERE tag = '#active') + */ +export function rewriteTagsMacro(sql: string): string { + return sql.replace(/TAGS\s*\(([^)]*)\)/gi, (_, argsStr: string) => { + // Extract all single-quoted string arguments + const tags = [...argsStr.matchAll(/'([^']*)'/g)].map(m => m[1]) + if (tags.length === 0) return 'TRUE' + if (tags.length === 1) { + return `path IN (SELECT path FROM tags WHERE tag = '${tags[0]}')` + } + return `path IN (${tags.map(tag => `SELECT path FROM tags WHERE tag = '${tag}'`).join(' INTERSECT ')})` + }) +} + +/** + * Recursively flattens a left-associative AND chain into a flat array of leaf nodes. + */ +function flattenAnd(node: any): any[] { + const opName = typeof node.operator === 'string' ? node.operator : node.operator?.name + if (node.type === 'binary_expr' && opName === 'AND') { + return [...flattenAnd(node.left), ...flattenAnd(node.right)] + } + return [node] +} + +/** + * If the node is a tag comparison (`tag = '#value'` or `tags.tag = '#value'`), + * returns the tag value string. Otherwise returns null. + */ +function extractTagValue(node: any): string | null { + if (node.type !== 'binary_expr' || node.operator !== '=') return null + const { left, right } = node + + const isTagCol = + (left.type === 'identifier' && left.name?.toLowerCase() === 'tag') || + (left.type === 'member_expr' && + left.object?.type === 'identifier' && left.object.name?.toLowerCase() === 'tags' && + left.property?.type === 'identifier' && left.property.name?.toLowerCase() === 'tag') + + if (!isTagCol) return null + if (right.type !== 'string_literal') return null + return right.value +} + +/** + * Auto-detects the common mistake of `tag = '#a' AND tag = '#b'` (which always returns zero + * results) and rewrites it to an INTERSECT subquery. Handles mixed conditions too, e.g. + * `tag = '#a' AND tag = '#b' AND name = 'test'` is correctly preserved as + * `path IN (...INTERSECT...) AND name = 'test'`. + * + * Does nothing if: + * - The SQL cannot be parsed + * - Fewer than 2 tag comparisons exist in any AND chain + * - The TAGS() macro was already used (no tag AND patterns remain) + */ +export function autoDetectTagAndPattern(sql: string): string { + let parsed: any + try { + parsed = parse(sql, { + dialect: 'sqlite', + includeSpaces: true, + includeComments: true, + includeNewlines: true, + includeRange: true, + paramTypes: ['@name'] + }) + } catch { + return sql // unparseable — leave unchanged + } + + interface Candidate { + start: number + end: number + tagValues: string[] + otherRanges: [number, number][] + } + + const candidates: Candidate[] = [] + + const visitor = cstVisitor({ + binary_expr: (node: any) => { + const opName = typeof node.operator === 'string' ? node.operator : node.operator?.name + if (opName !== 'AND' || !node.range) return + + const leaves = flattenAnd(node) + const tagValues: string[] = [] + const otherRanges: [number, number][] = [] + + for (const leaf of leaves) { + const tagVal = extractTagValue(leaf) + if (tagVal !== null) { + tagValues.push(tagVal) + } else if (leaf.range) { + otherRanges.push([leaf.range[0], leaf.range[1]]) + } + } + + if (tagValues.length < 2) return + + candidates.push({ start: node.range[0], end: node.range[1], tagValues, otherRanges }) + } + }) + + visitor(parsed) + + if (candidates.length === 0) return sql + + // Keep only the outermost candidates — inner AND subtrees are subsumed by their parents + const outermost = candidates.filter(c => + !candidates.some(other => + other !== c && other.start <= c.start && c.end <= other.end + ) + ) + + if (outermost.length === 0) return sql + + // Apply replacements from right to left so earlier positions remain valid + outermost.sort((a, b) => b.start - a.start) + + let result = sql + for (const { start, end, tagValues, otherRanges } of outermost) { + const intersectPart = + tagValues.length === 1 + ? `path IN (SELECT path FROM tags WHERE tag = '${tagValues[0]}')` + : `path IN (${tagValues.map(t => `SELECT path FROM tags WHERE tag = '${t}'`).join(' INTERSECT ')})` + + const otherParts = otherRanges.map(([s, e]) => result.slice(s, e).trim()) + const replacement = [intersectPart, ...otherParts].join(' AND ') + + result = result.slice(0, start) + replacement + result.slice(end) + } + + return result +} + +/** + * Function transforms SQL query and updates tables into actual names in the database. * @param query SQL query string * @param tableNames mappings of user-defined values (keys) and actual table names in the database (values) + * @param options optional behaviour flags * @returns returns object containing new query (sql) and all tables that has been mapped (mappedTables) */ -export const transformQuery = (query: string, tableNames: Record) => { - const cst = parse(query, { +export const transformQuery = ( + query: string, + tableNames: Record, + options?: { disableTagAutoDetection?: boolean } +) => { + // Pre-process TAGS() macro before handing off to the CST parser + let preprocessed = rewriteTagsMacro(query) + + // Auto-detect and rewrite implicit `tag='X' AND tag='Y'` patterns (unless disabled) + if (!options?.disableTagAutoDetection) { + preprocessed = autoDetectTagAndPattern(preprocessed) + } + + const cst = parse(preprocessed, { dialect: 'sqlite', includeSpaces: true, includeComments: true, @@ -17,7 +175,7 @@ export const transformQuery = (query: string, tableNames: Record }) const watchTables: string[] = [] - + const tableMapper = cstVisitor({ identifier: (identifier) => { if (tableNames[identifier.name]) { @@ -35,4 +193,4 @@ export const transformQuery = (query: string, tableNames: Record sql: show(cst), mappedTables: uniq(watchTables) } -} \ No newline at end of file +} diff --git a/src/modules/editor/sql/transformer.test.ts b/src/modules/editor/sql/transformer.test.ts index 7a89fd7..538075c 100644 --- a/src/modules/editor/sql/transformer.test.ts +++ b/src/modules/editor/sql/transformer.test.ts @@ -1,4 +1,4 @@ -import { transformQuery } from './sqlTransformer' +import { transformQuery, rewriteTagsMacro, autoDetectTagAndPattern } from './sqlTransformer' describe('SQL Transformer', () => { @@ -28,18 +28,18 @@ WHERE list IS NOT NULL`, }) it('should properly transform window function', () => { - const input = `WITH RECURSIVE -numbers AS ( -SELECT 1 AS num -UNION ALL -SELECT num + 1 -FROM numbers -WHERE num < 10 + const input = `WITH RECURSIVE +numbers AS ( +SELECT 1 AS num +UNION ALL +SELECT num + 1 +FROM numbers +WHERE num < 10 ) -SELECT -num, -SUM(num) OVER (ORDER BY num) AS running_total -FROM +SELECT +num, +SUM(num) OVER (ORDER BY num) AS running_total +FROM numbers` expect(transformQuery(input, { files: 'files' })).toEqual({ @@ -50,7 +50,7 @@ numbers` it('should properly transform query with parameters', () => { const input = `WITH data(a) AS ( - VALUES(1),(2),(3),(5),(6),(7) +\tVALUES(1),(2),(3),(5),(6),(7) ) SELECT * FROM data WHERE a >= @test` @@ -59,4 +59,149 @@ SELECT * FROM data WHERE a >= @test` mappedTables: [] }) }) -}) \ No newline at end of file +}) + +// --------------------------------------------------------------------------- +// TAGS() macro — rewrites multi-tag AND queries into INTERSECT subqueries +// --------------------------------------------------------------------------- + +describe('rewriteTagsMacro', () => { + it('rewrites a single tag to one IN subquery', () => { + const result = rewriteTagsMacro("SELECT * FROM files WHERE TAGS('#project')") + expect(result).toBe( + "SELECT * FROM files WHERE path IN (SELECT path FROM tags WHERE tag = '#project')" + ) + }) + + it('rewrites two tags to INTERSECT subquery', () => { + const result = rewriteTagsMacro("SELECT * FROM files WHERE TAGS('#project', '#active')") + expect(result).toBe( + "SELECT * FROM files WHERE path IN (SELECT path FROM tags WHERE tag = '#project' INTERSECT SELECT path FROM tags WHERE tag = '#active')" + ) + }) + + it('rewrites three tags to three-way INTERSECT', () => { + const result = rewriteTagsMacro("WHERE TAGS('#a', '#b', '#c')") + expect(result).toBe( + "WHERE path IN (SELECT path FROM tags WHERE tag = '#a' INTERSECT SELECT path FROM tags WHERE tag = '#b' INTERSECT SELECT path FROM tags WHERE tag = '#c')" + ) + }) + + it('rewrites TAGS() with no arguments to TRUE (no-op filter)', () => { + expect(rewriteTagsMacro('SELECT * FROM files WHERE TAGS()')).toBe( + 'SELECT * FROM files WHERE TRUE' + ) + }) + + it('is case-insensitive — lowercase tags() is also rewritten', () => { + const result = rewriteTagsMacro("SELECT * FROM files WHERE tags('#project')") + expect(result).toContain("path IN (SELECT path FROM tags WHERE tag = '#project')") + }) + + it('preserves other WHERE conditions alongside TAGS()', () => { + const result = rewriteTagsMacro("SELECT * FROM files WHERE TAGS('#project') AND name LIKE '%note%'") + expect(result).toContain("path IN (SELECT path FROM tags WHERE tag = '#project')") + expect(result).toContain("AND name LIKE '%note%'") + }) + + it('leaves SQL without TAGS() unchanged', () => { + const sql = "SELECT * FROM files WHERE name = 'hello'" + expect(rewriteTagsMacro(sql)).toBe(sql) + }) +}) + +// --------------------------------------------------------------------------- +// autoDetectTagAndPattern — transparently rewrites tag='X' AND tag='Y' +// --------------------------------------------------------------------------- + +describe('autoDetectTagAndPattern', () => { + it('rewrites two tag conditions joined by AND into INTERSECT', () => { + const sql = "SELECT * FROM files WHERE tag = '#a' AND tag = '#b'" + const result = autoDetectTagAndPattern(sql) + expect(result).toContain("path IN (SELECT path FROM tags WHERE tag = '#a' INTERSECT SELECT path FROM tags WHERE tag = '#b')") + }) + + it('rewrites three tag conditions into three-way INTERSECT', () => { + const sql = "SELECT * FROM files WHERE tag = '#a' AND tag = '#b' AND tag = '#c'" + const result = autoDetectTagAndPattern(sql) + expect(result).toContain("path IN (SELECT path FROM tags WHERE tag = '#a' INTERSECT SELECT path FROM tags WHERE tag = '#b' INTERSECT SELECT path FROM tags WHERE tag = '#c')") + }) + + it('accepts both tag and tags.tag column references', () => { + const sql = "SELECT * FROM files WHERE tags.tag = '#a' AND tag = '#b'" + const result = autoDetectTagAndPattern(sql) + expect(result).toContain('INTERSECT') + }) + + it('preserves non-tag conditions in mixed AND chains', () => { + const sql = "SELECT * FROM files WHERE tag = '#a' AND tag = '#b' AND name = 'test'" + const result = autoDetectTagAndPattern(sql) + expect(result).toContain("path IN (SELECT path FROM tags WHERE tag = '#a' INTERSECT SELECT path FROM tags WHERE tag = '#b')") + expect(result).toContain("name = 'test'") + }) + + it('leaves a single tag condition unchanged (no AND needed)', () => { + const sql = "SELECT * FROM files WHERE tag = '#a'" + expect(autoDetectTagAndPattern(sql)).toBe(sql) + }) + + it('leaves OR-joined tag conditions unchanged', () => { + const sql = "SELECT * FROM files WHERE tag = '#a' OR tag = '#b'" + expect(autoDetectTagAndPattern(sql)).toBe(sql) + }) + + it('leaves SQL without tag conditions unchanged', () => { + const sql = "SELECT * FROM files WHERE name = 'hello'" + expect(autoDetectTagAndPattern(sql)).toBe(sql) + }) + + it('returns original SQL when parsing fails', () => { + const invalid = 'this is not valid SQL @@##' + expect(autoDetectTagAndPattern(invalid)).toBe(invalid) + }) +}) + +// --------------------------------------------------------------------------- +// transformQuery — TAGS() macro integration +// --------------------------------------------------------------------------- + +describe('transformQuery — TAGS() macro integration', () => { + it('applies the TAGS() rewrite then runs normal table-name mapping', () => { + const result = transformQuery( + "SELECT * FROM files WHERE TAGS('#project')", + { files: 'files_vault_abc' } + ) + expect(result.sql).toContain('FROM files_vault_abc') + expect(result.sql).toContain("path IN (SELECT path FROM tags WHERE tag = '#project')") + }) + + it('TAGS() with two tags produces an INTERSECT subquery in the final SQL', () => { + const result = transformQuery( + "SELECT * FROM files WHERE TAGS('#project', '#status/backlog')", + {} + ) + expect(result.sql).toContain('INTERSECT') + expect(result.sql).toContain("tag = '#project'") + expect(result.sql).toContain("tag = '#status/backlog'") + }) + + it('auto-detects tag AND pattern and rewrites it', () => { + const result = transformQuery( + "SELECT * FROM files WHERE tag = '#project' AND tag = '#active'", + {} + ) + expect(result.sql).toContain('INTERSECT') + }) + + it('disableTagAutoDetection option suppresses auto-detection', () => { + const result = transformQuery( + "SELECT * FROM files WHERE tag = '#project' AND tag = '#active'", + {}, + { disableTagAutoDetection: true } + ) + // Should NOT rewrite — keep original AND pattern + expect(result.sql).not.toContain('INTERSECT') + expect(result.sql).toContain("tag = '#project'") + expect(result.sql).toContain("tag = '#active'") + }) +}) diff --git a/src/modules/settings/SQLSealSettingsTab.ts b/src/modules/settings/SQLSealSettingsTab.ts index 59e845d..9102f4e 100644 --- a/src/modules/settings/SQLSealSettingsTab.ts +++ b/src/modules/settings/SQLSealSettingsTab.ts @@ -10,6 +10,7 @@ export interface SQLSealSettings { enableSQLViewer: boolean; enableDynamicUpdates: boolean; enableSyntaxHighlighting: boolean; + disableTagAutoDetection: boolean; defaultView: 'grid' | 'markdown' | 'html'; gridItemsPerPage: number } @@ -21,6 +22,7 @@ export const DEFAULT_SETTINGS: SQLSealSettings = { enableSQLViewer: true, enableDynamicUpdates: true, enableSyntaxHighlighting: true, + disableTagAutoDetection: false, defaultView: 'grid', gridItemsPerPage: 20 }; @@ -75,6 +77,15 @@ export class SQLSealSettingsTab extends PluginSettingTab { this.display(); // this.callChanges() })); + new Setting(containerEl) + .setName('Disable Tag Auto-Detection') + .setDesc('By default SQLSeal automatically rewrites `tag = \'#a\' AND tag = \'#b\'` into an efficient INTERSECT query. Enable this to turn off that behaviour and write raw SQL yourself.') + .addToggle(toggle => toggle + .setValue(this.settings.get('disableTagAutoDetection')) + .onChange(async (value) => { + this.settings.set('disableTagAutoDetection', !!value) + this.display(); + })); containerEl.createEl('h3', { text: 'Views' }); From aa914d962c51fe1e9ceed503e2dc91618ff07f55 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Mon, 6 Apr 2026 12:46:09 +0100 Subject: [PATCH 2/2] chore: adding changeset --- .changeset/giant-times-roll.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/giant-times-roll.md diff --git a/.changeset/giant-times-roll.md b/.changeset/giant-times-roll.md new file mode 100644 index 0000000..be929d6 --- /dev/null +++ b/.changeset/giant-times-roll.md @@ -0,0 +1,5 @@ +--- +"sqlseal": minor +--- + +adding support for TAGS macro