From 2c4de10784f53fd8695d6eef4bdb6a9cac049db1 Mon Sep 17 00:00:00 2001 From: intraQ <251956840+intraq-dev-ai@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:31:17 +1000 Subject: [PATCH] Fix SQL Editor export row limit --- .../data-source/live-sql-query-types.ts | 2 +- .../modules/data-source/sql-query-engine.ts | 2 +- apps/web/src/modules/sql-editor/api.ts | 8 +- .../sql-editor/sql-editor-page-actions.ts | 24 ++++- .../modules/sql-editor/use-sql-editor-page.ts | 11 ++- docs/SQL_EDITOR.md | 11 +++ tests/smoke/sql-editor-export-limits.test.ts | 91 +++++++++++++++++++ 7 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 tests/smoke/sql-editor-export-limits.test.ts diff --git a/apps/api/src/modules/data-source/live-sql-query-types.ts b/apps/api/src/modules/data-source/live-sql-query-types.ts index 4ebc1d5..66cb465 100644 --- a/apps/api/src/modules/data-source/live-sql-query-types.ts +++ b/apps/api/src/modules/data-source/live-sql-query-types.ts @@ -4,7 +4,7 @@ import type { ClientConfig } from 'pg'; import type { DataSourceRecord } from './foundation-store.js'; export const DEFAULT_LIMIT = 100; -export const MAX_LIMIT = 1000; +export const MAX_LIMIT = 100_000; export const DEFAULT_QUERY_TIMEOUT_MS = 15_000; export const MAX_QUERY_TIMEOUT_MS = 300_000; export const MIN_QUERY_TIMEOUT_MS = 1_000; diff --git a/apps/api/src/modules/data-source/sql-query-engine.ts b/apps/api/src/modules/data-source/sql-query-engine.ts index 5d34bbc..e19137d 100644 --- a/apps/api/src/modules/data-source/sql-query-engine.ts +++ b/apps/api/src/modules/data-source/sql-query-engine.ts @@ -18,7 +18,7 @@ import type { } from './sql-query-types.js'; const DEFAULT_LIMIT = 100; -const MAX_LIMIT = 1000; +const MAX_LIMIT = 100_000; export type { SqlQueryCell, diff --git a/apps/web/src/modules/sql-editor/api.ts b/apps/web/src/modules/sql-editor/api.ts index 9ec9663..0eb1e8f 100644 --- a/apps/web/src/modules/sql-editor/api.ts +++ b/apps/web/src/modules/sql-editor/api.ts @@ -12,6 +12,9 @@ import type { SqlEditorMetadataTable } from './types'; +export const SQL_EDITOR_PREVIEW_ROW_LIMIT = 1_000; +export const SQL_EDITOR_EXPORT_ROW_LIMIT = 100_000; + export async function fetchSqlEditorSources(): Promise { const payload = await requestApi<{ dataSources: SqlEditorSource[] }>('/api/sql-editor/data-sources'); return payload.dataSources; @@ -29,12 +32,13 @@ export async function fetchSqlEditorSchema(dataSourceId: string): Promise = {} + parameterValues: Record = {}, + options: { defaultLimit?: number; maxLimit?: number } = {} ): Promise { return requestApi('/api/sql-editor/execute', { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ dataSourceId, parameterValues, query }) + body: JSON.stringify({ dataSourceId, parameterValues, query, ...options }) }); } diff --git a/apps/web/src/modules/sql-editor/sql-editor-page-actions.ts b/apps/web/src/modules/sql-editor/sql-editor-page-actions.ts index e487020..c9af9e5 100644 --- a/apps/web/src/modules/sql-editor/sql-editor-page-actions.ts +++ b/apps/web/src/modules/sql-editor/sql-editor-page-actions.ts @@ -1,4 +1,5 @@ import { normalizePivotConfig, resolvePivotSelection } from './pivot'; +import { executeSqlEditorQuery, SQL_EDITOR_EXPORT_ROW_LIMIT } from './api'; import { clearSqlEditorQueryHistory, saveSqlEditorQueryHistory, saveSqlEditorTabs } from './storage'; import { csvFileNameForBaseName, @@ -18,6 +19,7 @@ import { csvFromResult, defaultDateTokenForParameter, defaultQuery, + replaceParameters, syncSqlParameters } from './workflow'; @@ -212,11 +214,27 @@ export function createSqlEditorPageActions(state: SqlEditorPageState) { setQuery(next, false); } - function prepareCsv(): void { + async function prepareCsv(): Promise { if (!result.value) return; const baseName = selectedCustomSource.value?.name ?? selectedSource.value?.name ?? 'sql-results'; - downloadCsvFile(csvFromResult(result.value), csvFileNameForBaseName(baseName)); - status.value = 'CSV downloaded'; + status.value = `Preparing CSV export, capped at ${SQL_EDITOR_EXPORT_ROW_LIMIT.toLocaleString()} rows`; + try { + const exportQuery = result.value.query || replaceParameters(query.value, parameterValues.value); + const exportResult = await executeSqlEditorQuery( + selectedDataSourceId.value, + exportQuery, + parameterValues.value, + { + defaultLimit: SQL_EDITOR_EXPORT_ROW_LIMIT, + maxLimit: SQL_EDITOR_EXPORT_ROW_LIMIT + } + ); + downloadCsvFile(csvFromResult(exportResult), csvFileNameForBaseName(baseName)); + status.value = `CSV downloaded with ${exportResult.rowCount.toLocaleString()} row${exportResult.rowCount === 1 ? '' : 's'}`; + } catch (caught) { + error.value = caught instanceof Error ? caught.message : 'CSV export failed.'; + status.value = 'CSV export failed'; + } } function previousPage(): void { diff --git a/apps/web/src/modules/sql-editor/use-sql-editor-page.ts b/apps/web/src/modules/sql-editor/use-sql-editor-page.ts index 0ad78b2..6c34663 100644 --- a/apps/web/src/modules/sql-editor/use-sql-editor-page.ts +++ b/apps/web/src/modules/sql-editor/use-sql-editor-page.ts @@ -1,6 +1,7 @@ import { computed, nextTick, onMounted, watch } from 'vue'; import { executeSqlEditorQuery, + SQL_EDITOR_PREVIEW_ROW_LIMIT, fetchSqlEditorSchema, fetchSqlEditorSources, fetchSqlEditorSuggestions, @@ -160,7 +161,15 @@ export function useSqlEditorPage() { page.currentPage.value = 1; try { const executableQuery = replaceParameters(page.query.value, page.parameterValues.value); - page.result.value = await executeSqlEditorQuery(page.selectedDataSourceId.value, executableQuery, page.parameterValues.value); + page.result.value = await executeSqlEditorQuery( + page.selectedDataSourceId.value, + executableQuery, + page.parameterValues.value, + { + defaultLimit: SQL_EDITOR_PREVIEW_ROW_LIMIT, + maxLimit: SQL_EDITOR_PREVIEW_ROW_LIMIT + } + ); actions.syncPivotSelection(); page.lastSuccessfulRunSignature.value = page.currentRunSignature.value; actions.pushHistory(page.query.value); diff --git a/docs/SQL_EDITOR.md b/docs/SQL_EDITOR.md index 9084916..c107958 100644 --- a/docs/SQL_EDITOR.md +++ b/docs/SQL_EDITOR.md @@ -18,6 +18,17 @@ Use SQL Editor against databases you control or are authorized to query. Keep connection strings and database passwords in `.env` or deployment secrets, never in screenshots, examples, issues, or committed files. +## Row limits and export + +SQL Editor keeps interactive query previews bounded so the browser is not asked +to render very large result sets. + +- Query preview is capped at 1,000 rows. +- CSV export reruns the last successful query and is capped at 100,000 rows. +- If the SQL itself contains a smaller `LIMIT`, that query limit still applies. +- If the SQL contains a larger `LIMIT`, the app caps it to the relevant preview + or export maximum. + ## Demo path 1. Run the seeded app. diff --git a/tests/smoke/sql-editor-export-limits.test.ts b/tests/smoke/sql-editor-export-limits.test.ts new file mode 100644 index 0000000..0a62be4 --- /dev/null +++ b/tests/smoke/sql-editor-export-limits.test.ts @@ -0,0 +1,91 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { executeDataSourceSqlQuery } from '../../apps/api/src/modules/data-source/sql-query-engine.js'; +import { + executeSqlEditorQuery, + SQL_EDITOR_EXPORT_ROW_LIMIT, + SQL_EDITOR_PREVIEW_ROW_LIMIT +} from '../../apps/web/src/modules/sql-editor/api.js'; + +describe('SQL Editor row limits', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('uses explicit preview and export limits in SQL Editor API requests', async () => { + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + return new Response(JSON.stringify({ + success: true, + data: { + columns: ['id'], + rows: [{ id: 1 }], + rowCount: 1, + executionTime: 1, + dataSource: { id: body.dataSourceId, name: 'Sample', type: 'sample' }, + columnTypes: [{ name: 'id', type: 'number' }], + query: body.query, + requestBody: body + } + }), { status: 200, headers: { 'content-type': 'application/json' } }); + }); + vi.stubGlobal('fetch', fetchMock); + + await executeSqlEditorQuery('source-1', 'select * from sample_sales_model', {}, { + defaultLimit: SQL_EDITOR_PREVIEW_ROW_LIMIT, + maxLimit: SQL_EDITOR_PREVIEW_ROW_LIMIT + }); + await executeSqlEditorQuery('source-1', 'select * from sample_sales_model', {}, { + defaultLimit: SQL_EDITOR_EXPORT_ROW_LIMIT, + maxLimit: SQL_EDITOR_EXPORT_ROW_LIMIT + }); + + const previewBody = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body ?? '{}')) as Record; + const exportBody = JSON.parse(String(fetchMock.mock.calls[1]?.[1]?.body ?? '{}')) as Record; + + expect(previewBody.defaultLimit).toBe(1_000); + expect(previewBody.maxLimit).toBe(1_000); + expect(exportBody.defaultLimit).toBe(100_000); + expect(exportBody.maxLimit).toBe(100_000); + }); + + it('allows SQL Editor export-sized limits above the preview cap for sample data', () => { + const rows = Array.from({ length: 1_250 }, (_value, index) => ({ + id: index + 1, + revenue: index * 10 + })); + + const result = executeDataSourceSqlQuery({ + query: 'select * from sample_sales_model', + tempDataSource: { + id: 'source-1', + name: 'Sample Sales', + type: 'sample', + sourceType: 'source', + status: 'connected', + isSample: true, + config: {}, + settings: {}, + dictionary: {}, + tables: [{ + id: 'sample_sales_model', + name: 'sample_sales_model', + description: 'Sample sales model', + fields: [ + { name: 'id', type: 'number', description: 'ID', dictionaryDescription: 'ID' }, + { name: 'revenue', type: 'number', description: 'Revenue', dictionaryDescription: 'Revenue' } + ], + dictionary: {}, + isSelected: true, + sampleRows: rows + }] + }, + defaultLimit: 1_250, + maxLimit: SQL_EDITOR_EXPORT_ROW_LIMIT + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.rowCount).toBe(1_250); + expect(result.data.rows).toHaveLength(1_250); + }); +});