diff --git a/apps/web/src/backend/api/code-cell-execute.test.ts b/apps/web/src/backend/api/code-cell-execute.test.ts new file mode 100644 index 0000000..5f21d62 --- /dev/null +++ b/apps/web/src/backend/api/code-cell-execute.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from 'vitest'; +import { executeCodeCellNode } from './code-cell-node-exec'; +import { + clampCodeCellTimeout, + runCodeCellOnServer, + validateCodeCellRequest, +} from './code-cell-execute'; + +/** Value planted in APP_ENCRYPTION_KEY to prove an escaped cell cannot read it. */ +const SENTINEL_SECRET = 'sentinel-must-not-leak'; + +describe('validateCodeCellRequest', () => { + it('accepts a minimal js payload', () => { + const v = validateCodeCellRequest({ + body: 'return [];', + kind: 'js', + last: null, + vars: {}, + }); + expect(v.ok).toBe(true); + if (v.ok) { + expect(v.value.kind).toBe('js'); + expect(v.value.timeoutMs).toBeGreaterThan(0); + } + }); + + it('rejects bad kind / empty body', () => { + expect(validateCodeCellRequest({ body: '', kind: 'js' }).ok).toBe(false); + expect(validateCodeCellRequest({ body: 'return [];', kind: 'python' }).ok).toBe(false); + }); + + it('rejects shallow or malformed last/vars', () => { + expect( + validateCodeCellRequest({ + body: 'return [];', + kind: 'js', + last: { columns: [1], rows: [[1]], rowCount: 1 }, + }).ok + ).toBe(false); + expect( + validateCodeCellRequest({ + body: 'return [];', + kind: 'js', + last: null, + vars: { n: { kind: 'list' } }, + }).ok + ).toBe(false); + }); + + it('clamps timeout', () => { + expect(clampCodeCellTimeout(50)).toBe(100); + expect(clampCodeCellTimeout(999_999)).toBe(30_000); + }); +}); + +describe('executeCodeCellNode', () => { + it('runs async + lodash', async () => { + const r = await executeCodeCellNode({ + body: `import _ from 'lodash'; +const n = await Promise.resolve(3); +return _.map([n], (x) => ({ x })); +`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true, rowCount: 1 }); + if (r.ok) expect(r.rows).toEqual([[3]]); + }); + + it('supports await fetch via mock', async () => { + const realFetch = globalThis.fetch; + globalThis.fetch = (async () => + new Response(JSON.stringify({ hello: 'node' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) as typeof fetch; + try { + const r = await executeCodeCellNode({ + body: ` + const res = await fetch('https://example.test'); + const json = await res.json(); + return [{ status: res.status, hello: json.hello }]; + `, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[200, 'node']]); + } finally { + globalThis.fetch = realFetch; + } + }); +}); + +describe('runCodeCellOnServer (worker_threads sandbox)', () => { + async function run( + body: string, + kind: 'js' | 'ts', + timeoutMs = 5000 + ) { + const validated = validateCodeCellRequest({ + body, + kind, + last: null, + vars: {}, + maxRows: 10, + timeoutMs, + }); + expect(validated.ok).toBe(true); + if (!validated.ok) throw new Error(validated.error); + return runCodeCellOnServer(validated.value); + } + + it('transpiles TS and returns a grid', async () => { + // Worker cold-start (tsx) is slower under a full parallel vitest run. + const result = await run(`const n: number = 2;\nreturn [{ n: n * 3 }];`, 'ts', 20_000); + if (!result.ok) throw new Error(result.error); + expect(result.rows).toEqual([[6]]); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + }, 30_000); + + it('denies an escaped cell the server environment', async () => { + const prev = process.env.APP_ENCRYPTION_KEY; + process.env.APP_ENCRYPTION_KEY = SENTINEL_SECRET; + try { + // The AsyncFunction preamble shadows `process` lexically only — the Function + // constructor compiles in global scope and reaches the real one. The worker + // thread empties its own copy of process.env so the escape yields nothing. + const result = await run( + `const p = (function(){}).constructor('return process')();\n` + + `return [{ envKeys: Object.keys(p.env).length, secret: String(p.env.APP_ENCRYPTION_KEY) }];`, + 'js', + 20_000 + ); + if (!result.ok) throw new Error(result.error); + expect(result.rows).toEqual([[0, 'undefined']]); + // Parent env must stay intact (worker gets its own process.env copy). + expect(process.env.APP_ENCRYPTION_KEY).toBe(SENTINEL_SECRET); + } finally { + if (prev === undefined) delete process.env.APP_ENCRYPTION_KEY; + else process.env.APP_ENCRYPTION_KEY = prev; + } + }, 30_000); + + it('kills a cell that never yields', async () => { + const result = await run('while (true) {}\nreturn [];', 'js', 1000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/timed out/i); + }, 15_000); +}); diff --git a/apps/web/src/backend/api/code-cell-execute.ts b/apps/web/src/backend/api/code-cell-execute.ts new file mode 100644 index 0000000..45a9732 --- /dev/null +++ b/apps/web/src/backend/api/code-cell-execute.ts @@ -0,0 +1,207 @@ +/** + * POST /api/sql/code-cell — run a `-- @node` / `-- @nodets` cell on the + * FoxSchema Node backend (`kind` is `"js"` | `"ts"` on the wire). + */ + +import { Worker } from 'node:worker_threads'; +import { fileURLToPath } from 'node:url'; +import type { BrowserCodeCellKind } from '@foxschema/core'; +import { isCodeCellLast, isCodeCellVars } from '@foxschema/core'; +import type { CodeCellLast, CodeCellResult, CodeCellVars } from './code-cell-node-exec'; +import { clampMaxRows } from './sql-execute'; + +export const MAX_CODE_CELL_LENGTH = 100_000; +export const DEFAULT_CODE_CELL_TIMEOUT_MS = 10_000; +export const MAX_CODE_CELL_TIMEOUT_MS = 30_000; + +/** Wire language for Node cells (fence `node`/`nodets` already mapped client-side). */ +export type CodeCellKindLang = BrowserCodeCellKind; + +export type CodeCellRequestBody = { + body?: unknown; + kind?: unknown; + last?: unknown; + vars?: unknown; + maxRows?: unknown; + timeoutMs?: unknown; +}; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function clampCodeCellTimeout(v: unknown): number { + const n = typeof v === 'number' ? Math.floor(v) : Number.NaN; + if (!Number.isFinite(n)) return DEFAULT_CODE_CELL_TIMEOUT_MS; + return Math.min(Math.max(n, 100), MAX_CODE_CELL_TIMEOUT_MS); +} + +export type ValidatedCodeCell = { + body: string; + kind: CodeCellKindLang; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; + timeoutMs: number; +}; + +export function validateCodeCellRequest( + raw: CodeCellRequestBody +): { ok: true; value: ValidatedCodeCell } | { ok: false; error: string } { + if (typeof raw.body !== 'string' || !raw.body.trim()) { + return { ok: false, error: 'body is required' }; + } + if (raw.body.length > MAX_CODE_CELL_LENGTH) { + return { ok: false, error: `body must be under ${MAX_CODE_CELL_LENGTH} characters` }; + } + const kind: CodeCellKindLang | null = + raw.kind === 'js' || raw.kind === 'ts' ? raw.kind : null; + if (!kind) return { ok: false, error: 'kind must be "js" or "ts"' }; + const lastCandidate = raw.last ?? null; + if (!isCodeCellLast(lastCandidate)) { + return { ok: false, error: 'last must be null or { columns, rows, rowCount }' }; + } + const varsCandidate = raw.vars ?? {}; + if (!isCodeCellVars(varsCandidate)) { + return { ok: false, error: 'vars must be an object of variable shapes' }; + } + + return { + ok: true, + value: { + body: raw.body, + kind, + last: lastCandidate, + vars: varsCandidate, + maxRows: clampMaxRows(raw.maxRows), + timeoutMs: clampCodeCellTimeout(raw.timeoutMs), + }, + }; +} + +async function transpileTs(body: string): Promise { + const ts = await import('typescript'); + const out = ts.transpileModule(body, { + compilerOptions: { + target: ts.ScriptTarget.ES2020, + module: ts.ModuleKind.ESNext, + strict: false, + }, + reportDiagnostics: true, + }); + const errs = (out.diagnostics ?? []).filter((d) => d.category === ts.DiagnosticCategory.Error); + if (errs.length > 0) { + const msg = errs + .map((d) => ts.flattenDiagnosticMessageText(d.messageText, '\n')) + .join('; '); + throw new Error(msg || 'TypeScript transpile failed'); + } + return out.outputText; +} + +/** + * Cells run ONLY in a worker thread. There is deliberately no in-process + * fallback: the timeout is enforced by `worker.terminate()`, and a busy loop + * (`while (true) {}`) on the main thread cannot be interrupted — it would wedge + * the whole API server. The worker also runs with a scrubbed `process.env` + * (see code-cell-thread.ts), which an in-process run would not. If the thread + * cannot start, the cell fails closed. + */ +function runInWorkerThread(args: { + body: string; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; + timeoutMs: number; +}): Promise { + return new Promise((resolve) => { + let settled = false; + let worker: Worker | undefined; + let timer: ReturnType | undefined; + + const settle = (result: CodeCellResult) => { + if (settled) return; + settled = true; + if (timer !== undefined) clearTimeout(timer); + try { + worker?.terminate(); + } catch { + /* ignore */ + } + resolve(result); + }; + + const failed = (detail: string) => + settle({ + ok: false, + error: `Code cell sandbox unavailable: ${detail}`, + }); + + try { + // Always register tsx so .ts worker entry loads under vitest and plain node. + // Deduplicate if the parent already passed the same flag (tsx server). + const execArgv = [...process.execArgv]; + if (!execArgv.some((a) => a.includes('tsx'))) { + execArgv.push('--import', 'tsx/esm'); + } + worker = new Worker(fileURLToPath(new URL('./code-cell-thread.ts', import.meta.url)), { + workerData: { + body: args.body, + last: args.last, + vars: args.vars, + maxRows: args.maxRows, + }, + execArgv, + }); + } catch (error: unknown) { + failed(errorMessage(error)); + return; + } + + timer = setTimeout(() => { + settle({ + ok: false, + error: `Code cell timed out after ${args.timeoutMs}ms`, + }); + }, args.timeoutMs); + + worker.on('message', (msg: CodeCellResult) => settle(msg)); + worker.on('error', (error) => { + if (!settled) failed(errorMessage(error)); + }); + worker.on('exit', (code) => { + // terminate() after success/timeout often exits non-zero — ignore once settled. + if (!settled && code !== 0) failed(`worker exited with code ${code}`); + }); + }); +} + +/** + * Transpile (if needed) and execute a code cell on Node (worker_threads + timeout). + */ +export async function runCodeCellOnServer( + validated: ValidatedCodeCell +): Promise { + const started = Date.now(); + let body = validated.body; + try { + if (validated.kind === 'ts') { + body = await transpileTs(body); + } + } catch (error: unknown) { + return { + ok: false, + error: `TypeScript: ${errorMessage(error)}`, + durationMs: Date.now() - started, + }; + } + + const result = await runInWorkerThread({ + body, + last: validated.last, + vars: validated.vars, + maxRows: validated.maxRows, + timeoutMs: validated.timeoutMs, + }); + return { ...result, durationMs: Date.now() - started }; +} diff --git a/apps/web/src/backend/api/code-cell-node-exec.ts b/apps/web/src/backend/api/code-cell-node-exec.ts new file mode 100644 index 0000000..558ff96 --- /dev/null +++ b/apps/web/src/backend/api/code-cell-node-exec.ts @@ -0,0 +1,56 @@ +/** + * Node-side code-cell executor (async + fetch + allowlisted packages). + * Used by the worker thread and unit tests. + * + * The parsing/normalizing/sandbox machinery is shared with the browser + * executor via `@foxschema/core`; this module supplies only the Node globals + * preamble and the server's copies of the allowlisted packages. + */ + +import * as lodash from 'lodash-es'; +import * as dateFns from 'date-fns'; +import { + runCodeCellBody, + type CodeCellLast, + type CodeCellVars, + type CodeCellResult, +} from '@foxschema/core'; + +export type { CodeCellLast, CodeCellVars, CodeCellResult } from '@foxschema/core'; + +const MODULES: Record = { + lodash: lodash as object, + 'lodash-es': lodash as object, + 'date-fns': dateFns as object, +}; + +/** + * Shadow Node globals (`fetch` stays available for async HTTP). + * A guardrail against accidents, not a security boundary — see + * `code-cell-thread.ts` for what actually contains an escaped cell. + */ +const SANDBOX_PREAMBLE = ` +"use strict"; +var require = undefined; +var process = undefined; +var Buffer = undefined; +var __dirname = undefined; +var __filename = undefined; +var module = undefined; +var exports = undefined; +var global = undefined; +var globalThis = undefined; +`; + +/** + * Strip SQL `--` comments, resolve allowlisted imports, then run the body in + * an AsyncFunction sandbox. + */ +export function executeCodeCellNode(args: { + body: string; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; +}): Promise { + return runCodeCellBody({ ...args, preamble: SANDBOX_PREAMBLE, modules: MODULES }); +} diff --git a/apps/web/src/backend/api/code-cell-thread.ts b/apps/web/src/backend/api/code-cell-thread.ts new file mode 100644 index 0000000..e4d5446 --- /dev/null +++ b/apps/web/src/backend/api/code-cell-thread.ts @@ -0,0 +1,34 @@ +/** + * worker_threads entry for Node code cells. + * Receives workerData and posts a CodeCellResult. + */ +import { parentPort, workerData } from 'node:worker_threads'; +import { executeCodeCellNode, type CodeCellLast, type CodeCellVars } from './code-cell-node-exec'; + +// The AsyncFunction sandbox shadows `process` lexically, but a cell can still +// reach the real one via `(function(){}).constructor('return process')()` — the +// Function constructor compiles in global scope, so no `var` shadowing applies. +// Worker threads get their OWN copy of process.env, so emptying it here costs +// the parent nothing and denies an escaped cell the secrets it would otherwise +// read and POST out via `fetch` (APP_ENCRYPTION_KEY decrypts every saved DB +// password). Runs after imports, so nothing above has lost its config. +process.env = {}; +process.argv = process.argv.slice(0, 1); + +type Payload = { + body: string; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; +}; + +async function main() { + const data = workerData as Payload; + const result = await executeCodeCellNode(data); + parentPort?.postMessage(result); +} + +main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + parentPort?.postMessage({ ok: false, error: message }); +}); diff --git a/apps/web/src/backend/api/routes.ts b/apps/web/src/backend/api/routes.ts index 64352db..f4032f9 100644 --- a/apps/web/src/backend/api/routes.ts +++ b/apps/web/src/backend/api/routes.ts @@ -24,6 +24,11 @@ import { SignupModule } from '../modules/signup.module'; import { rateLimit } from './rate-limit'; import { runStatements, clampMaxRows, MAX_STATEMENTS, MAX_STATEMENT_LENGTH } from './sql-execute'; import { clampOffset } from './sql-page-wrap'; +import { + runCodeCellOnServer, + validateCodeCellRequest, + type CodeCellRequestBody, +} from './code-cell-execute'; import { getMetadataDbConfig, SUPPORTED_ENGINES, type DbEngine } from '../database/config'; import { createMetadataStore } from '../database/stores/registry'; import { keySchemeInfo } from '../cores/crypto'; @@ -408,6 +413,24 @@ export function createApiRoutes(connectionModule: ConnectionModule, connectionSt } }); + // SQL Editor Node code cells (`-- @node` / `-- @nodets`). No DB connection; + // runs allowlisted JS/TS with fetch in a worker_threads sandbox. + const codeCellLimiter = rateLimit({ windowMs: 60 * 1000, max: 30 }); + router.post('/sql/code-cell', codeCellLimiter, async (req: Request, res: Response) => { + const validated = validateCodeCellRequest(req.body as CodeCellRequestBody); + if (!validated.ok) { + res.status(400).json({ error: validated.error }); + return; + } + try { + const result = await runCodeCellOnServer(validated.value); + res.json(result); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : 'Code cell execution failed'; + res.status(500).json({ error: message }); + } + }); + router.post('/compare', async (req: Request, res: Response) => { const { source, target, scope } = req.body as { source: ConnectionRef; diff --git a/apps/web/src/frontend/api/sqlApi.test.ts b/apps/web/src/frontend/api/sqlApi.test.ts new file mode 100644 index 0000000..df60581 --- /dev/null +++ b/apps/web/src/frontend/api/sqlApi.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; +import { parseSqlStatementResult } from './sqlApi'; + +describe('parseSqlStatementResult', () => { + it('accepts ok and error payloads', () => { + expect( + parseSqlStatementResult({ + ok: true, + columns: ['a'], + rows: [[1]], + rowCount: 1, + truncated: false, + durationMs: 12, + }) + ).toEqual({ + ok: true, + value: { + ok: true, + columns: ['a'], + rows: [[1]], + rowCount: 1, + truncated: false, + hasNext: false, + durationMs: 12, + }, + }); + + expect( + parseSqlStatementResult({ ok: false, error: 'boom', durationMs: 3 }) + ).toEqual({ + ok: true, + value: { ok: false, error: 'boom', durationMs: 3 }, + }); + }); + + it('rejects malformed success grids', () => { + expect( + parseSqlStatementResult({ + ok: true, + columns: ['a'], + rows: [1], + rowCount: 1, + truncated: false, + durationMs: 1, + }).ok + ).toBe(false); + expect(parseSqlStatementResult({ ok: true, durationMs: 1 }).ok).toBe(false); + expect(parseSqlStatementResult({ ok: false, durationMs: 1 }).ok).toBe(false); + }); +}); diff --git a/apps/web/src/frontend/api/sqlApi.ts b/apps/web/src/frontend/api/sqlApi.ts index 9ebb2dd..c70b790 100644 --- a/apps/web/src/frontend/api/sqlApi.ts +++ b/apps/web/src/frontend/api/sqlApi.ts @@ -1,5 +1,6 @@ import { getApiBase } from './apiBase'; import type { ConnectionRef } from './schemaApi'; +import type { BrowserCodeCellKind, CodeCellLast, CodeCellVars } from '../lib/sql-splitter'; /** One statement's outcome from POST /sql/execute (mirrors backend sql-execute.ts). */ export type SqlStatementResult = @@ -41,3 +42,98 @@ export async function executeSql( if (!res.ok || !data.results) throw new Error(data.error || `Query failed (${res.status})`); return { results: data.results }; } + +/** Body for POST /sql/code-cell. Wire `kind` is language only (`js`|`ts`); Node vs browser is implied by the route. */ +export type ServerCodeCellPayload = { + body: string; + kind: BrowserCodeCellKind; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; + timeoutMs?: number; +}; + +function responseError(data: unknown): string | undefined { + if (data && typeof data === 'object' && 'error' in data) { + const err = (data as { error: unknown }).error; + if (typeof err === 'string' && err) return err; + } + return undefined; +} + +function isUnknownMatrix(value: unknown): value is unknown[][] { + return Array.isArray(value) && value.every((r) => Array.isArray(r)); +} + +/** Parse a `/sql/code-cell` JSON body into `SqlStatementResult` (rejects malformed success payloads). */ +export function parseSqlStatementResult( + data: unknown +): { ok: true; value: SqlStatementResult } | { ok: false; error: string } { + if (!data || typeof data !== 'object') { + return { ok: false, error: 'Invalid code cell response' }; + } + const o = data as Record; + if (typeof o.durationMs !== 'number' || !Number.isFinite(o.durationMs)) { + return { ok: false, error: 'Invalid code cell response (durationMs)' }; + } + if (o.ok === false) { + if (typeof o.error !== 'string' || !o.error) { + return { ok: false, error: 'Invalid code cell error response' }; + } + return { ok: true, value: { ok: false, error: o.error, durationMs: o.durationMs } }; + } + if (o.ok !== true) { + return { ok: false, error: responseError(data) || 'Invalid code cell response' }; + } + if (!Array.isArray(o.columns) || !o.columns.every((c) => typeof c === 'string')) { + return { ok: false, error: 'Invalid code cell response (columns)' }; + } + if (!isUnknownMatrix(o.rows)) { + return { ok: false, error: 'Invalid code cell response (rows)' }; + } + if (typeof o.rowCount !== 'number' || !Number.isFinite(o.rowCount)) { + return { ok: false, error: 'Invalid code cell response (rowCount)' }; + } + if (typeof o.truncated !== 'boolean') { + return { ok: false, error: 'Invalid code cell response (truncated)' }; + } + return { + ok: true, + value: { + ok: true, + columns: o.columns, + rows: o.rows, + rowCount: o.rowCount, + truncated: o.truncated, + hasNext: typeof o.hasNext === 'boolean' ? o.hasNext : false, + durationMs: o.durationMs, + }, + }; +} + +/** Run a `-- @node` / `-- @nodets` cell on the FoxSchema server. */ +export async function runCodeCellOnServer( + payload: ServerCodeCellPayload +): Promise { + const res = await fetch(`${getApiBase()}/sql/code-cell`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(payload), + }); + const text = await res.text(); + let data: unknown; + try { + data = JSON.parse(text); + } catch { + throw new Error(`Invalid response from server (${res.status}): ${text.slice(0, 200)}`); + } + if (!res.ok) { + throw new Error(responseError(data) || `Code cell failed (${res.status})`); + } + const parsed = parseSqlStatementResult(data); + if (!parsed.ok) { + throw new Error(parsed.error); + } + return parsed.value; +} diff --git a/apps/web/src/frontend/components/sql-editor/ResultsPanel.tsx b/apps/web/src/frontend/components/sql-editor/ResultsPanel.tsx index f4b4370..140e004 100644 --- a/apps/web/src/frontend/components/sql-editor/ResultsPanel.tsx +++ b/apps/web/src/frontend/components/sql-editor/ResultsPanel.tsx @@ -5,6 +5,7 @@ import type { ResultsLayout } from '../../store/sqlEditorTabLogic'; import { DataGrid, PANE_DEFAULT_H_PX, PANE_DEFAULT_PX, PANE_MIN_H_PX, PANE_MIN_PX } from './DataGrid'; import type { SqlStatementResult } from '../../api/sqlApi'; import { detectCodeCell } from '../../lib/codeCellRunner'; +import { CODE_CELL_KIND_LABEL } from '../../lib/sql-splitter'; interface Props { runs: CredentialRun[]; @@ -48,7 +49,7 @@ function bindAxisDrag(cursor: string, onMove: (ev: MouseEvent) => void): void { const statementLabel = (sql: string, index: number): string => { const cell = detectCodeCell(sql); if (cell) { - return `Query ${index + 1} · ${cell.kind === 'ts' ? 'TypeScript' : 'JavaScript'}`; + return `Query ${index + 1} · ${CODE_CELL_KIND_LABEL[cell.kind].long}`; } const compact = sql.replace(/\s+/g, ' ').trim(); return `Query ${index + 1} · ${compact.length > 48 ? compact.slice(0, 48) + '…' : compact}`; diff --git a/apps/web/src/frontend/components/sql-editor/SqlBookmarksPanel.tsx b/apps/web/src/frontend/components/sql-editor/SqlBookmarksPanel.tsx index cec7d17..478657e 100644 --- a/apps/web/src/frontend/components/sql-editor/SqlBookmarksPanel.tsx +++ b/apps/web/src/frontend/components/sql-editor/SqlBookmarksPanel.tsx @@ -1,6 +1,7 @@ import React, { useEffect, useRef, useState } from 'react'; -import { Pencil, Trash2 } from 'lucide-react'; +import { Pencil, Trash2, Sparkles } from 'lucide-react'; import { useSqlEditorStore } from '../../store/useSqlEditorStore'; +import { SQL_EDITOR_SAMPLE_BOOKMARKS } from '../../lib/sqlEditorSamples'; /** * Persist named SQL scripts. Save is on the sidebar section header; open loads @@ -11,9 +12,11 @@ export const SqlBookmarksPanel: React.FC = () => { const openBookmark = useSqlEditorStore((s) => s.openBookmark); const renameBookmark = useSqlEditorStore((s) => s.renameBookmark); const deleteBookmark = useSqlEditorStore((s) => s.deleteBookmark); + const installSampleBookmarks = useSqlEditorStore((s) => s.installSampleBookmarks); const [editingId, setEditingId] = useState(null); const [draft, setDraft] = useState(''); + const [sampleNote, setSampleNote] = useState(null); const inputRef = useRef(null); useEffect(() => { @@ -26,12 +29,42 @@ export const SqlBookmarksPanel: React.FC = () => { setEditingId(null); }; + const onInstallSamples = () => { + const n = installSampleBookmarks(); + setSampleNote( + n > 0 + ? `Added/updated ${n} sample bookmark${n === 1 ? '' : 's'}` + : 'Sample bookmarks already up to date' + ); + window.setTimeout(() => setSampleNote(null), 2500); + }; + + const sampleIds = new Set(SQL_EDITOR_SAMPLE_BOOKMARKS.map((s) => s.id)); + const hasAllSamples = SQL_EDITOR_SAMPLE_BOOKMARKS.every((s) => + bookmarks.some((b) => b.id === s.id) + ); + return (
+
+ + {sampleNote && ( + {sampleNote} + )} +
+ {bookmarks.length === 0 ? (

- Save a named script to reopen later. Rename the tab first so the bookmark keeps a clear - title. + Save a named script to reopen later, or add the built-in JS/TS/Node samples above.

) : (
    @@ -58,7 +91,11 @@ export const SqlBookmarksPanel: React.FC = () => { data-testid={`sql-bookmark-open-${b.id}`} title={b.sql.slice(0, 200) || '(empty)'} onClick={() => openBookmark(b.id)} - className="flex-1 min-w-0 text-left text-[13px] font-semibold text-slate-300 hover:text-cyan-300 truncate" + className={`flex-1 min-w-0 text-left text-[13px] font-semibold truncate ${ + sampleIds.has(b.id) + ? 'text-cyan-300/90 hover:text-cyan-200' + : 'text-slate-300 hover:text-cyan-300' + }`} > {b.title} diff --git a/apps/web/src/frontend/components/sql-editor/StatementStrip.tsx b/apps/web/src/frontend/components/sql-editor/StatementStrip.tsx index a273b19..2413913 100644 --- a/apps/web/src/frontend/components/sql-editor/StatementStrip.tsx +++ b/apps/web/src/frontend/components/sql-editor/StatementStrip.tsx @@ -4,8 +4,11 @@ import { Check, Copy } from 'lucide-react'; import { checkStatement, dmlLacksWhere, + isCodeCellKind, isMutatingDmlStatement, statementVerb, + CODE_CELL_KIND_LABEL, + type CodeCellKind, type SplitStatement, } from '../../lib/sql-splitter'; import { findVariableRefs, substituteVariables } from '../../lib/sql-variables'; @@ -46,6 +49,11 @@ const preview = (text: string, max = 120): string => { return compact.length > max ? compact.slice(0, max) + '…' : compact; }; +const codeCellBadge = (kind: CodeCellKind) => ({ + label: CODE_CELL_KIND_LABEL[kind].short, + title: `${CODE_CELL_KIND_LABEL[kind].long} code cell (-- @${kind} … -- @end)`, +}); + const DML_BADGE: Record = { update: 'UPD', delete: 'DEL', @@ -183,7 +191,8 @@ export const StatementStrip: React.FC = ({ statements, checked, onToggle, const status = checkStatement(stmt); const ok = status.level === 'ok'; const isChecked = checked.includes(i); - const codeKind = stmt.kind === 'js' || stmt.kind === 'ts' ? stmt.kind : null; + const codeKind = isCodeCellKind(stmt.kind) ? stmt.kind : null; + const codeBadge = codeKind ? codeCellBadge(codeKind) : null; const verb = codeKind ? null : statementVerb(stmt.text); const dmlBadge = safeMode && verb && isMutatingDmlStatement(stmt.text) ? DML_BADGE[verb] : null; @@ -227,12 +236,12 @@ export const StatementStrip: React.FC = ({ statements, checked, onToggle, > {ok ? '✓' : '⚠'} - {codeKind && ( + {codeBadge && ( - {codeKind === 'ts' ? 'TS' : 'JS'} + {codeBadge.label} )} {dmlBadge && ( diff --git a/apps/web/src/frontend/lib/codeCell.worker.ts b/apps/web/src/frontend/lib/codeCell.worker.ts index 5c1671e..cb042de 100644 --- a/apps/web/src/frontend/lib/codeCell.worker.ts +++ b/apps/web/src/frontend/lib/codeCell.worker.ts @@ -1,6 +1,6 @@ /// import { - executeCodeCellSync, + executeCodeCell, type CodeCellLast, type CodeCellResult, type CodeCellVars, @@ -19,12 +19,16 @@ export type CodeCellWorkerResponse = { id: number } & CodeCellResult; const ctx: DedicatedWorkerGlobalScope = self as unknown as DedicatedWorkerGlobalScope; ctx.onmessage = (ev: MessageEvent) => { - const msg = ev.data; - const result = executeCodeCellSync({ - body: msg.body, - last: msg.last, - vars: msg.vars, - maxRows: msg.maxRows, - }); - ctx.postMessage({ id: msg.id, ...result } satisfies CodeCellWorkerResponse); + const { id, body, last, vars, maxRows } = ev.data; + void executeCodeCell({ body, last, vars, maxRows }) + .catch((error: unknown) => ({ + // A rejection here (import prep, structured-clone of the result) would + // otherwise post nothing at all and leave the caller to time out with a + // misleading "timed out" message instead of the real error. + ok: false as const, + error: error instanceof Error ? error.message : String(error), + })) + .then((result) => { + ctx.postMessage({ id, ...result } satisfies CodeCellWorkerResponse); + }); }; diff --git a/apps/web/src/frontend/lib/codeCellExec.test.ts b/apps/web/src/frontend/lib/codeCellExec.test.ts index 3e9ab63..109b0c1 100644 --- a/apps/web/src/frontend/lib/codeCellExec.test.ts +++ b/apps/web/src/frontend/lib/codeCellExec.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { - executeCodeCellSync, + executeCodeCell, normalizeCodeCellReturn, sanitizeVarsForCodeCell, codeCellHasReturn, @@ -25,6 +25,19 @@ describe('sanitizeVarsForCodeCell', () => { expect(vars.ids).toEqual({ kind: 'list', values: [1, 2] }); expect(vars.t).toEqual({ kind: 'table', columns: ['a'], rows: [[1]] }); }); + + it('copies list and table values so cell mutations do not leak', () => { + const values = [1, 2]; + const rows = [[9]]; + const vars = sanitizeVarsForCodeCell([ + { name: 'ids', kind: 'list', values }, + { name: 't', kind: 'table', columns: ['a'], rows }, + ]); + (vars.ids as { values: unknown[] }).values.push(3); + (vars.t as { rows: unknown[][] }).rows[0]!.push(0); + expect(values).toEqual([1, 2]); + expect(rows).toEqual([[9]]); + }); }); describe('normalizeCodeCellReturn', () => { @@ -46,14 +59,64 @@ describe('normalizeCodeCellReturn', () => { ]); }); - it('rejects unsupported returns', () => { + it('accepts empty array as empty grid', () => { + expect(normalizeCodeCellReturn([], 10)).toEqual({ + ok: true, + columns: [], + rows: [], + rowCount: 0, + truncated: false, + }); + }); + + it('maps object rows under declared columns', () => { + const r = normalizeCodeCellReturn( + { + columns: ['id', 'name'], + rows: [{ id: 1, name: 'a', extra: 'x' }, { id: 2 }], + }, + 50 + ); + expect(r).toMatchObject({ ok: true, columns: ['id', 'name'], rowCount: 2 }); + if (r.ok) expect(r.rows).toEqual([ + [1, 'a'], + [2, undefined], + ]); + }); + + it('unions keys across object rows', () => { + const r = normalizeCodeCellReturn([{ a: 1 }, { b: 2 }], 10); + expect(r).toMatchObject({ ok: true, columns: ['a', 'b'] }); + if (r.ok) expect(r.rows).toEqual([ + [1, undefined], + [undefined, 2], + ]); + }); + + it('rejects null, primitives, and arrays of non-objects', () => { + expect(normalizeCodeCellReturn(null, 10).ok).toBe(false); + expect(normalizeCodeCellReturn(undefined, 10).ok).toBe(false); expect(normalizeCodeCellReturn(42, 10).ok).toBe(false); + expect(normalizeCodeCellReturn([1, 2, 3], 10).ok).toBe(false); + expect(normalizeCodeCellReturn({ columns: ['a'] }, 10).ok).toBe(false); }); }); -describe('executeCodeCellSync', () => { - it('maps last.rows into a new grid', () => { - const r = executeCodeCellSync({ +describe('codeCellHasReturn', () => { + it('ignores return inside strings, line comments, and block comments', () => { + expect(codeCellHasReturn(`const s = "return nothing";`)).toBe(false); + expect(codeCellHasReturn(`const s = 'return nothing';`)).toBe(false); + expect(codeCellHasReturn('const s = `return nothing`;')).toBe(false); + expect(codeCellHasReturn(`// return nowhere\nconst x = 1;`)).toBe(false); + expect(codeCellHasReturn(`/* return nowhere */\nconst x = 1;`)).toBe(false); + expect(codeCellHasReturn(`const s = "return nothing";\nreturn [];`)).toBe(true); + expect(codeCellHasReturn(`// return nowhere\nreturn [];`)).toBe(true); + }); +}); + +describe('executeCodeCell', () => { + it('maps last.rows into a new grid', async () => { + const r = await executeCodeCell({ body: `return { columns: ['id', 'n'], rows: last.rows.map((r) => [r[0], Number(r[0]) * 2]), @@ -69,8 +132,8 @@ describe('executeCodeCellSync', () => { ]); }); - it('allows local variables and for…of loops', () => { - const r = executeCodeCellSync({ + it('allows local variables and for…of loops', async () => { + const r = await executeCodeCell({ body: ` const out = []; const factor = 2; @@ -91,8 +154,79 @@ describe('executeCodeCellSync', () => { ]); }); - it('rejects cells without a return statement', () => { - const r = executeCodeCellSync({ + it('supports while loops, if branches, and nested helpers', async () => { + const r = await executeCodeCell({ + body: ` + function label(n) { + if (n % 2 === 0) return 'even'; + return 'odd'; + } + const out = []; + let i = 0; + while (i < last.rows.length) { + const id = Number(last.rows[i][0]); + out.push({ id, kind: label(id) }); + i++; + } + return out; + `, + last: { columns: ['id'], rows: [[1], [2], [3]], rowCount: 3 }, + vars: {}, + maxRows: 100, + }); + expect(r).toMatchObject({ ok: true, rowCount: 3 }); + if (r.ok) { + expect(r.columns).toEqual(['id', 'kind']); + expect(r.rows).toEqual([ + [1, 'odd'], + [2, 'even'], + [3, 'odd'], + ]); + } + }); + + it('passes through last as columns/rows', async () => { + const last = { columns: ['id'], rows: [[7]], rowCount: 1 }; + const r = await executeCodeCell({ + body: 'return last;', + last, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true, columns: ['id'], rowCount: 1 }); + if (r.ok) expect(r.rows).toEqual([[7]]); + }); + + it('handles null last by guarding in cell code', async () => { + const r = await executeCodeCell({ + body: `return last ? last.rows.map((r) => ({ id: r[0] })) : [];`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true, rowCount: 0 }); + }); + + it('reads scalar, list, and table vars', async () => { + const r = await executeCodeCell({ + body: `return { + columns: ['n', 'first', 'cell'], + rows: [[vars.n.value, vars.ids.values[0], vars.t.rows[0][0]]], + };`, + last: null, + vars: { + n: { kind: 'scalar', value: 9 }, + ids: { kind: 'list', values: [11, 22] }, + t: { kind: 'table', columns: ['a'], rows: [['x']] }, + }, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[9, 11, 'x']]); + }); + + it('rejects cells without a return statement', async () => { + const r = await executeCodeCell({ body: `const rows = last.rows.map((r) => ({ id: r[0] }));`, last: { columns: ['id'], rows: [[1]], rowCount: 1 }, vars: {}, @@ -102,32 +236,126 @@ describe('executeCodeCellSync', () => { if (!r.ok) expect(r.error).toMatch(/return statement/i); }); - it('ignores the word return inside strings when checking', () => { - expect(codeCellHasReturn(`const s = "return nothing";`)).toBe(false); - expect(codeCellHasReturn(`const s = "return nothing";\nreturn [];`)).toBe(true); + it('rejects empty body and empty-after-imports', async () => { + expect(await executeCodeCell({ + body: ' ', + last: null, + vars: {}, + maxRows: 10, + })).toMatchObject({ ok: false, error: expect.stringMatching(/empty/i) }); + + expect(await executeCodeCell({ + body: `import _ from 'lodash';\n`, + last: null, + vars: {}, + maxRows: 10, + })).toMatchObject({ ok: false, error: expect.stringMatching(/empty after imports/i) }); }); - it('reads vars and surfaces throws', () => { - const ok = executeCodeCellSync({ - body: `return { columns: ['v'], rows: [[vars.n.value]] };`, + it('resolves returned Promises', async () => { + const r = await executeCodeCell({ + body: `return Promise.resolve([{ id: 1 }]);`, last: null, - vars: { n: { kind: 'scalar', value: 9 } }, + vars: {}, maxRows: 10, }); - expect(ok).toMatchObject({ ok: true }); - if (ok.ok) expect(ok.rows).toEqual([[9]]); + expect(r).toMatchObject({ ok: true, rowCount: 1 }); + if (r.ok) expect(r.rows).toEqual([[1]]); + }); - const bad = executeCodeCellSync({ + it('surfaces thrown errors and syntax errors', async () => { + const thrown = await executeCodeCell({ body: `throw new Error('boom'); return null;`, last: null, vars: {}, maxRows: 10, }); - expect(bad).toMatchObject({ ok: false }); - if (!bad.ok) expect(bad.error).toMatch(/boom/); + expect(thrown).toMatchObject({ ok: false }); + if (!thrown.ok) expect(thrown.error).toMatch(/boom/); + + const syntax = await executeCodeCell({ + body: `return {;`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(syntax.ok).toBe(false); }); -}); + it('allows fetch and shadows window / document', async () => { + const r = await executeCodeCell({ + body: `return [{ + fetchType: typeof fetch, + windowType: typeof window, + documentType: typeof document, + }];`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) { + expect(r.columns).toEqual(['fetchType', 'windowType', 'documentType']); + expect(r.rows[0]?.[0]).toBe('function'); + expect(r.rows[0]?.[1]).toBe('undefined'); + expect(r.rows[0]?.[2]).toBe('undefined'); + } + }); + + it('awaits Promise returns and async/await', async () => { + const r = await executeCodeCell({ + body: ` + const n = await Promise.resolve(7); + return [{ n }]; + `, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[7]]); + }); + + it('supports await fetch via mock', async () => { + const realFetch = globalThis.fetch; + globalThis.fetch = (async () => + new Response(JSON.stringify({ ok: true, id: 42 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) as typeof fetch; + try { + const r = await executeCodeCell({ + body: ` + const res = await fetch('https://example.test/api'); + const json = await res.json(); + return [{ status: res.status, id: json.id }]; + `, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[200, 42]]); + } finally { + globalThis.fetch = realFetch; + } + }); + + it('enforces maxRows truncation on object-array returns', async () => { + const r = await executeCodeCell({ + body: `return last.rows.map((r) => ({ id: r[0] }));`, + last: { + columns: ['id'], + rows: [[1], [2], [3], [4]], + rowCount: 4, + }, + vars: {}, + maxRows: 2, + }); + expect(r).toMatchObject({ ok: true, rowCount: 2, truncated: true }); + if (r.ok) expect(r.rows).toEqual([[1], [2]]); + }); +}); describe('prepareCodeCellSource', () => { it('strips fence markers and @set lines', () => { @@ -136,10 +364,11 @@ describe('prepareCodeCellSource', () => { return last; -- @end`); expect(prepared).toMatchObject({ + ok: true, kind: 'js', body: 'return last;', }); - if (!('error' in prepared)) { + if (prepared.ok) { expect(prepared.directives).toEqual([{ mode: 'table', name: 'out' }]); } }); @@ -149,9 +378,68 @@ return last; -- @js return last; -- @end`); - expect(prepared).toMatchObject({ kind: 'js', body: 'return last;' }); - if (!('error' in prepared)) { + expect(prepared).toMatchObject({ ok: true, kind: 'js', body: 'return last;' }); + if (prepared.ok) { expect(prepared.directives).toEqual([{ mode: 'table', name: 'src' }]); } }); + + it('detects typescript fences and rejects plain SQL', () => { + const ts = prepareCodeCellSource(`-- @ts +const n: number = 1; +return [{ n }]; +-- @end`); + expect(ts).toMatchObject({ ok: true, kind: 'ts', body: 'const n: number = 1;\nreturn [{ n }];' }); + + expect(prepareCodeCellSource('SELECT 1;')).toEqual({ ok: false, error: 'Not a code cell' }); + }); + + it('merges leading and inner @set directives', () => { + const prepared = prepareCodeCellSource(`-- @set a = table +-- @js +-- @set b = table +return last; +-- @end`); + expect(prepared).toMatchObject({ ok: true, kind: 'js', body: 'return last;' }); + if (prepared.ok) { + expect(prepared.directives).toEqual([ + { mode: 'table', name: 'a' }, + { mode: 'table', name: 'b' }, + ]); + } + }); +}); + +describe('code cell result and input isolation', () => { + it('keeps columns that first appear after the sampling prefix', () => { + const objs = Array.from({ length: 60 }, (_, i) => + i < 55 ? { id: i } : { id: i, note: 'late' } + ); + const out = normalizeCodeCellReturn(objs, 200); + expect(out.ok).toBe(true); + if (out.ok) expect(out.columns).toEqual(['id', 'note']); + }); + + it('drops columns only past the row cap', () => { + const objs = [{ id: 1 }, { id: 2, note: 'kept' }, { id: 3, late: 'cut' }]; + const out = normalizeCodeCellReturn(objs, 2); + expect(out.ok).toBe(true); + if (out.ok) { + expect(out.columns).toEqual(['id', 'note']); + expect(out.truncated).toBe(true); + } + }); + + it('does not let a cell mutate the previous grid', async () => { + const prior = { columns: ['id'], rows: [[1], [2], [3]] as unknown[][], rowCount: 3 }; + const result = await executeCodeCell({ + body: 'last.rows.reverse(); last.rows.push([99]); return [{ n: last.rows.length }];', + last: prior, + vars: {}, + maxRows: 100, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.rows).toEqual([[4]]); + expect(prior.rows).toEqual([[1], [2], [3]]); + }); }); diff --git a/apps/web/src/frontend/lib/codeCellExec.ts b/apps/web/src/frontend/lib/codeCellExec.ts index 2a898bc..b4b0eda 100644 --- a/apps/web/src/frontend/lib/codeCellExec.ts +++ b/apps/web/src/frontend/lib/codeCellExec.ts @@ -1,42 +1,35 @@ /** - * Pure helpers for SQL Editor code cells (`-- @js` / `-- @ts`). - * The Worker and unit tests both call `executeCodeCellSync`. + * Browser-side executor for SQL Editor code cells (`-- @js` / `-- @ts`). + * The Web Worker and unit tests call `executeCodeCell`. Node fences + * (`-- @node` / `-- @nodets`) use the server executor instead. * - * Cells may use local `let`/`const`/`var`, functions, loops, and allowlisted - * `import`s (`lodash` / `lodash-es` / `date-fns`). Each cell is isolated - * (no shared helpers across cells). Must **return** a grid value. + * The parsing/normalizing/sandbox machinery is shared with the Node executor + * via `@foxschema/core`; this module supplies only the browser's globals + * preamble and bundled packages. + * + * Cells may use local `let`/`const`/`var`, functions, loops, `async`/`await`, + * `fetch`, and allowlisted `import`s (`lodash` / `lodash-es` / `date-fns`). + * Each cell is isolated (no shared helpers across cells). Must **return** a grid. */ -import { codeCellHasReturn } from './sql-splitter'; -import { prepareCodeCellImports } from './codeCellPackages'; - -export { codeCellHasReturn } from './sql-splitter'; - -export type CodeCellLast = { - columns: string[]; - rows: unknown[][]; - rowCount: number; -} | null; - -/** Structured-cloneable variable bag passed into cells (secrets omitted). */ -export type CodeCellVars = Record< - string, - | { kind: 'scalar'; value: unknown } - | { kind: 'list'; values: unknown[] } - | { kind: 'table'; columns: string[]; rows: unknown[][] } ->; - -export type CodeCellOk = { - ok: true; - columns: string[]; - rows: unknown[][]; - rowCount: number; - truncated: boolean; -}; - -export type CodeCellErr = { ok: false; error: string }; - -export type CodeCellResult = CodeCellOk | CodeCellErr; +import { + runCodeCellBody, + type CodeCellLast, + type CodeCellVars, + type CodeCellOk, + type CodeCellErr, + type CodeCellResult, +} from './sql-splitter'; +import { CODE_CELL_PACKAGE_MODULES } from './codeCellPackages'; + +export { codeCellHasReturn, normalizeCodeCellReturn } from './sql-splitter'; +export type { + CodeCellLast, + CodeCellVars, + CodeCellOk, + CodeCellErr, + CodeCellResult, +} from './sql-splitter'; type VarLike = { name: string; @@ -48,14 +41,6 @@ type VarLike = { rows?: unknown[][]; }; -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -function isPlainObject(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value); -} - /** Drop secret variables; shape the rest for the cell scope. */ export function sanitizeVarsForCodeCell(variables: VarLike[]): CodeCellVars { const out: CodeCellVars = {}; @@ -76,83 +61,9 @@ export function sanitizeVarsForCodeCell(variables: VarLike[]): CodeCellVars { return out; } -function normalizeObjectRows( - value: Record[], - maxRows: number -): CodeCellOk { - const colSet = new Set(); - for (const row of value.slice(0, 50)) { - for (const k of Object.keys(row)) colSet.add(k); - } - const columns = [...colSet]; - const truncated = value.length > maxRows; - const kept = truncated ? value.slice(0, maxRows) : value; - const rows = kept.map((r) => columns.map((c) => r[c])); - return { ok: true, columns, rows, rowCount: rows.length, truncated }; -} - -function normalizeColumnsRows( - columnsRaw: unknown[], - rowsRaw: unknown[], - maxRows: number -): CodeCellOk { - const columns = columnsRaw.map((c) => String(c)); - const truncated = rowsRaw.length > maxRows; - const kept = truncated ? rowsRaw.slice(0, maxRows) : rowsRaw; - const rows: unknown[][] = kept.map((r) => { - if (Array.isArray(r)) return columns.map((_, i) => r[i]); - if (isPlainObject(r)) return columns.map((c) => r[c]); - return columns.map(() => undefined); - }); - return { ok: true, columns, rows, rowCount: rows.length, truncated }; -} - -/** - * Normalize a cell return value into columns/rows. - * Accepts `{ columns, rows }` or an array of plain objects. - */ -export function normalizeCodeCellReturn( - value: unknown, - maxRows: number -): CodeCellOk | CodeCellErr { - if (value === null || value === undefined) { - return { - ok: false, - error: - 'Code cell must return a value — use return { columns, rows } or return [...objects]', - }; - } - - if (Array.isArray(value)) { - if (value.length === 0) { - return { ok: true, columns: [], rows: [], rowCount: 0, truncated: false }; - } - if (value.every(isPlainObject)) { - return normalizeObjectRows(value, maxRows); - } - return { - ok: false, - error: 'Array return must be an array of plain objects (or return { columns, rows })', - }; - } - - if (isPlainObject(value)) { - const { columns, rows } = value as { columns?: unknown; rows?: unknown }; - if (!Array.isArray(columns) || !Array.isArray(rows)) { - return { - ok: false, - error: 'Return { columns: string[], rows: unknown[][] } or an array of objects', - }; - } - return normalizeColumnsRows(columns, rows, maxRows); - } - - return { ok: false, error: `Unsupported return type: ${typeof value}` }; -} - +/** Shadow browser APIs except `fetch` (allowed for async HTTP). */ const SANDBOX_PREAMBLE = ` "use strict"; -var fetch = undefined; var XMLHttpRequest = undefined; var WebSocket = undefined; var indexedDB = undefined; @@ -166,57 +77,19 @@ var globalThis = undefined; `; /** - * Run JS cell body with `last`, `vars`, and allowlisted import bindings in scope. - * Callers must transpile TypeScript to JS before invoking. + * Run a JS cell body with `last`, `vars`, and allowlisted import bindings in + * scope. Supports `async`/`await` and `fetch`. Callers must transpile + * TypeScript first. */ -export function executeCodeCellSync(args: { +export function executeCodeCell(args: { body: string; last: CodeCellLast; vars: CodeCellVars; maxRows: number; -}): CodeCellResult { - const rawBody = args.body.trim(); - if (!rawBody) { - return { ok: false, error: 'Code cell is empty' }; - } - - const prepared = prepareCodeCellImports(rawBody); - if (!prepared.ok) { - return { ok: false, error: prepared.error }; - } - - const body = prepared.body.trim(); - if (!body) { - return { ok: false, error: 'Code cell is empty after imports' }; - } - if (!codeCellHasReturn(body)) { - return { - ok: false, - error: - 'Code cell must include a return statement — e.g. return { columns, rows } or return rows.map(...)', - }; - } - - const bindingNames = Object.keys(prepared.bindings); - for (const name of bindingNames) { - if (!/^[A-Za-z_$][\w$]*$/.test(name)) { - return { ok: false, error: `Invalid import binding name: ${name}` }; - } - } - const bindingValues = bindingNames.map((n) => prepared.bindings[n]); - - try { - // Sandboxed Function: last/vars + allowlisted import bindings only. - const fn = new Function('last', 'vars', ...bindingNames, `${SANDBOX_PREAMBLE}${body}`); - const raw = fn(args.last, args.vars, ...bindingValues); - if (raw != null && typeof (raw as Promise).then === 'function') { - return { - ok: false, - error: 'Async code cells are not supported yet (do not return a Promise)', - }; - } - return normalizeCodeCellReturn(raw, args.maxRows); - } catch (error: unknown) { - return { ok: false, error: errorMessage(error) }; - } +}): Promise { + return runCodeCellBody({ + ...args, + preamble: SANDBOX_PREAMBLE, + modules: CODE_CELL_PACKAGE_MODULES, + }); } diff --git a/apps/web/src/frontend/lib/codeCellPackages.test.ts b/apps/web/src/frontend/lib/codeCellPackages.test.ts index 07483be..6546505 100644 --- a/apps/web/src/frontend/lib/codeCellPackages.test.ts +++ b/apps/web/src/frontend/lib/codeCellPackages.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { parseCodeCellImports } from './codeCellPackages'; -import { executeCodeCellSync } from './codeCellExec'; +import { + parseCodeCellImports, + prepareCodeCellImports, + resolveCodeCellImportBindings, +} from './codeCellPackages'; +import { executeCodeCell } from './codeCellExec'; describe('parseCodeCellImports', () => { it('accepts allowlisted default / named / namespace imports', () => { @@ -16,6 +20,32 @@ return []; expect(parsed.bodyWithoutImports).toBe('return [];'); }); + it('allows blank lines and // comments before imports', () => { + const parsed = parseCodeCellImports(` +// helpers +import { map } from 'lodash-es'; + +return map([1], (n) => ({ n })); +`); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.specs).toHaveLength(1); + expect(parsed.bodyWithoutImports).toBe('return map([1], (n) => ({ n }));'); + }); + + it('supports named import aliases', () => { + const parsed = parseCodeCellImports( + `import { map as mapRows } from 'lodash';\nreturn mapRows([], (x) => x);` + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.specs[0]).toEqual({ + kind: 'named', + pkg: 'lodash', + names: [{ imported: 'map', local: 'mapRows' }], + }); + }); + it('rejects unknown packages', () => { const parsed = parseCodeCellImports(`import fs from 'fs';\nreturn [];`); expect(parsed.ok).toBe(false); @@ -27,17 +57,54 @@ return []; expect(parsed.ok).toBe(false); if (!parsed.ok) expect(parsed.error).toMatch(/top of the code cell/i); }); + + it('rejects mixed default+named imports', () => { + const parsed = parseCodeCellImports( + `import _, { map } from 'lodash';\nreturn [];` + ); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error).toMatch(/Mixed default\+named/i); + }); + + it('rejects invalid named import tokens', () => { + const parsed = parseCodeCellImports( + `import { map as } from 'lodash';\nreturn [];` + ); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error).toMatch(/Invalid named import/i); + }); }); -describe('executeCodeCellSync with imports + functions', () => { - it('runs lodash + local function', () => { +describe('resolveCodeCellImportBindings', () => { + it('rejects duplicate local bindings', () => { + const parsed = parseCodeCellImports( + `import _ from 'lodash';\nimport { map as _ } from 'lodash';\nreturn [];` + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + const resolved = resolveCodeCellImportBindings(parsed.specs); + expect(resolved.ok).toBe(false); + if (!resolved.ok) expect(resolved.error).toMatch(/Duplicate binding/i); + }); + + it('rejects unknown named exports', () => { + const prepared = prepareCodeCellImports( + `import { notARealExport } from 'lodash';\nreturn [];` + ); + expect(prepared.ok).toBe(false); + if (!prepared.ok) expect(prepared.error).toMatch(/not exported/i); + }); +}); + +describe('executeCodeCell with imports + functions', () => { + it('runs lodash + local function', async () => { const body = `import _ from 'lodash'; function doubleRow(r) { return { id: r[0], n: Number(r[0]) * 2 }; } return _.map(last.rows, doubleRow); `; - const r = executeCodeCellSync({ + const r = await executeCodeCell({ body, last: { columns: ['id'], rows: [[4], [6]], rowCount: 2 }, vars: {}, @@ -53,8 +120,27 @@ return _.map(last.rows, doubleRow); } }); - it('runs date-fns named import', () => { - const r = executeCodeCellSync({ + it('runs lodash-es named import with alias', async () => { + const r = await executeCodeCell({ + body: `import { groupBy as byKind } from 'lodash-es'; +const rows = [ + { kind: 'a', n: 1 }, + { kind: 'a', n: 2 }, + { kind: 'b', n: 3 }, +]; +const grouped = byKind(rows, 'kind'); +return [{ a: grouped.a.length, b: grouped.b.length }]; +`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[2, 1]]); + }); + + it('runs date-fns named import', async () => { + const r = await executeCodeCell({ body: `import { format } from 'date-fns'; return [{ stamp: format(new Date(Date.UTC(2020, 0, 2)), 'yyyy-MM-dd', { useAdditionalDayOfYearTokens: false }) }]; `, @@ -67,4 +153,71 @@ return [{ stamp: format(new Date(Date.UTC(2020, 0, 2)), 'yyyy-MM-dd', { useAddit expect(String(r.rows[0]?.[0])).toMatch(/^\d{4}-\d{2}-\d{2}$/); } }); + + it('runs date-fns namespace import', async () => { + const r = await executeCodeCell({ + body: `import * as df from 'date-fns'; +return [{ ok: typeof df.format === 'function' }]; +`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: true }); + if (r.ok) expect(r.rows).toEqual([[true]]); + }); + + it('fails closed when package is not allowlisted', async () => { + const r = await executeCodeCell({ + body: `import path from 'path';\nreturn [];`, + last: null, + vars: {}, + maxRows: 10, + }); + expect(r).toMatchObject({ ok: false }); + if (!r.ok) expect(r.error).toMatch(/allowlisted/i); + }); +}); + +describe('import parsing edge cases', () => { + it('does not treat an identifier starting with "import" as an import line', () => { + const parsed = parseCodeCellImports('importantRows = last.rows;\nreturn [{ n: 1 }];'); + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.specs).toHaveLength(0); + expect(parsed.bodyWithoutImports).toContain('importantRows'); + } + }); + + it('allows the word "import" inside a template literal', () => { + const parsed = parseCodeCellImports( + 'const doc = `\nimport x from "y"\n`;\nreturn [{ doc }];' + ); + expect(parsed.ok).toBe(true); + }); + + it('still rejects a real mid-body import', () => { + const parsed = parseCodeCellImports( + `const a = 1;\nimport { map } from 'lodash-es';\nreturn [];` + ); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error).toMatch(/must appear at the top/); + }); + + it('binds a local named after an Object.prototype key', () => { + // A plain `{}` accumulator reported `toString` / `valueOf` as duplicates. + const prepared = prepareCodeCellImports( + `import { map as toString } from 'lodash-es';\nreturn [];` + ); + expect(prepared.ok).toBe(true); + if (prepared.ok) expect(typeof prepared.bindings.toString).toBe('function'); + }); + + it('rejects an import that would shadow the cell scope', () => { + const prepared = prepareCodeCellImports( + `import { last } from 'lodash-es';\nreturn [{ n: 1 }];` + ); + expect(prepared.ok).toBe(false); + if (!prepared.ok) expect(prepared.error).toMatch(/reserved for the cell scope/); + }); }); diff --git a/apps/web/src/frontend/lib/codeCellPackages.ts b/apps/web/src/frontend/lib/codeCellPackages.ts index d2ddd6f..41064ea 100644 --- a/apps/web/src/frontend/lib/codeCellPackages.ts +++ b/apps/web/src/frontend/lib/codeCellPackages.ts @@ -1,16 +1,22 @@ /** - * Allowlisted packages for SQL Editor JS/TS code cells. - * Only these names may appear in `import … from '…'`. - * Cells stay isolated — imports are resolved per cell (no shared function registry). + * Allowlisted packages for SQL Editor JS/TS code cells, bundled for the browser. + * The import parser itself lives in `@foxschema/core` and is shared with the + * Node executor — this module only supplies the browser's module namespaces. */ import * as lodash from 'lodash-es'; import * as dateFns from 'date-fns'; - -export const CODE_CELL_ALLOWED_PACKAGES = ['lodash', 'lodash-es', 'date-fns'] as const; -export type CodeCellAllowedPackage = (typeof CODE_CELL_ALLOWED_PACKAGES)[number]; - -const ALLOWED = new Set(CODE_CELL_ALLOWED_PACKAGES); +import { + CODE_CELL_ALLOWED_PACKAGES, + parseCodeCellImports, + resolveCodeCellImportBindings as resolveBindings, + prepareCodeCellImports as prepareImports, + type CodeCellAllowedPackage, + type CodeCellImportSpec, +} from './sql-splitter'; + +export { CODE_CELL_ALLOWED_PACKAGES, parseCodeCellImports }; +export type { CodeCellAllowedPackage, CodeCellImportSpec }; /** Bundled module namespaces keyed by import specifier. */ export const CODE_CELL_PACKAGE_MODULES: Record = { @@ -19,187 +25,18 @@ export const CODE_CELL_PACKAGE_MODULES: Record = 'date-fns': dateFns as object, }; -type NamedBinding = { imported: string; local: string }; - -type ImportSpec = - | { kind: 'default'; local: string; pkg: string } - | { kind: 'namespace'; local: string; pkg: string } - | { kind: 'named'; names: NamedBinding[]; pkg: string }; - -const DEFAULT_IMPORT_RE = - /^import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; -const NAMESPACE_IMPORT_RE = - /^import\s+\*\s+as\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; -const NAMED_IMPORT_RE = - /^import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; -const MIXED_IMPORT_RE = - /^import\s+[A-Za-z_$][\w$]*\s*,\s*\{/; - -function parseNamedList(inner: string): NamedBinding[] | { error: string } { - const names: NamedBinding[] = []; - for (const part of inner.split(',')) { - const bit = part.trim(); - if (!bit) continue; - const asMatch = /^([A-Za-z_$][\w$]*)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(bit); - if (asMatch) { - names.push({ imported: asMatch[1]!, local: asMatch[2]! }); - continue; - } - if (/^[A-Za-z_$][\w$]*$/.test(bit)) { - names.push({ imported: bit, local: bit }); - continue; - } - return { error: `Invalid named import: ${bit}` }; - } - return names; -} - -function parseImportLine(trimmed: string): ImportSpec | { error: string } { - if (MIXED_IMPORT_RE.test(trimmed)) { - return { - error: 'Mixed default+named imports are not supported — use separate import lines', - }; - } - - const ns = NAMESPACE_IMPORT_RE.exec(trimmed); - if (ns) return { kind: 'namespace', local: ns[1]!, pkg: ns[2]! }; - - const named = NAMED_IMPORT_RE.exec(trimmed); - if (named) { - const names = parseNamedList(named[1]!); - if ('error' in names) return names; - return { kind: 'named', names, pkg: named[2]! }; - } - - const def = DEFAULT_IMPORT_RE.exec(trimmed); - if (def) return { kind: 'default', local: def[1]!, pkg: def[2]! }; - - return { error: `Unsupported import syntax: ${trimmed.slice(0, 80)}` }; -} - -/** - * Parse leading `import` lines, validate against the allowlist, and return the - * remaining body. Imports must be at the top (blank lines / // comments ok). - */ -export function parseCodeCellImports(body: string): - | { ok: true; specs: ImportSpec[]; bodyWithoutImports: string } - | { ok: false; error: string } { - const lines = body.split(/\r?\n/); - const specs: ImportSpec[] = []; - let i = 0; - - while (i < lines.length) { - const trimmed = lines[i]!.trim(); - if (trimmed === '' || trimmed.startsWith('//')) { - i++; - continue; - } - if (!trimmed.startsWith('import')) break; - - const parsed = parseImportLine(trimmed); - if ('error' in parsed) return { ok: false, error: parsed.error }; - if (!ALLOWED.has(parsed.pkg)) { - return { - ok: false, - error: `Package "${parsed.pkg}" is not allowlisted. Allowed: ${[...ALLOWED].join(', ')}`, - }; - } - specs.push(parsed); - i++; - } - - // Reject mid-body imports (after first non-import code line). - for (let j = i; j < lines.length; j++) { - const t = lines[j]!.trim(); - if (t.startsWith('import ') || t.startsWith('import{')) { - return { - ok: false, - error: 'import statements must appear at the top of the code cell', - }; - } - } - - return { - ok: true, - specs, - bodyWithoutImports: lines.slice(i).join('\n').trim(), - }; -} - -function moduleDefault(mod: object): unknown { - if (mod && typeof mod === 'object' && 'default' in mod) { - const def = (mod as { default: unknown }).default; - if (def != null) return def; - } - return mod; -} - -function setBinding( - bindings: Record, - local: string, - value: unknown -): { error: string } | null { - if (bindings[local] !== undefined) { - return { error: `Duplicate binding "${local}"` }; - } - bindings[local] = value; - return null; -} - -/** - * Build identifier → value bindings for `new Function` parameters. - */ +/** Build identifier → value bindings from the browser's bundled packages. */ export function resolveCodeCellImportBindings( - specs: ImportSpec[], + specs: CodeCellImportSpec[], modules: Record = CODE_CELL_PACKAGE_MODULES ): { ok: true; bindings: Record } | { ok: false; error: string } { - const bindings: Record = {}; - - for (const spec of specs) { - const mod = modules[spec.pkg]; - if (!mod) { - return { ok: false, error: `Package "${spec.pkg}" is not loaded` }; - } - - if (spec.kind === 'default') { - const err = setBinding(bindings, spec.local, moduleDefault(mod)); - if (err) return { ok: false, error: err.error }; - continue; - } - - if (spec.kind === 'namespace') { - const err = setBinding(bindings, spec.local, mod); - if (err) return { ok: false, error: err.error }; - continue; - } - - const rec = mod as Record; - for (const { imported, local } of spec.names) { - if (!(imported in rec)) { - return { - ok: false, - error: `"${imported}" is not exported by "${spec.pkg}"`, - }; - } - const err = setBinding(bindings, local, rec[imported]); - if (err) return { ok: false, error: err.error }; - } - } - - return { ok: true, bindings }; + return resolveBindings(specs, modules); } /** Parse + resolve imports in one step. */ -export function prepareCodeCellImports(body: string): - | { ok: true; body: string; bindings: Record } - | { ok: false; error: string } { - const parsed = parseCodeCellImports(body); - if (!parsed.ok) return parsed; - const resolved = resolveCodeCellImportBindings(parsed.specs); - if (!resolved.ok) return resolved; - return { - ok: true, - body: parsed.bodyWithoutImports, - bindings: resolved.bindings, - }; +export function prepareCodeCellImports( + body: string, + modules: Record = CODE_CELL_PACKAGE_MODULES +): { ok: true; body: string; bindings: Record } | { ok: false; error: string } { + return prepareImports(body, modules); } diff --git a/apps/web/src/frontend/lib/codeCellRunner.test.ts b/apps/web/src/frontend/lib/codeCellRunner.test.ts new file mode 100644 index 0000000..036dad9 --- /dev/null +++ b/apps/web/src/frontend/lib/codeCellRunner.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest'; +import { detectCodeCell, prepareCodeCellSource, runCodeCell } from './codeCellRunner'; + +describe('detectCodeCell', () => { + it('detects js and ts fences', () => { + expect(detectCodeCell(`-- @js\nreturn [];\n-- @end`)).toMatchObject({ + kind: 'js', + closed: true, + }); + expect(detectCodeCell(`-- @typescript\nreturn [];\n-- @end`)).toMatchObject({ + kind: 'ts', + closed: true, + }); + expect(detectCodeCell(`-- @node\nreturn [];\n-- @end`)).toMatchObject({ + kind: 'node', + closed: true, + }); + expect(detectCodeCell(`-- @nodets\nreturn [];\n-- @end`)).toMatchObject({ + kind: 'nodets', + closed: true, + }); + }); + + it('skips leading @set when detecting', () => { + const cell = detectCodeCell(`-- @set out = table\n-- @js\nreturn last;\n-- @end`); + expect(cell).toMatchObject({ kind: 'js', closed: true }); + expect(cell?.body).toContain('return last'); + }); + + it('returns null for plain SQL', () => { + expect(detectCodeCell('SELECT 1;')).toBeNull(); + }); + + it('marks unclosed fences', () => { + expect(detectCodeCell(`-- @js\nreturn [];`)).toMatchObject({ + kind: 'js', + closed: false, + }); + }); +}); + +describe('runCodeCell', () => { + it('runs a JS cell via sync fallback when Worker is unavailable', async () => { + const { result, directives } = await runCodeCell({ + statement: `-- @js +-- @set doubled = table +return last.rows.map((r) => ({ id: r[0], n: Number(r[0]) * 2 })); +-- @end`, + last: { columns: ['id'], rows: [[5]], rowCount: 1 }, + variables: [], + maxRows: 100, + }); + expect(directives).toEqual([{ mode: 'table', name: 'doubled' }]); + expect(result).toMatchObject({ + ok: true, + columns: ['id', 'n'], + rowCount: 1, + hasNext: false, + }); + if (result.ok) expect(result.rows).toEqual([[5, 10]]); + }); + + it('transpiles TypeScript cells before execution', async () => { + const { result } = await runCodeCell({ + statement: `-- @ts +const factor: number = 3; +const out = (last ? last.rows : []).map((r: unknown[]) => ({ + id: Number(r[0]), + n: Number(r[0]) * factor, +})); +return out; +-- @end`, + last: { columns: ['id'], rows: [[2], [4]], rowCount: 2 }, + variables: [], + maxRows: 100, + }); + if (!result.ok) { + throw new Error(`TS cell failed: ${result.error}`); + } + expect(result).toMatchObject({ ok: true, rowCount: 2 }); + expect(result.columns).toEqual(['id', 'n']); + expect(result.rows).toEqual([ + [2, 6], + [4, 12], + ]); + }); + + it('omits secret variables from the cell scope', async () => { + const { result } = await runCodeCell({ + statement: `-- @js +return [{ + hasToken: Object.prototype.hasOwnProperty.call(vars, 'token'), + n: vars.n.value, +}]; +-- @end`, + last: null, + variables: [ + { + id: '1', + name: 'token', + kind: 'scalar', + value: 'secret', + secret: true, + updatedAt: 1, + }, + { id: '2', name: 'n', kind: 'scalar', value: 42, updatedAt: 1 }, + ], + maxRows: 10, + }); + expect(result).toMatchObject({ ok: true }); + if (result.ok) expect(result.rows).toEqual([[false, 42]]); + }); + + it('returns an error for non-code statements', async () => { + const { result, directives } = await runCodeCell({ + statement: 'SELECT 1;', + last: null, + variables: [], + maxRows: 10, + }); + expect(directives).toEqual([]); + expect(result).toMatchObject({ ok: false, error: 'Not a code cell' }); + }); + + it('surfaces TypeScript transpile failures', async () => { + const { result } = await runCodeCell({ + statement: `-- @ts +const x: = 1; +return []; +-- @end`, + last: null, + variables: [], + maxRows: 10, + }); + // Some TS parse errors still emit JS; accept either transpile or runtime failure. + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.length).toBeGreaterThan(0); + } + }); + + it('keeps prepareCodeCellSource body free of fence markers', () => { + const prepared = prepareCodeCellSource(`-- @js +return [{ ok: true }]; +-- @end`); + expect(prepared).toMatchObject({ + ok: true, + kind: 'js', + body: 'return [{ ok: true }];', + directives: [], + }); + }); +}); diff --git a/apps/web/src/frontend/lib/codeCellRunner.ts b/apps/web/src/frontend/lib/codeCellRunner.ts index 68f4dfe..861ef5f 100644 --- a/apps/web/src/frontend/lib/codeCellRunner.ts +++ b/apps/web/src/frontend/lib/codeCellRunner.ts @@ -1,14 +1,23 @@ /** - * Run a SQL Editor JS/TS code cell. Prefers a Web Worker; falls back to sync - * execution when Worker is unavailable (unit tests / Node). + * Run a SQL Editor JS/TS/Node code cell. + * Browser fences (`-- @js` / `-- @ts`) use a Web Worker (in-process + * `executeCodeCell` fallback when Worker is unavailable); Node fences + * (`-- @node` / `-- @nodets`) POST to the FoxSchema server. */ -import type { SqlStatementResult } from '../api/sqlApi'; +import { runCodeCellOnServer, type SqlStatementResult } from '../api/sqlApi'; import type { SetDirective, SqlVariable } from './sql-variables'; import { parseSetDirectives } from './sql-variables'; -import { parseCodeCell } from './sql-splitter'; import { - executeCodeCellSync, + parseCodeCell, + stripFullLineSqlComments, + codeCellNeedsTs, + isNodeCodeCellKind, + nodeCodeCellWireKind, + type CodeCellKind, +} from './sql-splitter'; +import { + executeCodeCell, sanitizeVarsForCodeCell, type CodeCellLast, type CodeCellResult, @@ -16,12 +25,13 @@ import { } from './codeCellExec'; const DEFAULT_TIMEOUT_MS = 5_000; +const DEFAULT_NODE_TIMEOUT_MS = 10_000; let nextId = 1; let workerCtorPromise: Promise<(new () => Worker) | null> | null = null; export type RunCodeCellArgs = { - /** Full statement text including `-- @js` / `-- @end` (and optional `@set` lines). */ + /** Full statement text including fence markers (and optional `@set` lines). */ statement: string; last: CodeCellLast; variables: SqlVariable[]; @@ -30,27 +40,27 @@ export type RunCodeCellArgs = { }; /** - * Detect a fenced JS/TS cell, allowing leading `-- @set` above `-- @js`/`-- @ts`. + * Detect a fenced code cell, allowing leading `-- @set` above the fence. */ export function detectCodeCell( statement: string -): { kind: 'js' | 'ts'; body: string; closed: boolean } | null { +): { kind: CodeCellKind; body: string; closed: boolean } | null { const afterSets = parseSetDirectives(statement).sql; return parseCodeCell(afterSets) ?? parseCodeCell(statement); } -/** Prepare body + kind from a fenced statement (strips @set + fence markers). */ +/** Prepare body + kind from a fenced statement (strips @set, fence markers, full-line SQL `--` comments). */ export function prepareCodeCellSource(statement: string): - | { kind: 'js' | 'ts'; body: string; directives: SetDirective[] } - | { error: string } { - // Leading `-- @set` may sit above `-- @js` (reattachSetComments). + | { ok: true; kind: CodeCellKind; body: string; directives: SetDirective[] } + | { ok: false; error: string } { const leading = parseSetDirectives(statement); const cell = parseCodeCell(leading.sql); - if (!cell) return { error: 'Not a code cell' }; + if (!cell) return { ok: false, error: 'Not a code cell' }; const inner = parseSetDirectives(cell.body); return { + ok: true, kind: cell.kind, - body: inner.sql, + body: stripFullLineSqlComments(inner.sql), directives: [...leading.directives, ...inner.directives], }; } @@ -64,7 +74,7 @@ async function transpileTs(body: string): Promise { const out = ts.transpileModule(body, { compilerOptions: { target: ts.ScriptTarget.ES2020, - module: ts.ModuleKind.None, + module: ts.ModuleKind.ESNext, strict: false, }, reportDiagnostics: true, @@ -118,28 +128,33 @@ function runInWorker(args: { return new Promise((resolve) => { let settled = false; + let worker: Worker | undefined; + let timer: ReturnType | undefined; const id = nextId++; - let worker: Worker; - try { - worker = new WorkerCtor(); - } catch { - resolve(toStatementResult(executeCodeCellSync({ body, last, vars, maxRows }), started)); - return; - } const finish = (result: SqlStatementResult) => { if (settled) return; settled = true; - clearTimeout(timer); + if (timer !== undefined) clearTimeout(timer); try { - worker.terminate(); + worker?.terminate(); } catch { /* ignore */ } resolve(result); }; - const timer = setTimeout(() => { + try { + worker = new WorkerCtor(); + } catch { + // Worker construction failed — run in-process (no timeout enforcement here). + void executeCodeCell({ body, last, vars, maxRows }).then((r) => { + resolve(toStatementResult(r, started)); + }); + return; + } + + timer = setTimeout(() => { finish({ ok: false, error: `Code cell timed out after ${timeoutMs}ms`, @@ -165,7 +180,7 @@ function runInWorker(args: { } /** - * Execute a fenced JS/TS cell. Returns the same shape as SQL statement results. + * Execute a fenced code cell. Returns the same shape as SQL statement results. * Also returns parsed `@set` directives so the store can apply them. */ export async function runCodeCell( @@ -173,16 +188,43 @@ export async function runCodeCell( ): Promise<{ result: SqlStatementResult; directives: SetDirective[] }> { const started = Date.now(); const prepared = prepareCodeCellSource(args.statement); - if ('error' in prepared) { + if (!prepared.ok) { return { result: { ok: false, error: prepared.error, durationMs: Date.now() - started }, directives: [], }; } + const vars = sanitizeVarsForCodeCell(args.variables); + + // Node / Node-TS → server + if (isNodeCodeCellKind(prepared.kind)) { + const timeoutMs = args.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS; + try { + const result = await runCodeCellOnServer({ + body: prepared.body, + kind: nodeCodeCellWireKind(prepared.kind), + last: args.last, + vars, + maxRows: args.maxRows, + timeoutMs, + }); + return { result, directives: prepared.directives }; + } catch (error: unknown) { + return { + result: { + ok: false, + error: errorMessage(error), + durationMs: Date.now() - started, + }, + directives: prepared.directives, + }; + } + } + let body = prepared.body; try { - if (prepared.kind === 'ts') { + if (codeCellNeedsTs(prepared.kind)) { body = await transpileTs(body); } } catch (error: unknown) { @@ -196,14 +238,13 @@ export async function runCodeCell( }; } - const vars = sanitizeVarsForCodeCell(args.variables); const timeoutMs = args.timeoutMs ?? DEFAULT_TIMEOUT_MS; const execArgs = { body, last: args.last, vars, maxRows: args.maxRows }; const WorkerCtor = await loadWorkerCtor(); if (!WorkerCtor) { return { - result: toStatementResult(executeCodeCellSync(execArgs), started), + result: toStatementResult(await executeCodeCell(execArgs), started), directives: prepared.directives, }; } diff --git a/apps/web/src/frontend/lib/sql-splitter.ts b/apps/web/src/frontend/lib/sql-splitter.ts index 8484039..8092ee4 100644 --- a/apps/web/src/frontend/lib/sql-splitter.ts +++ b/apps/web/src/frontend/lib/sql-splitter.ts @@ -9,7 +9,44 @@ export { dmlLacksWhere, parseCodeCell, findCodeFences, + stripJsStringsAndComments, stripCodeFenceMarkers, codeCellHasReturn, + stripFullLineSqlComments, + isCodeCellKind, + isNodeCodeCellKind, + codeCellNeedsTs, + nodeCodeCellWireKind, + isCodeCellLast, + isCodeCellVars, + CODE_CELL_KIND_LABEL, +} from '@foxschema/core'; +export type { + SplitStatement, + StatementStatus, + StatementKind, + CodeCellKind, + BrowserCodeCellKind, + NodeCodeCellKind, + TsCodeCellKind, + CodeCellLast, + CodeCellVars, + CodeCellOk, + CodeCellErr, + CodeCellResult, +} from '@foxschema/core'; + +export { + CODE_CELL_ALLOWED_PACKAGES, + parseCodeCellImports, + resolveCodeCellImportBindings, + prepareCodeCellImports, + normalizeCodeCellReturn, + cloneCodeCellLast, + runCodeCellBody, +} from '@foxschema/core'; +export type { + CodeCellAllowedPackage, + CodeCellImportSpec, + RunCodeCellBodyArgs, } from '@foxschema/core'; -export type { SplitStatement, StatementStatus, StatementKind } from '@foxschema/core'; diff --git a/apps/web/src/frontend/lib/sqlEditorSamples.test.ts b/apps/web/src/frontend/lib/sqlEditorSamples.test.ts new file mode 100644 index 0000000..65721b7 --- /dev/null +++ b/apps/web/src/frontend/lib/sqlEditorSamples.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest'; +import { splitSqlStatements, checkStatement, parseCodeCell } from './sql-splitter'; +import { + SQL_EDITOR_SAMPLE_BOOKMARKS, + buildSampleBookmarks, +} from './sqlEditorSamples'; +import { prepareCodeCellSource, runCodeCell } from './codeCellRunner'; +import { executeCodeCell } from './codeCellExec'; + +describe('SQL Editor sample bookmarks', () => { + it('exposes stable ids and non-empty SQL', () => { + const ids = new Set(SQL_EDITOR_SAMPLE_BOOKMARKS.map((s) => s.id)); + expect(ids.size).toBe(SQL_EDITOR_SAMPLE_BOOKMARKS.length); + for (const s of SQL_EDITOR_SAMPLE_BOOKMARKS) { + expect(s.id).toMatch(/^sample-/); + expect(s.title).toMatch(/★ Sample/); + expect(s.sql.trim().length).toBeGreaterThan(20); + } + }); + + it('buildSampleBookmarks preserves ids for install/merge', () => { + const built = buildSampleBookmarks(123); + expect(built.map((b) => b.id)).toEqual( + SQL_EDITOR_SAMPLE_BOOKMARKS.map((s) => s.id) + ); + expect(built.every((b) => b.updatedAt === 123)).toBe(true); + }); + + it('each sample splits cleanly and closed code cells check ok', () => { + for (const sample of SQL_EDITOR_SAMPLE_BOOKMARKS) { + const stmts = splitSqlStatements(sample.sql); + expect(stmts.length, sample.id).toBeGreaterThan(0); + const codeCells = stmts.filter((s) => + s.kind === 'js' || s.kind === 'ts' || s.kind === 'node' || s.kind === 'nodets' + ); + expect(codeCells.length, sample.id).toBeGreaterThan(0); + for (const cell of codeCells) { + expect(cell.terminated, `${sample.id} missing @end`).toBe(true); + const check = checkStatement(cell); + expect(check.level, `${sample.id}: ${check.reasons.join('; ')}`).toBe('ok'); + } + } + }); + + it('runs the JS-only sample end-to-end', async () => { + const sample = SQL_EDITOR_SAMPLE_BOOKMARKS.find((s) => s.id === 'sample-js-pure-grid'); + expect(sample).toBeTruthy(); + const { result } = await runCodeCell({ + statement: sample!.sql, + last: null, + variables: [], + maxRows: 100, + }); + if (!result.ok) throw new Error(result.error); + expect(result).toMatchObject({ ok: true, rowCount: 1 }); + expect(result.rows).toEqual([[2, 1]]); + }); + + it('runs map-last body against a synthetic prior grid', async () => { + const sample = SQL_EDITOR_SAMPLE_BOOKMARKS.find((s) => s.id === 'sample-js-map-last'); + expect(sample).toBeTruthy(); + const stmts = splitSqlStatements(sample!.sql); + const js = stmts.find((s) => s.kind === 'js'); + expect(js).toBeTruthy(); + const prepared = prepareCodeCellSource(js!.text); + expect(prepared.ok).toBe(true); + if (!prepared.ok) return; + const cell = parseCodeCell(js!.text); + expect(cell?.kind).toBe('js'); + const r = await executeCodeCell({ + body: prepared.body, + last: { + columns: ['id', 'email'], + rows: [ + [1, 'alice@example.com'], + [2, 'bob@example.com'], + ], + rowCount: 2, + }, + vars: {}, + maxRows: 100, + }); + expect(r).toMatchObject({ ok: true, rowCount: 2 }); + if (r.ok) { + expect(r.rows).toEqual([ + [1, 'alice@example.com', 2], + [2, 'bob@example.com', 4], + ]); + } + }); + + it('runs the chain sample second cell even with a SQL-style -- comment', async () => { + const statement = `-- @js +-- Use last from the previous cell +return last.rows.map((r) => ({ id: r[0], domain: String(r[1]).split('@')[1] })); +-- @end +`; + const prepared = prepareCodeCellSource(statement); + expect(prepared).toMatchObject({ + ok: true, + kind: 'js', + body: "return last.rows.map((r) => ({ id: r[0], domain: String(r[1]).split('@')[1] }));", + }); + + const { result } = await runCodeCell({ + statement, + last: { + columns: ['id', 'email'], + rows: [[1, 'alice@example.com']], + rowCount: 1, + }, + variables: [], + maxRows: 100, + }); + if (!result.ok) throw new Error(result.error); + expect(result.rows).toEqual([[1, 'example.com']]); + }); +}); diff --git a/apps/web/src/frontend/lib/sqlEditorSamples.ts b/apps/web/src/frontend/lib/sqlEditorSamples.ts new file mode 100644 index 0000000..f91034c --- /dev/null +++ b/apps/web/src/frontend/lib/sqlEditorSamples.ts @@ -0,0 +1,203 @@ +/** + * Built-in SQL Editor sample bookmarks (browser JS/TS + Node code cells). + * Install via Bookmarks → "Add samples". Stable ids so re-install updates in place. + */ + +export type SqlEditorSample = { + /** Stable id — used as bookmark id when installed. */ + id: string; + title: string; + sql: string; +}; + +/** Cross-dialect demo rows (no user tables required). */ +const DEMO_PEOPLE_SQL = `SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'`; + +export const SQL_EDITOR_SAMPLE_BOOKMARKS: SqlEditorSample[] = [ + { + id: 'sample-js-map-last', + title: '★ Sample · JS map last rows', + sql: `${DEMO_PEOPLE_SQL}; + +-- @js +-- @set doubled = table +return { + columns: ['id', 'email', 'n'], + rows: last.rows.map((r) => [r[0], r[1], Number(r[0]) * 2]), +}; +-- @end +`, + }, + { + id: 'sample-js-loop-locals', + title: '★ Sample · JS loop + locals', + sql: `${DEMO_PEOPLE_SQL}; + +-- @js +const out = []; +const factor = 2; +for (const r of last.rows) { + const id = Number(r[0]); + out.push({ id, email: r[1], n: id * factor }); +} +return out; +-- @end +`, + }, + { + id: 'sample-js-lodash', + title: '★ Sample · JS lodash import', + sql: `${DEMO_PEOPLE_SQL}; + +-- @js +import _ from 'lodash'; + +function doubleRow(r) { + return { id: r[0], email: r[1], n: Number(r[0]) * 2 }; +} +return _.map(last.rows, doubleRow); +-- @end +`, + }, + { + id: 'sample-js-date-fns', + title: '★ Sample · JS date-fns', + sql: `SELECT '2020-01-02' AS day +UNION ALL +SELECT '2021-06-15'; + +-- @js +import { format, parseISO } from 'date-fns'; + +return last.rows.map((r) => ({ + day: r[0], + stamped: format(parseISO(String(r[0])), 'yyyy-MM-dd'), +})); +-- @end +`, + }, + { + id: 'sample-ts-typed', + title: '★ Sample · TS typed transform', + sql: `${DEMO_PEOPLE_SQL}; + +-- @ts +const factor: number = 3; +const out = (last ? last.rows : []).map((r: unknown[]) => ({ + id: Number(r[0]), + email: String(r[1]), + n: Number(r[0]) * factor, +})); +return out; +-- @end +`, + }, + { + id: 'sample-js-chain-cells', + title: '★ Sample · JS chain (SQL → JS → JS)', + sql: `${DEMO_PEOPLE_SQL}; + +-- @js +-- @set step1 = table +return last.rows.map((r) => ({ id: Number(r[0]), email: r[1] })); +-- @end + +-- @js +// Use last from the previous cell (and/or vars.step1 after @set). +const rows = last && last.rows.length ? last.rows : (vars.step1 ? vars.step1.rows : []); +return rows.map((r) => ({ + id: r[0], + email: r[1], + domain: String(r[1]).split('@')[1] || '', +})); +-- @end +`, + }, + { + id: 'sample-js-filter-while', + title: '★ Sample · JS filter + while', + sql: `${DEMO_PEOPLE_SQL}; + +-- @js +function isOdd(n) { + return n % 2 === 1; +} +const out = []; +let i = 0; +while (i < last.rows.length) { + const id = Number(last.rows[i][0]); + if (isOdd(id)) { + out.push({ id, email: last.rows[i][1], kind: 'odd' }); + } + i++; +} +return out; +-- @end +`, + }, + { + id: 'sample-js-pure-grid', + title: '★ Sample · JS only (no SQL)', + sql: `-- @js +import { groupBy } from 'lodash-es'; + +const rows = [ + { kind: 'a', n: 1 }, + { kind: 'a', n: 2 }, + { kind: 'b', n: 3 }, +]; +const grouped = groupBy(rows, 'kind'); +return [{ a: grouped.a.length, b: grouped.b.length }]; +-- @end +`, + }, + { + id: 'sample-js-async-fetch', + title: '★ Sample · JS async fetch', + sql: `-- @js +const res = await fetch('https://httpbin.org/get'); +const json = await res.json(); +return [{ + status: res.status, + url: json.url ?? '', + origin: json.origin ?? '', +}]; +-- @end +`, + }, + { + id: 'sample-node-async-fetch', + title: '★ Sample · Node async fetch', + sql: `-- @node +const res = await fetch('https://httpbin.org/get'); +const json = await res.json(); +return [{ + status: res.status, + url: json.url ?? '', + origin: json.origin ?? '', + runtime: 'node', +}]; +-- @end +`, + }, +]; + +export function buildSampleBookmarks(now = Date.now()): Array<{ + id: string; + title: string; + sql: string; + selectedConnectionIds: string[]; + updatedAt: number; +}> { + return SQL_EDITOR_SAMPLE_BOOKMARKS.map((s) => ({ + id: s.id, + title: s.title, + sql: s.sql, + selectedConnectionIds: [] as string[], + updatedAt: now, + })); +} diff --git a/apps/web/src/frontend/store/useSqlEditorStore.ts b/apps/web/src/frontend/store/useSqlEditorStore.ts index 1fa9a78..2e0bb73 100644 --- a/apps/web/src/frontend/store/useSqlEditorStore.ts +++ b/apps/web/src/frontend/store/useSqlEditorStore.ts @@ -5,6 +5,7 @@ import { loadSchema } from '../api/schemaApi'; import { isMutatingDmlStatement, isWriteStatement } from '../lib/sql-splitter'; import type { CodeCellLast } from '../lib/codeCellExec'; import { detectCodeCell, runCodeCell } from '../lib/codeCellRunner'; +import { buildSampleBookmarks } from '../lib/sqlEditorSamples'; import { applySetDirectives, exportVariables, @@ -195,6 +196,8 @@ interface SqlEditorState { openBookmark: (id: string) => void; renameBookmark: (id: string, title: string) => void; deleteBookmark: (id: string) => void; + /** Merge built-in JS/TS/Node sample bookmarks (by stable id). Returns how many were added/updated. */ + installSampleBookmarks: () => number; /** Create or overwrite a variable by name. Returns error string or null. */ upsertVariable: (input: { name: string; @@ -1093,6 +1096,24 @@ export const useSqlEditorStore = create()( }); }, + installSampleBookmarks: () => { + const samples = buildSampleBookmarks(); + const { bookmarks } = get(); + const sampleIds = new Set(samples.map((s) => s.id)); + const prevById = new Map(bookmarks.map((b) => [b.id, b])); + let touched = 0; + for (const sample of samples) { + const prev = prevById.get(sample.id); + if (!prev || prev.sql !== sample.sql || prev.title !== sample.title) { + touched += 1; + } + } + // Keep user bookmarks first; samples at the end so they stay easy to find. + const user = bookmarks.filter((b) => !sampleIds.has(b.id)); + set({ bookmarks: [...user, ...samples] }); + return touched; + }, + upsertVariable: (input) => { const name = normalizeVariableName(input.name); if (!isValidVariableName(name)) { diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index df061cb..68445a7 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -179,15 +179,28 @@ Tips: 200). Use **Next** / **Prev** on a result grid to page through more rows; visited pages stay cached in memory so going back does not re-query the server. Sibling result grids from the same Run sync vertical scroll by row index. -- **Code cells (JS / TS)** — mix SQL with local transforms in the same buffer. Fence - a cell with `-- @js` / `-- @ts` … `-- @end` (inner semicolons are fine). You can use - local `let`/`const`/`var`, **functions**, and loops (`for`, `while`, `for…of`). - Allowlisted **imports** (bundled, no CDN): `lodash`, `lodash-es`, `date-fns` — - put `import` lines at the top of the cell. Cells are **isolated** (imports/functions - do not carry to the next cell; use `last` / `vars` to pass data). The cell receives - `last` (previous statement’s grid) and `vars` (non-secret Variables). **You must - `return`** either `{ columns, rows }` or an array of plain objects. Python is not - available yet. +- **Code cells (JS / TS / Node)** — mix SQL with local transforms in the same buffer. Fence + a cell with `-- @js` / `-- @ts` … `-- @end` (runs in the browser; inner semicolons are fine) + or `-- @node` / `-- @nodets` … `-- @end` (runs on the FoxSchema **Node** server). You can use + local `let`/`const`/`var`, **functions**, loops (`for`, `while`, `for…of`), **`async`/`await`**, + and **`fetch`**. Allowlisted **imports** (bundled, no CDN): `lodash`, `lodash-es`, `date-fns` — + put `import` lines at the top of the cell. Prefer `//` comments inside cells (`--` is the JS + decrement operator). Cells are **isolated** (imports/functions do not carry to the next cell; + use `last` / `vars` to pass data). The cell receives `last` (previous statement’s grid) and + `vars` (non-secret Variables). **You must `return`** either `{ columns, rows }` or an array of + plain objects. Python is not available yet. + + In the SQL Editor sidebar, **Bookmarks → Add samples** installs ready-made ★ Sample + scripts (also under `docs/examples/sql-editor/`) so you can reopen them later. + + > **`-- @node` cells run code on the FoxSchema server.** They execute in a worker + > thread with a scrubbed environment and a hard timeout, but the JS sandbox is a + > guardrail against accidents, not a security boundary — a determined cell can still + > reach the network from the server (`fetch`) and burn CPU. On a personal/desktop + > install that is exactly the point. If you host FoxSchema for **multiple users**, + > treat the ability to run `-- @node` cells as equivalent to giving those users a + > shell on the server host, and only expose it to people you trust at that level. + > Browser cells (`-- @js` / `-- @ts`) run in the user's own tab and carry no such risk. ```sql SELECT id, email FROM user; @@ -202,6 +215,16 @@ Tips: -- @end ``` + Async + fetch (browser or Node): + + ```sql + -- @node + const res = await fetch('https://httpbin.org/get'); + const json = await res.json(); + return [{ status: res.status, url: json.url }]; + -- @end + ``` + - **Safe mode** — when on, write/DDL statements need an extra confirmation before run. Writes and DDL are allowed when you confirm them. Some dialects (e.g. SQLite / diff --git a/docs/examples/sql-editor/01-js-map-last.sql b/docs/examples/sql-editor/01-js-map-last.sql new file mode 100644 index 0000000..5bd69ac --- /dev/null +++ b/docs/examples/sql-editor/01-js-map-last.sql @@ -0,0 +1,13 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @js +-- @set doubled = table +return { + columns: ['id', 'email', 'n'], + rows: last.rows.map((r) => [r[0], r[1], Number(r[0]) * 2]), +}; +-- @end diff --git a/docs/examples/sql-editor/02-js-loop-locals.sql b/docs/examples/sql-editor/02-js-loop-locals.sql new file mode 100644 index 0000000..96fe920 --- /dev/null +++ b/docs/examples/sql-editor/02-js-loop-locals.sql @@ -0,0 +1,15 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @js +const out = []; +const factor = 2; +for (const r of last.rows) { + const id = Number(r[0]); + out.push({ id, email: r[1], n: id * factor }); +} +return out; +-- @end diff --git a/docs/examples/sql-editor/03-js-lodash.sql b/docs/examples/sql-editor/03-js-lodash.sql new file mode 100644 index 0000000..4b72d30 --- /dev/null +++ b/docs/examples/sql-editor/03-js-lodash.sql @@ -0,0 +1,14 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @js +import _ from 'lodash'; + +function doubleRow(r) { + return { id: r[0], email: r[1], n: Number(r[0]) * 2 }; +} +return _.map(last.rows, doubleRow); +-- @end diff --git a/docs/examples/sql-editor/04-js-date-fns.sql b/docs/examples/sql-editor/04-js-date-fns.sql new file mode 100644 index 0000000..f356ffd --- /dev/null +++ b/docs/examples/sql-editor/04-js-date-fns.sql @@ -0,0 +1,12 @@ +SELECT '2020-01-02' AS day +UNION ALL +SELECT '2021-06-15'; + +-- @js +import { format, parseISO } from 'date-fns'; + +return last.rows.map((r) => ({ + day: r[0], + stamped: format(parseISO(String(r[0])), 'yyyy-MM-dd'), +})); +-- @end diff --git a/docs/examples/sql-editor/05-ts-typed.sql b/docs/examples/sql-editor/05-ts-typed.sql new file mode 100644 index 0000000..4028ed8 --- /dev/null +++ b/docs/examples/sql-editor/05-ts-typed.sql @@ -0,0 +1,15 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @ts +const factor: number = 3; +const out = (last ? last.rows : []).map((r: unknown[]) => ({ + id: Number(r[0]), + email: String(r[1]), + n: Number(r[0]) * factor, +})); +return out; +-- @end diff --git a/docs/examples/sql-editor/06-js-chain-cells.sql b/docs/examples/sql-editor/06-js-chain-cells.sql new file mode 100644 index 0000000..9291e0a --- /dev/null +++ b/docs/examples/sql-editor/06-js-chain-cells.sql @@ -0,0 +1,20 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @js +-- @set step1 = table +return last.rows.map((r) => ({ id: Number(r[0]), email: r[1] })); +-- @end + +-- @js +// Use last from the previous cell (and/or vars.step1 after @set). +const rows = last && last.rows.length ? last.rows : (vars.step1 ? vars.step1.rows : []); +return rows.map((r) => ({ + id: r[0], + email: r[1], + domain: String(r[1]).split('@')[1] || '', +})); +-- @end diff --git a/docs/examples/sql-editor/07-js-filter-while.sql b/docs/examples/sql-editor/07-js-filter-while.sql new file mode 100644 index 0000000..c9d7c21 --- /dev/null +++ b/docs/examples/sql-editor/07-js-filter-while.sql @@ -0,0 +1,21 @@ +SELECT 1 AS id, 'alice@example.com' AS email +UNION ALL +SELECT 2, 'bob@example.com' +UNION ALL +SELECT 3, 'cara@example.com'; + +-- @js +function isOdd(n) { + return n % 2 === 1; +} +const out = []; +let i = 0; +while (i < last.rows.length) { + const id = Number(last.rows[i][0]); + if (isOdd(id)) { + out.push({ id, email: last.rows[i][1], kind: 'odd' }); + } + i++; +} +return out; +-- @end diff --git a/docs/examples/sql-editor/08-js-pure-grid.sql b/docs/examples/sql-editor/08-js-pure-grid.sql new file mode 100644 index 0000000..6a633f0 --- /dev/null +++ b/docs/examples/sql-editor/08-js-pure-grid.sql @@ -0,0 +1,11 @@ +-- @js +import { groupBy } from 'lodash-es'; + +const rows = [ + { kind: 'a', n: 1 }, + { kind: 'a', n: 2 }, + { kind: 'b', n: 3 }, +]; +const grouped = groupBy(rows, 'kind'); +return [{ a: grouped.a.length, b: grouped.b.length }]; +-- @end diff --git a/docs/examples/sql-editor/09-js-async-fetch.sql b/docs/examples/sql-editor/09-js-async-fetch.sql new file mode 100644 index 0000000..4aec77a --- /dev/null +++ b/docs/examples/sql-editor/09-js-async-fetch.sql @@ -0,0 +1,9 @@ +-- @js +const res = await fetch('https://httpbin.org/get'); +const json = await res.json(); +return [{ + status: res.status, + url: json.url ?? '', + origin: json.origin ?? '', +}]; +-- @end diff --git a/docs/examples/sql-editor/10-node-async-fetch.sql b/docs/examples/sql-editor/10-node-async-fetch.sql new file mode 100644 index 0000000..c868d72 --- /dev/null +++ b/docs/examples/sql-editor/10-node-async-fetch.sql @@ -0,0 +1,10 @@ +-- @node +const res = await fetch('https://httpbin.org/get'); +const json = await res.json(); +return [{ + status: res.status, + url: json.url ?? '', + origin: json.origin ?? '', + runtime: 'node', +}]; +-- @end diff --git a/docs/examples/sql-editor/README.md b/docs/examples/sql-editor/README.md new file mode 100644 index 0000000..4ad0818 --- /dev/null +++ b/docs/examples/sql-editor/README.md @@ -0,0 +1,12 @@ +# SQL Editor code-cell examples + +Copy any `.sql` file into the SQL Editor, or use **Bookmarks → Add samples** in the app +to install the same scripts as named bookmarks (★ Sample · …). + +They use `UNION ALL` demo rows so they run without a `user` table. + +| File | What it shows | +|------|----------------| +| `01`–`08` | Sync JS/TS transforms, lodash, date-fns, chaining | +| `09-js-async-fetch.sql` | Browser `-- @js` with `await fetch` | +| `10-node-async-fetch.sql` | Server `-- @node` with `await fetch` | diff --git a/packages/core/src/browser.ts b/packages/core/src/browser.ts index 9d7ab14..f69bd0c 100644 --- a/packages/core/src/browser.ts +++ b/packages/core/src/browser.ts @@ -34,12 +34,48 @@ export { dmlLacksWhere, parseCodeCell, findCodeFences, + stripJsStringsAndComments, stripCodeFenceMarkers, codeCellHasReturn, + stripFullLineSqlComments, + isCodeCellKind, + isNodeCodeCellKind, + codeCellNeedsTs, + nodeCodeCellWireKind, } from './modules/sql-splitter'; -export type { SplitStatement, StatementStatus, StatementKind } from './modules/sql-splitter'; +export type { + SplitStatement, + StatementStatus, + StatementKind, + CodeCellKind, + BrowserCodeCellKind, + NodeCodeCellKind, + TsCodeCellKind, +} from './modules/sql-splitter'; +export type { + CodeCellLast, + CodeCellVars, + CodeCellOk, + CodeCellErr, + CodeCellResult, +} from './modules/code-cell-types'; +export { isCodeCellLast, isCodeCellVars, CODE_CELL_KIND_LABEL } from './modules/code-cell-types'; export type { SqlDialect, CanonicalType, CanonicalBase, RenderedType } from './modules/sql-dialect.interface'; export { resolveDialect, DIALECT_MAP } from './modules/dialect-registry'; +export { + CODE_CELL_ALLOWED_PACKAGES, + parseCodeCellImports, + resolveCodeCellImportBindings, + prepareCodeCellImports, + normalizeCodeCellReturn, + cloneCodeCellLast, + runCodeCellBody, +} from './modules/code-cell-exec'; +export type { + CodeCellAllowedPackage, + CodeCellImportSpec, + RunCodeCellBodyArgs, +} from './modules/code-cell-exec'; export { buildConnectionString, withConnectionString, DEFAULT_PORTS } from './cores/connection-string'; export { PROVIDER_SETTINGS, getProviderSettings } from './providers/provider-settings'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 321fc64..dcf40fa 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -37,12 +37,48 @@ export { dmlLacksWhere, parseCodeCell, findCodeFences, + stripJsStringsAndComments, stripCodeFenceMarkers, codeCellHasReturn, + stripFullLineSqlComments, + isCodeCellKind, + isNodeCodeCellKind, + codeCellNeedsTs, + nodeCodeCellWireKind, } from './modules/sql-splitter'; -export type { SplitStatement, StatementStatus, StatementKind } from './modules/sql-splitter'; +export type { + SplitStatement, + StatementStatus, + StatementKind, + CodeCellKind, + BrowserCodeCellKind, + NodeCodeCellKind, + TsCodeCellKind, +} from './modules/sql-splitter'; +export type { + CodeCellLast, + CodeCellVars, + CodeCellOk, + CodeCellErr, + CodeCellResult, +} from './modules/code-cell-types'; +export { isCodeCellLast, isCodeCellVars, CODE_CELL_KIND_LABEL } from './modules/code-cell-types'; export type { SqlDialect, CanonicalType, CanonicalBase, RenderedType } from './modules/sql-dialect.interface'; export { resolveDialect, DIALECT_MAP } from './modules/dialect-registry'; +export { + CODE_CELL_ALLOWED_PACKAGES, + parseCodeCellImports, + resolveCodeCellImportBindings, + prepareCodeCellImports, + normalizeCodeCellReturn, + cloneCodeCellLast, + runCodeCellBody, +} from './modules/code-cell-exec'; +export type { + CodeCellAllowedPackage, + CodeCellImportSpec, + RunCodeCellBodyArgs, +} from './modules/code-cell-exec'; // Connection-string helpers export { buildConnectionString, withConnectionString, DEFAULT_PORTS } from './cores/connection-string'; diff --git a/packages/core/src/modules/code-cell-exec.ts b/packages/core/src/modules/code-cell-exec.ts new file mode 100644 index 0000000..a391120 --- /dev/null +++ b/packages/core/src/modules/code-cell-exec.ts @@ -0,0 +1,368 @@ +/** + * Runtime-agnostic SQL Editor code-cell executor. + * + * The browser (`-- @js` / `-- @ts`, Web Worker) and the Node backend + * (`-- @node` / `-- @nodets`, worker_threads) share everything here: the + * allowlisted-import parser, the return-value normalizer, and the + * AsyncFunction sandbox. Callers supply only what actually differs between the + * two runtimes — the bundled module namespaces and the globals preamble. + */ + +import { + codeCellHasReturn, + stripFullLineSqlComments, + stripJsStringsAndComments, +} from './sql-splitter'; +import type { + CodeCellLast, + CodeCellOk, + CodeCellErr, + CodeCellResult, + CodeCellVars, +} from './code-cell-types'; + +export const CODE_CELL_ALLOWED_PACKAGES = ['lodash', 'lodash-es', 'date-fns'] as const; +export type CodeCellAllowedPackage = (typeof CODE_CELL_ALLOWED_PACKAGES)[number]; + +const ALLOWED = new Set(CODE_CELL_ALLOWED_PACKAGES); + +type NamedBinding = { imported: string; local: string }; + +export type CodeCellImportSpec = + | { kind: 'default'; local: string; pkg: string } + | { kind: 'namespace'; local: string; pkg: string } + | { kind: 'named'; names: NamedBinding[]; pkg: string }; + +const DEFAULT_IMPORT_RE = /^import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; +const NAMESPACE_IMPORT_RE = + /^import\s+\*\s+as\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; +const NAMED_IMPORT_RE = /^import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]\s*;?\s*$/; +const MIXED_IMPORT_RE = /^import\s+[A-Za-z_$][\w$]*\s*,\s*\{/; + +/** + * Start of an `import` *statement*. The lookahead keeps identifiers that merely + * begin with "import" (`importantRows = …`) and dynamic `import(…)` out. + */ +const IMPORT_STMT_RE = /^import(?=[\s{*'"])/; + +/** Sandbox parameter names an import may not shadow. */ +const RESERVED_BINDINGS = new Set(['last', 'vars']); + +const IDENTIFIER_RE = /^[A-Za-z_$][\w$]*$/; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseNamedList(inner: string): NamedBinding[] | { error: string } { + const names: NamedBinding[] = []; + for (const part of inner.split(',')) { + const bit = part.trim(); + if (!bit) continue; + const asMatch = /^([A-Za-z_$][\w$]*)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(bit); + if (asMatch) { + names.push({ imported: asMatch[1]!, local: asMatch[2]! }); + continue; + } + if (IDENTIFIER_RE.test(bit)) { + names.push({ imported: bit, local: bit }); + continue; + } + return { error: `Invalid named import: ${bit}` }; + } + return names; +} + +function parseImportLine(trimmed: string): CodeCellImportSpec | { error: string } { + if (MIXED_IMPORT_RE.test(trimmed)) { + return { + error: 'Mixed default+named imports are not supported — use separate import lines', + }; + } + + const ns = NAMESPACE_IMPORT_RE.exec(trimmed); + if (ns) return { kind: 'namespace', local: ns[1]!, pkg: ns[2]! }; + + const named = NAMED_IMPORT_RE.exec(trimmed); + if (named) { + const names = parseNamedList(named[1]!); + if ('error' in names) return names; + return { kind: 'named', names, pkg: named[2]! }; + } + + const def = DEFAULT_IMPORT_RE.exec(trimmed); + if (def) return { kind: 'default', local: def[1]!, pkg: def[2]! }; + + return { error: `Unsupported import syntax: ${trimmed.slice(0, 80)}` }; +} + +/** + * Parse leading `import` lines, validate against the allowlist, and return the + * remaining body. Imports must be at the top (blank lines / `//` comments ok). + */ +export function parseCodeCellImports(body: string): + | { ok: true; specs: CodeCellImportSpec[]; bodyWithoutImports: string } + | { ok: false; error: string } { + const lines = body.split(/\r?\n/); + const specs: CodeCellImportSpec[] = []; + let i = 0; + + while (i < lines.length) { + const trimmed = lines[i]!.trim(); + if (trimmed === '' || trimmed.startsWith('//')) { + i++; + continue; + } + if (!IMPORT_STMT_RE.test(trimmed)) break; + + const parsed = parseImportLine(trimmed); + if ('error' in parsed) return { ok: false, error: parsed.error }; + if (!ALLOWED.has(parsed.pkg)) { + return { + ok: false, + error: `Package "${parsed.pkg}" is not allowlisted. Allowed: ${[...ALLOWED].join(', ')}`, + }; + } + specs.push(parsed); + i++; + } + + // Reject mid-body imports (after the first non-import code line). Scan with + // strings/comments stripped so an `import …` inside a template literal or a + // commented-out line is not mistaken for real code. + const rest = stripJsStringsAndComments(lines.slice(i).join('\n')); + for (const line of rest.split('\n')) { + if (IMPORT_STMT_RE.test(line.trim())) { + return { + ok: false, + error: 'import statements must appear at the top of the code cell', + }; + } + } + + return { ok: true, specs, bodyWithoutImports: lines.slice(i).join('\n').trim() }; +} + +function moduleDefault(mod: object): unknown { + if (mod && typeof mod === 'object' && 'default' in mod) { + const def = (mod as { default: unknown }).default; + if (def != null) return def; + } + return mod; +} + +function setBinding( + bindings: Record, + local: string, + value: unknown +): { error: string } | null { + if (RESERVED_BINDINGS.has(local)) { + return { + error: `"${local}" is reserved for the cell scope — import it under another name (import { ${local} as ${local}Fn } from …)`, + }; + } + // `bindings` is a null-prototype object, so hasOwn is the only membership + // test that matters — `in` / `!== undefined` on a plain object would report + // inherited keys (`toString`, `valueOf`) as duplicates. + if (Object.hasOwn(bindings, local)) { + return { error: `Duplicate binding "${local}"` }; + } + bindings[local] = value; + return null; +} + +/** Build identifier → value bindings for the sandbox parameters. */ +export function resolveCodeCellImportBindings( + specs: CodeCellImportSpec[], + modules: Record +): { ok: true; bindings: Record } | { ok: false; error: string } { + // Null prototype: a local named `toString` / `valueOf` / `__proto__` must be + // an ordinary key, not a collision with (or a write to) Object.prototype. + const bindings = Object.create(null) as Record; + + for (const spec of specs) { + const mod = modules[spec.pkg]; + if (!mod) return { ok: false, error: `Package "${spec.pkg}" is not loaded` }; + + if (spec.kind === 'default') { + const err = setBinding(bindings, spec.local, moduleDefault(mod)); + if (err) return { ok: false, error: err.error }; + continue; + } + + if (spec.kind === 'namespace') { + const err = setBinding(bindings, spec.local, mod); + if (err) return { ok: false, error: err.error }; + continue; + } + + const rec = mod as Record; + for (const { imported, local } of spec.names) { + if (!(imported in rec)) { + return { ok: false, error: `"${imported}" is not exported by "${spec.pkg}"` }; + } + const err = setBinding(bindings, local, rec[imported]); + if (err) return { ok: false, error: err.error }; + } + } + + return { ok: true, bindings }; +} + +/** Parse + resolve imports in one step. */ +export function prepareCodeCellImports( + body: string, + modules: Record +): { ok: true; body: string; bindings: Record } | { ok: false; error: string } { + const parsed = parseCodeCellImports(body); + if (!parsed.ok) return parsed; + const resolved = resolveCodeCellImportBindings(parsed.specs, modules); + if (!resolved.ok) return resolved; + return { ok: true, body: parsed.bodyWithoutImports, bindings: resolved.bindings }; +} + +function normalizeObjectRows(value: Record[], maxRows: number): CodeCellOk { + const truncated = value.length > maxRows; + const kept = truncated ? value.slice(0, maxRows) : value; + // Union the keys of every row that is actually rendered — sampling a fixed + // prefix silently drops columns that first appear in a later row. + const colSet = new Set(); + for (const row of kept) { + for (const k of Object.keys(row)) colSet.add(k); + } + const columns = [...colSet]; + const rows = kept.map((r) => columns.map((c) => r[c])); + return { ok: true, columns, rows, rowCount: rows.length, truncated }; +} + +function normalizeColumnsRows( + columnsRaw: unknown[], + rowsRaw: unknown[], + maxRows: number +): CodeCellOk { + const columns = columnsRaw.map((c) => String(c)); + const truncated = rowsRaw.length > maxRows; + const kept = truncated ? rowsRaw.slice(0, maxRows) : rowsRaw; + const rows: unknown[][] = kept.map((r) => { + if (Array.isArray(r)) return columns.map((_, i) => r[i]); + if (isPlainObject(r)) return columns.map((c) => r[c]); + return columns.map(() => undefined); + }); + return { ok: true, columns, rows, rowCount: rows.length, truncated }; +} + +/** + * Normalize a cell return value into columns/rows. + * Accepts `{ columns, rows }` or an array of plain objects. + */ +export function normalizeCodeCellReturn(value: unknown, maxRows: number): CodeCellOk | CodeCellErr { + if (value === null || value === undefined) { + return { + ok: false, + error: 'Code cell must return a value — use return { columns, rows } or return [...objects]', + }; + } + + if (Array.isArray(value)) { + if (value.length === 0) { + return { ok: true, columns: [], rows: [], rowCount: 0, truncated: false }; + } + if (value.every(isPlainObject)) return normalizeObjectRows(value, maxRows); + return { + ok: false, + error: 'Array return must be an array of plain objects (or return { columns, rows })', + }; + } + + if (isPlainObject(value)) { + const { columns, rows } = value as { columns?: unknown; rows?: unknown }; + if (!Array.isArray(columns) || !Array.isArray(rows)) { + return { + ok: false, + error: 'Return { columns: string[], rows: unknown[][] } or an array of objects', + }; + } + return normalizeColumnsRows(columns, rows, maxRows); + } + + return { ok: false, error: `Unsupported return type: ${typeof value}` }; +} + +/** + * Copy the previous grid before handing it to a cell. The worker paths get a + * structured clone for free, but an in-process run would otherwise pass the + * caller's own object — a cell doing `last.rows.reverse()` would silently + * rewrite the result grid already on screen. + */ +export function cloneCodeCellLast(last: CodeCellLast): CodeCellLast { + if (!last) return last; + return { + columns: [...last.columns], + rows: last.rows.map((r) => [...r]), + rowCount: last.rowCount, + }; +} + +// AsyncFunction so `await` and Promise returns work inside a cell. +const AsyncFunction = Object.getPrototypeOf(async function () {}) + .constructor as FunctionConstructor; + +export interface RunCodeCellBodyArgs { + body: string; + last: CodeCellLast; + vars: CodeCellVars; + maxRows: number; + /** + * Runtime globals to shadow inside the cell, e.g. `var process = undefined;`. + * A guardrail against accidents only — code compiled by the Function + * constructor runs in global scope and is not affected by this shadowing. + */ + preamble: string; + /** Allowlisted package namespaces for this runtime, keyed by import specifier. */ + modules: Record; +} + +/** + * Run a code-cell body with `last`, `vars`, and allowlisted import bindings in + * scope, then normalize whatever it returns into a grid. Callers must + * transpile TypeScript first. + */ +export async function runCodeCellBody(args: RunCodeCellBodyArgs): Promise { + // `--` is the decrement operator in JS, so a stray SQL-style comment line + // would be a syntax error; drop those before anything else looks at the body. + const rawBody = stripFullLineSqlComments(args.body).trim(); + if (!rawBody) return { ok: false, error: 'Code cell is empty' }; + + const prepared = prepareCodeCellImports(rawBody, args.modules); + if (!prepared.ok) return { ok: false, error: prepared.error }; + + const body = prepared.body.trim(); + if (!body) return { ok: false, error: 'Code cell is empty after imports' }; + if (!codeCellHasReturn(body)) { + return { + ok: false, + error: + 'Code cell must include a return statement — e.g. return { columns, rows } or return rows.map(...)', + }; + } + + const bindingNames = Object.keys(prepared.bindings); + for (const name of bindingNames) { + if (!IDENTIFIER_RE.test(name)) { + return { ok: false, error: `Invalid import binding name: ${name}` }; + } + } + const bindingValues = bindingNames.map((n) => prepared.bindings[n]); + + try { + const fn = new AsyncFunction('last', 'vars', ...bindingNames, `${args.preamble}${body}`); + const raw = await fn(cloneCodeCellLast(args.last), args.vars, ...bindingValues); + return normalizeCodeCellReturn(raw, args.maxRows); + } catch (error: unknown) { + return { ok: false, error: errorMessage(error) }; + } +} diff --git a/packages/core/src/modules/code-cell-types.test.ts b/packages/core/src/modules/code-cell-types.test.ts new file mode 100644 index 0000000..4c47ef1 --- /dev/null +++ b/packages/core/src/modules/code-cell-types.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { isCodeCellLast, isCodeCellVars } from './code-cell-types'; + +describe('isCodeCellLast', () => { + it('accepts null and well-formed grids', () => { + expect(isCodeCellLast(null)).toBe(true); + expect( + isCodeCellLast({ columns: ['a'], rows: [[1], [2]], rowCount: 2 }) + ).toBe(true); + }); + + it('rejects non-string columns or non-matrix rows', () => { + expect(isCodeCellLast({ columns: [1], rows: [[1]], rowCount: 1 })).toBe(false); + expect(isCodeCellLast({ columns: ['a'], rows: [1], rowCount: 1 })).toBe(false); + expect(isCodeCellLast({ columns: ['a'], rows: [[1]], rowCount: 'x' })).toBe(false); + expect(isCodeCellLast({})).toBe(false); + }); +}); + +describe('isCodeCellVars', () => { + it('accepts scalar / list / table shapes', () => { + expect( + isCodeCellVars({ + n: { kind: 'scalar', value: 1 }, + ids: { kind: 'list', values: [1, 2] }, + t: { kind: 'table', columns: ['a'], rows: [[9]] }, + }) + ).toBe(true); + expect(isCodeCellVars({})).toBe(true); + }); + + it('rejects missing payload fields', () => { + expect(isCodeCellVars({ n: { kind: 'scalar' } })).toBe(false); + expect(isCodeCellVars({ ids: { kind: 'list' } })).toBe(false); + expect(isCodeCellVars({ t: { kind: 'table', columns: ['a'] } })).toBe(false); + expect(isCodeCellVars({ bad: { kind: 'other' } })).toBe(false); + expect(isCodeCellVars([])).toBe(false); + }); +}); diff --git a/packages/core/src/modules/code-cell-types.ts b/packages/core/src/modules/code-cell-types.ts new file mode 100644 index 0000000..9d21efa --- /dev/null +++ b/packages/core/src/modules/code-cell-types.ts @@ -0,0 +1,92 @@ +/** + * Shared SQL Editor code-cell value shapes (browser + Node executors). + * Pure types + runtime predicates — safe for `@foxschema/core` browser builds. + */ + +/** + * Display names per fence kind. One table so the statement strip, the results + * header and any future surface cannot drift apart; the fence tag is the key + * itself (`-- @nodets` … `-- @end`). + */ +export const CODE_CELL_KIND_LABEL = { + js: { short: 'JS', long: 'JavaScript' }, + ts: { short: 'TS', long: 'TypeScript' }, + node: { short: 'NODE', long: 'Node.js' }, + nodets: { short: 'NODE-TS', long: 'Node TypeScript' }, +} as const satisfies Record; + +/** Previous statement grid injected as `last` (null when none). */ +export type CodeCellLast = { + columns: string[]; + rows: unknown[][]; + rowCount: number; +} | null; + +/** Structured-cloneable variable bag passed into cells (secrets omitted). */ +export type CodeCellVars = Record< + string, + | { kind: 'scalar'; value: unknown } + | { kind: 'list'; values: unknown[] } + | { kind: 'table'; columns: string[]; rows: unknown[][] } +>; + +export type CodeCellOk = { + ok: true; + columns: string[]; + rows: unknown[][]; + rowCount: number; + truncated: boolean; +}; + +export type CodeCellErr = { ok: false; error: string }; + +export type CodeCellResult = CodeCellOk | CodeCellErr; + +function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isUnknownArray(value: unknown): value is unknown[] { + return Array.isArray(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((c) => typeof c === 'string'); +} + +function isRowMatrix(value: unknown): value is unknown[][] { + return Array.isArray(value) && value.every((r) => Array.isArray(r)); +} + +/** True when `v` is null or a rectangular `{ columns, rows, rowCount }` grid. */ +export function isCodeCellLast(v: unknown): v is CodeCellLast { + if (v === null) return true; + if (!isPlainObject(v)) return false; + if (!isStringArray(v.columns)) return false; + if (!isRowMatrix(v.rows)) return false; + if (typeof v.rowCount !== 'number' || !Number.isFinite(v.rowCount)) return false; + return true; +} + +/** True when `v` is a vars bag with scalar/list/table field shapes. */ +export function isCodeCellVars(v: unknown): v is CodeCellVars { + if (!isPlainObject(v)) return false; + for (const val of Object.values(v)) { + if (!isPlainObject(val)) return false; + const kind = val.kind; + if (kind === 'scalar') { + if (!('value' in val)) return false; + continue; + } + if (kind === 'list') { + if (!isUnknownArray(val.values)) return false; + continue; + } + if (kind === 'table') { + if (!isStringArray(val.columns) || !isRowMatrix(val.rows)) return false; + continue; + } + return false; + } + return true; +} diff --git a/packages/core/src/modules/sql-splitter.test.ts b/packages/core/src/modules/sql-splitter.test.ts index 9103e53..7a2aa04 100644 --- a/packages/core/src/modules/sql-splitter.test.ts +++ b/packages/core/src/modules/sql-splitter.test.ts @@ -1,5 +1,17 @@ import { describe, it, expect } from 'vitest'; -import { splitSqlStatements, checkStatement, isWriteStatement, firstKeyword, extractTableAliases, isMutatingDmlStatement, dmlLacksWhere } from './sql-splitter'; +import { + splitSqlStatements, + checkStatement, + isWriteStatement, + statementVerb, + firstKeyword, + extractTableAliases, + isMutatingDmlStatement, + dmlLacksWhere, + parseCodeCell, + stripFullLineSqlComments, + codeCellHasReturn, +} from './sql-splitter'; describe('splitSqlStatements', () => { it('splits simple semicolon-terminated statements with line numbers', () => { @@ -85,6 +97,57 @@ return { columns: ['n'], rows: [[n]] }; -- @end`); expect(stmts).toHaveLength(1); expect(stmts[0]?.kind).toBe('ts'); + + const long = splitSqlStatements(`-- @typescript +return []; +-- @end`); + expect(long[0]?.kind).toBe('ts'); + + const jsLong = splitSqlStatements(`-- @javascript +return []; +-- @end`); + expect(jsLong[0]?.kind).toBe('js'); + }); + + it('supports -- @node / -- @nodets fences', () => { + const node = splitSqlStatements(`-- @node +return [{ ok: true }]; +-- @end`); + expect(node[0]?.kind).toBe('node'); + expect(node[0]?.terminated).toBe(true); + + const nodets = splitSqlStatements(`-- @nodets +const n: number = 1; +return [{ n }]; +-- @end`); + expect(nodets[0]?.kind).toBe('nodets'); + + const alias = splitSqlStatements(`-- @node-typescript +return []; +-- @end`); + expect(alias[0]?.kind).toBe('nodets'); + }); + + it('treats code cells as non-writes and reports js/ts/node verb', () => { + const js = splitSqlStatements(`-- @js\nreturn [];\n-- @end`)[0]!; + expect(isWriteStatement(js.text)).toBe(false); + expect(statementVerb(js.text)).toBe('js'); + + const ts = splitSqlStatements(`-- @ts\nreturn [];\n-- @end`)[0]!; + expect(isWriteStatement(ts.text)).toBe(false); + expect(statementVerb(ts.text)).toBe('ts'); + + const node = splitSqlStatements(`-- @node\nreturn [];\n-- @end`)[0]!; + expect(isWriteStatement(node.text)).toBe(false); + expect(statementVerb(node.text)).toBe('node'); + }); + + it('does not treat return inside comments as a real return', () => { + const stmts = splitSqlStatements(`-- @js +// return nowhere +const x = 1; +-- @end`); + expect(checkStatement(stmts[0]!).reasons.join()).toMatch(/return/i); }); it('warns when -- @end is missing', () => { @@ -113,6 +176,23 @@ return out; expect(checkStatement(stmts[0]!)).toEqual({ level: 'ok', reasons: [] }); }); + it('strips -- @end even when a trailing blank line follows', () => { + const parsed = parseCodeCell(`-- @js +return []; +-- @end +`); + expect(parsed).toMatchObject({ kind: 'js', closed: true, body: 'return [];' }); + }); + + it('stripFullLineSqlComments drops SQL -- lines that would break JS', () => { + expect( + stripFullLineSqlComments(`-- use last from prior cell +return last; +const n = a - b; +`) + ).toBe(`return last;\nconst n = a - b;\n`); + }); + it('preserves @set comments before a code fence via offsets', () => { const sql = `SELECT 1; -- @set src = table @@ -248,3 +328,27 @@ describe('extractTableAliases', () => { expect(quoted.u).toBe('ORDER'); }); }); + +describe('codeCellHasReturn with regex literals', () => { + it('sees a return after a regex that ends in an escaped slash', () => { + // `/\//` used to read as a `//` line comment, swallowing the return. + expect(codeCellHasReturn(String.raw`const p = String(x).split(/\//); return [{ p }];`)).toBe( + true + ); + expect(codeCellHasReturn(String.raw`const re = /^https:\/\//; return [{ ok: re.test(x) }];`)).toBe( + true + ); + }); + + it('still ignores return inside comments and strings', () => { + expect(codeCellHasReturn('// return []\nconst x = 1;')).toBe(false); + expect(codeCellHasReturn('/* return */ const x = 1;')).toBe(false); + expect(codeCellHasReturn(`const s = 'return'; const t = "return";`)).toBe(false); + expect(codeCellHasReturn('const t = `no return here`;')).toBe(false); + }); + + it('does not mistake division for a regex', () => { + expect(codeCellHasReturn('const n = a / b / c; return [{ n }];')).toBe(true); + expect(codeCellHasReturn('const n = a / b / c;')).toBe(false); + }); +}); diff --git a/packages/core/src/modules/sql-splitter.ts b/packages/core/src/modules/sql-splitter.ts index 8e8ef00..9836e69 100644 --- a/packages/core/src/modules/sql-splitter.ts +++ b/packages/core/src/modules/sql-splitter.ts @@ -12,11 +12,42 @@ * with internal semicolons split at each `;`; routine DDL belongs in the * migration flow, not the editor. * - * Code cells: `-- @js` / `-- @ts` … `-- @end` fences are opaque (inner `;` - * do not split) and emit a single statement with kind `'js'` | `'ts'`. + * Code cells: `-- @js` / `-- @ts` / `-- @node` / `-- @nodets` … `-- @end` + * fences are opaque (inner `;` do not split) and emit a single statement with + * the matching kind. */ -export type StatementKind = 'sql' | 'js' | 'ts'; +export type StatementKind = 'sql' | 'js' | 'ts' | 'node' | 'nodets'; + +/** Browser-executed fences (`-- @js` / `-- @ts`). Also the wire `kind` for Node cells. */ +export type BrowserCodeCellKind = 'js' | 'ts'; + +/** Server-executed fences (`-- @node` / `-- @nodets`). */ +export type NodeCodeCellKind = 'node' | 'nodets'; + +export type CodeCellKind = BrowserCodeCellKind | NodeCodeCellKind; + +/** Fences whose body is TypeScript (transpiled before run). */ +export type TsCodeCellKind = Extract; + +export function isCodeCellKind(kind: StatementKind | undefined): kind is CodeCellKind { + return kind === 'js' || kind === 'ts' || kind === 'node' || kind === 'nodets'; +} + +/** True when the fence runs on the FoxSchema Node backend. */ +export function isNodeCodeCellKind(kind: CodeCellKind | undefined): kind is NodeCodeCellKind { + return kind === 'node' || kind === 'nodets'; +} + +/** True when the cell body should be TypeScript-transpiled before run. */ +export function codeCellNeedsTs(kind: CodeCellKind | undefined): kind is TsCodeCellKind { + return kind === 'ts' || kind === 'nodets'; +} + +/** Map a Node fence kind to the POST /sql/code-cell wire `kind` (`js` | `ts`). */ +export function nodeCodeCellWireKind(kind: NodeCodeCellKind): BrowserCodeCellKind { + return codeCellNeedsTs(kind) ? 'ts' : 'js'; +} export interface SplitStatement { /** Statement text from its first non-whitespace character through its terminator. */ @@ -30,8 +61,8 @@ export interface SplitStatement { endLine: number; /** True when the statement ended with a `;` (SQL) or `-- @end` (code cell). */ terminated: boolean; - /** `'sql'` (default), or a fenced code cell. */ - kind?: StatementKind; + /** `'sql'` or a fenced code cell — always set by the splitter. */ + kind: StatementKind; } export interface StatementStatus { @@ -59,12 +90,16 @@ const WRITE_KEYWORDS = new Set([ // eslint-disable-next-line security/detect-unsafe-regex -- false positive: anchored at ^; optional bounded identifier const DOLLAR_TAG_RE = /^\$([A-Za-z_][A-Za-z0-9_]*)?\$/; -const FENCE_START_RE = /^[ \t]*--[ \t]*@(js|javascript|ts|typescript)[ \t]*$/i; +const FENCE_START_RE = + /^[ \t]*--[ \t]*@(node-typescript|node-ts|nodets|node|javascript|typescript|js|ts)[ \t]*$/i; const FENCE_END_RE = /^[ \t]*--[ \t]*@end[ \t]*$/i; -function kindFromFenceTag(tag: string): 'js' | 'ts' { +function kindFromFenceTag(tag: string): CodeCellKind { const t = tag.toLowerCase(); - return t === 'ts' || t === 'typescript' ? 'ts' : 'js'; + if (t === 'nodets' || t === 'node-ts' || t === 'node-typescript') return 'nodets'; + if (t === 'node') return 'node'; + if (t === 'ts' || t === 'typescript') return 'ts'; + return 'js'; } /** 1-based line number of the character at `index` (0 = start of file). */ @@ -86,16 +121,16 @@ function lineBounds(sql: string, from: number): { start: number; end: number; ne } interface CodeFenceRange { - kind: 'js' | 'ts'; + kind: CodeCellKind; start: number; end: number; closed: boolean; } /** - * Line-oriented scan for `-- @js` / `-- @ts` … `-- @end` fences. - * Fences are only recognized when the whole line matches (so SQL string - * literals containing the text are unlikely to false-positive). + * Line-oriented scan for `-- @js` / `-- @ts` / `-- @node` / `-- @nodets` … + * `-- @end` fences. Fences are only recognized when the whole line matches + * (so SQL string literals containing the text are unlikely to false-positive). */ export function findCodeFences(sql: string): CodeFenceRange[] { const fences: CodeFenceRange[] = []; @@ -135,13 +170,13 @@ export function findCodeFences(sql: string): CodeFenceRange[] { } /** - * True when `text` is (or starts as) a fenced JS/TS code cell. + * True when `text` is (or starts as) a fenced JS/TS/Node code cell. * Returns kind + body (fence markers stripped; leading `-- @set` may still * remain in body — callers typically run parseSetDirectives first). */ export function parseCodeCell( statement: string -): { kind: 'js' | 'ts'; body: string; closed: boolean } | null { +): { kind: CodeCellKind; body: string; closed: boolean } | null { const trimmed = statement.replace(/^\s+/, ''); const nl = trimmed.search(/\r?\n/); const firstLine = nl < 0 ? trimmed : trimmed.slice(0, nl); @@ -152,6 +187,10 @@ export function parseCodeCell( if (nl < 0) return { kind, body: '', closed: false }; const lines = trimmed.slice(nl).replace(/^\r?\n/, '').split(/\r?\n/); + // Trailing blank lines after `-- @end` are common; ignore them when closing. + while (lines.length > 0 && lines[lines.length - 1]!.trim() === '') { + lines.pop(); + } let closed = false; if (lines.length > 0 && FENCE_END_RE.test(lines[lines.length - 1]!)) { closed = true; @@ -163,19 +202,55 @@ export function parseCodeCell( /** Strip fence open/close lines from a full cell statement (after @set parse). */ export function stripCodeFenceMarkers( statement: string -): { kind: 'js' | 'ts'; body: string } | null { +): { kind: CodeCellKind; body: string } | null { const parsed = parseCodeCell(statement); return parsed ? { kind: parsed.kind, body: parsed.body } : null; } -/** Strip JS line/block comments and quoted strings for keyword scans. */ -function stripJsStringsAndComments(src: string): string { +/** + * Characters after which a `/` starts a regex literal rather than a division. + * Without this, `split(/\//)` reads as a `//` line comment and swallows the + * rest of the line — including a trailing `return`. + */ +const REGEX_ALLOWED_AFTER = new Set([ + '(', ',', '=', ':', '[', '!', '&', '|', '?', '{', '}', ';', '+', '-', '*', '%', '~', '^', '<', + '>', '\n', +]); + +/** Skip a regex literal starting at `src[i] === '/'`; returns the index after it. */ +function skipRegexLiteral(src: string, start: number): number { + let i = start + 1; + let inClass = false; + while (i < src.length) { + const c = src[i]!; + if (c === '\\') { + i += 2; + continue; + } + if (c === '\n') return i; // unterminated — treat as division after all + if (c === '[') inClass = true; + else if (c === ']') inClass = false; + else if (c === '/' && !inClass) return i + 1; + i++; + } + return i; +} + +/** Strip JS line/block comments, regex literals, and quoted strings for keyword scans. */ +export function stripJsStringsAndComments(src: string): string { let out = ''; let i = 0; const n = src.length; + // Last non-whitespace character emitted — decides regex-vs-division below. + let prev = ''; while (i < n) { const ch = src[i]!; const next = src[i + 1] ?? ''; + if (ch === '/' && (prev === '' || REGEX_ALLOWED_AFTER.has(prev)) && next !== '/' && next !== '*') { + i = skipRegexLiteral(src, i); + out += ' '; + continue; + } if (ch === '/' && next === '/') { i += 2; while (i < n && src[i] !== '\n') i++; @@ -190,6 +265,8 @@ function stripJsStringsAndComments(src: string): string { if (ch === "'" || ch === '"' || ch === '`') { const q = ch; out += ' '; + // A string is a value, so a following `/` is division, not a regex. + prev = 'x'; i++; while (i < n) { if (src[i] === '\\') { @@ -215,6 +292,7 @@ function stripJsStringsAndComments(src: string): string { continue; } out += ch; + if (!/\s/.test(ch)) prev = ch; i++; } return out; @@ -228,6 +306,19 @@ export function codeCellHasReturn(body: string): boolean { return /\breturn\b/.test(stripJsStringsAndComments(body)); } +/** + * Drop full-line SQL `-- …` comments from a code-cell body. + * Inside JS/TS, `--` is the decrement operator — a line like + * `-- use last` throws "Invalid left-hand side expression in prefix operation". + * Prefer `//` comments in cells; this strips accidental SQL-style ones at run time. + */ +export function stripFullLineSqlComments(body: string): string { + return body + .split(/\r?\n/) + .filter((line) => !/^[ \t]*--/.test(line)) + .join('\n'); +} + /** Split a SQL-only buffer into `;`-terminated statements (no code fences). */ function splitSqlOnly(sql: string): SplitStatement[] { const out: SplitStatement[] = []; @@ -438,13 +529,15 @@ export function checkStatement( ): StatementStatus { const cell = parseCodeCell(stmt.text); const kind = stmt.kind ?? cell?.kind ?? 'sql'; - if (kind === 'js' || kind === 'ts') { + if (isCodeCellKind(kind)) { const reasons: string[] = []; if (!stmt.terminated) { reasons.push('Missing -- @end for code cell'); } - // Strip leading `-- @set` lines inside the fence before the return check. - const body = (cell?.body ?? '').replace(/^(?:--\s*@set[^\n]*\n)+/i, ''); + // Strip full-line SQL `--` comments before the return check (`--` is + // decrement in JS). This covers `-- @set` directives too — they are + // full-line `--` comments, so no separate pass is needed. + const body = stripFullLineSqlComments(cell?.body ?? ''); if (body.trim() && !codeCellHasReturn(body)) { reasons.push('Code cell must include a return statement'); }