diff --git a/.opencode/skills/sql-translate/SKILL.md b/.opencode/skills/sql-translate/SKILL.md index 83931c42f7..321fdc2fa9 100644 --- a/.opencode/skills/sql-translate/SKILL.md +++ b/.opencode/skills/sql-translate/SKILL.md @@ -7,7 +7,7 @@ description: Translate SQL queries between database dialects (Snowflake, BigQuer ## Requirements **Agent:** builder or migrator (may write translated SQL to files) -**Tools used:** sql_translate, read, write, sql_validate +**Tools used:** sql_translate, read, write, altimate_core_validate Translate SQL queries from one database dialect to another using sqlglot's transpilation engine. @@ -36,7 +36,7 @@ Translate SQL queries from one database dialect to another using sqlglot's trans - Any warnings about lossy translations or features that need manual review 6. **Offer next steps** if applicable: - - Suggest running `sql_validate` on the translated SQL to verify syntax + - Suggest running `altimate_core_validate` on the translated SQL to verify syntax - Offer to write the translated SQL to a file - Offer to translate additional queries @@ -65,4 +65,4 @@ The user invokes this skill with optional dialect and SQL arguments: | Oracle | `oracle` | | Trino/Presto | `trino` / `presto` | -Use the tools: `sql_translate`, `read`, `write`, `sql_validate`. +Use the tools: `sql_translate`, `read`, `write`, `altimate_core_validate`. diff --git a/bun.lock b/bun.lock index e0f417e6f6..9797b47ab7 100644 --- a/bun.lock +++ b/bun.lock @@ -12,7 +12,6 @@ }, "devDependencies": { "@tsconfig/bun": "catalog:", - "@types/better-sqlite3": "7.6.13", "@types/pg": "8.18.0", "@typescript/native-preview": "catalog:", "husky": "9.1.7", @@ -42,7 +41,6 @@ "optionalDependencies": { "@databricks/sql": "^1.0.0", "@google-cloud/bigquery": "^8.0.0", - "better-sqlite3": "^11.0.0", "duckdb": "^1.0.0", "mssql": "^11.0.0", "mysql2": "^3.0.0", @@ -1286,7 +1284,7 @@ "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], - "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], "ci-info": ["ci-info@3.9.0", "", {}, "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ=="], @@ -3126,10 +3124,10 @@ "table-layout/typical": ["typical@7.3.0", "", {}, "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw=="], - "tar/chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], - "tar/yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], + "tar-fs/chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + "tar-stream/bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], "tedious/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], diff --git a/packages/opencode/src/agent/agent.ts b/packages/opencode/src/agent/agent.ts index b6363e8d6f..dcda6315eb 100644 --- a/packages/opencode/src/agent/agent.ts +++ b/packages/opencode/src/agent/agent.ts @@ -168,7 +168,7 @@ export namespace Agent { PermissionNext.fromConfig({ "*": "deny", // SQL read tools - sql_execute: "allow", sql_validate: "allow", sql_analyze: "allow", + sql_execute: "allow", altimate_core_validate: "allow", sql_analyze: "allow", sql_translate: "allow", sql_optimize: "allow", lineage_check: "allow", sql_explain: "allow", sql_format: "allow", sql_fix: "allow", sql_autocomplete: "allow", sql_diff: "allow", @@ -184,7 +184,7 @@ export namespace Agent { finops_unused_resources: "allow", finops_role_grants: "allow", finops_role_hierarchy: "allow", finops_user_roles: "allow", // Core tools - altimate_core_validate: "allow", altimate_core_check: "allow", + altimate_core_check: "allow", altimate_core_rewrite: "allow", // Read-only file access read: "allow", grep: "allow", glob: "allow", diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index 1efb790e69..cfebbdad52 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -2,7 +2,7 @@ You are altimate-code in analyst mode — a read-only data exploration agent. You CANNOT modify any files or execute destructive SQL. You can only: - Execute SELECT queries (enforced by AltimateCore read-only mode) via `sql_execute` -- Validate and lint SQL via `sql_validate` +- Validate and lint SQL via `altimate_core_validate` - Analyze SQL for anti-patterns and optimization opportunities via `sql_analyze` - Inspect database schemas via `schema_inspect` - Check column-level lineage via `lineage_check` @@ -41,7 +41,7 @@ Remember: your users are hired to generate insights, not warehouse bills. Every - /cost-report — Snowflake cost analysis with optimization suggestions - /query-optimize — Query optimization with anti-pattern detection - /sql-translate — Cross-dialect SQL translation with warnings (analysis only; writing translated files requires the builder agent) -- /impact-analysis — Downstream impact analysis using lineage + manifest +- /dbt-analyze — Downstream impact analysis using lineage + manifest - /lineage-diff — Compare column lineage between SQL versions - /data-viz — Build interactive data visualizations, dashboards, charts, and analytics views from query results Note: Skills that write files (/generate-tests, /model-scaffold, /yaml-config, /dbt-docs, /medallion-patterns, /incremental-logic) require the builder agent. diff --git a/packages/opencode/src/altimate/prompts/builder.txt b/packages/opencode/src/altimate/prompts/builder.txt index b2a63e14c5..d4a880869a 100644 --- a/packages/opencode/src/altimate/prompts/builder.txt +++ b/packages/opencode/src/altimate/prompts/builder.txt @@ -10,7 +10,7 @@ You are altimate-code in builder mode — a data engineering agent specializing You have full read/write access to the project. You can: - Create and modify dbt models, SQL files, and YAML configs - Execute SQL against connected warehouses via `sql_execute` -- Validate SQL with AltimateCore via `altimate_core_validate` (syntax + schema references) +- Validate SQL syntax and schema references via `altimate_core_validate` - Analyze SQL for anti-patterns and performance issues via `sql_analyze` - Inspect database schemas via `schema_inspect` - Search schemas by natural language via `schema_search` @@ -41,7 +41,7 @@ altimate-dbt info # Project metadata When writing SQL: - Always run `sql_analyze` to check for anti-patterns before finalizing queries -- Validate SQL with `sql_validate` before executing against a warehouse +- Validate SQL with `altimate_core_validate` before executing against a warehouse - Use `schema_inspect` to understand table structures before writing queries - Prefer CTEs over subqueries for readability - Include column descriptions in dbt YAML files diff --git a/packages/opencode/test/altimate/sql-validation-adversarial.test.ts b/packages/opencode/test/altimate/sql-validation-adversarial.test.ts new file mode 100644 index 0000000000..ca1f21446d --- /dev/null +++ b/packages/opencode/test/altimate/sql-validation-adversarial.test.ts @@ -0,0 +1,980 @@ +/** + * Adversarial, user-perspective, and extended E2E tests for SQL validation. + * + * Covers: + * 1. Adversarial inputs to sql-classify (bypass attempts, encoding tricks, edge cases) + * 2. User-perspective: sql_execute tool with mocked ctx.ask (permission flow, error messages) + * 3. E2E: full pipeline through Dispatcher with realistic scenarios + * 4. Stress: concurrent validation, large payloads, rapid-fire calls + */ + +import { describe, expect, test, beforeAll, afterAll, mock } from "bun:test" + +// Mock DuckDB driver so sql.execute tests don't need native duckdb +mock.module("@altimateai/drivers/duckdb", () => ({ + connect: async () => ({ + execute: async (sql: string) => ({ + columns: ["result"], + rows: [["ok"]], + row_count: 1, + truncated: false, + }), + connect: async () => {}, + close: async () => {}, + schemas: async () => [], + tables: async () => [], + columns: async () => [], + }), +})) + +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import { registerAll } from "../../src/altimate/native/altimate-core" +import { registerAllSql } from "../../src/altimate/native/sql/register" +import { registerAll as registerConnections } from "../../src/altimate/native/connections/register" +import * as Registry from "../../src/altimate/native/connections/registry" +import { classify, classifyMulti, classifyAndCheck } from "../../src/altimate/tools/sql-classify" +import { SqlExecuteTool } from "../../src/altimate/tools/sql-execute" +import { Instance } from "../../src/project/instance" +import { SessionID, MessageID } from "../../src/session/schema" +import { tmpdir } from "../fixture/fixture" + +beforeAll(() => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + registerAll() + registerAllSql() + registerConnections() +}) +afterAll(() => { + delete process.env.ALTIMATE_TELEMETRY_DISABLED +}) + +// Shared test context for tool execution +const baseCtx = { + sessionID: SessionID.make("ses_test_adversarial"), + messageID: MessageID.make(""), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [] as any[], + metadata: () => {}, + ask: async () => {}, +} + +// ========================================================================== +// 1. ADVERSARIAL: sql-classify bypass attempts +// ========================================================================== + +describe("Adversarial: sql-classify bypass attempts", () => { + // --- SQL comment injection --- + + test("line comment after SELECT hides nothing", () => { + const r = classifyAndCheck("SELECT 1 -- DROP DATABASE prod") + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("block comment with DROP keyword is still just a SELECT", () => { + const r = classifyAndCheck("SELECT /* DROP DATABASE */ 1") + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("DROP DATABASE inside line comment is NOT blocked", () => { + // The comment makes it not a real statement + const r = classifyAndCheck("SELECT 1 -- DROP DATABASE production") + expect(r.blocked).toBe(false) + }) + + test("block comment surrounding entire DROP is not a statement", () => { + const r = classifyAndCheck("/* DROP DATABASE production */") + expect(r.blocked).toBe(false) + }) + + test("real DROP after comment-looking text IS blocked", () => { + const r = classifyAndCheck("SELECT 1;\nDROP DATABASE production") + expect(r.blocked).toBe(true) + }) + + // --- String literal injection --- + + test("DROP DATABASE inside string literal is not blocked", () => { + const r = classifyAndCheck("SELECT 'DROP DATABASE production' AS label") + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("TRUNCATE inside string literal is not blocked", () => { + const r = classifyAndCheck("INSERT INTO logs VALUES ('TRUNCATE TABLE users happened')") + expect(r.queryType).toBe("write") + expect(r.blocked).toBe(false) // string literal, not real TRUNCATE + }) + + test("DROP SCHEMA inside double-quoted identifier is not blocked", () => { + const r = classifyAndCheck('SELECT "DROP SCHEMA" FROM metadata') + expect(r.blocked).toBe(false) + }) + + // --- Unicode and encoding --- + + test("unicode identifiers classify correctly", () => { + const r = classifyAndCheck('SELECT * FROM "日本語テーブル"') + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("emoji in string literal doesn't break classifier", () => { + const r = classifyAndCheck("SELECT '🔥💀🎉' AS emoji") + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("mixed-case DROP DATABASE is still blocked", () => { + const r = classifyAndCheck("DrOp DaTaBaSe production") + expect(r.blocked).toBe(true) + }) + + test("mixed-case TRUNCATE TABLE is still blocked", () => { + const r = classifyAndCheck("TrUnCaTe TaBlE users") + expect(r.blocked).toBe(true) + }) + + // --- Whitespace and formatting tricks --- + + test("extra whitespace around DROP DATABASE still blocked", () => { + const r = classifyAndCheck(" DROP DATABASE prod ") + expect(r.blocked).toBe(true) + }) + + test("newlines before DROP DATABASE still blocked", () => { + const r = classifyAndCheck("\n\n\nDROP DATABASE prod") + expect(r.blocked).toBe(true) + }) + + test("tab characters in DROP SCHEMA still blocked", () => { + const r = classifyAndCheck("DROP\tSCHEMA\tpublic") + expect(r.blocked).toBe(true) + }) + + // --- Multi-statement bypass attempts --- + + test("benign SELECT before DROP DATABASE still blocked", () => { + const r = classifyAndCheck("SELECT 1; DROP DATABASE prod") + expect(r.blocked).toBe(true) + }) + + test("benign SELECT before TRUNCATE still blocked", () => { + const r = classifyAndCheck("SELECT 1; TRUNCATE TABLE users") + expect(r.blocked).toBe(true) + }) + + test("multiple DROPs all blocked", () => { + const r = classifyAndCheck("DROP DATABASE a; DROP SCHEMA b; TRUNCATE c") + expect(r.blocked).toBe(true) + }) + + test("write buried between reads is still write", () => { + const r = classifyAndCheck("SELECT 1; INSERT INTO t VALUES (1); SELECT 2") + expect(r.queryType).toBe("write") + }) + + // --- Edge case inputs --- + + test("null-like string doesn't crash", () => { + const r = classifyAndCheck("null") + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + }) + + test("undefined-like string doesn't crash", () => { + const r = classifyAndCheck("undefined") + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + }) + + test("single semicolon doesn't crash (may throw parse error)", () => { + // altimate-core may throw a parse error on bare semicolons — that's acceptable + try { + const r = classifyAndCheck(";") + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + } catch (e: any) { + expect(e.message || String(e)).toBeTruthy() // Error is clear, not a segfault + } + }) + + test("multiple semicolons don't crash (may throw parse error)", () => { + try { + const r = classifyAndCheck(";;;") + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + } catch (e: any) { + expect(e.message || String(e)).toBeTruthy() + } + }) + + test("only whitespace doesn't crash", () => { + const r = classifyAndCheck(" \n\t\n ") + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + }) + + test("only comments are not blocked", () => { + const r = classifyAndCheck("-- this is a comment\n/* block */") + expect(r.blocked).toBe(false) + }) + + test("extremely long SELECT (50KB) doesn't crash", () => { + const cols = Array.from({ length: 5000 }, (_, i) => `col_${i}`).join(", ") + const r = classifyAndCheck(`SELECT ${cols} FROM big_table`) + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + test("extremely long INSERT (50KB) is classified as write", () => { + const vals = Array.from({ length: 5000 }, (_, i) => `(${i}, 'val_${i}')`).join(", ") + const r = classifyAndCheck(`INSERT INTO big_table VALUES ${vals}`) + expect(r.queryType).toBe("write") + expect(r.blocked).toBe(false) + }) + + // --- Nested CTE bypass attempts --- + + test("CTE hiding a DROP in its body still blocked", () => { + const r = classifyAndCheck( + "WITH temp AS (SELECT 1) DROP DATABASE prod", + ) + expect(r.blocked).toBe(true) + }) + + test("deeply nested subquery is still read", () => { + const r = classifyAndCheck( + "SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT 1) a) b) c", + ) + expect(r.queryType).toBe("read") + expect(r.blocked).toBe(false) + }) + + // --- Database-specific syntax --- + + test("PostgreSQL COPY is classified as write", () => { + const r = classifyAndCheck("COPY users TO '/tmp/users.csv' CSV HEADER") + expect(r.queryType).toBe("write") + }) + + test("BEGIN TRANSACTION is classified as write", () => { + const r = classifyAndCheck("BEGIN TRANSACTION") + expect(r.queryType).toBe("write") + }) + + test("COMMIT is classified as write", () => { + const r = classifyAndCheck("COMMIT") + expect(r.queryType).toBe("write") + }) + + test("ROLLBACK is classified as write", () => { + const r = classifyAndCheck("ROLLBACK") + expect(r.queryType).toBe("write") + }) + + // --- EXPLAIN variants --- + + test("EXPLAIN SELECT is classified as write (ambiguous)", () => { + const r = classifyAndCheck("EXPLAIN SELECT * FROM users") + expect(r.queryType).toBe("write") + expect(r.blocked).toBe(false) + }) + + test("EXPLAIN ANALYZE SELECT is classified as write (ambiguous)", () => { + const r = classifyAndCheck("EXPLAIN ANALYZE SELECT * FROM users") + expect(r.queryType).toBe("write") + expect(r.blocked).toBe(false) + }) +}) + +// ========================================================================== +// 2. USER PERSPECTIVE: sql_execute tool with mocked permission flow +// ========================================================================== + +describe("User perspective: sql_execute permission flow", () => { + test("read query executes without asking permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await tool.execute({ query: "SELECT 1", limit: 10 }, testCtx) + expect(askRequests.length).toBe(0) // No permission asked + }, + }) + }) + + test("write query asks for sql_execute_write permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await tool.execute( + { query: "INSERT INTO users VALUES (1, 'test')", limit: 100 }, + testCtx, + ) + expect(askRequests.length).toBe(1) + expect(askRequests[0].permission).toBe("sql_execute_write") + expect(askRequests[0].metadata.queryType).toBe("write") + }, + }) + }) + + test("write query permission request includes truncated query pattern", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + const longQuery = "UPDATE users SET name = '" + "x".repeat(300) + "'" + await tool.execute({ query: longQuery, limit: 100 }, testCtx) + // Pattern should be truncated to 200 chars + expect(askRequests[0].patterns[0].length).toBeLessThanOrEqual(200) + }, + }) + }) + + test("DROP DATABASE throws immediately without asking permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await expect( + tool.execute({ query: "DROP DATABASE production", limit: 100 }, testCtx), + ).rejects.toThrow("blocked for safety") + // Permission was NOT asked — it threw before reaching ctx.ask + expect(askRequests.length).toBe(0) + }, + }) + }) + + test("TRUNCATE throws immediately without asking permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await expect( + tool.execute({ query: "TRUNCATE TABLE users", limit: 100 }, testCtx), + ).rejects.toThrow("blocked for safety") + expect(askRequests.length).toBe(0) + }, + }) + }) + + test("DROP SCHEMA throws with clear user-facing message", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + try { + await tool.execute( + { query: "DROP SCHEMA public CASCADE", limit: 100 }, + baseCtx, + ) + expect(true).toBe(false) // Should not reach here + } catch (e: any) { + expect(e.message).toContain("DROP DATABASE") + expect(e.message).toContain("DROP SCHEMA") + expect(e.message).toContain("TRUNCATE") + expect(e.message).toContain("cannot be overridden") + } + }, + }) + }) + + test("multi-statement with DROP hidden after SELECT still throws", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + await expect( + tool.execute( + { query: "SELECT 1; DROP DATABASE prod", limit: 100 }, + baseCtx, + ), + ).rejects.toThrow("blocked for safety") + }, + }) + }) + + test("DDL operations (CREATE TABLE) ask permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await tool.execute( + { query: "CREATE TABLE new_table (id INT, name TEXT)", limit: 100 }, + testCtx, + ) + expect(askRequests.length).toBe(1) + expect(askRequests[0].permission).toBe("sql_execute_write") + }, + }) + }) + + test("CTE with only SELECT does NOT ask permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await tool.execute( + { + query: "WITH cte AS (SELECT 1 AS id) SELECT * FROM cte", + limit: 100, + }, + testCtx, + ) + expect(askRequests.length).toBe(0) + }, + }) + }) + + test("CTE with INSERT DOES ask permission", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const askRequests: any[] = [] + const testCtx = { + ...baseCtx, + ask: async (req: any) => { askRequests.push(req) }, + } + await tool.execute( + { + query: "WITH cte AS (SELECT 1 AS id) INSERT INTO target SELECT * FROM cte", + limit: 100, + }, + testCtx, + ) + expect(askRequests.length).toBe(1) + expect(askRequests[0].permission).toBe("sql_execute_write") + }, + }) + }) + + test("successful query returns formatted output with row count", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const result = await tool.execute( + { query: "SELECT 1", limit: 100 }, + baseCtx, + ) + expect(result.title).toContain("SQL:") + expect(result.metadata).toHaveProperty("rowCount") + expect(result.metadata).toHaveProperty("truncated") + }, + }) + }) + + test("title truncates long queries to 60 chars", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + Registry.setConfigs({ test: { type: "duckdb", path: ":memory:" } }) + const tool = await SqlExecuteTool.init() + const longQuery = "SELECT " + Array(100).fill("col").join(", ") + " FROM table" + const result = await tool.execute( + { query: longQuery, limit: 100 }, + baseCtx, + ) + expect(result.title).toContain("...") + expect(result.title.length).toBeLessThanOrEqual(70) // "SQL: " + 60 + "..." + }, + }) + }) +}) + +// ========================================================================== +// 3. E2E: Full validation pipeline with realistic user scenarios +// ========================================================================== + +describe("E2E: realistic user scenarios through validation pipeline", () => { + test("data analyst runs a safe reporting query", async () => { + const sql = ` + SELECT + department, + COUNT(*) AS employee_count, + AVG(salary) AS avg_salary, + MAX(salary) AS max_salary + FROM employees + WHERE hire_date >= '2024-01-01' + GROUP BY department + ORDER BY avg_salary DESC + LIMIT 20 + ` + // Step 1: Analyze + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult.success).toBe(true) + expect(analyzeResult.issue_count).toBe(0) + + // Step 2: Validate + const validateResult = await Dispatcher.call("altimate_core.validate", { sql }) + expect(validateResult).toHaveProperty("success") + + // Step 3: Classify + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + }) + + test("data engineer writes a migration query — goes through permission", async () => { + const sql = ` + ALTER TABLE orders ADD COLUMN discount_pct DECIMAL(5,2) DEFAULT 0.00 + ` + // Analyze + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult).toHaveProperty("success") + + // Classify + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(false) // ALTER is not hard-blocked, just needs permission + }) + + test("user accidentally tries DROP DATABASE — hard blocked before any execution", async () => { + const sql = "DROP DATABASE analytics_prod" + + // Classify immediately catches it + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(true) + + // Validation still runs (doesn't crash) + const validateResult = await Dispatcher.call("altimate_core.validate", { sql }) + expect(validateResult).toHaveProperty("success") + }) + + test("complex CTE with window functions passes full pipeline", async () => { + const sql = ` + WITH ranked_sales AS ( + SELECT + product_id, + region, + SUM(amount) AS total_sales, + ROW_NUMBER() OVER (PARTITION BY region ORDER BY SUM(amount) DESC) AS rank + FROM sales + WHERE sale_date BETWEEN '2024-01-01' AND '2024-12-31' + GROUP BY product_id, region + ) + SELECT product_id, region, total_sales, rank + FROM ranked_sales + WHERE rank <= 5 + ORDER BY region, rank + ` + const [analyze, validate] = await Promise.all([ + Dispatcher.call("sql.analyze", { sql }), + Dispatcher.call("altimate_core.validate", { sql }), + ]) + expect(analyze).toHaveProperty("success") + expect(validate).toHaveProperty("success") + + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + }) + + test("SQL with anti-patterns is flagged by analyzer but still allowed to execute", async () => { + const sql = "SELECT * FROM users, orders" + + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult.issue_count).toBeGreaterThan(0) // Cartesian product / SELECT * + + // But classification still allows it (it's a read) + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + }) + + test("SQL injection attempt is caught by safety scan", async () => { + const sql = "SELECT * FROM users WHERE id = '1' OR '1'='1'" + + const checkResult = await Dispatcher.call("altimate_core.check", { sql }) + expect(checkResult).toHaveProperty("data") + const data = checkResult.data as Record + expect(data.safety).toBeDefined() + + // Also test is_safe directly + const safetyResult = await Dispatcher.call("altimate_core.is_safe", { sql }) + expect(safetyResult).toHaveProperty("success") + }) + + test("user query with schema context gets accurate validation", async () => { + const sql = "SELECT id, name, email FROM customers WHERE active = true LIMIT 100" + const schema_context = { + customers: { + id: "INTEGER", + name: "VARCHAR", + email: "VARCHAR", + active: "BOOLEAN", + created_at: "TIMESTAMP", + }, + } + + const [validate, check, analyze] = await Promise.all([ + Dispatcher.call("altimate_core.validate", { sql, schema_context }), + Dispatcher.call("altimate_core.check", { sql, schema_context }), + Dispatcher.call("sql.analyze", { sql, schema_context }), + ]) + + expect(validate.success).toBe(true) + expect(check.success).toBe(true) + expect(analyze.issue_count).toBe(0) + }) + + test("MERGE statement needs permission and passes through full pipeline", async () => { + const sql = ` + MERGE INTO target_table t + USING source_table s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET t.name = s.name, t.updated_at = CURRENT_TIMESTAMP + WHEN NOT MATCHED THEN INSERT (id, name, updated_at) VALUES (s.id, s.name, CURRENT_TIMESTAMP) + ` + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult).toHaveProperty("success") + + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(false) // MERGE is write but not hard-blocked + }) +}) + +// ========================================================================== +// 4. E2E: Dispatcher error recovery and resilience +// ========================================================================== + +describe("E2E: error recovery and resilience", () => { + test("malformed SQL doesn't crash any pipeline stage", async () => { + const malformedQueries = [ + "SELCT * FORM users", + "INSERT INTO", + "UPDATE SET WHERE", + ")))(((", + "SELECT 'unclosed string", + "SELECT * FROM `backtick`table`", + ] + + for (const sql of malformedQueries) { + // Dispatcher handlers catch errors and return result shapes + const validate = await Dispatcher.call("altimate_core.validate", { sql }) + expect(validate).toHaveProperty("success") + expect(validate).toHaveProperty("data") + + const check = await Dispatcher.call("altimate_core.check", { sql }) + expect(check).toHaveProperty("success") + expect(check).toHaveProperty("data") + + const analyze = await Dispatcher.call("sql.analyze", { sql }) + expect(analyze).toHaveProperty("success") + expect(analyze).toHaveProperty("issues") + + // Classify may throw on truly unparseable SQL — that's acceptable + try { + const classified = classifyAndCheck(sql) + expect(classified).toHaveProperty("queryType") + expect(classified).toHaveProperty("blocked") + } catch (e: any) { + // Parse error from altimate-core is acceptable, not a crash + expect(e.message || String(e)).toBeTruthy() + } + } + }) + + test("empty input handled gracefully across all pipeline stages", async () => { + const [validate, check, analyze] = await Promise.all([ + Dispatcher.call("altimate_core.validate", { sql: "" }), + Dispatcher.call("altimate_core.check", { sql: "" }), + Dispatcher.call("sql.analyze", { sql: "" }), + ]) + + expect(validate).toHaveProperty("success") + expect(check).toHaveProperty("success") + expect(analyze).toHaveProperty("success") + + const classified = classifyAndCheck("") + expect(classified.queryType).toBe("read") + expect(classified.blocked).toBe(false) + }) + + test("null/undefined schema_context handled gracefully", async () => { + const r1 = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT 1", + schema_context: null as any, + }) + expect(r1).toHaveProperty("success") + + const r2 = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT 1", + schema_context: undefined as any, + }) + expect(r2).toHaveProperty("success") + }) + + test("very large schema_context doesn't crash", async () => { + const schema: Record> = {} + for (let i = 0; i < 100; i++) { + const cols: Record = {} + for (let j = 0; j < 50; j++) { + cols[`col_${j}`] = "VARCHAR" + } + schema[`table_${i}`] = cols + } + + const r = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT col_0 FROM table_0", + schema_context: schema, + }) + expect(r).toHaveProperty("success") + }) +}) + +// ========================================================================== +// 5. Stress: concurrent and rapid-fire validation +// ========================================================================== + +describe("Stress: concurrent validation", () => { + test("20 concurrent validate calls all return results", async () => { + const queries = Array.from( + { length: 20 }, + (_, i) => `SELECT ${i} AS num, 'query_${i}' AS label`, + ) + const results = await Promise.all( + queries.map((sql) => + Dispatcher.call("altimate_core.validate", { sql }), + ), + ) + expect(results.length).toBe(20) + for (const r of results) { + expect(r).toHaveProperty("success") + expect(r).toHaveProperty("data") + } + }) + + test("20 concurrent classify calls all return results", () => { + const queries = Array.from({ length: 20 }, (_, i) => + i % 3 === 0 + ? `SELECT ${i}` + : i % 3 === 1 + ? `INSERT INTO t VALUES (${i})` + : `DROP DATABASE db_${i}`, + ) + const results = queries.map((q) => classifyAndCheck(q)) + expect(results.length).toBe(20) + + // Verify correct classification for each type + for (let i = 0; i < 20; i++) { + if (i % 3 === 0) { + expect(results[i].queryType).toBe("read") + expect(results[i].blocked).toBe(false) + } else if (i % 3 === 1) { + expect(results[i].queryType).toBe("write") + expect(results[i].blocked).toBe(false) + } else { + expect(results[i].blocked).toBe(true) + } + } + }) + + test("mixed concurrent pipeline calls (analyze + validate + check)", async () => { + const sql = "SELECT id, name FROM users WHERE id = 1" + + const promises = Array.from({ length: 5 }, () => + Promise.all([ + Dispatcher.call("sql.analyze", { sql }), + Dispatcher.call("altimate_core.validate", { sql }), + Dispatcher.call("altimate_core.check", { sql }), + ]), + ) + + const batches = await Promise.all(promises) + expect(batches.length).toBe(5) + + for (const [analyze, validate, check] of batches) { + expect(analyze).toHaveProperty("success") + expect(validate).toHaveProperty("success") + expect(check).toHaveProperty("success") + } + }) + + test("rapid sequential classify doesn't accumulate errors", () => { + for (let i = 0; i < 200; i++) { + const sql = + i % 4 === 0 + ? "SELECT 1" + : i % 4 === 1 + ? "INSERT INTO t VALUES (1)" + : i % 4 === 2 + ? "DROP DATABASE x" + : "" + const r = classifyAndCheck(sql) + expect(r).toHaveProperty("queryType") + expect(r).toHaveProperty("blocked") + } + }) +}) + +// ========================================================================== +// 6. E2E: Dialect-specific queries through the full pipeline +// ========================================================================== + +describe("E2E: dialect-specific SQL through pipeline", () => { + test("Snowflake QUALIFY clause", async () => { + const sql = ` + SELECT * + FROM orders + QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY order_date DESC) = 1 + ` + const r = await Dispatcher.call("altimate_core.validate", { sql }) + expect(r).toHaveProperty("success") + + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + }) + + test("BigQuery STRUCT and ARRAY syntax", async () => { + const sql = "SELECT STRUCT(1 AS id, 'test' AS name) AS s" + const r = await Dispatcher.call("altimate_core.validate", { sql }) + expect(r).toHaveProperty("success") + + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + }) + + test("PostgreSQL RETURNING clause", async () => { + const sql = "INSERT INTO users (name) VALUES ('test') RETURNING id, name" + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(false) + }) + + test("CREATE TABLE AS SELECT", async () => { + const sql = "CREATE TABLE summary AS SELECT region, SUM(amount) FROM sales GROUP BY region" + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(false) // DDL, not hard-blocked + }) + + test("INSERT ... ON CONFLICT (upsert)", async () => { + const sql = ` + INSERT INTO users (id, name) VALUES (1, 'test') + ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name + ` + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("write") + expect(blocked).toBe(false) + }) + + test("multiple dialects validate without crashing", async () => { + const dialectQueries = [ + "SELECT TOP 10 * FROM users", // SQL Server + "SELECT * FROM users LIMIT 10", // PostgreSQL/MySQL + "SELECT * FROM users FETCH FIRST 10 ROWS ONLY", // ANSI SQL + "SELECT * FROM users SAMPLE (10)", // Snowflake + ] + + for (const sql of dialectQueries) { + const r = await Dispatcher.call("altimate_core.validate", { sql }) + expect(r).toHaveProperty("success") + expect(r).toHaveProperty("data") + } + }) +}) + +// ========================================================================== +// 7. E2E: translate and optimize tools in the pipeline +// ========================================================================== + +describe("E2E: translate and optimize through pipeline", () => { + test("sql.translate is callable and returns result shape", async () => { + const r = await Dispatcher.call("sql.translate", { + sql: "SELECT IFNULL(name, 'unknown') FROM users", + from_dialect: "snowflake", + to_dialect: "postgres", + }) + expect(r).toHaveProperty("success") + }) + + test("sql.optimize is callable and returns suggestions", async () => { + const r = await Dispatcher.call("sql.optimize", { + sql: "SELECT * FROM users WHERE id IN (SELECT user_id FROM orders)", + }) + expect(r).toHaveProperty("success") + }) + + test("translated SQL maintains correct classification", async () => { + // A SELECT stays a SELECT after translation + const translated = await Dispatcher.call("sql.translate", { + sql: "SELECT NVL(name, 'unknown') FROM users LIMIT 10", + from_dialect: "snowflake", + to_dialect: "postgres", + }) + + if (translated.success && translated.data?.sql) { + const { queryType, blocked } = classifyAndCheck(translated.data.sql as string) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + } + }) +}) diff --git a/packages/opencode/test/altimate/sql-validation-e2e.test.ts b/packages/opencode/test/altimate/sql-validation-e2e.test.ts new file mode 100644 index 0000000000..4d8b06cf89 --- /dev/null +++ b/packages/opencode/test/altimate/sql-validation-e2e.test.ts @@ -0,0 +1,777 @@ +/** + * End-to-end tests for SQL validation tools. + * + * Verifies: + * 1. Tool names in prompts match actual registered tools + * 2. Agent permissions reference real tools (no phantom `sql_validate`) + * 3. altimate_core_validate works end-to-end via Dispatcher + * 4. altimate_core_check composite pipeline works end-to-end + * 5. sql.analyze composite pipeline works end-to-end + * 6. Pre-execution protocol tools are callable (sql_analyze → altimate_core_validate → sql_execute) + * 7. sql-classify correctly gates sql_execute + * 8. Analyst and builder agent permissions are consistent with their prompts + */ + +import { describe, expect, test, beforeAll, afterAll } from "bun:test" +import path from "path" +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import { registerAll } from "../../src/altimate/native/altimate-core" +import { registerAllSql } from "../../src/altimate/native/sql/register" +import { classifyAndCheck } from "../../src/altimate/tools/sql-classify" +import { Instance } from "../../src/project/instance" +import { Agent } from "../../src/agent/agent" +import { PermissionNext } from "../../src/permission/next" +import { tmpdir } from "../fixture/fixture" + +// Disable telemetry +beforeAll(() => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + registerAll() + registerAllSql() +}) +afterAll(() => { delete process.env.ALTIMATE_TELEMETRY_DISABLED }) + +// --------------------------------------------------------------------------- +// 1. Tool Name Consistency — prompts reference only real tools +// --------------------------------------------------------------------------- + +describe("Tool name consistency in prompts", () => { + test("builder prompt does NOT reference phantom sql_validate", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const builder = await Agent.get("builder") + expect(builder).toBeDefined() + expect(builder!.prompt).not.toContain("sql_validate") + }, + }) + }) + + test("analyst prompt does NOT reference phantom sql_validate", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + expect(analyst!.prompt).not.toContain("sql_validate") + }, + }) + }) + + test("builder prompt references altimate_core_validate (the real tool)", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const builder = await Agent.get("builder") + expect(builder).toBeDefined() + expect(builder!.prompt).toContain("altimate_core_validate") + }, + }) + }) + + test("analyst prompt references altimate_core_validate (the real tool)", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + expect(analyst!.prompt).toContain("altimate_core_validate") + }, + }) + }) + + test("builder prompt contains pre-execution protocol with correct tool names", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const builder = await Agent.get("builder") + expect(builder).toBeDefined() + // Pre-Execution Protocol references: + expect(builder!.prompt).toContain("sql_analyze") + expect(builder!.prompt).toContain("altimate_core_validate") + expect(builder!.prompt).toContain("sql_execute") + // The protocol section itself + expect(builder!.prompt).toContain("Pre-Execution Protocol") + }, + }) + }) +}) + +// --------------------------------------------------------------------------- +// 2. Agent Permissions — reference only real tools +// --------------------------------------------------------------------------- + +describe("Agent permissions reference real tools", () => { + function evalPerm(agent: Agent.Info | undefined, permission: string): PermissionNext.Action | undefined { + if (!agent) return undefined + return PermissionNext.evaluate(permission, "*", agent.permission).action + } + + test("analyst allows altimate_core_validate (not phantom sql_validate)", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + // The real tool must be allowed + expect(evalPerm(analyst, "altimate_core_validate")).toBe("allow") + }, + }) + }) + + test("analyst allows all documented SQL validation tools", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + + // SQL read tools from agent.ts + const expectedAllowed = [ + "sql_execute", + "altimate_core_validate", + "sql_analyze", + "sql_translate", + "sql_optimize", + "lineage_check", + "sql_explain", + "sql_format", + "sql_fix", + "sql_autocomplete", + "sql_diff", + // Core tools + "altimate_core_check", + "altimate_core_rewrite", + ] + + for (const tool of expectedAllowed) { + expect(evalPerm(analyst, tool)).toBe("allow") + } + }, + }) + }) + + test("analyst denies write operations", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + expect(evalPerm(analyst, "sql_execute_write")).toBe("deny") + expect(evalPerm(analyst, "edit")).toBe("deny") + expect(evalPerm(analyst, "write")).toBe("deny") + }, + }) + }) + + test("builder has sql_execute_write as ask (not allow or deny)", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const builder = await Agent.get("builder") + expect(builder).toBeDefined() + expect(evalPerm(builder, "sql_execute_write")).toBe("ask") + }, + }) + }) + + test("no agent permissions reference sql_validate (phantom tool)", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const agents = await Agent.list() + for (const agent of agents) { + // Serialize the permission ruleset and check for sql_validate + const serialized = JSON.stringify(agent.permission) + expect(serialized).not.toContain('"sql_validate"') + } + }, + }) + }) +}) + +// --------------------------------------------------------------------------- +// 3. altimate_core_validate — end-to-end via Dispatcher +// --------------------------------------------------------------------------- + +describe("altimate_core_validate e2e", () => { + test("SELECT 1 returns AltimateCoreResult shape", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT 1", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + expect(typeof result.success).toBe("boolean") + expect(typeof result.data).toBe("object") + }) + + test("valid query with schema context returns success: true", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT id, name FROM users", + schema_context: { + users: { id: "INTEGER", name: "VARCHAR", email: "VARCHAR" }, + }, + }) + expect(result.success).toBe(true) + expect(result.data.valid).not.toBe(false) + }) + + test("query referencing unknown table without schema still returns result", async () => { + // Without schema context, table references can't be verified — + // the handler returns valid=false which maps to success=false + const result = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT id, name FROM users WHERE id = 1", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + }) + + test("CTE query returns result shape", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "WITH cte AS (SELECT 1 AS id) SELECT * FROM cte", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + }) + + test("multi-statement SQL is accepted", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT 1; SELECT 2", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + }) + + test("heavily malformed SQL returns result (not a crash)", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "NOT SQL AT ALL ))) {{{{", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + }) + + test("empty SQL returns result (not a crash)", async () => { + const result = await Dispatcher.call("altimate_core.validate", { sql: "" }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + }) + + test("validates with SchemaDefinition format", async () => { + const result = await Dispatcher.call("altimate_core.validate", { + sql: "SELECT id FROM orders", + schema_context: { + version: "1", + dialect: "generic", + database: null, + schema_name: null, + tables: { + orders: { + columns: [ + { name: "id", type: "INT", nullable: false }, + { name: "amount", type: "DECIMAL", nullable: true }, + ], + }, + }, + }, + }) + expect(result).toHaveProperty("success") + }) +}) + +// --------------------------------------------------------------------------- +// 4. altimate_core_check — composite pipeline e2e +// --------------------------------------------------------------------------- + +describe("altimate_core_check e2e", () => { + test("clean query returns all-pass result", async () => { + const result = await Dispatcher.call("altimate_core.check", { + sql: "SELECT id FROM users WHERE id = 1", + }) + expect(result.success).toBe(true) + expect(result.data).toHaveProperty("validation") + expect(result.data).toHaveProperty("lint") + expect(result.data).toHaveProperty("safety") + }) + + test("result has expected structure", async () => { + const result = await Dispatcher.call("altimate_core.check", { + sql: "SELECT * FROM users", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("data") + const data = result.data as Record + + // validation section + expect(data.validation).toHaveProperty("valid") + + // lint section + expect(data.lint).toBeDefined() + + // safety section + expect(data.safety).toBeDefined() + }) + + test("SQL injection pattern is flagged by safety scan", async () => { + const result = await Dispatcher.call("altimate_core.check", { + sql: "SELECT * FROM users WHERE id = 1 OR 1=1", + }) + expect(result).toHaveProperty("data") + const data = result.data as Record + // The safety scan should detect the tautology + expect(data.safety).toBeDefined() + }) + + test("check with schema context works", async () => { + const result = await Dispatcher.call("altimate_core.check", { + sql: "SELECT id, name FROM customers", + schema_context: { + customers: { id: "INTEGER", name: "VARCHAR" }, + }, + }) + expect(result).toHaveProperty("success") + expect(result.data).toHaveProperty("validation") + }) +}) + +// --------------------------------------------------------------------------- +// 5. sql.analyze — composite lint + semantics + safety +// --------------------------------------------------------------------------- + +describe("sql.analyze e2e", () => { + test("clean query returns no issues", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECT id, name FROM users WHERE id = 1", + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("issues") + expect(result).toHaveProperty("issue_count") + expect(result).toHaveProperty("confidence") + expect(result.confidence_factors).toContain("lint") + expect(result.confidence_factors).toContain("semantics") + expect(result.confidence_factors).toContain("safety") + }) + + test("SELECT * triggers lint finding", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECT * FROM users", + }) + expect(result).toHaveProperty("issues") + // SELECT * should be caught by lint + const selectStarIssue = result.issues.find( + (i: any) => i.type === "lint" && (i.message?.includes("SELECT *") || i.message?.includes("select_star") || i.message?.toLowerCase?.().includes("star")), + ) + // At minimum, issues array should exist and analyzer should not crash + expect(Array.isArray(result.issues)).toBe(true) + }) + + test("cartesian product is detected", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECT * FROM users, orders", + }) + expect(result).toHaveProperty("issues") + expect(Array.isArray(result.issues)).toBe(true) + // Should detect cartesian product + expect(result.issue_count).toBeGreaterThan(0) + }) + + test("result structure matches SqlAnalyzeResult type", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECT 1", + }) + // SqlAnalyzeResult shape + expect(typeof result.success).toBe("boolean") + expect(Array.isArray(result.issues)).toBe(true) + expect(typeof result.issue_count).toBe("number") + expect(typeof result.confidence).toBe("string") + expect(Array.isArray(result.confidence_factors)).toBe(true) + }) + + test("analyze with schema context", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECT id, name FROM customers WHERE customer_id = 1", + schema_context: { + customers: { customer_id: "INTEGER", id: "INTEGER", name: "VARCHAR" }, + }, + }) + expect(result).toHaveProperty("success") + expect(result).toHaveProperty("issues") + }) + + test("malformed SQL doesn't crash analyzer", async () => { + const result = await Dispatcher.call("sql.analyze", { + sql: "SELECTT FORM", + }) + expect(result).toHaveProperty("success") + // Should handle gracefully — either error field or empty issues + expect(result).toHaveProperty("issues") + }) +}) + +// --------------------------------------------------------------------------- +// 6. Pre-Execution Protocol — tools called in sequence +// --------------------------------------------------------------------------- + +describe("Pre-execution protocol e2e", () => { + test("step 1: sql_analyze runs on the query", async () => { + const sql = "SELECT * FROM orders JOIN customers ON orders.customer_id = customers.id" + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult).toHaveProperty("success") + expect(analyzeResult).toHaveProperty("issues") + expect(analyzeResult).toHaveProperty("issue_count") + }) + + test("step 2: altimate_core_validate catches syntax errors", async () => { + const sql = "SELECT id, name FROM users WHERE id = 1" + const validateResult = await Dispatcher.call("altimate_core.validate", { sql }) + expect(validateResult).toHaveProperty("success") + expect(validateResult).toHaveProperty("data") + }) + + test("step 3: classify determines if permission check is needed", () => { + // Read queries skip permission + const readResult = classifyAndCheck("SELECT * FROM users") + expect(readResult.queryType).toBe("read") + expect(readResult.blocked).toBe(false) + + // Write queries need permission + const writeResult = classifyAndCheck("INSERT INTO users VALUES (1, 'test')") + expect(writeResult.queryType).toBe("write") + expect(writeResult.blocked).toBe(false) + + // Destructive queries are hard-blocked + const destructiveResult = classifyAndCheck("DROP DATABASE production") + expect(destructiveResult.blocked).toBe(true) + }) + + test("full protocol sequence: analyze → validate → classify", async () => { + const sql = "SELECT o.order_id, c.name FROM orders o JOIN customers c ON o.customer_id = c.id WHERE o.amount > 100" + + // Step 1: Analyze for anti-patterns + const analyzeResult = await Dispatcher.call("sql.analyze", { sql }) + expect(analyzeResult).toHaveProperty("success") + expect(analyzeResult).toHaveProperty("issue_count") + + // Step 2: Validate syntax + const validateResult = await Dispatcher.call("altimate_core.validate", { sql }) + expect(validateResult).toHaveProperty("success") + + // Step 3: Classify for permission gating + const { queryType, blocked } = classifyAndCheck(sql) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + + // All three steps complete without errors — query is safe to execute + }) + + test("protocol catches issues: analyze flags problems, validate catches syntax", async () => { + // Query with anti-patterns + const badSql = "SELECT * FROM users, orders" + const analyzeResult = await Dispatcher.call("sql.analyze", { sql: badSql }) + // Should detect issues (SELECT * and/or cartesian product) + expect(analyzeResult.issue_count).toBeGreaterThan(0) + + // Query with syntax issues + const validateResult = await Dispatcher.call("altimate_core.validate", { + sql: "SELECTT id FORM users", + }) + expect(validateResult).toHaveProperty("success") + expect(validateResult).toHaveProperty("data") + }) +}) + +// --------------------------------------------------------------------------- +// 7. sql-classify gates sql_execute correctly +// --------------------------------------------------------------------------- + +describe("sql-classify correctly gates execution", () => { + test("SELECT queries pass without permission check", () => { + const queries = [ + "SELECT 1", + "SELECT * FROM users", + "WITH cte AS (SELECT 1) SELECT * FROM cte", + "SELECT id, name FROM orders WHERE status = 'active'", + ] + for (const q of queries) { + const { queryType, blocked } = classifyAndCheck(q) + expect(queryType).toBe("read") + expect(blocked).toBe(false) + } + }) + + test("DML queries require permission", () => { + const queries = [ + "INSERT INTO users VALUES (1, 'test')", + "UPDATE users SET name = 'new'", + "DELETE FROM users WHERE id = 1", + "MERGE INTO target USING source ON target.id = source.id WHEN MATCHED THEN UPDATE SET target.name = source.name", + ] + for (const q of queries) { + const { queryType, blocked } = classifyAndCheck(q) + expect(queryType).toBe("write") + expect(blocked).toBe(false) + } + }) + + test("DDL queries require permission", () => { + const queries = [ + "CREATE TABLE new_table (id INT)", + "ALTER TABLE users ADD COLUMN email TEXT", + "DROP TABLE users", + ] + for (const q of queries) { + const { queryType } = classifyAndCheck(q) + expect(queryType).toBe("write") + } + }) + + test("destructive queries are hard-blocked", () => { + const queries = [ + "DROP DATABASE production", + "DROP SCHEMA public", + "TRUNCATE TABLE users", + "TRUNCATE users", + "drop database mydb", + "drop schema analytics", + ] + for (const q of queries) { + const { blocked } = classifyAndCheck(q) + expect(blocked).toBe(true) + } + }) + + test("multi-statement with destructive query is blocked", () => { + const { blocked } = classifyAndCheck("SELECT 1; DROP DATABASE prod") + expect(blocked).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// 8. Dispatcher registration — all SQL validation methods exist +// --------------------------------------------------------------------------- + +describe("SQL validation Dispatcher methods are registered", () => { + test("altimate_core.validate is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.validate")).toBe(true) + }) + + test("altimate_core.check is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.check")).toBe(true) + }) + + test("altimate_core.lint is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.lint")).toBe(true) + }) + + test("altimate_core.safety is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.safety")).toBe(true) + }) + + test("altimate_core.semantics is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.semantics")).toBe(true) + }) + + test("altimate_core.fix is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.fix")).toBe(true) + }) + + test("altimate_core.grade is registered", () => { + expect(Dispatcher.hasNativeHandler("altimate_core.grade")).toBe(true) + }) + + test("sql.analyze composite is registered", () => { + expect(Dispatcher.hasNativeHandler("sql.analyze")).toBe(true) + }) + + test("sql.translate composite is registered", () => { + expect(Dispatcher.hasNativeHandler("sql.translate")).toBe(true) + }) + + test("sql.optimize composite is registered", () => { + expect(Dispatcher.hasNativeHandler("sql.optimize")).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// 9. Skill files — no phantom tool references +// --------------------------------------------------------------------------- + +describe("Skill files reference real tools", () => { + // Skills live at the repo root, not packages/opencode + const repoRoot = path.resolve(process.cwd(), "../..") + + test("sql-translate skill references altimate_core_validate, not sql_validate", async () => { + const skillPath = path.join( + repoRoot, + ".opencode/skills/sql-translate/SKILL.md", + ) + const content = await Bun.file(skillPath).text() + expect(content).not.toContain("sql_validate") + expect(content).toContain("altimate_core_validate") + }) + + test("sql-review skill references real tool names", async () => { + const skillPath = path.join( + repoRoot, + ".opencode/skills/sql-review/SKILL.md", + ) + const content = await Bun.file(skillPath).text() + expect(content).not.toContain("sql_validate") + // Should reference the actual tools + expect(content).toContain("altimate_core_check") + expect(content).toContain("altimate_core_grade") + expect(content).toContain("sql_analyze") + }) +}) + +// --------------------------------------------------------------------------- +// 10. Prompt skill references match actual skill directories +// --------------------------------------------------------------------------- + +describe("Prompt skill references match actual skills", () => { + const repoRoot = path.resolve(process.cwd(), "../..") + const fs = require("fs") + + function getSkillDirs(): string[] { + const skillsDir = path.join(repoRoot, ".opencode/skills") + if (!fs.existsSync(skillsDir)) return [] + return fs + .readdirSync(skillsDir, { withFileTypes: true }) + .filter((d: any) => d.isDirectory()) + .map((d: any) => d.name) + } + + function extractSkillRefs(text: string): string[] { + // Match /skill-name patterns (e.g., /dbt-analyze, /sql-review) + const matches = text.match(/\/([a-z][a-z0-9-]+)/g) || [] + return [...new Set(matches.map((m) => m.slice(1)))] + } + + // Known non-skill slash references to exclude + const NON_SKILL_REFS = new Set([ + "tmp", "dev", "null", "etc", "bin", "usr", "home", "var", + "opencode", "sql", "dbt", "api", "v1", "v2", + ]) + + test("analyst 'Skills Available' section only lists skills that exist", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + + const skillDirs = getSkillDirs() + + // Extract only the "Skills Available" section (before the "Note:" line) + const prompt = analyst!.prompt + const skillsSectionMatch = prompt.match( + /## Skills Available[^\n]*\n([\s\S]*?)(?=\nNote:|## )/, + ) + if (!skillsSectionMatch) return // No skills section found — nothing to check + + const skillsSection = skillsSectionMatch[1] + const refs = extractSkillRefs(skillsSection) + .filter((r) => !NON_SKILL_REFS.has(r)) + + for (const ref of refs) { + expect(skillDirs).toContain(ref) + } + }, + }) + }) + + test("analyst prompt does NOT reference phantom /impact-analysis", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const analyst = await Agent.get("analyst") + expect(analyst).toBeDefined() + expect(analyst!.prompt).not.toContain("/impact-analysis") + // Should reference the real skill + expect(analyst!.prompt).toContain("/dbt-analyze") + }, + }) + }) + + test("builder prompt skill references match actual skills", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const builder = await Agent.get("builder") + expect(builder).toBeDefined() + expect(builder!.prompt).not.toContain("/impact-analysis") + expect(builder!.prompt).toContain("/dbt-analyze") + }, + }) + }) +}) + +// --------------------------------------------------------------------------- +// 11. Cross-cutting: validate + check + analyze all agree on same SQL +// --------------------------------------------------------------------------- + +describe("Validation tools agree on results", () => { + test("all tools accept valid SQL with schema context without errors", async () => { + // Use LIMIT to avoid MISSING_LIMIT lint finding + const sql = "SELECT id, name FROM users WHERE id = 1 LIMIT 10" + const schema_context = { + users: { id: "INTEGER", name: "VARCHAR" }, + } + + const [validate, check, analyze] = await Promise.all([ + Dispatcher.call("altimate_core.validate", { sql, schema_context }), + Dispatcher.call("altimate_core.check", { sql, schema_context }), + Dispatcher.call("sql.analyze", { sql, schema_context }), + ]) + + // validate: should be valid with schema + expect(validate.success).toBe(true) + + // check: validation section should pass + expect(check.success).toBe(true) + const checkData = check.data as Record + expect(checkData.validation?.valid).not.toBe(false) + + // analyze: should have 0 issues (clean query with LIMIT) + expect(analyze.issue_count).toBe(0) + }) + + test("all tools handle complex CTE query", async () => { + const sql = ` + WITH monthly_revenue AS ( + SELECT + DATE_TRUNC('month', order_date) AS month, + SUM(amount) AS revenue + FROM orders + GROUP BY 1 + ) + SELECT month, revenue + FROM monthly_revenue + ORDER BY month DESC + LIMIT 12 + ` + + const [validate, check, analyze] = await Promise.all([ + Dispatcher.call("altimate_core.validate", { sql }), + Dispatcher.call("altimate_core.check", { sql }), + Dispatcher.call("sql.analyze", { sql }), + ]) + + // All should succeed without crashing + expect(validate).toHaveProperty("success") + expect(check).toHaveProperty("success") + expect(analyze).toHaveProperty("success") + }) +})