diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..70966aa --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,82 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build and Test Commands + +```bash +# Build the project +cargo build + +# Run all tests +cargo test + +# Run a specific test module +cargo test --test mod basic # Basic end-to-end tests +cargo test --test mod comparison # Comparison operator tests +cargo test --test mod join # JOIN tests +cargo test --test mod aggregate # Aggregate function tests + +# Run a single test by name +cargo test test_name_here + +# Run with verbose output +cargo test -- --nocapture + +# Run the CLI directly +cargo run -- -s "SELECT * FROM data" data.csv + +# Run in interactive REPL mode +cargo run -- --interactive data.csv +``` + +## Architecture Overview + +Sqawk is an SQL-based CLI tool for processing delimiter-separated files (CSV, TSV, etc.). It loads files into in-memory tables, executes SQL queries, and optionally writes results back. + +### Core Components + +- **`main.rs`**: Entry point orchestrating the pipeline: CLI parsing → file loading → SQL execution → output → optional writeback +- **`database.rs`**: Central store for in-memory tables, manages table definitions and schemas +- **`table.rs`**: In-memory table representation with columns, rows, and projection capabilities +- **`sql_executor.rs`**: SQL parsing (via `sqlparser` crate) and execution against in-memory tables + +### SQL Execution (VM-based) + +SQL execution uses a bytecode VM inspired by SQLite's architecture: +1. SQL is parsed via the `sqlparser` crate +2. The AST is compiled to bytecode (`src/vm/compiler.rs`) +3. The VM engine executes the bytecode (`src/vm/engine.rs`) + +### VM Module (`src/vm/`) + +- `bytecode.rs`: Defines bytecode instruction set +- `compiler.rs`: Compiles SQL AST to bytecode +- `engine.rs`: Executes bytecode instructions + +### File Handling + +- **`file_handler.rs`**: Manages loading files into database tables +- **`csv_handler.rs`**: Standard CSV file parsing (comma-separated) +- **`delim_handler.rs`**: Custom delimiter support via `-F` flag (TSV, colon-separated, etc.) + +### SQL Features + +- `aggregate.rs`: COUNT, SUM, AVG, MIN, MAX functions +- `join.rs`: Cross join and INNER JOIN implementations +- `string_functions.rs`: UPPER, LOWER, TRIM, SUBSTR, REPLACE + +### Safe Writeback Model + +By default, all modifications remain in memory only. The `--write` flag must be explicitly provided to save changes back to source files. Only tables modified by INSERT/UPDATE/DELETE are written. + +## Test Organization + +Tests are in `tests/` organized by functionality: +- `basic/`, `comparison/`, `join/`, `join_on/`, `order_by/`, `update/` +- `aggregate/`, `alias/`, `distinct/`, `group_by/`, `limit_offset/` +- `delimiter/`, `csv_handler/`, `string_functions/`, `repl/` +- `helpers/`: Test utilities +- `common.rs`: Shared test helpers including REPL script runner + +Tests use `assert_cmd` for CLI testing and `tempfile` for temporary test data. diff --git a/Cargo.lock b/Cargo.lock index f894954..4e5e4b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -28,54 +28,53 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", - "once_cell", - "windows-sys", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "assert_cmd" -version = "2.0.17" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" +checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" dependencies = [ "anstyle", "bstr", - "doc-comment", "libc", "predicates", "predicates-core", @@ -85,21 +84,21 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "regex-automata", @@ -108,9 +107,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -120,9 +119,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "clap" -version = "4.5.37" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", "clap_derive", @@ -130,9 +129,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstream", "anstyle", @@ -142,9 +141,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -154,42 +153,42 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "clipboard-win" -version = "5.4.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" dependencies = [ "error-code", ] [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "csv" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ "csv-core", "itoa", "ryu", - "serde", + "serde_core", ] [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] @@ -200,12 +199,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "endian-type" version = "0.1.2" @@ -214,12 +207,12 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -242,7 +235,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -256,14 +249,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", "r-efi", - "wasi", + "wasip2", ] [[package]] @@ -274,48 +267,57 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "log" -version = "0.4.27" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memmap2" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] [[package]] name = "nibble_vec" @@ -359,6 +361,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "predicates" version = "3.1.3" @@ -391,27 +399,27 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radix_trie" @@ -425,9 +433,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -437,9 +445,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -448,21 +456,21 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -484,29 +492,38 @@ dependencies = [ "unicode-segmentation", "unicode-width", "utf8parse", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -515,24 +532,25 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "sqawk" -version = "0.1.1" +version = "0.8.0" dependencies = [ "anyhow", "assert_cmd", "clap", "csv", + "libc", + "memmap2", "predicates", "regex", "rustyline", "sqlparser", "tempfile", - "thiserror", ] [[package]] @@ -552,9 +570,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.101" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -563,15 +581,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.1" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -580,31 +598,11 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" @@ -614,9 +612,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "utf8parse" @@ -634,14 +632,20 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.59.0" @@ -651,6 +655,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -716,10 +729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" diff --git a/Cargo.toml b/Cargo.toml index 74e40a9..d6c0cc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,34 +1,46 @@ [package] name = "sqawk" -version = "0.1.1" +version = "0.8.0" edition = "2021" authors = ["Jeff Garzik "] description = "An SQL-based command-line tool for processing delimiter-separated files (CSV, TSV, etc.), inspired by awk" license = "MIT" repository = "https://github.com/jgarzik/sqawk" homepage = "https://github.com/jgarzik/sqawk" -documentation = "https://github.com/jgarzik/sqawk" readme = "README.md" keywords = ["csv", "tsv", "sql", "delimited", "awk"] -categories = ["command-line-utilities", "database", "text-processing"] +categories = ["command-line-utilities", "text-processing", "parser-implementations"] # Minimum version of Rust required -rust-version = "1.65.0" +rust-version = "1.70.0" +exclude = ["tests/", "doc/", ".github/", "CLAUDE.md"] [dependencies] -clap = { version = "4.4", features = ["derive"] } +clap = { version = "4", features = ["derive"] } sqlparser = "0.36" -csv = "1.2" -anyhow = "1.0" -thiserror = "1.0" -regex = "1.9" +csv = "1" +anyhow = "1" +regex = "1" +memmap2 = "0.9" +libc = "0.2" +tempfile = "3" -rustyline = "15.0.0" +rustyline = "15" [dev-dependencies] -assert_cmd = "2.0" -predicates = "3.0" -tempfile = "3.8" +assert_cmd = "2" +predicates = "3" + +[profile.release] +lto = true + +[profile.profiling] +inherits = "release" +debug = true [[bin]] name = "sqawk" path = "src/main.rs" + +[[bin]] +name = "tsq" +path = "src/tsq.rs" diff --git a/README.md b/README.md index b159131..9ecb126 100644 --- a/README.md +++ b/README.md @@ -4,42 +4,17 @@ [![Docs.rs](https://docs.rs/sqawk/badge.svg)](https://docs.rs/sqawk) [![MIT licensed](https://img.shields.io/crates/l/sqawk.svg)](./LICENSE) -Sqawk is an SQL-based command-line tool for processing delimiter-separated files (CSV, TSV, etc.), inspired by the classic `awk` command. It loads data into in-memory tables, executes SQL queries against these tables, and writes the results back to the console or files. +Sqawk is an SQL-based command-line tool for processing delimiter-separated files (CSV, TSV, etc.), inspired by the classic `awk` command. It loads data into in-memory tables, executes SQL queries, and optionally writes results back to files. ## Features -- **Powerful SQL Query Engine** - - Support for SELECT, INSERT, UPDATE, and DELETE operations - - WHERE clause filtering with comparison operators - - DISTINCT keyword for removing duplicate rows - - ORDER BY for sorting results (ASC/DESC) - - Column aliases with the AS keyword - - Aggregate functions (COUNT, SUM, AVG, MIN, MAX) - - GROUP BY for data aggregation - -- **Multi-Table Operations** - - Cross joins between tables - - INNER JOIN with ON conditions for precise join criteria - - Support for joining multiple tables - - Table-qualified column names - -- **Smart Data Handling** - - Automatic type inference (Integer, Float, Boolean, String) - - Type coercion for comparisons - - Null value support - -- **File Format Support** - - Process CSV, TSV, and custom-delimited files - - Custom field separator support with -F option (like awk) - - Fast in-memory execution - - Process multiple files in a single command - - Table name customization - - Chain multiple SQL statements - -- **Safe Operation** - - Doesn't modify files without explicit request (--write flag) - - Only writes back tables that were modified - - Verbose mode for operation transparency +- **SQL Query Engine** - SELECT, INSERT, UPDATE, DELETE with WHERE, ORDER BY, GROUP BY, HAVING, LIMIT/OFFSET +- **Joins** - INNER, LEFT, RIGHT, FULL OUTER, and CROSS joins with ON conditions +- **Aggregates** - COUNT, SUM, AVG, MIN, MAX with GROUP BY support +- **Functions** - String (UPPER, LOWER, SUBSTR, REPLACE, etc.), math (ABS, ROUND, etc.), date/time +- **File Formats** - CSV, TSV, and custom delimiters; headerless files via `--tabledef` +- **Safe by Default** - Files unchanged unless `--write` flag is specified +- **Interactive REPL** - Explore data interactively with `-i` flag ## Installation @@ -47,116 +22,32 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files cargo install sqawk ``` -## Usage - -### Basic SELECT query +## Quick Examples ```sh -sqawk -s "SELECT * FROM data" data.csv -``` +# Query a CSV file +sqawk -s "SELECT name, salary FROM employees WHERE department = 'Engineering'" employees.csv -This loads `data.csv` into an in-memory table called "data" and performs a SELECT query. +# Join two files +sqawk -s "SELECT u.name, o.total FROM users u JOIN orders o ON u.id = o.user_id" users.csv orders.csv -### Filtering data with WHERE clause +# Aggregate data +sqawk -s "SELECT department, AVG(salary) FROM employees GROUP BY department" employees.csv -```sh -sqawk -s "SELECT * FROM employees WHERE salary > 50000" employees.csv -``` - -### Updating rows - -```sh -sqawk -s "UPDATE data SET status = 'active' WHERE id = 5" data.csv --write +# Modify and save +sqawk -s "UPDATE data SET status = 'archived' WHERE year < 2020" data.csv --write ``` -This updates the status field to 'active' for rows with id = 5 and saves the changes back to data.csv. - -### Deleting rows - -```sh -sqawk -s "DELETE FROM data WHERE id = 5" data.csv --write -``` - -This removes rows with id = 5 and saves the changes back to data.csv. - -### Multiple operations - -```sh -sqawk -s "UPDATE data SET status = 'inactive' WHERE last_login < '2023-01-01'" -s "DELETE FROM data WHERE status = 'inactive' AND last_login < '2022-01-01'" -s "SELECT * FROM data" data.csv --write -``` - -This executes multiple SQL statements in sequence: first marking recent inactive accounts, then removing very old inactive accounts, and finally showing the results. - -### Multiple files - -```sh -sqawk -s "SELECT * FROM users" -s "SELECT * FROM orders" users.csv orders.csv -``` - -### Finding unique values with DISTINCT - -```sh -# Get unique values from a single column -sqawk -s "SELECT DISTINCT category FROM products" products.csv - -# Get unique combinations of columns -sqawk -s "SELECT DISTINCT department, role FROM employees" employees.csv - -# Use DISTINCT with ORDER BY for sorted unique values -sqawk -s "SELECT DISTINCT region FROM customers ORDER BY region" customers.csv -``` - -### Join tables with INNER JOIN - -```sh -# Join users and orders using INNER JOIN with ON condition -sqawk -s "SELECT users.name, orders.product_id, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id" users.csv orders.csv - -# Join with additional WHERE filtering -sqawk -s "SELECT users.name, orders.product_id, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id WHERE orders.product_id > 100" users.csv orders.csv - -# Using DISTINCT with JOINs to find unique customer-product pairs -sqawk -s "SELECT DISTINCT users.name, products.name FROM users INNER JOIN orders ON users.id = orders.user_id INNER JOIN products ON orders.product_id = products.product_id" users.csv orders.csv products.csv -``` - -### Custom field separators - -```sh -# Process a tab-delimited file (TSV) -sqawk -F '\t' -s "SELECT * FROM employees WHERE salary > 70000" employees.tsv - -# Process a colon-delimited file -sqawk -F ':' -s "SELECT id, name, email FROM contacts" contacts.txt -``` - -### Verbose mode - -```sh -sqawk -s "SELECT * FROM data WHERE value > 100" data.csv -v -``` - -### Write mode - -```sh -sqawk -s "DELETE FROM data WHERE status = 'expired'" data.csv --write -``` - -By default, sqawk doesn't modify input files. Use the `--write` flag to save changes back to the original files. - ## Documentation -For more detailed information, see: - -- [User Guide](doc/user_guide.md) - Complete guide to installing and using Sqawk -- [SQL Language Reference](doc/sql_reference.md) - Comprehensive guide to Sqawk's SQL dialect -- [In-Memory Database Architecture](doc/database.md) - Technical details about the database implementation (for developers) +- [User Guide](doc/user_guide.md) - Installation, CLI options, and examples +- [SQL Reference](doc/sql_reference.md) - Complete SQL syntax and functions +- [Database Architecture](doc/database.md) - Technical internals (for contributors) ## License -Licensed under the MIT License ([LICENSE](LICENSE) or http://opensource.org/licenses/MIT). +MIT License - see [LICENSE](LICENSE) -## Contribution +## Contributing -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in the work by you shall be licensed as MIT, without any additional -terms or conditions. \ No newline at end of file +Contributions welcome. Any contribution submitted for inclusion shall be licensed as MIT. diff --git a/doc/database.md b/doc/database.md index 7ef3cc9..e8c9ebd 100644 --- a/doc/database.md +++ b/doc/database.md @@ -8,7 +8,9 @@ Sqawk implements a lightweight in-memory database engine that supports SQL opera 1. [Data Model](#data-model) 2. [Architecture](#architecture) -3. [Performance Considerations](#performance-considerations) +3. [File I/O Patterns](#file-io-patterns) +4. [Storage Backend Architecture](#storage-backend-architecture) +5. [Performance Considerations](#performance-considerations) ## Data Model @@ -86,45 +88,124 @@ The file handling system consists of multiple components: - Supports tab, colon, pipe, and other custom separators - Preserves the original delimiter and format during writeback +## File I/O Patterns + +### Atomic File Writes + +When saving modified tables back to disk (via the `--write` flag), Sqawk uses an atomic write pattern to ensure data integrity: + +1. **Write to Temporary File**: Data is first written to a temporary file in the same directory as the target file +2. **Sync to Disk**: The temporary file is explicitly synced (`fsync`) to ensure data is durably stored on the physical device +3. **Atomic Rename**: The temporary file is renamed to the target filename using an atomic rename operation + +This pattern provides several safety guarantees: + +- **Crash Safety**: If the process crashes during writing, the original file remains intact +- **Disk Full Protection**: If the disk runs out of space, the original file is preserved +- **All-or-Nothing Semantics**: Either the complete new file exists, or the original file is unchanged + +The temporary file is created in the same directory as the target file to ensure the rename operation is atomic (same filesystem requirement on POSIX systems). + +### Delimiter Consistency + +Sqawk maintains delimiter consistency between input and output: + +- When a file is loaded with a specific delimiter (via `-F` flag or file extension detection), that delimiter is preserved in the table metadata +- Query results are output using the same delimiter as the source table +- When files are written back, the original delimiter format is preserved + +This ensures that TSV files remain TSV, colon-separated files remain colon-separated, and so on. The principle of least surprise: output format matches input format unless explicitly overridden. + +### Output Modes + +Sqawk supports two primary output modes: + +1. **Stdout Output**: Query results (SELECT) are written to standard output using the source table's delimiter +2. **File Writeback**: Modified tables (via INSERT, UPDATE, DELETE) are written back to their source files when `--write` is specified + +For stdout output, the data is streamed directly without buffering concerns. For file writeback, the atomic write pattern described above is used. + +## Storage Backend Architecture + +Sqawk supports multiple storage backends for table row data, enabling efficient handling of different data sources: + +### Memory Storage (Default for stdin) + +The `MemoryStorage` backend stores all row data in heap-allocated memory: +- Used when reading from stdin or pipes +- All string values are owned (heap-allocated) +- Supports all operations including INSERT, UPDATE, DELETE +- Data is mutable throughout the session + +### Mmap Storage (Default for on-disk files) + +The `MmapStorage` backend uses memory-mapped files for zero-copy access: +- Automatically used when loading files from disk +- String values borrow directly from the mmap'd region (zero allocation) +- Read-only by default to maintain zero-copy efficiency +- Converts to MemoryStorage on first write operation + +### Automatic Backend Selection + +The storage backend is selected automatically based on the data source: +- **On-disk files**: Uses mmap for zero-copy loading +- **Stdin/pipes**: Uses traditional in-memory storage +- **Write operations**: Mmap tables convert to memory on first modification + +### Write Operation Behavior + +When a table with mmap storage is modified (INSERT, UPDATE, DELETE): +1. All rows are copied to heap memory (converting Cow::Borrowed to Cow::Owned) +2. The storage backend switches from Mmap to Memory +3. Subsequent operations use the in-memory copy +4. The original file remains unchanged until `--write` is specified + +This design provides: +- Zero-allocation loading for read-only queries +- Transparent handling of write operations +- Memory efficiency for large read-only datasets + ## Performance Considerations Sqawk's in-memory database is optimized for: - **Fast Loading**: Delimiter-separated files are parsed directly into memory +- **Zero-Copy Access**: Memory-mapped files enable direct access without allocation overhead - **Format Flexibility**: Support for CSV, TSV, and custom-delimited files - **Efficient Filtering**: WHERE clauses are applied in a single pass - **Low Memory Overhead**: Simple data structures minimize memory usage - **Zero Configuration**: No setup required, works directly with files in various formats For larger datasets, consider: -- The entire dataset must fit in memory +- The entire dataset must fit in memory (or be addressable via mmap) - Complex queries may require multiple passes over the data -- Write operations create new copies of the data in memory +- Write operations on mmap tables trigger a copy to memory - Custom delimiters may have slightly different performance characteristics than standard CSV +- Read-only operations on large files benefit most from mmap storage ## Join Implementation -The database engine includes support for combining data from multiple tables through joins: - -### Join Engine Design +The database engine supports combining data from multiple tables through SQL-standard join operations. -- **Cross Join Implementation**: Creates a Cartesian product of all rows -- **Filter-Based Joins**: Uses WHERE conditions for relationship-based filtering -- **Multi-table Support**: Handles joining multiple tables in sequence +### Supported Join Types -### Column Naming Strategy +- **INNER JOIN**: Returns rows when there is a match in both tables +- **LEFT OUTER JOIN**: Returns all rows from the left table, with matched rows from the right (or NULLs) +- **RIGHT OUTER JOIN**: Returns all rows from the right table, with matched rows from the left (or NULLs) +- **FULL OUTER JOIN**: Returns all rows from both tables, with NULLs where no match exists +- **CROSS JOIN**: Returns the Cartesian product of both tables -To maintain clarity when working with multiple tables: +### Join Syntax -- **Qualified Naming**: Columns are prefixed with their table names -- **Consistent Referencing**: A consistent naming convention is applied in multi-table operations -- **Qualified References**: Column references include table qualifiers in conditions +- **ON constraints**: Supported for specifying join conditions (`JOIN t2 ON t1.id = t2.id`) +- **USING constraints**: Not supported +- **Multi-table joins**: Supports chaining 3+ tables in a single query ### Technical Implementation -- The join operation first creates a cross product, then applies filters -- Tables are processed in the order specified -- The column naming system ensures disambiguation in result sets +- Joins are compiled to VM bytecode for efficient execution +- Nested loop algorithm with appropriate NULL handling for outer joins +- Column references use table qualifiers for disambiguation in result sets - Type coercion rules are applied consistently in join conditions ## Current System Limitations @@ -132,7 +213,6 @@ To maintain clarity when working with multiple tables: The database engine has several architectural limitations: - **No Index Structure**: All operations perform full table scans -- **Limited Join Capabilities**: Advanced join syntax not implemented - **No Transaction Support**: Operations are applied immediately with no rollback capability - **Schema Flexibility**: Types are inferred rather than enforced - **No Constraints System**: Referential integrity not enforced diff --git a/doc/sql_reference.md b/doc/sql_reference.md index 71e7231..47ef66f 100644 --- a/doc/sql_reference.md +++ b/doc/sql_reference.md @@ -1,809 +1,196 @@ -# Sqawk SQL Language Reference - -## Introduction - -Sqawk provides a powerful SQL-like query language for processing delimiter-separated files. While CSV (comma-separated values) is the default format, Sqawk also supports other delimiter-separated formats like TSV (tab-separated values) and custom delimiters. This document serves as a reference for the SQL dialect supported by Sqawk and explains its syntax, features, and limitations. - -## Table of Contents - -1. [Introduction](#introduction) -2. [Table Names and File Specification](#table-names-and-file-specification) -3. [File Formats and Field Separators](#file-formats-and-field-separators) -4. [Chaining SQL Statements](#chaining-sql-statements) -5. [Data Types](#data-types) -6. [SQL Statement Types](#sql-statement-types) -7. [CREATE TABLE Statement](#create-table-statement) - - [Basic Syntax](#create-table-basic-syntax) - - [Data Types](#create-table-data-types) - - [Table Location and Format](#table-location-and-format) - - [Custom Delimiters](#custom-delimiters) -8. [SELECT Statement](#select-statement) - - [Basic Syntax](#basic-syntax) - - [Column Selection](#column-selection) - - [Column Aliases](#column-aliases) - - [WHERE Clause](#where-clause) - - [String Functions](#string-functions) - - [ORDER BY Clause](#order-by-clause) - - [LIMIT and OFFSET Clauses](#limit-and-offset-clauses) - - [Aggregate Functions](#aggregate-functions) - - [GROUP BY Clause](#group-by-clause) - - [HAVING Clause](#having-clause) -9. [Multi-Table Operations (Joins)](#multi-table-operations-joins) -10. [INSERT Statement](#insert-statement) -11. [UPDATE Statement](#update-statement) -12. [DELETE Statement](#delete-statement) -13. [Limitations](#limitations) -14. [Writeback Behavior](#writeback-behavior) - -## Table Names and File Specification - -When using Sqawk, delimiter-separated files are loaded as in-memory tables. By default, the table name is derived from the filename (without the extension). You can also explicitly specify a table name: - -```bash -# Default table name: "sample" -sqawk -s "SELECT * FROM sample" sample.csv - -# Explicitly named table: "users" -sqawk -s "SELECT * FROM users" users=sample.csv -``` - -This naming flexibility allows you to: -- Use meaningful table names that differ from filenames -- Work with multiple files that would otherwise have the same derived table name -- Create more readable SQL queries with domain-specific table names - -## File Formats and Field Separators - -Sqawk can work with various delimiter-separated file formats, not just standard CSV files. The default behavior is to treat files as comma-separated values (CSV), but you can specify a custom field separator using the `-F` option: - -```bash -# Process a tab-delimited file (TSV) -sqawk -F '\t' -s "SELECT * FROM employees WHERE salary > 70000" employees.tsv - -# Process a colon-delimited file -sqawk -F ':' -s "SELECT id, name, email FROM contacts" contacts.txt - -# Process a pipe-delimited file -sqawk -F '|' -s "SELECT * FROM data" data.txt -``` - -The `-F` option is similar to awk's field separator option, allowing Sqawk to handle a wide variety of text file formats. This capability is particularly useful when working with: - -- Tab-delimited files (TSV) -- Exports from various systems that use custom delimiters -- Fixed-width files converted to a delimiter format -- Log files with specific field separators - -When using the `-F` option, Sqawk will: -1. Parse the file using the specified delimiter instead of commas -2. Automatically detect and preserve the header row for column names -3. Perform the same type inference and SQL operations as with CSV files -4. Write back to the original format when using the `--write` flag - -### File Format Detection - -Sqawk uses the following logic to determine which file format handler to use: -- If the `-F` option is specified, the file is treated as a custom delimiter-separated file -- Files with a `.csv` extension are treated as standard CSV files -- Other file extensions without a specified delimiter are treated as tab-delimited by default - -### Comment Support in CSV Files - -Sqawk supports comment lines in CSV files. Lines that begin with a comment character (typically '#') are ignored during processing: - -```csv -# This is a comment and will be ignored -id,name,age -1,Alice,32 -# Another comment -2,Bob,25 -``` - -Comment support is particularly useful for: -- Adding metadata or documentation within the file -- Temporarily excluding rows from processing -- Adding version information or data provenance details - -### Error Recovery Options - -When processing CSV or other delimiter-separated files, Sqawk can handle malformed rows in several ways: - -- **Strict Mode**: By default, malformed rows (those with too few or too many fields) cause an error -- **Recovery Mode**: With appropriate options, Sqawk can: - - Skip malformed rows entirely - - Pad malformed rows with NULL values if they have too few fields - - Truncate malformed rows if they have too many fields - -This error recovery capability is especially useful when working with imperfect data sources where strict conformance to the expected format isn't guaranteed. - -## Chaining SQL Statements - -Sqawk allows you to execute multiple SQL statements in sequence, with each statement operating on the result of the previous ones: - -```bash -# Execute two SQL statements in sequence -sqawk -s "DELETE FROM users WHERE inactive = true" -s "SELECT * FROM users" users.csv -``` - -This allows for complex operations such as: -- Modifying data and then viewing the results -- Performing multi-step data transformations -- Running sequential operations like cleanup and then analysis - -Each statement executes against the in-memory state after the previous statement's execution. +# Sqawk SQL Reference ## Data Types -Sqawk supports the following data types: - -| Type | Description | Example | Storage | -|------|-------------|---------|---------| -| `Null` | Missing or null value | NULL | Special variant | -| `Integer` | 64-bit signed integer | 42 | i64 | -| `Float` | 64-bit floating point | 3.14 | f64 | -| `String` | UTF-8 text | "hello" | String | -| `Boolean` | True/false value | true | bool | - -### Type Inference - -When loading data from delimiter-separated files, Sqawk automatically infers the most appropriate type for each value: - -1. First tries to parse as an `Integer` -2. If that fails, tries to parse as a `Float` -3. If that fails, tries to parse as a `Boolean` (values like true/false, yes/no, 1/0) -4. If all else fails, stores the value as a `String` -5. Empty values are stored as `Null` - -This dynamic type inference provides flexibility when working with delimiter-separated data, which typically doesn't include explicit type information. The same type inference logic applies to all file formats, whether they are CSV files, TSV files, or files with custom delimiters. - -### Type Coercion in Comparisons - -Sqawk implements SQL-like type coercion rules when comparing values: - -- **NULL Values**: - - NULL equals NULL - - NULL is less than any other value - - Comparisons between NULL and non-NULL values generally evaluate to false - -- **Numeric Comparisons**: - - `Integer` and `Float` values can be compared directly - - When comparing different numeric types, integers are converted to floats - -- **Same-Type Comparisons**: - - Strings are compared lexicographically (dictionary order) - - Boolean values follow false < true - -- **Different-Type Comparisons**: - - Types follow a strict precedence order: NULL < Boolean < Number < String - - This means: - - Boolean values are less than any numeric or string value - - Numbers (both Integer and Float) are less than any String value - - Strings are greater than all other types - -This type precedence system is particularly important for operations like `MIN()` and `MAX()` and when sorting values with `ORDER BY`. - -## SQL Statement Types - -Sqawk currently supports the following SQL statement types: - -| Statement | Description | Example | -|-----------|-------------|---------| -| `SELECT` | Query data from tables | `SELECT * FROM users WHERE age > 30` | -| `CREATE TABLE` | Create a new table with a defined schema | `CREATE TABLE users (id INT, name TEXT, age INT)` | -| `INSERT` | Add new rows to tables | `INSERT INTO users VALUES (4, 'Dave', 28)` | -| `UPDATE` | Modify existing rows in tables | `UPDATE users SET age = 29 WHERE name = 'Dave'` | -| `DELETE` | Remove rows from tables | `DELETE FROM users WHERE age < 18` | - -## CREATE TABLE Statement - -### Create Table Basic Syntax - -```sql -CREATE TABLE table_name ( - column1 data_type, - column2 data_type, - ... -) [LOCATION 'file_path'] - [STORED AS file_format] - [WITH (option_name='option_value', ...)] -``` - -The CREATE TABLE statement allows you to define a new table with a specified schema. The statement requires: +| Type | Aliases | +|------|---------| +| `INTEGER` | `INT` | +| `REAL` | `FLOAT`, `DOUBLE` | +| `TEXT` | `STRING` | +| `BOOLEAN` | `BOOL` | +| `NULL` | | -- A table name -- One or more column definitions with their data types -- Optional LOCATION clause to specify where the table should be stored -- Optional STORED AS clause to specify the file format -- Optional WITH clause to specify additional table properties +Type inference on load: Integer → Float → Boolean → String. Empty values become NULL. -Example of a basic CREATE TABLE statement: +## SELECT ```sql -CREATE TABLE users ( - id INT, - name TEXT, - email TEXT, - age INT, - salary FLOAT -) -``` - -### Create Table Data Types - -Sqawk supports the following data types in CREATE TABLE statements: - -| Data Type | Description | Example | -|-----------|-------------|---------| -| `INT` or `INTEGER` | 64-bit signed integer | `id INT` | -| `FLOAT` or `REAL` | 64-bit floating point | `salary FLOAT` | -| `TEXT` or `STRING` | UTF-8 text | `name TEXT` | -| `BOOLEAN` | True/false value | `active BOOLEAN` | - -When defining columns, you must specify a data type for each column. This type information is used when inserting data into the table and for data validation. - -```sql -CREATE TABLE products ( - product_id INT, - name TEXT, - description TEXT, - price FLOAT, - in_stock BOOLEAN -) -``` - -### Table Location and Format - -You can specify a location for the table's data file using the LOCATION keyword, followed by a path string: - -```sql -CREATE TABLE sales ( - id INT, - date TEXT, - amount FLOAT -) LOCATION './data/sales.csv' -``` - -Currently, Sqawk only supports the TEXTFILE format via the STORED AS clause: - -```sql -CREATE TABLE events ( - event_id INT, - timestamp TEXT, - type TEXT -) LOCATION './data/events.csv' STORED AS TEXTFILE -``` - -### Custom Delimiters - -You can specify a custom delimiter for the table using the WITH clause: - -```sql -CREATE TABLE logs ( - log_id INT, - timestamp TEXT, - level TEXT, - message TEXT -) LOCATION './data/logs.tsv' - STORED AS TEXTFILE - WITH (DELIMITER='\t') -``` - -This is particularly useful when working with tab-delimited files, semicolon-delimited files, or other custom formats. - -Complete example with all options: - -```sql -CREATE TABLE financial_data ( - account_id INT, - transaction_date TEXT, - amount FLOAT, - category TEXT -) LOCATION './data/financial.csv' - STORED AS TEXTFILE - WITH (DELIMITER=',') -``` - -The CREATE TABLE statement only defines the table's structure - it doesn't load or modify any data. After creating a table, you can insert data into it using the INSERT statement. - -## SELECT Statement - -### Basic Syntax - -```sql -SELECT [DISTINCT] [column_list | *] -FROM table_name [, table_name2, ...] +SELECT [DISTINCT] column_list | * +FROM table [alias] [, table2 ...] +[JOIN ...] [WHERE condition] -[GROUP BY column_list] +[GROUP BY columns] [HAVING condition] [ORDER BY column [ASC|DESC], ...] -[LIMIT count [OFFSET skip_count]] +[LIMIT n [OFFSET m]] ``` -### DISTINCT Keyword - -The `DISTINCT` keyword eliminates duplicate rows from the result set: - -```sql --- Return unique combinations of name and age -SELECT DISTINCT name, age FROM users - --- Return unique department values -SELECT DISTINCT department FROM employees - --- Can be used with aggregate functions -SELECT COUNT(DISTINCT department) AS unique_departments FROM employees -``` - -When using DISTINCT: -- Rows are considered identical only if all selected column values match exactly -- The comparison uses the same type system as the rest of SQL operations -- DISTINCT is applied after WHERE filtering but before ORDER BY -- DISTINCT can be used with JOINs to find unique combinations across tables -- DISTINCT is particularly useful for finding unique values or removing redundant results - -DISTINCT has two different applications in SQL: - -1. **Query-level DISTINCT** - Applied to the entire result set: - ```sql - SELECT DISTINCT department, location FROM employees - ``` - -2. **Aggregate function DISTINCT** - Applied to the values within an aggregate function: - ```sql - SELECT COUNT(DISTINCT department) FROM employees - ``` - -In the second case, the COUNT operation is performed only on unique department values, rather than counting duplicates. - ### Column Selection -You can select specific columns or use a wildcard: - ```sql --- Select specific columns -SELECT name, age FROM users - --- Select all columns -SELECT * FROM users - --- Select with column qualification (table names) -SELECT users.name, orders.date FROM users, orders +SELECT * -- all columns +SELECT col1, col2 -- specific columns +SELECT table.col -- qualified +SELECT col AS alias -- aliased +SELECT col alias -- alias without AS ``` -### Column Aliases +### WHERE Operators -You can provide alternative names for columns using the `AS` keyword: +| Operator | Example | +|----------|---------| +| `=`, `!=`, `<>` | `col = 5` | +| `<`, `>`, `<=`, `>=` | `col > 10` | +| `AND`, `OR`, `NOT` | `a > 1 AND b < 5` | +| `IS NULL`, `IS NOT NULL` | `col IS NULL` | +| `LIKE`, `ILIKE` | `name LIKE 'A%'` | +| `BETWEEN` | `col BETWEEN 1 AND 10` | +| `IN` | `col IN (1, 2, 3)` | +| `IN (SELECT ...)` | `id IN (SELECT id FROM t)` | -```sql --- Rename columns in the result -SELECT name AS employee_name, age AS employee_age FROM employees - --- Alias can be used without the AS keyword -SELECT name employee_name, age employee_age FROM employees - --- Aggregate functions can also have aliases -SELECT COUNT(*) AS total_count, AVG(salary) AS average_salary FROM employees -``` - -Column aliases are particularly useful when: -- Making result column names more descriptive -- Renaming complex expressions -- Disambiguating columns with the same name from different tables -- Using in combination with ORDER BY to sort by aliased columns - -Aliases defined in the SELECT clause can be referenced in the ORDER BY clause: - -```sql --- Sort by the aliased column 'years' -SELECT name, age AS years FROM employees ORDER BY years DESC -``` - -### WHERE Clause - -The `WHERE` clause filters rows based on conditions: - -```sql --- Equality -SELECT * FROM users WHERE name = 'Alice' - --- Inequality -SELECT * FROM users WHERE age != 30 - --- Greater than -SELECT * FROM users WHERE age > 25 - --- Less than -SELECT * FROM users WHERE age < 40 - --- Greater than or equal to -SELECT * FROM users WHERE age >= 18 - --- Less than or equal to -SELECT * FROM users WHERE age <= 65 -``` - -### String Functions - -Sqawk supports the following string functions for manipulating and comparing text data in WHERE clauses: - -| Function | Description | Example | -|----------|-------------|---------| -| `UPPER(str)` | Converts a string to uppercase | `SELECT * FROM users WHERE UPPER(name) = 'ALICE'` | -| `LOWER(str)` | Converts a string to lowercase | `SELECT * FROM users WHERE LOWER(email) = 'alice@example.com'` | -| `TRIM(str)` | Removes leading and trailing whitespace | `SELECT * FROM users WHERE TRIM(username) = 'alice'` | -| `SUBSTR(str, start[, length])` | Extracts a substring | `SELECT * FROM users WHERE SUBSTR(email, 1, 5) = 'alice'` | -| `REPLACE(str, find, replace)` | Replaces all occurrences of a substring | `SELECT * FROM users WHERE REPLACE(email, '@example.com', '') = 'alice'` | - -These string functions can be used in WHERE clauses to filter rows based on string manipulations: - -```sql --- Case-insensitive equality using UPPER or LOWER -SELECT * FROM users WHERE UPPER(name) = 'ALICE' - --- Working with substrings -SELECT * FROM emails WHERE SUBSTR(email, -4) = '.com' - --- Find users with trimmed whitespace -SELECT * FROM users WHERE TRIM(username) = 'alice' - --- Replace parts of strings for comparison -SELECT * FROM contacts WHERE REPLACE(phone, '-', '') = '1234567890' - --- Combining string functions -SELECT * FROM users WHERE UPPER(SUBSTR(name, 1, 1)) = 'A' -``` - -String functions can be nested and combined for more complex string operations. They are particularly useful for: - -- Case-insensitive searching -- Pattern matching when working with text data -- Data cleaning and normalization -- Extracting portions of strings for comparison - -> **Note:** Currently, string functions are only supported in WHERE clauses and cannot be used directly in the SELECT clause for projection. This limitation is documented in the [Limitations](#limitations) section. - -### ORDER BY Clause - -The `ORDER BY` clause sorts results by one or more columns: +### CASE Expression ```sql --- Ascending order (default) -SELECT * FROM users ORDER BY age - --- Descending order -SELECT * FROM users ORDER BY age DESC - --- Multiple columns with different directions -SELECT * FROM users ORDER BY age ASC, name DESC - --- Order by aliased columns -SELECT name AS employee_name, age AS years FROM users ORDER BY years DESC +CASE WHEN cond THEN result [WHEN ...] [ELSE default] END +CASE expr WHEN val THEN result [WHEN ...] [ELSE default] END ``` -### LIMIT and OFFSET Clauses - -The `LIMIT` and `OFFSET` clauses control the number of rows returned by a query: - -```sql --- Return only the first 10 rows -SELECT * FROM users LIMIT 10 - --- Skip the first 5 rows and return the next 10 -SELECT * FROM users LIMIT 10 OFFSET 5 - --- Combine with ORDER BY for pagination -SELECT * FROM users ORDER BY age DESC LIMIT 10 OFFSET 20 -``` - -LIMIT and OFFSET are particularly useful for: -- Pagination of large result sets -- Retrieving "top N" results when combined with ORDER BY -- Sampling data from large tables -- Creating efficient user interfaces that load data incrementally - -Important characteristics: -- LIMIT accepts a positive integer specifying the maximum number of rows to return -- OFFSET (optional) specifies the number of rows to skip before starting to return rows -- Both clauses are applied after all other query operations (WHERE, GROUP BY, ORDER BY, etc.) -- If OFFSET is greater than or equal to the number of rows after filtering, an empty result set is returned -- LIMIT with a value of 0 will return an empty result set - ### Aggregate Functions -Sqawk supports the following aggregate functions: +| Function | Description | +|----------|-------------| +| `COUNT(*)`, `COUNT(col)`, `COUNT(DISTINCT col)` | Row/value count | +| `SUM(col)` | Sum of values | +| `AVG(col)` | Average | +| `MIN(col)` | Minimum | +| `MAX(col)` | Maximum | -| Function | Description | Example | -|----------|-------------|---------| -| `COUNT(*)` | Count rows | `SELECT COUNT(*) FROM users` | -| `SUM(column)` | Sum values in column | `SELECT SUM(salary) FROM employees` | -| `AVG(column)` | Average of values in column | `SELECT AVG(age) FROM users` | -| `MIN(column)` | Minimum value in column | `SELECT MIN(salary) FROM employees` | -| `MAX(column)` | Maximum value in column | `SELECT MAX(age) FROM users` | - -Aggregate functions can be used with column aliases: - -```sql -SELECT COUNT(*) AS count, SUM(salary) AS total_salary, AVG(age) AS avg_age FROM employees -``` +### String Functions -Aggregate functions can also be used with `WHERE` clauses to filter input rows: +| Function | Description | +|----------|-------------| +| `UPPER(s)` | Uppercase | +| `LOWER(s)` | Lowercase | +| `TRIM(s)` | Remove leading/trailing whitespace | +| `SUBSTR(s, start [, len])` | Substring (1-indexed) | +| `SUBSTRING(s FROM start [FOR len])` | Substring (alternate syntax) | +| `REPLACE(s, from, to)` | Replace occurrences | +| `CONCAT(s1, s2, ...)` | Concatenate strings | +| `LENGTH(s)` | String length | +| `LEFT(s, n)` | First n characters | +| `RIGHT(s, n)` | Last n characters | +| `LPAD(s, len, pad)` | Left-pad to length | +| `RPAD(s, len, pad)` | Right-pad to length | -```sql -SELECT COUNT(*) AS count, AVG(salary) AS avg_salary FROM employees WHERE department = 'Engineering' -``` +### Math Functions -### GROUP BY Clause +| Function | Description | +|----------|-------------| +| `ABS(n)` | Absolute value | +| `ROUND(n)` | Round to nearest integer | +| `CEIL(n)`, `CEILING(n)` | Round up | +| `FLOOR(n)` | Round down | -The `GROUP BY` clause allows you to group rows that have the same values in specified columns and apply aggregate functions to each group: +### Arithmetic -```sql -SELECT column1, column2, aggregate_function(column3) -FROM table_name -GROUP BY column1, column2 -``` +`+`, `-`, `*`, `/`, `%` (modulo) -Examples of using GROUP BY: +### Date/Time Functions -```sql --- Group by a single column -SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary -FROM employees -GROUP BY department - --- Group by multiple columns -SELECT department, location, COUNT(*) AS employee_count -FROM employees -GROUP BY department, location - --- Group by with ORDER BY -SELECT department, COUNT(*) AS employee_count, SUM(salary) AS total_salary -FROM employees -GROUP BY department -ORDER BY total_salary DESC -``` +| Function | Description | +|----------|-------------| +| `NOW()`, `CURRENT_TIMESTAMP` | Current timestamp | +| `CURRENT_DATE` | Current date | +| `CURRENT_TIME` | Current time | +| `DATE(expr)` | Extract date | +| `TIME(expr)` | Extract time | -GROUP BY can be used with all aggregate functions (COUNT, SUM, AVG, MIN, MAX) and can be combined with column aliases: +## JOIN ```sql -SELECT department, - COUNT(*) AS count, - SUM(salary) AS total_salary, - AVG(salary) AS avg_salary, - MIN(salary) AS min_salary, - MAX(salary) AS max_salary -FROM employees -GROUP BY department -``` - -Rules and behavior: -- All columns in the SELECT clause that are not in aggregate functions must be included in the GROUP BY clause -- Column aliases defined in the SELECT clause cannot be used in the GROUP BY clause (but they can be used in ORDER BY) -- GROUP BY columns are always included in the result set -- NULL values in GROUP BY columns are treated as a single group +-- Cross join (cartesian product) +SELECT * FROM t1, t2 -### HAVING Clause +-- Inner join +SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id -The `HAVING` clause filters grouped results based on conditions, similar to how the WHERE clause filters individual rows: +-- Outer joins +SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id +SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id +SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id -```sql -SELECT column1, column2, aggregate_function(column3) -FROM table_name -GROUP BY column1, column2 -HAVING condition +-- Multiple joins +SELECT * FROM t1 + JOIN t2 ON t1.id = t2.t1_id + JOIN t3 ON t2.id = t3.t2_id ``` -The HAVING clause is applied after groups are formed and aggregate functions are calculated, whereas the WHERE clause is applied before grouping. - -Examples of using HAVING: +## Set Operations ```sql --- Filter groups based on aggregate results -SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary -FROM employees -GROUP BY department -HAVING COUNT(*) > 5 - --- Filter groups using multiple conditions -SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary -FROM employees -GROUP BY department -HAVING COUNT(*) > 5 AND AVG(salary) > 60000 - --- Combine WHERE and HAVING clauses -SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary -FROM employees -WHERE location = 'New York' -GROUP BY department -HAVING AVG(salary) > 70000 +SELECT ... UNION SELECT ... -- combined, deduplicated +SELECT ... UNION ALL SELECT ... -- combined, with duplicates +SELECT ... INTERSECT SELECT ... -- rows in both +SELECT ... EXCEPT SELECT ... -- rows in first but not second ``` -Key characteristics of the HAVING clause: -- Applied after GROUP BY and aggregate calculations -- Can reference aggregate functions -- Can contain arithmetic operations (e.g., `HAVING AVG(salary) * 1.1 > 75000`) -- Can be combined with other clauses like ORDER BY and LIMIT -- HAVING without GROUP BY treats the entire table as a single group - -## Multi-Table Operations (Joins) - -### Cross Joins - -A cross join creates a Cartesian product of all rows from both tables: - -```sql -SELECT * FROM users, orders -``` - -This returns all possible combinations of rows from the `users` and `orders` tables. - -### Inner Joins - -Sqawk supports two inner join syntax styles: - -#### 1. Using WHERE Conditions: +## INSERT ```sql -SELECT * FROM users, orders WHERE users.id = orders.user_id +INSERT INTO table VALUES (v1, v2, ...) +INSERT INTO table (col1, col2) VALUES (v1, v2) +INSERT INTO table SELECT ... FROM other_table ``` -This returns only the rows where a user ID in the `users` table matches a user_id in the `orders` table. - -#### 2. Using INNER JOIN ... ON Syntax: +## UPDATE ```sql -SELECT * FROM users INNER JOIN orders ON users.id = orders.user_id -``` - -The explicit JOIN...ON syntax provides a clearer structure for complex joins and better distinguishes the join criteria from other filtering conditions. - -### Combining Joins with WHERE Clauses - -When using explicit JOIN...ON syntax, you can still add WHERE clauses to filter the joined results: - -```sql -SELECT users.name, orders.product_id -FROM users INNER JOIN orders ON users.id = orders.user_id -WHERE orders.product_id > 100 -``` - -This returns only rows that satisfy both the join condition AND the additional WHERE filter. - -### Multi-Table Joins - -You can join multiple tables using either syntax: - -```sql --- Using traditional WHERE joins -SELECT users.name, products.name, orders.date -FROM users, orders, products -WHERE users.id = orders.user_id AND products.product_id = orders.product_id - --- Or using explicit JOIN...ON syntax -SELECT users.name, products.name, orders.date -FROM users -INNER JOIN orders ON users.id = orders.user_id -INNER JOIN products ON products.product_id = orders.product_id +UPDATE table SET col1 = val1 [, col2 = val2, ...] +[WHERE condition] ``` -### Column Naming in Joins - -In join results, columns are qualified with their table names to avoid ambiguity: +## DELETE ```sql -SELECT users.name AS user_name, orders.date AS order_date -FROM users INNER JOIN orders ON users.id = orders.user_id +DELETE FROM table [WHERE condition] ``` -## INSERT Statement - -### Basic Syntax +## CREATE TABLE ```sql -INSERT INTO table_name VALUES (value1, value2, ...) +CREATE TABLE name ( + col1 TYPE, + col2 TYPE, + ... +) +[LOCATION 'path'] +[STORED AS TEXTFILE] +[WITH (DELIMITER='...')] ``` -The `INSERT` statement adds a new row to the table: - ```sql -INSERT INTO users VALUES (5, 'Eve', 42) +CREATE TABLE name AS SELECT ... FROM ... ``` -All columns must be provided in the correct order. - -## UPDATE Statement - -### Basic Syntax +## DROP TABLE ```sql -UPDATE table_name SET column1 = value1 [, column2 = value2, ...] [WHERE condition] +DROP TABLE name +DROP TABLE IF EXISTS name ``` -The `UPDATE` statement modifies existing rows: +## ALTER TABLE ```sql --- Update specific rows -UPDATE users SET age = 21 WHERE name = 'Alice' - --- Update all rows -UPDATE users SET active = true +ALTER TABLE name ADD COLUMN col_name TYPE ``` -## DELETE Statement - -### Basic Syntax +## TRUNCATE ```sql -DELETE FROM table_name [WHERE condition] +TRUNCATE TABLE name ``` -The `DELETE` statement removes rows from a table: - -```sql --- Delete specific rows -DELETE FROM users WHERE age < 18 - --- Delete all rows -DELETE FROM users -``` +## Writeback -## Limitations - -Current limitations of Sqawk's SQL implementation: - -- **Table Operations**: - - No indices are used; all operations scan the entire table - - No schema enforcement or constraints - - No views - -- **Join Operations**: - - INNER JOIN with ON conditions is supported - - No outer joins (LEFT, RIGHT, or FULL OUTER) are supported - -- **Query Features**: - - WHERE clauses support a variety of expressions including comparisons, logical operators, and string functions - - String functions are only supported in WHERE clauses, not in SELECT clauses for projection - -- **Error Handling**: - - Errors are reported with detailed messages and context - - CSV parsing errors include line numbers to help locate issues - - No subqueries - - No window functions - - No common table expressions (CTEs) - -- **Data Manipulation**: - - No transactions (BEGIN, COMMIT, ROLLBACK) - - INSERT must provide values for all columns - - No column specifications in INSERT (must follow table column order) - -**Supported Features**: -- Column aliases (AS keyword) -- DISTINCT keyword for removing duplicate rows -- ORDER BY with ascending/descending sorting -- LIMIT and OFFSET for pagination and result set control -- Aggregate functions (COUNT, SUM, AVG, MIN, MAX) -- GROUP BY clause for data aggregation -- HAVING clause for filtering grouped results -- Arithmetic operations in expressions (addition, subtraction, multiplication, division) -- Multi-column sorting -- Table-qualified column names -- Cross joins and inner joins through both WHERE conditions and INNER JOIN...ON syntax -- Support for custom field separators with -F option -- Compatible with CSV, TSV, and custom-delimited files - -## Writeback Behavior - -Modified tables are only written back to their source files if: -- The `--write` (or `-w`) flag is explicitly provided -- The table was actually modified by an SQL operation (INSERT, UPDATE, DELETE) - -When writing data back: -- The original file format (CSV, TSV, or custom delimiter) is preserved -- Header rows are maintained -- Column order is preserved -- Data types are formatted appropriately based on the original values - -Without the `--write` flag, source files remain untouched regardless of operations performed. This allows for exploratory data analysis without the risk of modifying source files. - ---- - -*This document describes the SQL dialect supported by Sqawk as of the current version. Future versions may add additional capabilities.* \ No newline at end of file +Modifications (INSERT, UPDATE, DELETE) remain in-memory unless `--write` flag is specified. diff --git a/doc/user_guide.md b/doc/user_guide.md index 41a9ed0..2cd0f94 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -16,6 +16,7 @@ - [Interactive Mode (-i)](#interactive-mode--i) - [Write Flag (--write)](#write-flag---write) - [Field Separator Option (-F)](#field-separator-option--f) + - [Table Definition Option (--tabledef)](#table-definition-option---tabledef) - [Verbose Mode (-v)](#verbose-mode--v) - [Help (--help)](#help---help) 5. [Working with Files](#working-with-files) @@ -24,12 +25,16 @@ - [Handling Multiple Files](#handling-multiple-files) - [File Writeback Behavior](#file-writeback-behavior) - [Defining Schemas with CREATE TABLE](#defining-schemas-with-create-table) -6. [Common Usage Patterns](#common-usage-patterns) - - [Data Exploration](#data-exploration) - - [Data Cleanup](#data-cleanup) - - [Data Transformation](#data-transformation) - - [Joining Data from Multiple Files](#joining-data-from-multiple-files) - - [Generating Reports](#generating-reports) +6. [Examples](#examples) + - [Basic Queries](#basic-queries) + - [Aggregation and Grouping](#aggregation-and-grouping) + - [Distinct Values](#distinct-values) + - [Joins](#joins) + - [Data Modification](#data-modification) + - [String Functions](#string-functions) + - [System Files](#system-files---tabledef) + - [Output Redirection](#output-redirection) + - [Interactive REPL Session](#interactive-repl-session) 7. [Working with Large Files](#working-with-large-files) 8. [Troubleshooting](#troubleshooting) 9. [Appendices](#appendices) @@ -260,6 +265,34 @@ Notes on field separators: - Common separators include tab (`\t`), comma (`,`), colon (`:`), and pipe (`|`) - The specified separator is also used when writing back to files +### Table Definition Option (--tabledef) + +The `--tabledef` option allows you to define column names for files that don't have header rows, such as system files like `/etc/passwd`: + +```sh +# Process /etc/passwd with meaningful column names +sqawk -F: --tabledef=passwd:username,password,uid,gid,gecos,home,shell \ + -s "SELECT username, home FROM passwd WHERE uid >= 1000" \ + passwd=/etc/passwd +``` + +Format: `--tabledef=table_name:col1,col2,col3,...` + +This is useful for: +- System files like `/etc/passwd`, `/etc/group`, `/etc/hosts` +- Log files with fixed column formats +- Any file without a header row + +Multiple table definitions can be provided: + +```sh +sqawk -F: \ + --tabledef=passwd:username,password,uid,gid,gecos,home,shell \ + --tabledef=group:groupname,password,gid,members \ + -s "SELECT username, groupname FROM passwd, group WHERE passwd.gid = group.gid" \ + passwd=/etc/passwd group=/etc/group +``` + ### Verbose Mode (-v) The verbose mode provides additional information about the operations being performed: @@ -484,168 +517,148 @@ Example of safe write behavior: sqawk -s "UPDATE data SET category = lookup.category FROM lookup WHERE data.code = lookup.code" -s "SELECT * FROM data" data.csv lookup.csv --write ``` -## Common Usage Patterns - -### Data Exploration +## Examples -Quickly analyze and explore data files: +### Basic Queries ```sh -# Count the number of records +# Count records sqawk -s "SELECT COUNT(*) FROM data" data.csv -# Get basic statistics -sqawk -s "SELECT MIN(value) AS min, MAX(value) AS max, AVG(value) AS avg FROM data" data.csv +# Filter rows +sqawk -s "SELECT * FROM data WHERE status = 'active'" data.csv + +# Sort results +sqawk -s "SELECT * FROM data ORDER BY date DESC" data.csv -# See distribution by category -sqawk -s "SELECT category, COUNT(*) FROM data GROUP BY category ORDER BY COUNT(*) DESC" data.csv +# Limit output +sqawk -s "SELECT * FROM data LIMIT 10" data.csv +``` + +### Aggregation and Grouping + +```sh +# Basic statistics +sqawk -s "SELECT MIN(value), MAX(value), AVG(value) FROM data" data.csv + +# Group by with count +sqawk -s "SELECT category, COUNT(*) FROM data GROUP BY category" data.csv + +# Multiple aggregates +sqawk -s "SELECT region, COUNT(*) AS orders, SUM(amount) AS total + FROM orders GROUP BY region ORDER BY total DESC" orders.csv +``` -# Find unique values in a column -sqawk -s "SELECT DISTINCT category FROM data ORDER BY category" data.csv +### Distinct Values + +```sh +# Unique values in a column +sqawk -s "SELECT DISTINCT category FROM data" data.csv # Count unique values -sqawk -s "SELECT COUNT(DISTINCT category) AS unique_categories FROM data" data.csv +sqawk -s "SELECT COUNT(DISTINCT category) FROM data" data.csv -# Find unique combinations of columns +# Unique combinations sqawk -s "SELECT DISTINCT department, role FROM employees" employees.csv ``` -**Using the Interactive REPL for Data Exploration:** - -The interactive REPL mode is especially powerful for iterative data exploration: +### Joins ```sh -# Launch the REPL with your data files -sqawk -i sales.csv customers.csv products.csv +# Inner join +sqawk -s "SELECT u.name, o.date FROM users u + INNER JOIN orders o ON u.id = o.user_id" users.csv orders.csv + +# Left join (include all users, even without orders) +sqawk -s "SELECT u.name, o.date FROM users u + LEFT JOIN orders o ON u.id = o.user_id" users.csv orders.csv + +# Three-table join +sqawk -s "SELECT u.name, p.name AS product, o.date + FROM users u + INNER JOIN orders o ON u.id = o.user_id + INNER JOIN products p ON o.product_id = p.id" \ + users.csv orders.csv products.csv ``` -Once in the REPL, you can rapidly explore your data: +### Data Modification -``` -# Check available tables -sqawk> .tables -customers products sales +```sh +# Update values +sqawk -s "UPDATE data SET status = 'archived' WHERE date < '2023-01-01'" data.csv --write -# Examine table structure -sqawk> .schema sales -CREATE TABLE sales ( - id INTEGER, - customer_id INTEGER, - product_id INTEGER, - date TEXT, - quantity INTEGER, - amount FLOAT -); +# Delete rows +sqawk -s "DELETE FROM data WHERE status = 'expired'" data.csv --write -# Start with a simple exploration -sqawk> SELECT COUNT(*) FROM sales; -count -1250 +# Insert new rows +sqawk -s "INSERT INTO data VALUES (100, 'New Item', 'active')" data.csv --write +``` -# Drill down into specific segments -sqawk> SELECT date, SUM(amount) FROM sales GROUP BY date ORDER BY date DESC LIMIT 5; -date,sum -2023-12-15,12580.75 -2023-12-14,9845.50 -2023-12-13,11267.25 -2023-12-12,8976.00 -2023-12-11,10432.50 - -# Join tables to get a richer view -sqawk> SELECT c.name, COUNT(*) AS order_count, SUM(s.amount) AS total_spent - FROM customers c - JOIN sales s ON c.id = s.customer_id - GROUP BY c.name - ORDER BY total_spent DESC - LIMIT 3; -name,order_count,total_spent -Enterprise Corp,42,58750.25 -Acme Inc,38,45620.75 -Global Services,35,42180.50 -``` - -The REPL enables a more natural workflow for data analysis, allowing you to: -- Build queries incrementally -- See immediate results -- Refine and adjust as you go -- Explore relationships between tables -- Maintain context across multiple queries - -### Data Cleanup - -Clean and transform data files: +### String Functions ```sh -# Remove duplicate records using Sqawk's DISTINCT keyword -sqawk -s "SELECT DISTINCT * FROM data" data.csv > deduped_data.csv - -# Extract only unique combinations of name and email -sqawk -s "SELECT DISTINCT name, email FROM contacts" contacts.csv > unique_contacts.csv +# Case conversion +sqawk -s "SELECT UPPER(name), LOWER(email) FROM contacts" contacts.csv -# Delete rows with missing values -sqawk -s "DELETE FROM data WHERE email IS NULL OR email = ''" data.csv --write +# Substring extraction +sqawk -s "SELECT SUBSTR(date, 1, 7) AS month FROM transactions" transactions.csv -# Fix casing issues -sqawk -s "UPDATE data SET name = UPPER(name)" data.csv --write +# String replacement +sqawk -s "UPDATE data SET phone = REPLACE(phone, '-', '')" data.csv --write ``` -### Data Transformation +### System Files (--tabledef) + +```sh +# Query /etc/passwd +sqawk -F: --tabledef=passwd:user,pass,uid,gid,gecos,home,shell \ + -s "SELECT user, home FROM passwd WHERE uid >= 1000" \ + passwd=/etc/passwd + +# Query /etc/hosts +sqawk --tabledef=hosts:ip,hostname \ + -s "SELECT * FROM hosts WHERE ip LIKE '192.168.%'" \ + hosts=/etc/hosts +``` -Transform data for analysis or export: +### Output Redirection ```sh -# Extract subset of columns -sqawk -s "SELECT id, name, email FROM contacts" contacts.csv > minimal_contacts.csv +# Export filtered data to new file +sqawk -s "SELECT * FROM data WHERE region = 'North'" data.csv > north_data.csv -# Reshape data by filtering and sorting -sqawk -s "SELECT * FROM data WHERE region = 'North' ORDER BY date DESC" data.csv > north_region_latest.csv +# Deduplicate to new file +sqawk -s "SELECT DISTINCT * FROM data" data.csv > deduped.csv -# Create derived columns -sqawk -s "SELECT id, name, salary, salary * 0.3 AS bonus FROM employees" employees.csv +# Convert delimiter (CSV to TSV) +sqawk -s "SELECT * FROM data" data.csv | sqawk -F, -s "SELECT * FROM stdin" > data.tsv ``` -### Joining Data from Multiple Files - -Combine data from different files: +### Interactive REPL Session ```sh -# Simple join between two files -sqawk -s "SELECT users.name, orders.product_id, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id" users.csv orders.csv - -# Three-way join with filtering -sqawk -s "SELECT users.name AS customer, products.name AS product, orders.date - FROM users - INNER JOIN orders ON users.id = orders.user_id - INNER JOIN products ON orders.product_id = products.product_id - WHERE orders.date > '2023-01-01'" - users.csv orders.csv products.csv - -# Using DISTINCT with JOINs to find unique customer-product pairs -sqawk -s "SELECT DISTINCT users.name, products.name - FROM users - INNER JOIN orders ON users.id = orders.user_id - INNER JOIN products ON orders.product_id = products.product_id" - users.csv orders.csv products.csv +sqawk -i sales.csv customers.csv ``` -### Generating Reports +``` +sqawk> .tables +customers sales -Create summary reports from data: +sqawk> .schema sales +CREATE TABLE sales (id INTEGER, customer_id INTEGER, amount FLOAT); -```sh -# Sales summary by region -sqawk -s "SELECT region, COUNT(*) AS order_count, SUM(amount) AS total_sales - FROM orders - GROUP BY region - ORDER BY total_sales DESC" - orders.csv +sqawk> SELECT COUNT(*) FROM sales; +count +1250 + +sqawk> SELECT c.name, SUM(s.amount) AS total + FROM customers c JOIN sales s ON c.id = s.customer_id + GROUP BY c.name ORDER BY total DESC LIMIT 3; +name,total +Enterprise Corp,58750.25 +Acme Inc,45620.75 -# Monthly trends -sqawk -s "SELECT SUBSTR(date, 1, 7) AS month, COUNT(*) AS transaction_count - FROM transactions - GROUP BY month - ORDER BY month" - transactions.csv +sqawk> .exit ``` ## Working with Large Files @@ -725,7 +738,7 @@ Sqawk loads all data into memory, which provides excellent performance but requi - Use the `-v` verbose flag to see the exact SQL being executed - Verify SQL statement syntax, particularly quotes, parentheses, and required clauses -6. **Special characters in files**: +9. **Special characters in files**: - For files with quotes or special characters, Sqawk follows CSV escaping rules - If encountering parsing issues, check for malformed CSV data diff --git a/src/aggregate.rs b/src/aggregate.rs index abf10d7..84a9f99 100644 --- a/src/aggregate.rs +++ b/src/aggregate.rs @@ -213,7 +213,7 @@ mod tests { Value::Integer(10), Value::Null, Value::Integer(20), - Value::String("test".to_string()), + Value::String("test".to_string().into()), ]; let count = AggregateFunction::Count.execute(&values).unwrap(); @@ -276,7 +276,7 @@ mod tests { let values = vec![ Value::Integer(30), Value::Float(5.5), - Value::String("abc".to_string()), + Value::String("abc".to_string().into()), Value::Null, ]; @@ -302,11 +302,11 @@ mod tests { let values = vec![ Value::Integer(30), Value::Float(50.5), - Value::String("xyz".to_string()), + Value::String("xyz".to_string().into()), Value::Null, ]; let max = AggregateFunction::Max.execute(&values).unwrap(); - assert_eq!(max, Value::String("xyz".to_string())); + assert_eq!(max, Value::String("xyz".to_string().into())); } } diff --git a/src/capacity.rs b/src/capacity.rs new file mode 100644 index 0000000..aa490ed --- /dev/null +++ b/src/capacity.rs @@ -0,0 +1,136 @@ +//! Capacity hints for pre-allocating collections +//! +//! These constants define initial capacities for various collections +//! to reduce reallocation overhead. Values are based on typical usage patterns. + +// ============================================================================= +// Row and Column Capacities +// ============================================================================= + +/// Default capacity for row vectors (number of fields per row). +/// Most CSV/TSV files have fewer than 64 columns. +pub const DEFAULT_ROW_CAPACITY: usize = 16; + +/// Default capacity for column definition vectors. +pub const DEFAULT_COLUMN_CAPACITY: usize = 16; + +// ============================================================================= +// VM Engine Capacities +// ============================================================================= + +/// Default capacity for VM register vectors. +/// Most programs use fewer than 64 registers. +pub const DEFAULT_REGISTER_CAPACITY: usize = 32; + +/// Default capacity for cursor HashMap. +/// Most queries use 1-4 cursors. +pub const DEFAULT_CURSOR_CAPACITY: usize = 4; + +/// Default capacity for sorter HashMap. +/// Most queries have 0-2 sorters. +pub const DEFAULT_SORTER_CAPACITY: usize = 2; + +/// Default capacity for accumulator HashMap. +/// Most aggregate queries have 1-8 accumulators. +pub const DEFAULT_ACCUMULATOR_CAPACITY: usize = 4; + +/// Default capacity for result rows vector. +/// Pre-allocate for small result sets; grows as needed. +pub const DEFAULT_RESULT_CAPACITY: usize = 64; + +/// Default capacity for pending modifications vector. +pub const DEFAULT_MODIFICATIONS_CAPACITY: usize = 4; + +// ============================================================================= +// Compiler Capacities +// ============================================================================= + +/// Default capacity for instruction vectors in programs. +pub const DEFAULT_INSTRUCTION_CAPACITY: usize = 64; + +// ============================================================================= +// Table/Database Capacities +// ============================================================================= + +/// Default capacity for table HashMap in database. +/// Most sessions work with 1-8 tables. +pub const DEFAULT_TABLE_CAPACITY: usize = 8; + +// ============================================================================= +// Sorter Capacities +// ============================================================================= + +/// Default capacity for sorter row vectors. +/// Grows dynamically, but start with reasonable size. +pub const DEFAULT_SORTER_ROWS_CAPACITY: usize = 256; + +/// Default capacity for sort key vectors. +pub const DEFAULT_SORT_KEYS_CAPACITY: usize = 4; + +// ============================================================================= +// File-based Estimation +// ============================================================================= + +/// Average bytes per row estimate for CSV files. +/// Used to estimate row count from file size. +pub const ESTIMATED_BYTES_PER_ROW: usize = 128; + +/// Minimum row capacity for file-based estimation. +pub const MIN_ESTIMATED_ROW_CAPACITY: usize = 64; + +/// Maximum row capacity for file-based estimation. +/// Prevents excessive pre-allocation for very large files. +pub const MAX_ESTIMATED_ROW_CAPACITY: usize = 10_000_000; + +/// Estimate row count from file size (for mmap pre-allocation). +/// Returns a capacity hint, not an exact count. +#[inline] +pub fn estimate_row_count(file_size: usize) -> usize { + let estimate = file_size / ESTIMATED_BYTES_PER_ROW; + estimate.clamp(MIN_ESTIMATED_ROW_CAPACITY, MAX_ESTIMATED_ROW_CAPACITY) +} + +// ============================================================================= +// Headerless File Detection +// ============================================================================= + +/// Check if a line is a comment (starts with #). +#[inline] +pub fn is_comment_line(line: &[u8]) -> bool { + line.starts_with(b"#") +} + +/// Check if a field looks like data rather than a header name. +/// Used to auto-detect headerless files like /etc/passwd. +#[inline] +pub fn is_data_field(field: &str) -> bool { + field.starts_with('/') || // Path + field == "*" || // Password placeholder + field == "root" || // Common username + field == "nobody" || // Common username + field.parse::().is_ok() // Numeric ID +} + +/// Check if a row of fields looks like data rather than headers. +pub fn is_likely_data_row>(fields: &[S]) -> bool { + fields.iter().any(|f| is_data_field(f.as_ref())) +} + +/// Generate alphabetical column names (a, b, c, ..., z, aa, ab, ...). +pub fn generate_alpha_columns(count: usize) -> Vec { + (0..count) + .map(|i| { + let mut name = String::new(); + let mut n = i; + loop { + name.insert(0, (b'a' + (n % 26) as u8) as char); + n /= 26; + if n == 0 { + break; + } + n -= 1; + } + name + }) + .collect() +} diff --git a/src/cli.rs b/src/cli.rs index 51aa25c..9f32ba9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -96,13 +96,6 @@ pub struct SqawkArgs { /// and their original file format and delimiters will be preserved. #[clap(short = 'w', long, help = "Write changes back to input files")] pub write: bool, - - /// Use VM-based SQL execution engine - /// - /// When enabled, uses the bytecode-based virtual machine execution engine - /// instead of the default direct execution engine. - #[clap(long, help = "Use VM-based SQL execution engine")] - pub vm: bool, } /// Parse command-line arguments into the SqawkArgs structure diff --git a/src/config.rs b/src/config.rs index 167dc4c..dca9aa0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,9 +23,6 @@ pub struct AppConfig { /// Whether to write changes back to files write_changes: bool, - - /// Whether to use the VM execution engine - use_vm: bool, } impl AppConfig { @@ -41,14 +38,12 @@ impl AppConfig { field_separator: Option, table_definitions: Vec, write_changes: bool, - use_vm: bool, ) -> Self { Self { verbose, field_separator, table_definitions, write_changes, - use_vm, } } @@ -76,9 +71,4 @@ impl AppConfig { pub fn set_write_changes(&mut self, write: bool) { self.write_changes = write; } - - /// Get whether to use VM execution engine - pub fn use_vm(&self) -> bool { - self.use_vm - } } diff --git a/src/csv_handler.rs b/src/csv_handler.rs index 31b7ff4..8bf8cf0 100644 --- a/src/csv_handler.rs +++ b/src/csv_handler.rs @@ -16,6 +16,8 @@ use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; +use tempfile::NamedTempFile; + use crate::error::{SqawkError, SqawkResult}; use crate::table::{Table, Value}; @@ -43,6 +45,9 @@ impl CsvHandler { /// Save a table to a CSV file /// + /// Uses atomic write pattern: writes to a temporary file first, then renames + /// into place. This preserves the original file if writing fails midway. + /// /// # Arguments /// * `table` - The table to save /// * `file_path` - The path to the file @@ -50,25 +55,44 @@ impl CsvHandler { /// # Returns /// * `SqawkResult<()>` - Result of the operation pub fn save_csv(&self, table: &Table, file_path: &Path) -> SqawkResult<()> { - // Create a CSV writer - let file = File::create(file_path).map_err(SqawkError::IoError)?; - let mut writer = csv::Writer::from_writer(file); + // Create temp file in the same directory as target (required for atomic rename) + let parent = file_path.parent().unwrap_or(Path::new(".")); + let temp_file = NamedTempFile::new_in(parent).map_err(SqawkError::IoError)?; - // Write header row - writer - .write_record(table.columns()) - .map_err(SqawkError::CsvError)?; - - // Write data rows - for row in table.rows() { - let string_values: Vec = row.iter().map(|value| value.to_string()).collect(); + // Write data in a scope so the writer is dropped before we call persist + { + // Create a CSV writer writing to the temp file + let mut writer = csv::Writer::from_writer(temp_file.as_file()); + // Write header row writer - .write_record(&string_values) + .write_record(table.columns()) .map_err(SqawkError::CsvError)?; + + // Write data rows + for row in table.rows() { + let string_values: Vec = + row.iter().map(|value| value.to_string()).collect(); + + writer + .write_record(&string_values) + .map_err(SqawkError::CsvError)?; + } + + // Flush the CSV writer before it's dropped + writer.flush().map_err(SqawkError::IoError)?; } - writer.flush().map_err(SqawkError::IoError)?; + // Sync to disk before rename to ensure data durability + temp_file + .as_file() + .sync_all() + .map_err(SqawkError::IoError)?; + + // Atomically rename temp file to target path + temp_file + .persist(file_path) + .map_err(|e| SqawkError::IoError(e.error))?; Ok(()) } @@ -111,9 +135,12 @@ impl CsvHandler { let file = File::open(&file_path)?; let reader = BufReader::new(file); + // has_headers is true by default - first row is column names + let has_headers = true; + // Create a CSV reader with enhanced options let mut csv_reader = csv::ReaderBuilder::new() - .has_headers(true) + .has_headers(has_headers) .comment(Some(b'#')) // Support comment lines starting with # // Enable flexible mode only if error recovery is requested .flexible(true) // Always use flexible mode to allow for skipping errors @@ -121,7 +148,7 @@ impl CsvHandler { // Get headers or use custom column names if provided let headers = if let Some(columns) = custom_columns { - // Use the provided custom column names + // Use the provided custom column names (first row will be data) columns } else { // Use column names from the CSV header row @@ -146,33 +173,39 @@ impl CsvHandler { let mut skipped_rows = 0; let mut row_number = 0; + // Reusable row buffer to avoid per-row allocations + let column_count = table.column_count(); + let mut row_buffer = Vec::with_capacity(column_count); + for result in csv_reader.records() { row_number += 1; match result { Ok(record) => { - if should_recover && record.len() != table.column_count() { - // In recovery mode, handle rows with different column counts - let mut row = Vec::new(); + // Clear buffer for reuse + row_buffer.clear(); - // For each column in our table - for i in 0..table.column_count() { + if should_recover && record.len() != column_count { + // In recovery mode, handle rows with different column counts + for i in 0..column_count { if i < record.len() { // Use the value if it exists - row.push(Value::from(record.get(i).unwrap_or(""))); + row_buffer.push(Value::from(record.get(i).unwrap_or(""))); } else { // Pad with null values if we need more - row.push(Value::Null); + row_buffer.push(Value::Null); } } - // Now we have a properly sized row, add it without validation - table.add_row_recovery(row)?; + // Now we have a properly sized row, add it by cloning from buffer + table.add_row_from_slice(&row_buffer)?; } else { // Normal path - convert record to a row of values and validate - let row = record.iter().map(Value::from).collect(); + for field in record.iter() { + row_buffer.push(Value::from(field)); + } // This call can fail if the columns don't match and we're not in recovery mode - if let Err(e) = table.add_row(row) { + if let Err(e) = table.add_row_from_slice(&row_buffer) { if should_recover { // If we're in recovery mode, log and continue skipped_rows += 1; diff --git a/src/database.rs b/src/database.rs index 309b9f3..78d22b2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -8,6 +8,7 @@ //! - Storing all tables with their names //! - Providing a unified interface for table operations +use crate::capacity::DEFAULT_TABLE_CAPACITY; use crate::config::AppConfig; use crate::error::{SqawkError, SqawkResult}; use crate::table::Table; @@ -46,7 +47,7 @@ impl Database { /// Create a new, empty database pub fn new() -> Self { Database { - tables: HashMap::new(), + tables: HashMap::with_capacity(DEFAULT_TABLE_CAPACITY), } } diff --git a/src/delim_handler.rs b/src/delim_handler.rs index 3e55a1a..d28df44 100644 --- a/src/delim_handler.rs +++ b/src/delim_handler.rs @@ -15,9 +15,12 @@ //! to use the specified delimiter instead of commas. use std::fs::File; -use std::io::BufReader; +use std::io::{BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; +use tempfile::NamedTempFile; + +use crate::capacity::{generate_alpha_columns, is_likely_data_row}; use crate::error::{SqawkError, SqawkResult}; use crate::table::{Table, Value}; @@ -51,6 +54,9 @@ impl DelimHandler { /// Save a table to a delimiter-separated file /// + /// Uses atomic write pattern: writes to a temporary file first, then renames + /// into place. This preserves the original file if writing fails midway. + /// /// # Arguments /// * `table` - The table to save /// * `file_path` - The path to the file @@ -64,27 +70,41 @@ impl DelimHandler { file_path: &Path, delimiter: &str, ) -> SqawkResult<()> { - use std::fs::File; - use std::io::{BufWriter, Write}; + // Create temp file in the same directory as target (required for atomic rename) + let parent = file_path.parent().unwrap_or(Path::new(".")); + let temp_file = NamedTempFile::new_in(parent).map_err(SqawkError::IoError)?; - // Open the file for writing - let file = File::create(file_path).map_err(SqawkError::IoError)?; - let mut writer = BufWriter::new(file); + // Write data in a scope so the writer is dropped before we call persist + { + // Create a buffered writer for the temp file + let mut writer = BufWriter::new(temp_file.as_file()); - // Write the header row - let header = table.columns().join(delimiter); - writeln!(writer, "{}", header).map_err(SqawkError::IoError)?; + // Write the header row + let header = table.columns().join(delimiter); + writeln!(writer, "{}", header).map_err(SqawkError::IoError)?; - // Write data rows - for row in table.rows() { - let row_values: Vec = row.iter().map(|value| value.to_string()).collect(); + // Write data rows + for row in table.rows() { + let row_values: Vec = row.iter().map(|value| value.to_string()).collect(); - let row_str = row_values.join(delimiter); - writeln!(writer, "{}", row_str).map_err(SqawkError::IoError)?; + let row_str = row_values.join(delimiter); + writeln!(writer, "{}", row_str).map_err(SqawkError::IoError)?; + } + + // Flush the buffered writer before it's dropped + writer.flush().map_err(SqawkError::IoError)?; } - // Flush and close the writer - writer.flush().map_err(SqawkError::IoError)?; + // Sync to disk before rename to ensure data durability + temp_file + .as_file() + .sync_all() + .map_err(SqawkError::IoError)?; + + // Atomically rename temp file to target path + temp_file + .persist(file_path) + .map_err(|e| SqawkError::IoError(e.error))?; Ok(()) } @@ -140,10 +160,13 @@ impl DelimHandler { ))); }; + // has_headers is true by default - first row is column names + let has_headers = true; + // Create a CSV reader with custom delimiter // Also add support for comment lines (starting with #) for system files like /etc/passwd let mut csv_reader = csv::ReaderBuilder::new() - .has_headers(true) + .has_headers(has_headers) .delimiter(delimiter_byte) .comment(Some(b'#')) // Support for comment lines starting with # .flexible(true) // Allow for variable number of fields @@ -152,44 +175,19 @@ impl DelimHandler { // If custom column names are provided, use them // Otherwise detect/generate headers based on file content let headers = if let Some(columns) = custom_columns { - // Use the provided custom column names + // Use the provided custom column names (first row will be treated as data) columns } else { // No custom column names, generate or detect from file match csv_reader.headers().map_err(SqawkError::CsvError) { Ok(header_row) => { - // Check if the first row looks like data rather than headers - // This helps with system files like /etc/passwd that don't have headers - let is_likely_data = header_row.iter().any(|field| { - // Common indicators that a field is data, not a header - field.starts_with('/') || // Path - field == "*" || // Password placeholder - field == "root" || // Common username - field == "nobody" || // Common username - field.parse::().is_ok() // Numeric ID - }); - - if is_likely_data { - // Generate alphabetical column names (a, b, c, etc.) - (0..header_row.len()) - .map(|i| { - // Convert number to alphabetical column name (a, b, ..., z, aa, ab, ...) - let mut name = String::new(); - let mut n = i; - loop { - name.insert(0, (b'a' + (n % 26) as u8) as char); - n /= 26; - if n == 0 { - break; - } - n -= 1; // Adjust for the shift from 0-based to 1-based - } - name - }) - .collect::>() + let fields: Vec<&str> = header_row.iter().collect(); + if is_likely_data_row(&fields) { + // First row is data, generate a,b,c column names + generate_alpha_columns(fields.len()) } else { // Use the headers as they are - header_row.iter().map(|s| s.to_string()).collect::>() + fields.iter().map(|s| s.to_string()).collect() } } Err(_) => { @@ -198,23 +196,7 @@ impl DelimHandler { let first_record = record_iter.into_iter().next(); if let Some(Ok(record)) = first_record { - // Generate alphabetical column names (a, b, c, etc.) - (0..record.len()) - .map(|i| { - // Convert number to alphabetical column name (a, b, ..., z, aa, ab, ...) - let mut name = String::new(); - let mut n = i; - loop { - name.insert(0, (b'a' + (n % 26) as u8) as char); - n /= 26; - if n == 0 { - break; - } - n -= 1; // Adjust for the shift from 0-based to 1-based - } - name - }) - .collect::>() + generate_alpha_columns(record.len()) } else { // Fallback to a minimal set if we can't determine field count vec!["a".to_string()] @@ -231,14 +213,20 @@ impl DelimHandler { delimiter.to_string(), ); - // Read rows + // Read rows with reusable buffer to avoid per-row allocations + let column_count = table.column_count(); + let mut row_buffer = Vec::with_capacity(column_count); + for result in csv_reader.records() { let record = result.map_err(SqawkError::CsvError)?; - // Convert record to a row of values - let row = record.iter().map(Value::from).collect(); + // Clear buffer for reuse and fill with values + row_buffer.clear(); + for field in record.iter() { + row_buffer.push(Value::from(field)); + } - table.add_row(row)?; + table.add_row_from_slice(&row_buffer)?; } Ok(table) diff --git a/src/error.rs b/src/error.rs index 260bb99..ac72add 100644 --- a/src/error.rs +++ b/src/error.rs @@ -4,11 +4,9 @@ //! It provides a comprehensive error handling system that categorizes //! different failure modes, supports error propagation, and supplies //! helpful error messages to users. -//! -//! The module uses thiserror to minimize boilerplate code and create -//! a consistent error handling approach throughout the codebase. -use thiserror::Error; +use std::error::Error; +use std::fmt; /// SqawkError represents all possible errors that can occur in the sqawk application /// @@ -21,18 +19,15 @@ use thiserror::Error; /// /// Each variant includes descriptive error messages to help users understand /// and troubleshoot problems. -#[derive(Error, Debug)] +#[derive(Debug)] pub enum SqawkError { /// Error during file system operations (reading/writing files) - #[error("I/O error: {0}")] - IoError(#[from] std::io::Error), + IoError(std::io::Error), /// Error while parsing or processing delimited file data - #[error("File parsing error: {0}")] - CsvError(#[from] csv::Error), + CsvError(csv::Error), /// Enhanced CSV parsing error with file location information - #[error("CSV parse error in {file} at line {line}: {error}")] CsvParseError { file: String, line: usize, @@ -40,58 +35,93 @@ pub enum SqawkError { }, /// Error during SQL query parsing with sqlparser - #[error("SQL parsing error: {0}")] - SqlParseError(#[from] sqlparser::parser::ParserError), + SqlParseError(sqlparser::parser::ParserError), /// Error when a referenced table doesn't exist - #[error("Table '{0}' not found")] TableNotFound(String), /// Error when trying to create a table that already exists - #[error("Table '{0}' already exists")] TableAlreadyExists(String), /// Error when a file doesn't exist - #[error("File not found: {0}")] FileNotFound(String), /// Error when a table doesn't have an associated file path - #[error("Table '{0}' has no associated file path")] NoFilePath(String), /// Error when a referenced column doesn't exist in a table - #[error("Column '{0}' not found")] ColumnNotFound(String), /// Error for invalid file=table specifications - #[error("Invalid file specification: {0}")] InvalidFileSpec(String), /// Error for SQL features that aren't implemented yet - #[error("Unsupported SQL feature: {0}")] UnsupportedSqlFeature(String), - /// Error for type mismatches or conversion failures - #[error("Type error: {0}")] - TypeError(String), - /// Error for semantically invalid SQL queries - #[error("Invalid SQL query: {0}")] InvalidSqlQuery(String), - /// Error for division by zero in arithmetic operations - #[error("Division by zero")] - DivideByZero, - - /// Error for invalid function arguments - #[error("Invalid function arguments: {0}")] - InvalidFunctionArguments(String), - /// Error in VM execution - #[error("VM execution error: {0}")] VmError(String), } +impl fmt::Display for SqawkError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SqawkError::IoError(e) => write!(f, "I/O error: {}", e), + SqawkError::CsvError(e) => write!(f, "File parsing error: {}", e), + SqawkError::CsvParseError { file, line, error } => { + write!(f, "CSV parse error in {} at line {}: {}", file, line, error) + } + SqawkError::SqlParseError(e) => write!(f, "SQL parsing error: {}", e), + SqawkError::TableNotFound(name) => write!(f, "Table '{}' not found", name), + SqawkError::TableAlreadyExists(name) => write!(f, "Table '{}' already exists", name), + SqawkError::FileNotFound(path) => write!(f, "File not found: {}", path), + SqawkError::NoFilePath(name) => { + write!(f, "Table '{}' has no associated file path", name) + } + SqawkError::ColumnNotFound(name) => write!(f, "Column '{}' not found", name), + SqawkError::InvalidFileSpec(spec) => write!(f, "Invalid file specification: {}", spec), + SqawkError::UnsupportedSqlFeature(feature) => { + write!(f, "Unsupported SQL feature: {}", feature) + } + SqawkError::InvalidSqlQuery(msg) => write!(f, "Invalid SQL query: {}", msg), + SqawkError::VmError(msg) => write!(f, "VM execution error: {}", msg), + } + } +} + +impl Error for SqawkError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + SqawkError::IoError(e) => Some(e), + SqawkError::CsvError(e) => Some(e), + SqawkError::SqlParseError(e) => Some(e), + _ => None, + } + } +} + +// From implementations for automatic error conversion (replaces #[from]) + +impl From for SqawkError { + fn from(err: std::io::Error) -> Self { + SqawkError::IoError(err) + } +} + +impl From for SqawkError { + fn from(err: csv::Error) -> Self { + SqawkError::CsvError(err) + } +} + +impl From for SqawkError { + fn from(err: sqlparser::parser::ParserError) -> Self { + SqawkError::SqlParseError(err) + } +} + // Custom implementation of PartialEq for SqawkError // This implementation only compares the variant names, not their content // This is useful for testing, where we want to check if an error is of the right type @@ -111,12 +141,7 @@ impl PartialEq for SqawkError { (SqawkError::ColumnNotFound(_), SqawkError::ColumnNotFound(_)) => true, (SqawkError::InvalidFileSpec(_), SqawkError::InvalidFileSpec(_)) => true, (SqawkError::UnsupportedSqlFeature(_), SqawkError::UnsupportedSqlFeature(_)) => true, - (SqawkError::TypeError(_), SqawkError::TypeError(_)) => true, (SqawkError::InvalidSqlQuery(_), SqawkError::InvalidSqlQuery(_)) => true, - (SqawkError::DivideByZero, SqawkError::DivideByZero) => true, - (SqawkError::InvalidFunctionArguments(_), SqawkError::InvalidFunctionArguments(_)) => { - true - } (SqawkError::VmError(_), SqawkError::VmError(_)) => true, // If variants are different, they are not equal _ => false, diff --git a/src/file_handler.rs b/src/file_handler.rs index 0be06a7..79ef5d8 100644 --- a/src/file_handler.rs +++ b/src/file_handler.rs @@ -14,6 +14,8 @@ use crate::csv_handler::CsvHandler; use crate::database::Database; use crate::delim_handler::DelimHandler; use crate::error::{SqawkError, SqawkResult}; +use crate::storage::mmap::MmapStorage; +use crate::storage::Storage; use crate::table::Table; /// Enum representing different file formats supported by sqawk @@ -117,19 +119,24 @@ impl FileHandler { let file_path_str = file_path.to_string_lossy().to_string(); // First, check if the table already exists in the database - // This could happen if it was defined through CLI table definitions - let existing_schema; + // This could happen if it was defined through CLI table definitions (--tabledef) + let predefined_columns: Option>; { // Create a temporary scope for the database borrow let db = self.database_mut(); - existing_schema = db.has_table(&table_name); + predefined_columns = if db.has_table(&table_name) { + // Get the predefined column names to use instead of auto-detected ones + db.get_table(&table_name).ok().map(|t| t.columns().to_vec()) + } else { + None + }; } // Show verbose output if needed let verbose = self.config.verbose(); - if existing_schema && verbose { + if predefined_columns.is_some() && verbose { println!( - "Table '{}' already exists in database, loading data into existing schema", + "Table '{}' has predefined schema, using those column names", table_name ); } @@ -140,21 +147,55 @@ impl FileHandler { // Column definitions now come exclusively from Database // No need for custom columns logic here anymore - // Create the table based on the format - let table = match format { - FileFormat::Csv => { - // Load the table from the CSV file, no custom columns since we use Database schemas - self.csv_handler.load_csv(file_spec, None, None)? + // Determine delimiter for this file + // If -F flag is provided, always use it (overrides format detection) + let delimiter_str = if let Some(sep) = self.config.field_separator() { + sep + } else { + match format { + FileFormat::Csv => ",".to_string(), + FileFormat::Delimited => "\t".to_string(), } - FileFormat::Delimited => { - let delimiter = self - .config - .field_separator() - .unwrap_or_else(|| "\t".to_string()); - // No custom columns since we use Database schemas - self.delim_handler - .load_delimited(file_spec, &delimiter, None)? + }; + let delimiter_byte = delimiter_str.as_bytes()[0]; + + // Try to use mmap for on-disk files (zero-copy loading) + // Only use mmap if the file exists and is a regular file + let table = if file_path.is_file() { + // Use memory-mapped storage for zero-copy access + // Pass predefined columns so mmap treats first row as data when appropriate + match MmapStorage::open_with_columns( + &file_path, + delimiter_byte, + predefined_columns.clone(), + ) { + Ok(mmap_storage) => { + if verbose { + println!("Using mmap storage for table '{}'", table_name); + } + let columns = mmap_storage.columns().to_vec(); + Table::with_storage( + &table_name, + columns, + Some(file_path.clone()), + delimiter_str, + Storage::Mmap(mmap_storage), + ) + } + Err(e) => { + // Fall back to regular loading if mmap fails + if verbose { + println!( + "Mmap failed for '{}', falling back to regular loading: {}", + table_name, e + ); + } + self.load_file_regular(file_spec, &format, &predefined_columns)? + } } + } else { + // Stdin or other non-file source - use regular loading + self.load_file_regular(file_spec, &format, &predefined_columns)? }; // Now that we have the table, we can update the database without borrowing conflicts @@ -162,12 +203,15 @@ impl FileHandler { // Create a new scope for database operations let db = self.database_mut(); - // Handle existing schema if needed - if existing_schema { - // For now, we'll replace it - in the future we might want to handle this - // more gracefully with schema validation and merging + // Handle existing schema if needed (from --tabledef) + if predefined_columns.is_some() { + // Remove the placeholder table created by --tabledef + // We're replacing it with the actual table containing data if verbose { - println!("Replacing existing table '{}' with loaded data", table_name); + println!( + "Replacing predefined schema for '{}' with loaded data", + table_name + ); } db.remove_table(&table_name); } @@ -179,6 +223,32 @@ impl FileHandler { Ok(Some((table_name, file_path_str))) } + /// Load a file using the regular (non-mmap) method + /// + /// This is the fallback method used when mmap is not available or fails. + fn load_file_regular( + &self, + file_spec: &str, + format: &FileFormat, + predefined_columns: &Option>, + ) -> SqawkResult { + match format { + FileFormat::Csv => { + // Load the table from the CSV file + self.csv_handler + .load_csv(file_spec, predefined_columns.clone(), None) + } + FileFormat::Delimited => { + let delimiter = self + .config + .field_separator() + .unwrap_or_else(|| "\t".to_string()); + self.delim_handler + .load_delimited(file_spec, &delimiter, predefined_columns.clone()) + } + } + } + /// Parse a file specification into a table name and path /// /// # Arguments @@ -240,55 +310,6 @@ impl FileHandler { db.get_table(table_name) } - /// Get a mutable reference to a table by name - /// - /// # Arguments - /// * `table_name` - Name of the table to retrieve - /// - /// # Returns - /// * `SqawkResult<&mut Table>` - Mutable reference to the requested table - pub fn get_table_mut(&mut self, table_name: &str) -> SqawkResult<&mut Table> { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler - let db = unsafe { &mut *self.database }; - db.get_table_mut(table_name) - } - - /// Add a table to the collection - /// - /// # Arguments - /// * `name` - Name of the table - /// * `table` - Table to add - /// - /// # Returns - /// * `SqawkResult<()>` - Result of the operation - pub fn add_table(&mut self, name: String, mut table: Table) -> SqawkResult<()> { - // Check if the table has a file path before adding and log information - if let Some(path) = table.file_path() { - if self.config.verbose() { - println!("Adding table '{}' with file path: {:?}", name, path); - } - - // Make absolute path if necessary (needed for CREATE TABLE with relative paths) - if !path.is_absolute() { - // Get current directory - if let Ok(mut cur_dir) = std::env::current_dir() { - // Join with the relative path - cur_dir.push(path.clone()); - if self.config.verbose() { - println!("Converting to absolute path: {:?}", cur_dir); - } - // Update the file path in the table - table.set_file_path(cur_dir); - } - } - } else if self.config.verbose() { - println!("Adding table '{}' with NO file path", name); - } - - // Add the table to the database - self.database_mut().add_table(name, table) - } - /// Get all table names /// /// # Returns diff --git a/src/lib.rs b/src/lib.rs index 04139c3..6c4d398 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ //! It facilitates SQL querying against in-memory CSV tables. pub mod aggregate; +pub mod capacity; pub mod cli; pub mod config; pub mod csv_handler; @@ -13,6 +14,6 @@ pub mod error; pub mod file_handler; pub mod repl; pub mod sql_executor; -pub mod string_functions; +pub mod storage; pub mod table; pub mod vm; diff --git a/src/main.rs b/src/main.rs index 11f5d98..f29eaf2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,7 @@ //! 5. Save modified tables back to disk if requested mod aggregate; +mod capacity; mod cli; mod config; mod csv_handler; @@ -36,7 +37,7 @@ mod error; mod file_handler; mod repl; mod sql_executor; -mod string_functions; +mod storage; mod table; mod vm; @@ -76,7 +77,6 @@ fn main() -> Result<()> { args.field_separator.clone(), // Field separator for tables args.tabledef.clone(), // Table column definitions args.write, // Whether to write changes to files - args.vm, // Whether to use VM execution engine ); // Configure diagnostics output if verbose mode is enabled (-v flag) @@ -134,26 +134,11 @@ fn main() -> Result<()> { // Process each SQL statement in the order specified on the command line // This allows operations like: UPDATE -> DELETE -> SELECT to see the effects for sql in &args.sql { - // Log the SQL being executed in verbose mode - if config.verbose() { - if args.vm { - println!("Executing SQL with VM: {sql}"); - } else { - println!("Executing SQL: {sql}"); - } - } - // Execute the SQL statement against the in-memory tables // The result may be a table (for SELECT) or None (for UPDATE, DELETE, INSERT) - let result = if args.vm { - sql_executor - .execute_vm(sql) - .with_context(|| format!("Failed to execute SQL with VM: {sql}"))? - } else { - sql_executor - .execute(sql) - .with_context(|| format!("Failed to execute SQL: {sql}"))? - }; + let result = sql_executor + .execute(sql) + .with_context(|| format!("Failed to execute SQL: {sql}"))?; // Step 4: Output results to stdout (for SELECT queries) match result { diff --git a/src/repl.rs b/src/repl.rs index 9519fdb..26b22d6 100644 --- a/src/repl.rs +++ b/src/repl.rs @@ -143,9 +143,27 @@ impl Highlighter for CommandCompleter { impl Validator for CommandCompleter { fn validate( &self, - _ctx: &mut validate::ValidationContext, + ctx: &mut validate::ValidationContext, ) -> rustyline::Result { - Ok(validate::ValidationResult::Valid(None)) + let input = ctx.input().trim(); + + // Empty input is valid (just press enter) + if input.is_empty() { + return Ok(validate::ValidationResult::Valid(None)); + } + + // Dot commands are complete on a single line + if input.starts_with('.') { + return Ok(validate::ValidationResult::Valid(None)); + } + + // SQL statements must end with semicolon + if input.ends_with(';') { + return Ok(validate::ValidationResult::Valid(None)); + } + + // Otherwise, we need more input (multiline SQL) + Ok(validate::ValidationResult::Incomplete) } } diff --git a/src/sql_executor.rs b/src/sql_executor.rs index cbf1c2e..e78542a 100644 --- a/src/sql_executor.rs +++ b/src/sql_executor.rs @@ -1,15 +1,11 @@ //! SQL execution module for sqawk //! -//! This module handles parsing and executing SQL statements against in-memory tables. It provides: +//! This module provides a thin wrapper around the VM execution engine for SQL processing. +//! It handles: //! -//! - SQL statement parsing using the sqlparser crate with a generic SQL dialect -//! - Execution logic for SELECT, INSERT, UPDATE, and DELETE statements -//! - Support for multi-table operations including cross joins and inner joins -//! - Column alias handling and resolution for both regular columns and aggregate functions -//! - ORDER BY implementation with multi-column support and configurable sort direction -//! - LIMIT and OFFSET support for pagination and result set control -//! - WHERE clause evaluation using a robust type system with SQL-like comparison semantics -//! - Tracking of modified tables for selective write-back to their original files +//! - SQL statement execution via the bytecode VM engine +//! - Tracking of modified tables for selective write-back +//! - Table management operations for the REPL interface //! //! The module implements a non-destructive approach, modifying only in-memory tables //! until explicitly requested to save changes back to the original files. @@ -17,23 +13,18 @@ use std::collections::HashSet; use anyhow::Result; -use sqlparser::ast::{ - Assignment, ColumnDef as SqlColumnDef, Expr, FileFormat as SqlFileFormat, Join as SqlJoin, - JoinConstraint, JoinOperator, ObjectName, Query, Select, SelectItem, SetExpr, SqlOption, - Statement, TableFactor, TableWithJoins, Value as SqlValue, -}; -use sqlparser::dialect::HiveDialect; -use sqlparser::parser::Parser; -use crate::aggregate::AggregateFunction; +use crate::capacity::DEFAULT_TABLE_CAPACITY; use crate::config::AppConfig; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; use crate::file_handler::FileHandler; -use crate::string_functions::StringFunction; -use crate::table::{ColumnDefinition, DataType, SortDirection, Table, Value}; +use crate::table::DataType; /// SQL statement executor +/// +/// This executor wraps the VM execution engine and provides additional functionality +/// for tracking modified tables and integrating with the REPL interface. pub struct SqlExecutor<'a> { /// Database for storing and accessing tables database: &'a mut Database, @@ -47,8 +38,8 @@ pub struct SqlExecutor<'a> { /// Application configuration for global settings config: AppConfig, - /// Number of affected rows from the last statement - affected_row_count: usize, + /// Number of rows affected by the last DML statement + affected_rows: usize, } impl<'a> SqlExecutor<'a> { @@ -61,13 +52,13 @@ impl<'a> SqlExecutor<'a> { SqlExecutor { database, file_handler, - modified_tables: HashSet::new(), + modified_tables: HashSet::with_capacity(DEFAULT_TABLE_CAPACITY), config: config.clone(), - affected_row_count: 0, + affected_rows: 0, } } - /// Execute an SQL statement with VM engine + /// Execute an SQL statement using the VM engine /// /// This method delegates execution to the VM-based bytecode engine, which: /// 1. Parses the SQL using sqlparser @@ -79,3334 +70,34 @@ impl<'a> SqlExecutor<'a> { /// /// # Returns /// * `SqawkResult>` - Result of the operation, possibly containing a table - pub fn execute_vm(&mut self, sql: &str) -> SqawkResult> { + pub fn execute(&mut self, sql: &str) -> SqawkResult> { if self.config.verbose() { - println!("Using VM execution engine for SQL: {}", sql); - } - - // Delegate to the VM module's execute_vm function - crate::vm::execute_vm(sql, self.database, self.config.verbose()) - } - - /// Execute a parsed SQL statement directly with the VM - /// This bypasses the regular SQL executor logic for VM execution - fn execute_vm_stmt(&self, stmt: &Statement) -> SqawkResult> { - // Convert the statement back to SQL text - // This is inefficient but simplifies integration - let sql = format!("{}", stmt); - - if self.config.verbose() { - println!("VM Engine executing statement: {}", sql); - } - - // Pass directly to VM engine - crate::vm::execute_vm(&sql, self.database, self.config.verbose()) - } - - /// Get the number of rows affected by the last executed statement - pub fn get_affected_row_count(&self) -> SqawkResult { - Ok(self.affected_row_count) - } - - /// Execute an SQL statement - /// - /// Returns Some(Table) for SELECT queries, None for other statements. - pub fn execute(&mut self, sql: &str) -> SqawkResult> { - // For CREATE TABLE statements with LOCATION, we need to use a dialect that - // properly supports the LOCATION clause - HiveDialect is made for this - let dialect = HiveDialect {}; // Hive dialect is specifically designed for LOCATION clauses - - if self.config.verbose() { - println!("Executing SQL: {}", sql); - } - - let statements = Parser::parse_sql(&dialect, sql).map_err(SqawkError::SqlParseError)?; - - if statements.is_empty() { - return Err(SqawkError::InvalidSqlQuery( - "No SQL statements found".to_string(), - )); - } - - // Execute each statement - let mut result = None; - for statement in statements { - // We've handled CREATE TABLE with LOCATION properly now, no need for extra debug logging here - - result = self.execute_statement(statement)?; - } - - Ok(result) - } - - /// Execute a single SQL statement - /// - /// This is the primary entry point for SQL execution in the Sqawk engine. It serves - /// as a dispatcher that: - /// 1. Examines the SQL statement type - /// 2. Routes to the appropriate specialized handler: - /// - SELECT: execute_query() - Returns a virtual result table - /// - INSERT: execute_insert() - Adds new rows to a table - /// - UPDATE: execute_update() - Modifies existing rows based on criteria - /// - DELETE: execute_delete() - Removes rows from a table based on criteria - /// 3. Tracks operation status including affected row counts - /// 4. Formats the appropriate return value based on operation type - /// - /// The function centralizes error handling and ensures consistent behavior across - /// all SQL operations. For data manipulation operations (INSERT/UPDATE/DELETE), - /// it also marks affected tables as modified for later write operations. - /// - /// # Arguments - /// * `statement` - The parsed SQL statement to execute (from sqlparser) - /// - /// # Returns - /// * `Ok(Some(Table))` for SELECT queries with the result set - /// * `Ok(None)` for other statement types (INSERT, UPDATE, DELETE) - /// * `Err` if the statement cannot be executed or contains unsupported features - fn execute_statement(&mut self, statement: Statement) -> SqawkResult> { - // If VM mode is enabled, route all statements through the VM engine - if self.config.use_vm() { - // Use our VM implementation instead of the regular SQL executor - return self.execute_vm_stmt(&statement); - } - - // Otherwise use the regular SQL executor - match statement { - Statement::Query(query) => self.execute_query(*query), - Statement::Insert { - table_name, - columns, - source, - .. - } => { - // For INSERT, we count affected rows as the number of rows inserted - // Currently we only support VALUES, so that's the number of value lists - let Query { body, .. } = &*source; - if let SetExpr::Values(values) = &**body { - // Count the number of rows that will be inserted - self.affected_row_count = values.rows.len(); - } else { - // If not using VALUES, we'll set affected rows later - self.affected_row_count = 0; - } - - self.execute_insert(table_name, columns, source)?; - - if self.config.verbose() { - eprintln!("Inserted {} rows", self.affected_row_count); - } - Ok(None) - } - Statement::Update { - table, - assignments, - selection, - .. - } => { - let updated_count = self.execute_update(table, assignments, selection)?; - // Store the affected row count for .changes command - self.affected_row_count = updated_count; - - if self.config.verbose() { - eprintln!("Updated {} rows", updated_count); - } - Ok(None) - } - Statement::Delete { - from, selection, .. - } => { - if from.len() != 1 { - return Err(SqawkError::UnsupportedSqlFeature( - "DELETE with multiple tables is not supported".to_string(), - )); - } - let table_with_joins = &from[0]; - let deleted_count = self.execute_delete(table_with_joins, selection)?; - // Store the affected row count for .changes command - self.affected_row_count = deleted_count; - - if self.config.verbose() { - eprintln!("Deleted {} rows", deleted_count); - } - Ok(None) - } - Statement::CreateTable { - name, - columns, - file_format, - location, - hive_formats, - with_options, - .. - } => { - // Print complete debug information about the parsed CREATE TABLE statement - if self.config.verbose() { - println!("Parsed CREATE TABLE statement:"); - println!(" Table name: {:?}", name); - println!( - " LOCATION clause: {:?}", - hive_formats.as_ref().and_then(|hf| hf.location.as_ref()) - ); - println!(" File format: {:?}", file_format); - println!(" WITH options: {:?}", with_options); - println!(" Columns: {:?}", columns.len()); - } - - // In sqlparser, the LOCATION clause is stored in the hive_formats field - // even when using non-Hive dialects like GenericDialect - let actual_location = if let Some(hf) = hive_formats.as_ref() { - hf.location.clone() - } else { - // Fallback to direct location field (unlikely to be used) - location.clone() - }; - - self.execute_create_table( - name, - columns, - file_format, - actual_location, - with_options, - )?; - if self.config.verbose() { - eprintln!("Table created successfully"); - } - Ok(None) - } - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported SQL statement: {:?}", - statement - ))), - } - } - - /// Execute a SQL SELECT query - /// - /// This function implements the core SQL SELECT processing workflow by: - /// 1. Analyzing the query structure to determine its complexity - /// 2. Detecting the presence of aggregate functions - /// 3. Routing to specialized handlers based on query characteristics: - /// - Simple queries use standard row-by-row processing - /// - Aggregate queries require grouping and aggregate function evaluation - /// - DISTINCT queries require duplicate elimination - /// - ORDER BY requires result sorting - /// - LIMIT/OFFSET requires pagination handling - /// - /// The function serves as a dispatcher that examines query features and directs - /// to the appropriate specialized query handlers. It handles the full SQL logical - /// processing order: FROM/JOIN → WHERE → GROUP BY → HAVING → SELECT → DISTINCT → - /// ORDER BY → LIMIT/OFFSET. - /// - /// # Arguments - /// * `query` - The parsed Query object containing the SELECT statement - /// - /// # Returns - /// * `Ok(Some(Table))` with the query results as a new virtual table - /// * `Ok(None)` for empty result sets or certain operations - /// * `Err` if the query is invalid or contains unsupported features - fn execute_query(&self, query: Query) -> SqawkResult> { - match *query.body { - SetExpr::Select(ref select) => { - // Handle special case of SELECT without FROM (e.g., SELECT 1) - if select.from.is_empty() { - return Err(SqawkError::InvalidSqlQuery( - "SELECT query must have at least one table".to_string(), - )); - } - - // Process the FROM clause to get a table or join result - let source_table = self.process_from_clause(&select.from)?; - - // Check if the query contains any aggregate functions - let has_aggregates = self.contains_aggregate_functions(&select.projection); - - // Process based on whether we have aggregates or not - if has_aggregates { - if self.config.verbose() { - eprintln!("Applying aggregate functions"); - } - self.execute_aggregate_query(source_table, select, &query) - } else { - self.execute_simple_query(source_table, select, &query) - } - } - _ => Err(SqawkError::UnsupportedSqlFeature( - "Only simple SELECT statements are supported".to_string(), - )), - } - } - - /// Executes a SQL query containing aggregate functions - /// - /// This function implements specialized processing for SQL queries that use aggregate - /// functions (SUM, AVG, COUNT, MIN, MAX). It follows the standard SQL logical - /// processing order with focus on grouping operations: - /// - /// 1. FROM/JOIN → Create working table (already provided as source_table) - /// 2. WHERE → Pre-filter rows before grouping - /// 3. GROUP BY → Organize rows into groups based on specified columns - /// 4. Aggregation → Apply aggregate functions to each group - /// 5. HAVING → Filter groups based on aggregate results - /// 6. SELECT → Construct result rows from group values and aggregates - /// 7. DISTINCT → Eliminate duplicate result rows (if specified) - /// 8. ORDER BY → Sort the final results - /// 9. LIMIT/OFFSET → Apply pagination - /// - /// The implementation handles complex aggregation scenarios including: - /// - Mixing aggregate and non-aggregate columns (requires GROUP BY) - /// - Applying aggregates to the entire table when no GROUP BY is present - /// - Filtering groups with HAVING based on aggregate results - /// - Complex expressions in aggregate functions (e.g., SUM(price * quantity)) - /// - /// # Arguments - /// * `source_table` - The input table created from FROM/JOIN processing - /// * `select` - The SELECT statement with projections, WHERE, GROUP BY, and HAVING - /// * `query` - The full query object with DISTINCT, ORDER BY, LIMIT, and OFFSET - /// - /// # Returns - /// * `Ok(Some(Table))` with the aggregated results as a new table - /// * `Err` if any step in aggregation processing fails - fn execute_aggregate_query( - &self, - source_table: Table, - select: &Select, - query: &Query, - ) -> SqawkResult> { - // Apply WHERE clause before aggregation - let filtered_table = self.apply_where_clause_if_present(source_table, &select.selection)?; - - // Apply GROUP BY if present, otherwise apply simple aggregation - let result_table = if !select.group_by.is_empty() { - if self.config.verbose() { - eprintln!("Applying GROUP BY"); - } - SqlExecutor::apply_grouped_aggregate_functions( - &select.projection, - &filtered_table, - &select.group_by, - )? - } else { - self.apply_aggregate_functions(&select.projection, &filtered_table)? - }; - - // Apply HAVING if present (only after GROUP BY) - let result_after_having = if let Some(having_expr) = &select.having { - if self.config.verbose() { - eprintln!("Applying HAVING"); - } - self.apply_having_clause(result_table, having_expr)? - } else { - result_table - }; - - // Apply post-processing steps (DISTINCT, ORDER BY, LIMIT, OFFSET) - let final_result = self.apply_post_processing_steps(result_after_having, select, query)?; - Ok(Some(final_result)) - } - - /// Executes a simple (non-aggregate) SQL SELECT query - /// - /// This function implements the core SQL processing logic for queries without - /// aggregate functions (SUM, COUNT, etc.). It follows the standard SQL logical - /// processing order: - /// - /// 1. FROM/JOIN → Create working table (already provided as source_table) - /// 2. WHERE → Filter rows that don't match the selection criteria - /// 3. SELECT → Extract only the requested columns (projection) - /// 4. DISTINCT → Eliminate duplicate rows if requested - /// 5. ORDER BY → Sort the results based on specified columns - /// 6. LIMIT/OFFSET → Apply row count limitations and pagination - /// - /// The implementation handles column references, aliases, and expressions in - /// both the WHERE clause and projection list. Each step transforms the working - /// table until the final result set is produced. - /// - /// # Arguments - /// * `source_table` - The input table created from FROM/JOIN processing - /// * `select` - The SELECT statement details (projection, where clause) - /// * `query` - The complete query object (DISTINCT, ORDER BY, LIMIT/OFFSET) - /// - /// # Returns - /// * `Ok(Some(Table))` with the fully processed query results - /// * `Err` if any step in query processing fails (invalid columns, type errors, etc.) - fn execute_simple_query( - &self, - source_table: Table, - select: &Select, - query: &Query, - ) -> SqawkResult> { - // For non-aggregate queries, use the normal column resolution - let column_specs = self.resolve_select_items(&select.projection, &source_table)?; - - // Apply WHERE clause before projection - let filtered_table = self.apply_where_clause_if_present(source_table, &select.selection)?; - - // Apply projection to get only the requested columns with aliases - let result_table = filtered_table.project_with_aliases(&column_specs)?; - - // Apply post-processing steps (DISTINCT, ORDER BY, LIMIT, OFFSET) - let final_result = self.apply_post_processing_steps(result_table, select, query)?; - Ok(Some(final_result)) - } - - /// Helper function to apply WHERE clause if present - /// - /// Conditionally applies a WHERE clause filter to a table if the clause exists. - /// This allows for uniform handling of tables with and without filtering. - /// - /// # Arguments - /// * `table` - The source table to filter - /// * `selection` - Optional WHERE clause expression - /// - /// # Returns - /// * A new filtered table if WHERE clause is present - /// * The original table unchanged if WHERE clause is not present - fn apply_where_clause_if_present( - &self, - table: Table, - selection: &Option, - ) -> SqawkResult
{ - if let Some(where_clause) = selection { - if self.config.verbose() { - eprintln!("WHERE comparison"); - } - self.apply_where_clause(table, where_clause) - } else { - // If no WHERE clause, just use the table as is - Ok(table) - } - } - - /// Applies final SQL query post-processing steps: DISTINCT, ORDER BY, LIMIT/OFFSET - /// - /// This function implements the final stages of SQL query processing according to - /// SQL's logical execution order. It handles operations that take place after the - /// core query execution (FROM/JOIN, WHERE, GROUP BY, HAVING, projection) has completed: - /// - /// Processing sequence: - /// 1. DISTINCT - Eliminates duplicate rows from the result set - /// - Compares all column values for exact matches - /// - Preserves only the first occurrence of each unique row - /// 2. ORDER BY - Sorts the result set based on specified columns - /// - Supports ascending and descending sort directions - /// - Handles multi-column sorting (primary, secondary, tertiary keys, etc.) - /// - Maintains stable sort order for equivalent values - /// 3. LIMIT/OFFSET - Applies pagination to the sorted results - /// - LIMIT: Restricts the number of rows returned - /// - OFFSET: Skips the specified number of initial rows - /// - /// Each step is applied conditionally, only if the corresponding clause exists - /// in the SQL statement. This maintains efficiency for queries that don't need - /// all post-processing operations. - /// - /// # Arguments - /// * `table` - The working table after core query processing, ready for post-processing - /// * `select` - The SELECT statement containing DISTINCT clause information - /// * `query` - The full Query object with ORDER BY, LIMIT, and OFFSET clauses - /// - /// # Returns - /// * `SqawkResult
` - The final query result table after all post-processing - fn apply_post_processing_steps( - &self, - mut table: Table, - select: &Select, - query: &Query, - ) -> SqawkResult
{ - // Apply DISTINCT if present - if select.distinct.is_some() { - if self.config.verbose() { - eprintln!("Applying DISTINCT"); - } - table = table.distinct()?; - } - - // Apply ORDER BY if present - if !query.order_by.is_empty() { - if self.config.verbose() { - eprintln!("Applying ORDER BY"); - } - table = self.apply_order_by(table, &query.order_by)?; - } - - // Apply LIMIT and OFFSET if present - if query.limit.is_some() || query.offset.is_some() { - if self.config.verbose() { - eprintln!("Applying LIMIT/OFFSET"); - } - table = self.apply_limit_offset(table, query)?; - } - - Ok(table) - } - - /// Apply LIMIT and OFFSET clauses to a table - /// - /// This function extracts limit and offset values from the SQL query, - /// and applies them to the table using the Table.limit() method. - /// - /// # Arguments - /// * `table` - The table to apply limit and offset to - /// * `query` - The SQL query containing limit and offset clauses - /// - /// # Returns - /// * A new table with limit and offset applied - fn apply_limit_offset(&self, table: Table, query: &Query) -> SqawkResult
{ - // Extract LIMIT value (default to all rows if not specified) - let limit = if let Some(limit_expr) = &query.limit { - match limit_expr { - // Parse the limit value from the SQL expression - sqlparser::ast::Expr::Value(SqlValue::Number(n, _)) => match n.parse::() { - Ok(val) => val, - Err(_) => { - return Err(SqawkError::InvalidSqlQuery(format!( - "Invalid LIMIT value: {}", - n - ))); - } - }, - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only constant numeric values are supported for LIMIT".to_string(), - )); - } - } - } else { - // If no LIMIT is specified, use usize::MAX to effectively get all rows - usize::MAX - }; - - // Extract OFFSET value (default to 0 if not specified) - let offset = if let Some(offset_clause) = &query.offset { - match &offset_clause.value { - sqlparser::ast::Expr::Value(SqlValue::Number(n, _)) => match n.parse::() { - Ok(val) => val, - Err(_) => { - return Err(SqawkError::InvalidSqlQuery(format!( - "Invalid OFFSET value: {}", - n - ))); - } - }, - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only constant numeric values are supported for OFFSET".to_string(), - )); - } - } - } else { - // If no OFFSET is specified, use 0 - 0 - }; - - // Apply the limit and offset to the table - table.limit(limit, offset) - } - - /// Apply an ORDER BY clause to sort the result table - /// - /// # Arguments - /// * `table` - The table to sort - /// * `order_by` - The ORDER BY expressions from the SQL query - /// - /// # Returns - /// * A new sorted table - fn apply_order_by( - &self, - table: Table, - order_by: &[sqlparser::ast::OrderByExpr], - ) -> SqawkResult
{ - // Convert ORDER BY expressions to column indices and sort directions - let mut sort_columns = Vec::new(); - - for order_expr in order_by { - // Extract the column index for this ORDER BY expression - let col_idx = match &order_expr.expr { - Expr::Identifier(ident) => { - // Try to find the column by exact name first - match table.column_index(&ident.value) { - Some(idx) => idx, - None => { - // Try with qualified names (table.column) lookup - let mut found = false; - let mut idx = 0; - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&format!(".{}", ident.value)) { - found = true; - idx = i; - break; - } - } - - if found { - idx - } else { - return Err(SqawkError::ColumnNotFound(ident.value.clone())); - } - } - } - } - // Handle qualified column reference: table.column - Expr::CompoundIdentifier(idents) => { - if idents.len() != 2 { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple qualified column references (table.column) are supported in ORDER BY".to_string(), - )); - } - - let table_name = &idents[0].value; - let column_name = &idents[1].value; - let qualified_name = format!("{}.{}", table_name, column_name); - - match table.column_index(&qualified_name) { - Some(idx) => idx, - None => { - return Err(SqawkError::ColumnNotFound(qualified_name)); - } - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple column references are supported in ORDER BY".to_string(), - )); - } - }; - - // Determine sort direction (ASC/DESC) - let direction = match order_expr.asc { - // If asc is None or Some(true), use Ascending - None | Some(true) => SortDirection::Ascending, - // If asc is Some(false), use Descending - Some(false) => SortDirection::Descending, - }; - - sort_columns.push((col_idx, direction)); - } - - // Sort the table using the calculated columns and directions - table.sort(sort_columns) - } - - /// Process the FROM clause of a SQL query, including all types of table joins - /// - /// This function implements the first step in SQL logical processing order by: - /// 1. Identifying and loading the base table (first table in FROM clause) - /// 2. Determining join types (INNER, CROSS) from SQL syntax - /// 3. Building the proper join conditions from ON clauses or WHERE conditions - /// 4. Executing the join operations to create unified working table - /// 5. Preserving column origins for later reference qualification - /// - /// The function handles multiple join syntax forms in SQL: - /// - Explicit INNER JOIN with ON clause: `table1 JOIN table2 ON condition` - /// - Explicit CROSS JOIN: `table1 CROSS JOIN table2` - /// - Implicit CROSS JOIN with comma: `table1, table2` - /// - Multi-way joins combining any of the above forms - /// - /// Each join produces a temporary working table that combines columns from both - /// source tables, maintaining column name qualification for later reference. - /// - /// # Arguments - /// * `from` - Array of FROM clause items from the SELECT statement - /// - /// # Returns - /// * `SqawkResult
` containing the joined result table with all columns - /// properly qualified for later processing steps - fn process_from_clause(&self, from: &[TableWithJoins]) -> SqawkResult
{ - // Start with the first table in the FROM clause - let first_table_with_joins = &from[0]; - let first_table_name = self.get_table_name(first_table_with_joins)?; - let mut result_table = self.file_handler.get_table(&first_table_name)?.clone(); - - // Handle any joins in the first TableWithJoins - if !first_table_with_joins.joins.is_empty() { - result_table = - self.process_table_joins(&result_table, &first_table_with_joins.joins)?; - } - - // If there are multiple tables in the FROM clause, join them - // This is the CROSS JOIN case for "FROM table1, table2, ..." - if from.len() > 1 { - if self.config.verbose() { - eprintln!("Processing multiple tables in FROM clause as CROSS JOINs"); - } - for table_with_joins in &from[1..] { - let right_table_name = self.get_table_name(table_with_joins)?; - let right_table = self.file_handler.get_table(&right_table_name)?; - - // Cross join with the current result table - result_table = result_table.cross_join(right_table)?; - - // Process any joins on this table - if !table_with_joins.joins.is_empty() { - result_table = - self.process_table_joins(&result_table, &table_with_joins.joins)?; - } - } - } - - Ok(result_table) - } - - /// Process joins for a table - /// - /// This function processes a list of explicit JOIN clauses for a table. - /// It iterates through each join specification, resolves the right table, - /// and applies the appropriate join operation based on the join type. - /// - /// Supported join types: - /// - CROSS JOIN: Returns all combinations of rows from both tables - /// - INNER JOIN with ON condition: Returns only rows that match the join condition - /// - /// # Arguments - /// * `left_table` - The left table for the join operations - /// * `joins` - Array of SQL join specifications to process - /// - /// # Returns - /// * A new table resulting from applying all join operations sequentially - /// Process explicit JOIN clauses in a SQL statement - /// - /// This function handles the various types of JOIN operations in a SQL statement, - /// including INNER JOIN with ON conditions and implicit CROSS JOINs. - /// - /// # Arguments - /// * `left_table` - The left (base) table for the join operations - /// * `joins` - Array of JOIN clauses to process sequentially - /// - /// # Returns - /// * A new table containing the results of all join operations - /// * `Err` if any JOIN syntax is unsupported or tables can't be found - /// - /// # Implementation Details - /// Joins are processed sequentially, with each join building on the result - /// of the previous one. The implementation follows these steps: - /// 1. Clone the left table as the starting point - /// 2. For each join specification: - /// a. Extract the right table name - /// b. Retrieve the right table from the file handler - /// c. Apply the appropriate join algorithm based on the join type - fn process_table_joins(&self, left_table: &Table, joins: &[SqlJoin]) -> SqawkResult
{ - // Start with a clone of the left table as our working result - let mut result_table = left_table.clone(); - - // Process each join clause sequentially - for join in joins { - // Log join type in verbose mode for debugging - if self.config.verbose() { - eprintln!("Join type: {:?}", join.join_operator); - } - - // Extract the right table name from the join specification - // This handles simple table references like "TableName" but not complex expressions - let right_table_name = match &join.relation { - TableFactor::Table { name, .. } => name - .0 - .iter() - .map(|i| i.value.clone()) - .collect::>() - .join("."), - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple table references are supported in joins".to_string(), - )) - } - }; - - // Fetch the right table from the loaded tables collection - let right_table = self.file_handler.get_table(&right_table_name)?; - - // Apply different join algorithms based on join type and constraints - match &join.join_operator { - // Handle JOINs without ON conditions (CROSS JOINs) - // Note: In sqlparser 0.36, the lack of a constraint (JoinConstraint::None) - // indicates the absence of an ON clause, which we treat as a CROSS JOIN - JoinOperator::FullOuter(JoinConstraint::None) - | JoinOperator::Inner(JoinConstraint::None) - | JoinOperator::LeftOuter(JoinConstraint::None) - | JoinOperator::RightOuter(JoinConstraint::None) => { - // Create a Cartesian product of the tables (all possible row combinations) - result_table = result_table.cross_join(right_table)?; - } - - // Handle INNER JOIN with ON condition - JoinOperator::Inner(JoinConstraint::On(expr)) => { - if self.config.verbose() { - eprintln!("Processing INNER JOIN with ON condition: {:?}", expr); - } - - // Use inner_join with a closure that evaluates the ON condition - // for each potential row combination from the Cartesian product - result_table = result_table.inner_join(right_table, |row, table| { - // This closure evaluates the ON condition for each row in the cross-join result - self.evaluate_condition(expr, row, table) - })?; - } - - // Handle JOIN types that are not yet supported - // Future enhancement: Implement LEFT, RIGHT, and FULL OUTER JOINs - JoinOperator::LeftOuter(JoinConstraint::On(_)) - | JoinOperator::RightOuter(JoinConstraint::On(_)) - | JoinOperator::FullOuter(JoinConstraint::On(_)) => { - return Err(SqawkError::UnsupportedSqlFeature( - "LEFT, RIGHT and FULL OUTER JOIN with ON conditions not yet supported" - .to_string(), - )); - } - - // USING constraints or other constraints are not supported - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only INNER JOIN with ON conditions or CROSS JOIN is supported".to_string(), - )); - } - } - } - - Ok(result_table) - } - - /// Execute a SQL INSERT statement - /// - /// This function implements the SQL INSERT operation by: - /// 1. Identifying the target table - /// 2. Processing the column specifications (if provided) - /// 3. Evaluating the source query to obtain values - /// 4. Validating value types against column definitions - /// 5. Adding new rows to the table structure - /// 6. Tracking the table as modified for later write operations - /// 7. Tracking the number of affected rows for reporting - /// - /// It supports inserting into a subset of columns (others filled with NULL) and - /// can insert multiple rows in a single operation. The implementation performs - /// type validation to ensure data consistency. - /// - /// # Arguments - /// * `table_name` - The name of the table to insert into - /// * `columns` - Optional list of columns to insert into (empty means all columns) - /// * `source` - The query source containing values to insert (VALUES clause or sub-query) - /// - /// # Returns - /// * `Ok(())` if the insert was successful - /// * `Err` if the table doesn't exist or the values don't match the columns - fn execute_insert( - &mut self, - table_name: sqlparser::ast::ObjectName, - columns: Vec, - source: Box, - ) -> SqawkResult<()> { - // Get the target table name - let table_name = table_name - .0 - .into_iter() - .map(|i| i.value) - .collect::>() - .join("."); - - // Check if the table exists - let column_count = { - let table = self.file_handler.get_table(&table_name)?; - table.column_count() - }; - - // Extract column indices if specified - let column_indices = if !columns.is_empty() { - let table = self.file_handler.get_table(&table_name)?; - columns - .iter() - .map(|ident| { - table - .column_index(&ident.value) - .ok_or(SqawkError::ColumnNotFound(ident.value.clone())) - }) - .collect::, _>>()? - } else { - (0..column_count).collect() - }; - - // Get values to insert - match *source.body { - SetExpr::Values(values) => { - // Process each row of values - for value_row in &values.rows { - if value_row.len() != column_indices.len() { - return Err(SqawkError::InvalidSqlQuery(format!( - "INSERT statement has {} values but {} columns were specified", - value_row.len(), - column_indices.len() - ))); - } - - // Create a full row with NULL values - let mut row = vec![Value::Null; column_count]; - - // Fill in the specified columns - for (i, expr) in value_row.iter().enumerate() { - let col_idx = column_indices[i]; - row[col_idx] = self.evaluate_expr(expr)?; - } - - // Add the row to the table - let table = self.file_handler.get_table_mut(&table_name)?; - table.add_row(row)?; - } - - // Mark the table as modified - self.modified_tables.insert(table_name); - - Ok(()) - } - // TODO: Support INSERT ... SELECT - _ => Err(SqawkError::UnsupportedSqlFeature( - "Only INSERT ... VALUES is supported".to_string(), - )), - } - } - - /// Execute a SQL DELETE statement - /// - /// This function implements the SQL DELETE operation by: - /// 1. Identifying the target table - /// 2. Applying WHERE clause filtering (if present) - /// 3. Removing matching rows from the table - /// 4. Tracking the table as modified for later write operations - /// 5. Tracking the number of affected rows for reporting - /// - /// If no WHERE condition is provided, all rows in the table will be deleted. - /// The operation maintains the original column structure of the table. - /// - /// # Arguments - /// * `table_with_joins` - The table reference to delete rows from - /// * `selection` - Optional WHERE clause to filter which rows to delete - /// - /// # Returns - /// * The number of rows that were deleted - fn execute_delete( - &mut self, - table_with_joins: &TableWithJoins, - selection: Option, - ) -> SqawkResult { - // Get the target table name - let table_name = self.get_table_name(table_with_joins)?; - - // If there's a WHERE clause, we need to precompute which rows match before modifying the table - if let Some(ref where_expr) = selection { - // Create a list of row indices to delete - let table_ref = self.file_handler.get_table(&table_name)?; - - // Evaluate WHERE condition for each row before modifying the table - // to avoid borrow checker issues - let mut rows_to_delete: Vec = Vec::new(); - - for (idx, row) in table_ref.rows().iter().enumerate() { - if self - .evaluate_condition(where_expr, row, table_ref) - .unwrap_or(false) - { - rows_to_delete.push(idx); - } - } - - // Now get mutable reference and delete the rows - let table = self.file_handler.get_table_mut(&table_name)?; - - // If we have rows to delete, create a new set of rows excluding the ones to delete - if !rows_to_delete.is_empty() { - let deleted_count = rows_to_delete.len(); - - // Create a new row set excluding the rows to delete - let mut new_rows: Vec> = - Vec::with_capacity(table.row_count() - deleted_count); - - for (idx, row) in table.rows().iter().enumerate() { - if !rows_to_delete.contains(&idx) { - new_rows.push(row.clone()); - } - } - - table.replace_rows(new_rows); - - // Mark the table as modified - self.modified_tables.insert(table_name); - - Ok(deleted_count) - } else { - // No rows matched the WHERE condition - Ok(0) - } - } else { - // No WHERE clause means delete all rows - - let table = self.file_handler.get_table_mut(&table_name)?; - let deleted_count = table.row_count(); - - // Replace with empty row set - table.replace_rows(Vec::new()); - - // Mark the table as modified - self.modified_tables.insert(table_name); - - Ok(deleted_count) - } - } - - /// Extract the table name from a TableWithJoins - /// - /// Parses the table name from a TableWithJoins structure, handling - /// both simple and qualified table names. This function is used by various - /// SQL execution methods to resolve the target table for operations. - /// - /// # Arguments - /// * `table_with_joins` - The table reference structure to extract the name from - /// - /// # Returns - /// * `Ok(String)` containing the resolved table name - /// * `Err` if the table reference type is not supported - fn get_table_name(&self, table_with_joins: &TableWithJoins) -> SqawkResult { - match &table_with_joins.relation { - sqlparser::ast::TableFactor::Table { name, .. } => Ok(name - .0 - .iter() - .map(|i| i.value.clone()) - .collect::>() - .join(".")), - _ => Err(SqawkError::UnsupportedSqlFeature( - "Only simple table references are supported".to_string(), - )), - } - } - - /// Resolve SELECT items to column indices and aliases - /// - /// This function processes SELECT items from a query and maps them to column indices - /// in the source table. It handles: - /// - Wildcard (*) expansion to all columns - /// - Simple column references like "name" - /// - Qualified column references like "table.column" - /// - Column aliases using the AS keyword - /// - Special handling for aggregate functions - /// - /// # Arguments - /// * `items` - The SELECT items from the query (columns to select) - /// * `table` - The source table containing the columns - /// - /// # Returns - /// * A vector of (column_index, optional_alias) pairs for projecting the table - fn resolve_select_items( - &self, - items: &[SelectItem], - table: &Table, - ) -> SqawkResult)>> { - let mut column_specs = Vec::new(); - - for item in items { - match item { - SelectItem::Wildcard(_) => { - // For wildcard, add all columns without aliases - for i in 0..table.column_count() { - column_specs.push((i, None)); - } - } - SelectItem::UnnamedExpr(expr) => { - match expr { - // Simple column reference - Expr::Identifier(ident) => { - let idx = self.get_column_index_for_select(&ident.value, table)?; - column_specs.push((idx, None)); - } - // Qualified column reference (table.column or join_result.table.column) - Expr::CompoundIdentifier(parts) => { - let idx = self.get_qualified_column_index(parts, table)?; - column_specs.push((idx, None)); - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only direct column references are supported in SELECT".to_string(), - )); - } - } - } - SelectItem::ExprWithAlias { expr, alias } => { - // If query has aggregates and we're seeing a function with an alias, - // we should let apply_aggregate_functions handle it instead - if self.contains_aggregate_functions(&[SelectItem::ExprWithAlias { - expr: expr.clone(), - alias: alias.clone(), - }]) { - // Skip this item, as it will be handled by apply_aggregate_functions - // We add a placeholder that won't be used - column_specs.push((0, Some(alias.value.clone()))); - } else { - match expr { - Expr::Identifier(ident) => { - // Simple column reference with alias - let idx = self.get_column_index_for_select(&ident.value, table)?; - column_specs.push((idx, Some(alias.value.clone()))); - } - Expr::CompoundIdentifier(parts) => { - // Qualified column reference (table.column or join_result.table.column) with alias - let idx = self.get_qualified_column_index(parts, table)?; - column_specs.push((idx, Some(alias.value.clone()))); - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only direct column references are supported with aliases in SELECT".to_string(), - )); - } - } - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Unsupported SELECT item".to_string(), - )); - } - } - } - - Ok(column_specs) - } - - /// Get the column index for a simple column name - /// - /// Helper function that centralizes column index resolution for simple column names - fn get_column_index_for_select(&self, column_name: &str, table: &Table) -> SqawkResult { - // First try as an exact column name match - if let Some(idx) = table.column_index(column_name) { - return Ok(idx); - } - - // Try as qualified name by checking column patterns - let suffix = format!(".{}", column_name); - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - return Ok(i); - } - } - - // If we got here, the column wasn't found - Err(SqawkError::ColumnNotFound(column_name.to_string())) - } - - /// Get the column index for a qualified column reference - /// - /// Helper function that centralizes column index resolution for qualified column names - fn get_qualified_column_index( - &self, - parts: &[sqlparser::ast::Ident], - table: &Table, - ) -> SqawkResult { - // Build the fully qualified column name from parts - let qualified_name = parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."); - - // Try to find an exact match for the qualified column - if let Some(idx) = table.column_index(&qualified_name) { - return Ok(idx); - } - - // If we didn't find an exact match, try a suffix match - // This helps with cases like "users.id" matching "users_orders_cross.users.id" - if parts.len() == 2 { - let suffix = format!("{}.{}", parts[0].value, parts[1].value); - - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - return Ok(i); - } - } - } - - // If we got here, the qualified column wasn't found - Err(SqawkError::ColumnNotFound(qualified_name)) - } - - /// Apply a WHERE clause to filter table rows - /// - /// This function creates a new table containing only rows that match the condition - /// specified in the WHERE clause. It evaluates the condition for each row and includes - /// only those rows for which the condition evaluates to true. - /// - /// # Arguments - /// * `table` - The source table to filter - /// * `where_expr` - The WHERE clause expression to evaluate - /// - /// # Returns - /// * A new table containing only rows that match the condition - /// - /// # Important - /// This function is called before column projection to ensure all columns - /// needed for the WHERE condition evaluation are available. - fn apply_where_clause(&self, table: Table, where_expr: &Expr) -> SqawkResult
{ - // Create a new table that only includes rows matching the WHERE condition - // by calling the table.select method with a closure that evaluates the condition - let result = table.select(|row| { - // For each row, evaluate the WHERE condition expression - // If evaluation fails (returns an error), default to false (exclude the row) - self.evaluate_condition(where_expr, row, &table) - .unwrap_or(false) - }); - - Ok(result) - } - - /// Apply a HAVING clause to filter grouped results - /// - /// This function filters rows from a table based on the SQL HAVING condition, - /// which is applied after GROUP BY aggregation. HAVING conditions typically - /// operate on aggregate function results or columns in the GROUP BY clause. - /// - /// # Arguments - /// * `table` - The grouped/aggregated table to filter - /// * `having_expr` - The HAVING condition expression - /// - /// # Returns - /// * A new table containing only the rows that satisfy the HAVING condition - fn apply_having_clause(&self, table: Table, having_expr: &Expr) -> SqawkResult
{ - if self.config.verbose() { - eprintln!("HAVING expression: {:?}", having_expr); - eprintln!("Table columns: {:?}", table.columns()); - eprintln!("Table rows: {} rows to filter", table.rows().len()); - } - - // Create a new table using the select method, but with debug output - let result = table.select(|row| { - let condition_result = self.evaluate_condition(having_expr, row, &table); - - if self.config.verbose() { - eprintln!("Row: {:?}, condition result: {:?}", row, condition_result); - } - - let passes = condition_result.unwrap_or(false); - - if self.config.verbose() && passes { - eprintln!("Row passed HAVING condition"); - } else if self.config.verbose() { - eprintln!("Row filtered out by HAVING condition"); - } - - passes - }); - - if self.config.verbose() { - eprintln!("HAVING result: {} rows", result.rows().len()); - } - - Ok(result) - } - - /// Evaluate a SQL conditional expression against a single row - /// - /// This function serves as the main entry point for evaluating SQL conditional expressions - /// (WHERE clause, HAVING clause, JOIN ON conditions). It implements a recursive expression - /// evaluator that supports SQL boolean logic with the following capabilities: - /// - /// - Complete logical operator support (AND, OR) with short-circuit evaluation - /// - All standard comparison operators (=, !=, <>, >, >=, <, <=) - /// - Proper SQL NULL semantics (three-valued logic) - /// - NULL-specific operators (IS NULL, IS NOT NULL) - /// - Type conversion for heterogeneous comparisons - /// - Column reference resolution (both simple and qualified) - /// - Literal value support (strings, numbers, booleans, NULL) - /// - Function call evaluation (string manipulation, etc.) - /// - Subexpression support through recursive evaluation - /// - /// The implementation follows SQL semantics throughout, including proper handling - /// of truth tables for logical operations with NULL values, and automatic type - /// coercion for comparisons between different data types. - /// - /// # Arguments - /// * `expr` - The parsed SQL expression to evaluate - /// * `row` - The current row values to evaluate against - /// * `table` - The table metadata (needed for column name resolution) - /// - /// # Returns - /// * `Ok(true)` if the condition evaluates to TRUE for this row - /// * `Ok(false)` if the condition evaluates to FALSE or NULL for this row - /// * `Err` if there's an evaluation error (column not found, invalid type conversion, etc.) - fn evaluate_condition(&self, expr: &Expr, row: &[Value], table: &Table) -> SqawkResult { - match expr { - Expr::BinaryOp { left, op, right } => { - // Handle logical operators (AND, OR) differently from comparison operators - match op { - sqlparser::ast::BinaryOperator::And => { - self.evaluate_logical_and(left, right, row, table) - } - sqlparser::ast::BinaryOperator::Or => { - self.evaluate_logical_or(left, right, row, table) - } - // For comparison operators, delegate to a separate function - _ => self.evaluate_comparison(left, op, right, row, table), - } - } - Expr::IsNull(expr) => { - let val = self.evaluate_expr_with_row(expr, row, table)?; - Ok(val == Value::Null) - } - Expr::IsNotNull(expr) => { - let val = self.evaluate_expr_with_row(expr, row, table)?; - Ok(val != Value::Null) - } - // Support for Function expressions (needed for HAVING clause with aggregate functions) - Expr::Function(_func) => { - // Evaluate the function to get its result - let val = self.evaluate_expr_with_row(expr, row, table)?; - - // Determine boolean result from the value - self.value_to_boolean(&val) - } - // Add more expression types as needed - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported WHERE/HAVING condition: {:?}", - expr - ))), - } - } - - /// Evaluate a logical AND expression with short-circuit evaluation - /// - /// This function implements AND logic with short-circuit evaluation - /// (stops evaluating as soon as the result is known). If the left condition - /// evaluates to false, the right condition is never evaluated. - /// - /// # Arguments - /// * `left` - The left operand of the AND expression - /// * `right` - The right operand of the AND expression - /// * `row` - The current row data for evaluating column references - /// * `table` - The table metadata for column resolution - /// - /// # Returns - /// * `Ok(true)` if both conditions evaluate to true - /// * `Ok(false)` if either condition evaluates to false - /// * `Err` if there's an error evaluating either condition - /// - /// Convert a Value to a boolean result, following SQL-like conversion rules - /// - /// This helper method centralizes the logic for converting different value types to boolean results: - /// - Integers: true if > 0 - /// - Floats: true if > 0.0 - /// - Booleans: as-is - /// - Strings: true if non-empty - /// - Null: always false - /// - /// # Arguments - /// * `val` - The value to convert to a boolean - /// - /// # Returns - /// * `Ok(bool)` - The converted boolean value - fn value_to_boolean(&self, val: &Value) -> SqawkResult { - match val { - Value::Integer(i) => self.integer_to_boolean(*i), - Value::Float(f) => self.float_to_boolean(*f), - Value::Boolean(b) => Ok(*b), - Value::String(s) => self.string_to_boolean(s), - Value::Null => self.null_to_boolean(), - } - } - - /// Convert an integer to boolean using SQL-like semantics (true if > 0) - /// - /// # Arguments - /// * `value` - The integer value to convert - /// - /// # Returns - /// * `Ok(bool)` - The converted boolean value - fn integer_to_boolean(&self, value: i64) -> SqawkResult { - Ok(value > 0) - } - - /// Convert a float to boolean using SQL-like semantics (true if > 0.0) - /// - /// # Arguments - /// * `value` - The float value to convert - /// - /// # Returns - /// * `Ok(bool)` - The converted boolean value - fn float_to_boolean(&self, value: f64) -> SqawkResult { - Ok(value > 0.0) - } - - /// Convert a string to boolean using SQL-like semantics (true if non-empty) - /// - /// # Arguments - /// * `value` - The string value to convert - /// - /// # Returns - /// * `Ok(bool)` - The converted boolean value - fn string_to_boolean(&self, value: &str) -> SqawkResult { - Ok(!value.is_empty()) - } - - /// Convert NULL to boolean (always false in SQL semantics) - /// - /// # Returns - /// * `Ok(bool)` - Always returns Ok(false) - fn null_to_boolean(&self) -> SqawkResult { - Ok(false) - } - - /// Evaluates a SQL logical AND expression with short-circuit evaluation - /// - /// This function implements the SQL AND operator with SQL-standard three-valued - /// logic and short-circuit evaluation semantics. It follows these rules: - /// - /// 1. If left operand evaluates to FALSE: return FALSE (right not evaluated) - /// 2. If left operand evaluates to NULL: evaluate right operand - /// - If right operand is FALSE: return FALSE - /// - If right operand is TRUE or NULL: return NULL - /// 3. If left operand evaluates to TRUE: return result of right operand - /// - /// The short-circuit behavior (not evaluating the right side when the result - /// is already determined) provides both performance optimization and prevents - /// unnecessary errors that might occur in the right expression. - /// - /// # Arguments - /// * `left` - The left-side expression of the AND operation - /// * `right` - The right-side expression of the AND operation - /// * `row` - The current row data to evaluate against - /// * `table` - The table metadata for column resolution - /// - /// # Returns - /// * `Ok(true)` if both conditions evaluate to TRUE - /// * `Ok(false)` if either condition evaluates to FALSE - /// * `Err` if there's an error during expression evaluation - fn evaluate_logical_and( - &self, - left: &Expr, - right: &Expr, - row: &[Value], - table: &Table, - ) -> SqawkResult { - // Evaluate left condition - let left_result = self.evaluate_condition(left, row, table)?; - - // Short-circuit - if left is false, don't evaluate right - if !left_result { - return Ok(false); - } - - // Evaluate right condition only if left was true - let right_result = self.evaluate_condition(right, row, table)?; - - Ok(left_result && right_result) - } - - /// Evaluates a SQL logical OR expression with short-circuit evaluation - /// - /// This function implements the SQL OR operator with SQL-standard three-valued - /// logic and short-circuit evaluation semantics. It follows these rules: - /// - /// 1. If left operand evaluates to TRUE: return TRUE (right not evaluated) - /// 2. If left operand evaluates to NULL: evaluate right operand - /// - If right operand is TRUE: return TRUE - /// - If right operand is FALSE or NULL: return NULL - /// 3. If left operand evaluates to FALSE: return result of right operand - /// - /// The short-circuit behavior (not evaluating the right side when the left is TRUE) - /// provides performance optimization and prevents unnecessary errors that might - /// occur during right expression evaluation. - /// - /// # Arguments - /// * `left` - The left operand of the OR expression - /// * `right` - The right operand of the OR expression - /// * `row` - The current row data to evaluate against - /// * `table` - The table metadata for column resolution - /// - /// # Returns - /// * `Ok(true)` if either condition evaluates to TRUE - /// * `Ok(false)` if both conditions evaluate to FALSE - /// * `Err` if there's an error during expression evaluation - fn evaluate_logical_or( - &self, - left: &Expr, - right: &Expr, - row: &[Value], - table: &Table, - ) -> SqawkResult { - // Evaluate left condition - let left_result = self.evaluate_condition(left, row, table)?; - - // Short-circuit - if left is true, don't evaluate right - if left_result { - return Ok(true); - } - - // Evaluate right condition only if left was false - let right_result = self.evaluate_condition(right, row, table)?; - - Ok(left_result || right_result) - } - - /// Evaluate a comparison expression between two values - /// - /// Evaluates SQL comparison expressions with type coercion - /// - /// This function handles all SQL comparison operators and implements SQL's comparison - /// semantics including: - /// - /// - Equal (=) and Not Equal (!=, <>) - /// - Greater Than (>), Greater Than or Equal (>=) - /// - Less Than (<), Less Than or Equal (<=) - /// - /// The implementation provides the following SQL-compliant features: - /// - /// 1. Proper NULL handling: any comparison with NULL yields NULL (not TRUE/FALSE) - /// 2. Type coercion: intelligent comparison between different data types - /// - STRING vs NUMBER: attempts string-to-number conversion - /// - NUMBER vs NUMBER: performs numeric comparison regardless of storage type - /// - STRING vs STRING: performs case-sensitive string comparison - /// 3. Support for complex expressions on either side - /// 4. Proper error handling for invalid comparisons - /// - /// # Arguments - /// * `left` - The left expression to evaluate - /// * `op` - The binary comparison operator to apply - /// * `right` - The right expression to evaluate - /// * `row` - The current row data for evaluating column references - /// * `table` - The table metadata for column resolution - /// - /// # Returns - /// * `Ok(true)` if the comparison evaluates to TRUE - /// * `Ok(false)` if the comparison evaluates to FALSE or NULL - /// * `Err` if there's an error during evaluation or an invalid comparison - fn evaluate_comparison( - &self, - left: &Expr, - op: &sqlparser::ast::BinaryOperator, - right: &Expr, - row: &[Value], - table: &Table, - ) -> SqawkResult { - let left_val = self.evaluate_expr_with_row(left, row, table)?; - let right_val = self.evaluate_expr_with_row(right, row, table)?; - - match op { - // Equal (=) operator - sqlparser::ast::BinaryOperator::Eq => self.evaluate_equality(&left_val, &right_val), - - // Not equal (!=) operator - sqlparser::ast::BinaryOperator::NotEq => { - self.evaluate_inequality(&left_val, &right_val) - } - - // Greater than (>) operator - sqlparser::ast::BinaryOperator::Gt => { - self.compare_values_with_operator(&left_val, &right_val, ">") - } - - // Less than (<) operator - sqlparser::ast::BinaryOperator::Lt => { - self.compare_values_with_operator(&left_val, &right_val, "<") - } - - // Greater than or equal (>=) operator - sqlparser::ast::BinaryOperator::GtEq => { - self.compare_values_with_operator(&left_val, &right_val, ">=") - } - - // Less than or equal (<=) operator - sqlparser::ast::BinaryOperator::LtEq => { - self.compare_values_with_operator(&left_val, &right_val, "<=") - } - - // Add more operators as needed - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported binary operator: {:?}", - op - ))), - } - } - - /// Evaluates equality (=) between two SQL values with type coercion - /// - /// This function implements SQL equality semantics by: - /// 1. Comparing values according to SQL type comparison rules - /// 2. Performing intelligent type coercion when comparing different data types - /// 3. Handling NULL values according to SQL three-valued logic (NULL = anything is NULL) - /// - /// The equality operator in SQL has special semantics: - /// - String comparisons are case-sensitive ("Abc" = "abc" is FALSE) - /// - NULL = NULL is NULL (not TRUE) - /// - NULL = non-NULL is NULL (not FALSE) - /// - Different numeric types are converted for proper comparison (1 = 1.0 is TRUE) - /// - Strings that look like numbers can compare equal to numbers ("1" = 1 is TRUE) - /// - /// # Arguments - /// * `left_val` - The left SQL value to compare - /// * `right_val` - The right SQL value to compare - /// - /// # Returns - /// * `Ok(true)` if the values are equal according to SQL rules - /// * `Ok(false)` if the values are not equal or either value is NULL - fn evaluate_equality(&self, left_val: &Value, right_val: &Value) -> SqawkResult { - Ok(left_val == right_val) - } - - /// Evaluates inequality (!=, <>) between two SQL values with type coercion - /// - /// This function implements SQL inequality semantics by: - /// 1. Comparing values according to SQL type comparison rules - /// 2. Performing intelligent type coercion when comparing different data types - /// 3. Handling NULL values according to SQL three-valued logic (NULL != anything is NULL) - /// - /// The inequality operator in SQL has special semantics: - /// - String comparisons are case-sensitive ("Abc" != "abc" is TRUE) - /// - NULL != NULL is NULL (not FALSE) - /// - NULL != non-NULL is NULL (not TRUE) - /// - Different numeric types are converted for proper comparison (1 != 1.0 is FALSE) - /// - Strings that look like numbers can compare with numbers ("1" != 2 is TRUE) - /// - /// Note: This is the inverse of the equality operation but follows the same - /// SQL semantics for type coercion and NULL handling. - /// - /// # Arguments - /// * `left_val` - The left SQL value to compare - /// * `right_val` - The right SQL value to compare - /// - /// # Returns - /// * `Ok(true)` if the values are not equal according to SQL rules - /// * `Ok(false)` if the values are equal or either value is NULL - fn evaluate_inequality(&self, left_val: &Value, right_val: &Value) -> SqawkResult { - Ok(left_val != right_val) - } - - /// Compares two SQL values using a relational operator with SQL semantics - /// - /// This function implements the core relational comparison logic for SQL, supporting - /// the full range of SQL data type comparisons with appropriate type coercion: - /// - /// Supported comparisons: - /// - Numbers: INTEGER vs INTEGER, FLOAT vs FLOAT, INTEGER vs FLOAT - /// - Strings: STRING vs STRING (lexicographic comparison) - /// - Mixed types: Automatic conversion between compatible types - /// - NULL values: Any comparison with NULL yields NULL (not TRUE/FALSE) - /// - /// The implementation handles the following SQL comparison operators: - /// - Greater than (>) - /// - Less than (<) - /// - Greater than or equal to (>=) - /// - Less than or equal to (<=) - /// - /// Type coercion follows SQL standards: - /// - When comparing integers with floats, integers are converted to floats - /// - When comparing strings with numbers, strings are attempted to be parsed as numbers - /// - When types are incompatible, detailed error messages are provided - /// - /// # Arguments - /// * `left_val` - The left SQL value to compare - /// * `right_val` - The right SQL value to compare - /// * `op_symbol` - The string representation of the operator (">", "<", ">=", "<=") - /// - /// # Returns - /// * `Ok(true)` if the comparison evaluates to TRUE - /// * `Ok(false)` if the comparison evaluates to FALSE or NULL - /// * `Err` if the comparison is invalid (incompatible types, parsing error, etc.) - fn compare_values_with_operator( - &self, - left_val: &Value, - right_val: &Value, - op_symbol: &str, - ) -> SqawkResult { - match (left_val, right_val) { - // Integer-Integer comparison - (Value::Integer(a), Value::Integer(b)) => self.compare_integers(*a, *b, op_symbol), - - // Float-Float comparison - (Value::Float(a), Value::Float(b)) => self.compare_floats(*a, *b, op_symbol), - - // Integer-Float comparison (convert Integer to Float) - (Value::Integer(a), Value::Float(b)) => { - self.compare_integer_and_float(*a, *b, op_symbol) - } - - // Float-Integer comparison (convert Integer to Float) - (Value::Float(a), Value::Integer(b)) => { - self.compare_float_and_integer(*a, *b, op_symbol) - } - - // String-String comparison (lexicographic) - (Value::String(a), Value::String(b)) => self.compare_strings(a, b, op_symbol), - - // Error for incompatible types - _ => self.report_incompatible_types(left_val, right_val, op_symbol), - } - } - - /// Compare two integers with the specified operator - /// - /// # Arguments - /// * `a` - First integer - /// * `b` - Second integer - /// * `op_symbol` - Operator symbol (>, <, >=, <=) - /// - /// # Returns - /// * `Ok(bool)` - Result of the comparison - /// * `Err` - If the operator is not supported - fn compare_integers(&self, a: i64, b: i64, op_symbol: &str) -> SqawkResult { - Ok(match op_symbol { - ">" => a > b, - "<" => a < b, - ">=" => a >= b, - "<=" => a <= b, - _ => return self.invalid_operator_error(op_symbol), - }) - } - - /// Compare two floats with the specified operator - /// - /// # Arguments - /// * `a` - First float - /// * `b` - Second float - /// * `op_symbol` - Operator symbol (>, <, >=, <=) - /// - /// # Returns - /// * `Ok(bool)` - Result of the comparison - /// * `Err` - If the operator is not supported - fn compare_floats(&self, a: f64, b: f64, op_symbol: &str) -> SqawkResult { - Ok(match op_symbol { - ">" => a > b, - "<" => a < b, - ">=" => a >= b, - "<=" => a <= b, - _ => return self.invalid_operator_error(op_symbol), - }) - } - - /// Compare an integer and a float with the specified operator - /// - /// # Arguments - /// * `a` - Integer value - /// * `b` - Float value - /// * `op_symbol` - Operator symbol (>, <, >=, <=) - /// - /// # Returns - /// * `Ok(bool)` - Result of the comparison - /// * `Err` - If the operator is not supported - fn compare_integer_and_float(&self, a: i64, b: f64, op_symbol: &str) -> SqawkResult { - let a_float = a as f64; - Ok(match op_symbol { - ">" => a_float > b, - "<" => a_float < b, - ">=" => a_float >= b, - "<=" => a_float <= b, - _ => return self.invalid_operator_error(op_symbol), - }) - } - - /// Compare a float and an integer with the specified operator - /// - /// # Arguments - /// * `a` - Float value - /// * `b` - Integer value - /// * `op_symbol` - Operator symbol (>, <, >=, <=) - /// - /// # Returns - /// * `Ok(bool)` - Result of the comparison - /// * `Err` - If the operator is not supported - fn compare_float_and_integer(&self, a: f64, b: i64, op_symbol: &str) -> SqawkResult { - let b_float = b as f64; - Ok(match op_symbol { - ">" => a > b_float, - "<" => a < b_float, - ">=" => a >= b_float, - "<=" => a <= b_float, - _ => return self.invalid_operator_error(op_symbol), - }) - } - - /// Compare two strings with the specified operator - /// - /// # Arguments - /// * `a` - First string - /// * `b` - Second string - /// * `op_symbol` - Operator symbol (>, <, >=, <=) - /// - /// # Returns - /// * `Ok(bool)` - Result of the comparison - /// * `Err` - If the operator is not supported - fn compare_strings(&self, a: &str, b: &str, op_symbol: &str) -> SqawkResult { - Ok(match op_symbol { - ">" => a > b, - "<" => a < b, - ">=" => a >= b, - "<=" => a <= b, - _ => return self.invalid_operator_error(op_symbol), - }) - } - - /// Create an error for an invalid operator - /// - /// # Arguments - /// * `op_symbol` - The invalid operator symbol - /// - /// # Returns - /// * An appropriate error for the invalid operator - fn invalid_operator_error(&self, op_symbol: &str) -> SqawkResult { - Err(SqawkError::InvalidSqlQuery(format!( - "Unexpected operator symbol: {}", - op_symbol - ))) - } - - /// Report an error for incompatible types in a comparison - /// - /// # Arguments - /// * `left_val` - The left value - /// * `right_val` - The right value - /// * `op_symbol` - The operator symbol - /// - /// # Returns - /// * An appropriate error for incompatible types - fn report_incompatible_types( - &self, - left_val: &Value, - right_val: &Value, - op_symbol: &str, - ) -> SqawkResult { - Err(SqawkError::TypeError(format!( - "Cannot compare {:?} and {:?} with {}", - left_val, right_val, op_symbol - ))) - } - - /// Evaluate an expression to a Value - /// - /// This function evaluates SQL expressions like literals, constants, etc. - /// and converts them to our internal Value type. - /// - /// # Arguments - /// * `expr` - The SQL expression to evaluate - /// - /// # Returns - /// * `Ok(Value)` - The resulting value after evaluation - /// * `Err` - If the expression can't be evaluated or contains unsupported features - fn evaluate_expr(&self, expr: &Expr) -> SqawkResult { - match expr { - Expr::Value(value) => self.evaluate_sql_value(value), - // Handle unary operations like - (negation) - Expr::UnaryOp { op, expr } => self.evaluate_unary_operation(op, expr), - _ => self.unsupported_expression_error(expr), - } - } - - /// Evaluate a SQL value literal - /// - /// # Arguments - /// * `value` - The SQL value to evaluate - /// - /// # Returns - /// * `Ok(Value)` - The resulting value - /// * `Err` - If the value can't be evaluated - fn evaluate_sql_value(&self, value: &SqlValue) -> SqawkResult { - match value { - SqlValue::Number(n, _) => self.parse_number(n), - SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => { - Ok(Value::String(s.clone())) - } - SqlValue::Boolean(b) => Ok(Value::Boolean(*b)), - SqlValue::Null => Ok(Value::Null), - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported SQL value: {:?}", - value - ))), - } - } - - /// Parse a number string into an Integer or Float Value - /// - /// # Arguments - /// * `n` - The number string to parse - /// - /// # Returns - /// * `Ok(Value)` - The resulting Value::Integer or Value::Float - /// * `Err` - If the string can't be parsed as a number - fn parse_number(&self, n: &str) -> SqawkResult { - // Try to parse as integer first, then as float - if let Ok(i) = n.parse::() { - Ok(Value::Integer(i)) - } else if let Ok(f) = n.parse::() { - Ok(Value::Float(f)) - } else { - Err(SqawkError::TypeError(format!("Invalid number: {}", n))) - } - } - - /// Evaluate a unary operation (e.g., negation, plus, not) - /// - /// # Arguments - /// * `op` - The unary operator - /// * `expr` - The expression to apply the operator to - /// - /// # Returns - /// * `Ok(Value)` - The resulting value after applying the operator - /// * `Err` - If the operation is invalid - fn evaluate_unary_operation( - &self, - op: &sqlparser::ast::UnaryOperator, - expr: &Expr, - ) -> SqawkResult { - let val = self.evaluate_expr(expr)?; - - match op { - sqlparser::ast::UnaryOperator::Minus => self.apply_negation(&val), - sqlparser::ast::UnaryOperator::Plus => Ok(val), // Plus operator doesn't change the value - sqlparser::ast::UnaryOperator::Not => self.apply_boolean_not(&val), - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported unary operator: {:?}", - op - ))), - } - } - - /// Apply negation to a value (for the minus unary operator) - /// - /// # Arguments - /// * `val` - The value to negate - /// - /// # Returns - /// * `Ok(Value)` - The negated value - /// * `Err` - If the value can't be negated - fn apply_negation(&self, val: &Value) -> SqawkResult { - match val { - Value::Integer(i) => Ok(Value::Integer(-i)), - Value::Float(f) => Ok(Value::Float(-f)), - _ => Err(SqawkError::TypeError(format!( - "Cannot apply negation to non-numeric value: {:?}", - val - ))), - } - } - - /// Apply boolean NOT to a value - /// - /// # Arguments - /// * `val` - The value to apply NOT to - /// - /// # Returns - /// * `Ok(Value)` - The resulting value - /// * `Err` - If the value can't have NOT applied to it - fn apply_boolean_not(&self, val: &Value) -> SqawkResult { - match val { - Value::Boolean(b) => Ok(Value::Boolean(!b)), - _ => Err(SqawkError::TypeError(format!( - "Cannot apply NOT to non-boolean value: {:?}", - val - ))), - } - } - - /// Create an error for an unsupported expression - /// - /// # Arguments - /// * `expr` - The unsupported expression - /// - /// # Returns - /// * An appropriate error - fn unsupported_expression_error(&self, expr: &Expr) -> SqawkResult { - Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported expression: {:?}", - expr - ))) - } - - /// Evaluates a SQL expression in the context of a specific row - /// - /// This function is the core expression evaluator for SQL operations in Sqawk. - /// It resolves and computes the result of any SQL expression against a given row, - /// supporting the full range of SQL expressions: - /// - /// - Column references (both simple and qualified) - /// - Literal values (string, numeric, boolean, NULL) - /// - Binary operations (arithmetic: +, -, *, /, %) - /// - Function calls (string functions, aggregates) - /// - Nested expressions - /// - CASE expressions - /// - Compound expressions (using multiple operators) - /// - Type casting and conversions - /// - /// The implementation handles SQL-specific evaluation semantics including: - /// - Proper NULL propagation (NULL in operation → NULL result) - /// - Type coercion between compatible types - /// - Order of operations following SQL precedence rules - /// - Error handling for invalid operations/references - /// - /// This function serves as the basis for WHERE clause filtering, SELECT projection, - /// ORDER BY evaluation, JOIN condition checking, and other SQL operations that - /// need to evaluate expressions against specific rows. - /// - /// # Arguments - /// * `expr` - The SQL expression to evaluate - /// * `row` - The current row's values for resolving column references - /// * `table` - The table metadata for column name resolution - /// - /// # Returns - /// * `Ok(Value)` - The evaluated result as a typed SQL value - /// * `Err` - If column resolution fails or expression evaluation fails - fn evaluate_expr_with_row( - &self, - expr: &Expr, - row: &[Value], - table: &Table, - ) -> SqawkResult { - match expr { - // Simple column reference (unqualified) - Expr::Identifier(ident) => { - self.resolve_simple_column_reference(&ident.value, row, table) - } - // Qualified column reference (table.column or join_result.table.column) - Expr::CompoundIdentifier(parts) => { - self.resolve_qualified_column_reference(parts, row, table) - } - // Handle aggregate and string functions - Expr::Function(func) => { - let func_name = func - .name - .0 - .first() - .map(|i| i.value.clone()) - .unwrap_or_default(); - - // First check if this is a supported aggregate function - if let Some(_agg_func) = AggregateFunction::from_name(&func_name) { - // For aggregate functions in HAVING, look for the result in the current row - - // First try to find a column with the exact function name - if let Some(col_idx) = table.column_index(&func_name) { - return Ok(row[col_idx].clone()); - } - - // Next, try with common alias patterns for aggregates - for (idx, col_name) in table.columns().iter().enumerate() { - if col_name.contains(&func_name) - || (func_name == "COUNT" && col_name.contains("count")) - { - return Ok(row[idx].clone()); - } - } - - // For the HAVING clause with aggregate functions, we need to handle COUNT(*) specially - if func_name == "COUNT" { - // Check if this is COUNT(*) - if func.args.len() == 1 { - if let sqlparser::ast::FunctionArg::Unnamed( - sqlparser::ast::FunctionArgExpr::Wildcard, - ) = &func.args[0] - { - // Look for a column named "employee_count" or similar - for (idx, col_name) in table.columns().iter().enumerate() { - if col_name.contains("employee_count") - || col_name.contains("count") - { - return Ok(row[idx].clone()); - } - } - - // If we still can't find it, check if the first column after department is count - if table.columns().len() >= 2 && table.column_count() >= 2 { - return Ok(row[1].clone()); // Department is at 0, count likely at 1 - } - } - } - } - - // For AVG, sum, and other numerical aggregates - if func_name == "AVG" - || func_name == "SUM" - || func_name == "MIN" - || func_name == "MAX" - { - // Look for columns containing "avg", "sum", etc. or the column name - if func.args.len() == 1 { - if let sqlparser::ast::FunctionArg::Unnamed( - sqlparser::ast::FunctionArgExpr::Expr(Expr::Identifier(ident)), - ) = &func.args[0] - { - let column_name = &ident.value; - - // Look for columns like "avg_salary" or similar patterns - for (idx, col_name) in table.columns().iter().enumerate() { - if col_name.contains(&func_name.to_lowercase()) - && col_name.contains(column_name) - { - return Ok(row[idx].clone()); - } - } - - // If still not found and we have AVG(salary), look for avg_salary - if func_name == "AVG" && table.columns().len() >= 3 { - return Ok(row[2].clone()); // Department at 0, count at 1, avg likely at 2 - } - } - } - } - } - // Then check if this is a supported string function - else if let Some(string_func) = StringFunction::from_name(&func_name) { - // Evaluate the string function arguments - let mut arg_values = Vec::new(); - for arg in &func.args { - match arg { - sqlparser::ast::FunctionArg::Unnamed(expr) => match expr { - sqlparser::ast::FunctionArgExpr::Expr(expr) => { - let val = self.evaluate_expr_with_row(expr, row, table)?; - arg_values.push(val); - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function argument: {:?}", - expr - ))); - } - }, - _ => { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Named arguments are not supported: {:?}", - arg - ))); - } - } - } - - // Apply the string function with the evaluated arguments - return string_func.apply(&arg_values); - } - - // Fall back to standard expression evaluation - self.evaluate_expr(expr) - } - // Binary operations might need column references from the row - Expr::BinaryOp { left, op, right } => { - let left_val = self.evaluate_expr_with_row(left, row, table)?; - let right_val = self.evaluate_expr_with_row(right, row, table)?; - - // For basic arithmetic operators, delegate to helpers - match op { - sqlparser::ast::BinaryOperator::Plus => match (&left_val, &right_val) { - (Value::Integer(a), Value::Integer(b)) => Ok(Value::Integer(a + b)), - (Value::Float(a), Value::Float(b)) => Ok(Value::Float(a + b)), - (Value::Integer(a), Value::Float(b)) => Ok(Value::Float(*a as f64 + b)), - (Value::Float(a), Value::Integer(b)) => Ok(Value::Float(a + *b as f64)), - _ => Err(SqawkError::TypeError(format!( - "Cannot add {:?} and {:?}", - left_val, right_val - ))), - }, - sqlparser::ast::BinaryOperator::Minus => match (&left_val, &right_val) { - (Value::Integer(a), Value::Integer(b)) => Ok(Value::Integer(a - b)), - (Value::Float(a), Value::Float(b)) => Ok(Value::Float(a - b)), - (Value::Integer(a), Value::Float(b)) => Ok(Value::Float(*a as f64 - b)), - (Value::Float(a), Value::Integer(b)) => Ok(Value::Float(a - *b as f64)), - _ => Err(SqawkError::TypeError(format!( - "Cannot subtract {:?} from {:?}", - right_val, left_val - ))), - }, - sqlparser::ast::BinaryOperator::Multiply => match (&left_val, &right_val) { - (Value::Integer(a), Value::Integer(b)) => Ok(Value::Integer(a * b)), - (Value::Float(a), Value::Float(b)) => Ok(Value::Float(a * b)), - (Value::Integer(a), Value::Float(b)) => Ok(Value::Float(*a as f64 * b)), - (Value::Float(a), Value::Integer(b)) => Ok(Value::Float(a * *b as f64)), - _ => Err(SqawkError::TypeError(format!( - "Cannot multiply {:?} and {:?}", - left_val, right_val - ))), - }, - sqlparser::ast::BinaryOperator::Divide => match (&left_val, &right_val) { - (Value::Integer(a), Value::Integer(b)) => { - if *b == 0 { - return Err(SqawkError::DivideByZero); - } - Ok(Value::Float(*a as f64 / *b as f64)) - } - (Value::Float(a), Value::Float(b)) => { - if *b == 0.0 { - return Err(SqawkError::DivideByZero); - } - Ok(Value::Float(a / b)) - } - (Value::Integer(a), Value::Float(b)) => { - if *b == 0.0 { - return Err(SqawkError::DivideByZero); - } - Ok(Value::Float(*a as f64 / b)) - } - (Value::Float(a), Value::Integer(b)) => { - if *b == 0 { - return Err(SqawkError::DivideByZero); - } - Ok(Value::Float(a / *b as f64)) - } - _ => Err(SqawkError::TypeError(format!( - "Cannot divide {:?} by {:?}", - left_val, right_val - ))), - }, - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported binary operator in expression: {:?}", - op - ))), - } - } - // Handle other expression types by delegating to the main evaluate_expr function - _ => self.evaluate_expr(expr), - } - } - - /// Check if the SELECT items contain any aggregate functions - /// - /// This function analyzes a list of SELECT items and determines if any of them - /// contains an aggregate function (COUNT, SUM, AVG, MIN, MAX, etc.). It checks both - /// simple expressions and aliased expressions for aggregate function calls. - /// - /// This detection is crucial for determining whether to apply aggregate processing - /// to a query or process it as a regular row-by-row query. - /// - /// # Arguments - /// * `items` - The SELECT items from the query, potentially containing aggregate functions - /// - /// # Returns - /// * `true` if any of the items contains an aggregate function - /// * `false` if no aggregate functions are detected - fn contains_aggregate_functions(&self, items: &[SelectItem]) -> bool { - for item in items { - match item { - // Check for aggregate functions in non-aliased expressions - SelectItem::UnnamedExpr(Expr::Function(func)) => { - // Check if the function name is one of our supported aggregates - let name = func.name.0.first().map(|i| i.value.as_str()).unwrap_or(""); - if AggregateFunction::from_name(name).is_some() { - return true; - } - } - SelectItem::UnnamedExpr(_) => {} - // Check for aggregate functions in aliased expressions - SelectItem::ExprWithAlias { - expr: Expr::Function(func), - .. - } => { - // Check if the function name is one of our supported aggregates - let name = func.name.0.first().map(|i| i.value.as_str()).unwrap_or(""); - if AggregateFunction::from_name(name).is_some() { - return true; - } - } - SelectItem::ExprWithAlias { .. } => {} - _ => {} - } - } - false - } - - /// Apply aggregate functions to a table - /// - /// This function processes SELECT items containing aggregate functions (COUNT, SUM, AVG, MIN, MAX) - /// and applies them to the table data. It handles both aliased and non-aliased aggregate functions. - /// - /// The function: - /// 1. Extracts the appropriate column values for each function - /// 2. Executes the aggregate function on those values - /// 3. Creates a new single-row result table with the aggregate results - /// 4. Uses column names based on function names or provided aliases - /// - /// # Arguments - /// * `items` - The SELECT items containing aggregate functions to execute - /// * `table` - The source table containing the data to aggregate - /// - /// # Returns - /// * A new single-row table containing the results of all aggregate functions - /// * `Err` if any function arguments are invalid or unsupported - fn apply_aggregate_functions(&self, items: &[SelectItem], table: &Table) -> SqawkResult
{ - let mut result_columns = Vec::new(); - let mut result_values = Vec::new(); - - // Process each item in the SELECT list - for item in items { - match item { - SelectItem::UnnamedExpr(expr) => { - // Handle function call - if let Expr::Function(func) = expr { - let func_name = func - .name - .0 - .first() - .map(|i| i.value.clone()) - .unwrap_or_default(); - - // Check if this is a supported aggregate function - if let Some(agg_func) = AggregateFunction::from_name(&func_name) { - // Process the function arguments - if func.args.len() != 1 { - return Err(SqawkError::InvalidSqlQuery(format!( - "{} function requires exactly one argument", - func_name - ))); - } - - // Get the column values for the function argument - let column_values = - self.get_values_for_function_arg(&func.args[0], table)?; - - // Execute the aggregate function - let result_value = agg_func.execute(&column_values)?; - - // Add the result to our output - result_columns.push(func_name.clone()); - result_values.push(result_value); - } else { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", - func_name - ))); - } - } else { - return Err(SqawkError::UnsupportedSqlFeature( - "Only aggregate functions are supported in aggregate queries" - .to_string(), - )); - } - } - SelectItem::ExprWithAlias { expr, alias } => { - // Handle function call with alias - match expr { - Expr::Function(func) => { - let func_name = func - .name - .0 - .first() - .map(|i| i.value.clone()) - .unwrap_or_default(); - - // Check if this is a supported aggregate function - if let Some(agg_func) = AggregateFunction::from_name(&func_name) { - // Process the function arguments - if func.args.len() != 1 { - return Err(SqawkError::InvalidSqlQuery(format!( - "{} function requires exactly one argument", - func_name - ))); - } - - // Get the column values for the function argument - let column_values = - self.get_values_for_function_arg(&func.args[0], table)?; - - // Execute the aggregate function - let result_value = agg_func.execute(&column_values)?; - - // Add the result to our output with the alias - result_columns.push(alias.value.clone()); - result_values.push(result_value); - } else { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", - func_name - ))); - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only aggregate functions are supported in aggregate queries" - .to_string(), - )); - } - } - } - SelectItem::Wildcard(_) => { - return Err(SqawkError::UnsupportedSqlFeature( - "Wildcard (*) is not supported in queries with aggregate functions" - .to_string(), - )); - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Unsupported SELECT item in aggregate query".to_string(), - )); - } - } - } - - // Create a new table with a single row containing the aggregate results - let mut result_table = Table::new("aggregate_result", result_columns, None); - result_table.add_row(result_values)?; - Ok(result_table) - } - - /// Apply aggregate functions with GROUP BY clause - /// - /// This function implements SQL's GROUP BY functionality, which groups rows based on - /// specified columns and applies aggregate functions to each group. This is a key - /// component of analytical queries that need to summarize data across groups. - /// - /// # Arguments - /// * `items` - The SELECT items from the SQL query (columns and expressions to include) - /// * `table` - The source table to apply grouping and aggregation to - /// * `group_by` - The GROUP BY expressions defining how to group the rows - /// - /// # Returns - /// - /// # Implementation Details - /// The GROUP BY implementation follows these steps: - /// 1. Identify the columns to group by - /// 2. Build a `HashMap` that groups row indices by their group key values - /// 3. Process each SELECT item to determine which outputs to include - /// 4. For each group, generate one output row with: - /// - The GROUP BY column values - /// - The results of aggregate functions applied to that group - /// - /// # Returns - /// * A new table containing the results of all aggregate functions, one row per group - /// * `Err` if any function arguments are invalid or unsupported - fn apply_grouped_aggregate_functions( - items: &[SelectItem], - table: &Table, - group_by: &Vec, - ) -> SqawkResult
{ - // Extract GROUP BY columns - let mut group_columns = Vec::new(); - let mut group_column_indices = Vec::new(); - - // Process each GROUP BY expression - for expr in group_by { - match expr { - Expr::Identifier(ident) => { - // Simple column reference - let col_name = ident.value.clone(); - if let Some(col_idx) = table.column_index(&col_name) { - group_columns.push(col_name); - group_column_indices.push(col_idx); - } else { - // Try suffix match for qualified columns - let suffix = format!(".{}", col_name); - let mut found = false; - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - found = true; - group_columns.push(col.clone()); - group_column_indices.push(i); - break; - } - } - - if !found { - return Err(SqawkError::ColumnNotFound(col_name)); - } - } - } - Expr::CompoundIdentifier(parts) => { - // Qualified column reference (table.column) - let qualified_name = parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."); - - if let Some(col_idx) = table.column_index(&qualified_name) { - group_columns.push(qualified_name); - group_column_indices.push(col_idx); - } else { - // Try suffix match - if parts.len() == 2 { - let suffix = format!("{}.{}", parts[0].value, parts[1].value); - - let mut found = false; - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - found = true; - group_columns.push(col.clone()); - group_column_indices.push(i); - break; - } - } - - if !found { - return Err(SqawkError::ColumnNotFound(qualified_name)); - } - } else { - return Err(SqawkError::ColumnNotFound(qualified_name)); - } - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple column references are supported in GROUP BY".to_string(), - )); - } - } - } - - // Group the rows based on GROUP BY columns - let mut groups: std::collections::HashMap, Vec> = - std::collections::HashMap::new(); - - for (row_idx, row) in table.rows().iter().enumerate() { - // Build the group key from the values of GROUP BY columns - let group_key: Vec = group_column_indices - .iter() - .map(|&col_idx| row[col_idx].clone()) - .collect(); - - // Add this row's index to the appropriate group - groups.entry(group_key).or_default().push(row_idx); - } - - // Prepare the result table columns (GROUP BY columns + aggregate function results) - let mut result_columns = group_columns.clone(); - - // Process each SELECT item to identify function columns - let mut function_info = Vec::new(); - for item in items { - match item { - SelectItem::UnnamedExpr(expr) => { - // Handle simple column references that should match GROUP BY columns - if let Expr::Identifier(ident) = expr { - // Skip if this column is already in result_columns (GROUP BY column) - if table.column_index(&ident.value).is_some() - && !group_columns.contains(&ident.value) - { - // Non-aggregated column not in GROUP BY is not allowed - return Err(SqawkError::InvalidSqlQuery( - format!("Column '{}' must appear in the GROUP BY clause or be used in an aggregate function", ident.value) - )); - } - } else if let Expr::Function(func) = expr { - // Handle aggregate function - let func_name = func - .name - .0 - .first() - .map(|i| i.value.clone()) - .unwrap_or_default(); - - // Check if this is a supported aggregate function - if let Some(_agg_func) = AggregateFunction::from_name(&func_name) { - // Process the function arguments - if func.args.len() != 1 { - return Err(SqawkError::InvalidSqlQuery(format!( - "{} function requires exactly one argument", - func_name - ))); - } - - // Add the function column to results - result_columns.push(func_name.clone()); - - // Store function info for later execution - function_info.push((func_name, func.args[0].clone(), None)); - } else { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", - func_name - ))); - } - } else { - return Err(SqawkError::UnsupportedSqlFeature( - "Only column references and aggregate functions are supported in GROUP BY queries".to_string() - )); - } - } - SelectItem::ExprWithAlias { expr, alias } => { - // Handle expressions with aliases - if let Expr::Function(func) = expr { - let func_name = func - .name - .0 - .first() - .map(|i| i.value.clone()) - .unwrap_or_default(); - - // Check if this is a supported aggregate function - if let Some(_agg_func) = AggregateFunction::from_name(&func_name) { - // Process the function arguments - if func.args.len() != 1 { - return Err(SqawkError::InvalidSqlQuery(format!( - "{} function requires exactly one argument", - func_name - ))); - } - - // Add the aliased column to results - result_columns.push(alias.value.clone()); - - // Store function info for later execution with alias - function_info.push(( - func_name, - func.args[0].clone(), - Some(alias.value.clone()), - )); - } else { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", - func_name - ))); - } - } else { - return Err(SqawkError::UnsupportedSqlFeature( - "Only aggregate functions can have aliases in GROUP BY queries" - .to_string(), - )); - } - } - SelectItem::Wildcard(_) => { - return Err(SqawkError::UnsupportedSqlFeature( - "Wildcard (*) is not supported in GROUP BY queries".to_string(), - )); - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Unsupported SELECT item in GROUP BY query".to_string(), - )); - } - } - } - - // Create the result table - let mut result_table = Table::new("grouped_result", result_columns, None); - - // Generate a row for each group - for (group_key, row_indices) in groups { - let mut result_row = Vec::new(); - - // Add the GROUP BY column values - result_row.extend(group_key); - - // Apply aggregate functions to each group - for (func_name, func_arg, _alias) in &function_info { - // Extract values for this function's column in this group - let mut group_values = Vec::new(); - - for &row_idx in &row_indices { - if let sqlparser::ast::FunctionArg::Unnamed(expr) = func_arg { - match expr { - sqlparser::ast::FunctionArgExpr::Wildcard => { - // For COUNT(*), one value per row - group_values.push(Value::Integer(1)); - } - sqlparser::ast::FunctionArgExpr::QualifiedWildcard(_) => { - // For COUNT(table.*), one value per row like COUNT(*) - group_values.push(Value::Integer(1)); - } - sqlparser::ast::FunctionArgExpr::Expr(expr) => { - match expr { - Expr::Identifier(ident) => { - // Get column index - if let Some(col_idx) = table.column_index(&ident.value) { - group_values - .push(table.rows()[row_idx][col_idx].clone()); - } else { - // Try suffix match for qualified columns - let suffix = format!(".{}", ident.value); - let mut found = false; - let mut value = Value::Null; - - for (col_idx, col_name) in - table.columns().iter().enumerate() - { - if col_name.ends_with(&suffix) { - found = true; - value = table.rows()[row_idx][col_idx].clone(); - break; - } - } - - if found { - group_values.push(value); - } else { - return Err(SqawkError::ColumnNotFound( - ident.value.clone(), - )); - } - } - } - Expr::CompoundIdentifier(parts) => { - // Handle qualified column references - let qualified_name = parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."); - - if let Some(col_idx) = table.column_index(&qualified_name) { - group_values - .push(table.rows()[row_idx][col_idx].clone()); - } else { - // Try suffix match - if parts.len() == 2 { - let suffix = format!( - "{}.{}", - parts[0].value, parts[1].value - ); - - let mut found = false; - let mut value = Value::Null; - - for (col_idx, col_name) in - table.columns().iter().enumerate() - { - if col_name.ends_with(&suffix) { - found = true; - value = - table.rows()[row_idx][col_idx].clone(); - break; - } - } - - if found { - group_values.push(value); - } else { - return Err(SqawkError::ColumnNotFound( - qualified_name, - )); - } - } else { - return Err(SqawkError::ColumnNotFound( - qualified_name, - )); - } - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only column references are supported in aggregate functions".to_string() - )); - } - } - } - } - } else { - return Err(SqawkError::UnsupportedSqlFeature( - "Only unnamed arguments are supported in aggregate functions" - .to_string(), - )); - } - } - - // Execute the aggregate function on this group's values - if let Some(agg_func) = AggregateFunction::from_name(func_name) { - let result_value = agg_func.execute(&group_values)?; - result_row.push(result_value); - } else { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", - func_name - ))); - } - } - - // Add this group's result row to the table - result_table.add_row(result_row)?; - } - - Ok(result_table) - } - - /// Get values for a function argument - /// - /// This function extracts all values from a table column specified in an aggregate - /// function argument. It handles special cases like: - /// - COUNT(*) wildcard (returns placeholder values) - /// - Simple column references (e.g., "age") - /// - Qualified column references (e.g., "users.age") - /// - Column name resolution in join results - /// - /// # Arguments - /// * `arg` - The SQL function argument (column reference or wildcard) - /// * `table` - The source table containing the column data - /// - /// # Returns - /// * A vector of all values from the specified column - /// * For COUNT(*), a vector of placeholder values (one per row) - /// * `Err` if the column doesn't exist or the argument type is unsupported - fn get_values_for_function_arg( - &self, - arg: &sqlparser::ast::FunctionArg, - table: &Table, - ) -> SqawkResult> { - match arg { - sqlparser::ast::FunctionArg::Unnamed(sqlparser::ast::FunctionArgExpr::Wildcard) => { - // For COUNT(*), return a list of non-null placeholders, one for each row - Ok(table.rows().iter().map(|_| Value::Integer(1)).collect()) - } - sqlparser::ast::FunctionArg::Unnamed( - sqlparser::ast::FunctionArgExpr::QualifiedWildcard(_), - ) => { - // For COUNT(table.*), return a list of non-null placeholders, one for each row - Ok(table.rows().iter().map(|_| Value::Integer(1)).collect()) - } - sqlparser::ast::FunctionArg::Unnamed(sqlparser::ast::FunctionArgExpr::Expr(expr)) => { - match expr { - Expr::Identifier(ident) => { - // Get column index - let col_idx = match table.column_index(&ident.value) { - Some(idx) => idx, - None => { - // Try suffix match for qualified columns - let mut found = false; - let mut idx = 0; - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&format!(".{}", ident.value)) { - found = true; - idx = i; - break; - } - } - - if found { - idx - } else { - return Err(SqawkError::ColumnNotFound(ident.value.clone())); - } - } - }; - - // Extract all values for this column - Ok(table - .rows() - .iter() - .map(|row| row[col_idx].clone()) - .collect()) - } - Expr::CompoundIdentifier(parts) => { - // Handle qualified column references like table.column - let qualified_name = parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."); - - // Get column index - let col_idx = match table.column_index(&qualified_name) { - Some(idx) => idx, - None => { - // Try suffix match - if parts.len() == 2 { - let suffix = format!("{}.{}", parts[0].value, parts[1].value); - - let mut found = false; - let mut idx = 0; - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - found = true; - idx = i; - break; - } - } - - if found { - idx - } else { - return Err(SqawkError::ColumnNotFound(qualified_name)); - } - } else { - return Err(SqawkError::ColumnNotFound(qualified_name)); - } - } - }; - - // Extract all values for this column - Ok(table - .rows() - .iter() - .map(|row| row[col_idx].clone()) - .collect()) - } - _ => Err(SqawkError::UnsupportedSqlFeature( - "Only column references are supported in aggregate functions".to_string(), - )), - } - } - _ => Err(SqawkError::UnsupportedSqlFeature( - "Unsupported function argument type".to_string(), - )), - } - } - - /// Resolves a simple (unqualified) column reference like 'name' in SQL expressions - /// - /// This function implements SQL column name resolution semantics for unqualified - /// column references. It follows these resolution rules in sequence: - /// - /// 1. First attempt: Exact match with a column name in the table - /// 2. Second attempt: Match as suffix of qualified column names (e.g., 'name' matches 'table1.name') - /// 3. If multiple matches found in step 2, use the first match (left-most table in the FROM clause) - /// - /// The column resolution logic is critical for SQL's natural join behavior - /// and for handling simple column references in queries involving multiple tables. - fn resolve_simple_column_reference( - &self, - column_name: &str, - row: &[Value], - table: &Table, - ) -> SqawkResult { - // First try as an exact column name match - if let Some(idx) = table.column_index(column_name) { - return self.get_row_value_at_index(idx, row); - } - - // Try to find a matching qualified column (e.g., for "name", match "table.name") - let suffix = format!(".{}", column_name); - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - return self.get_row_value_at_index(i, row); - } - } - - // If we got here, the column wasn't found - Err(SqawkError::ColumnNotFound(column_name.to_string())) - } - - /// Resolves a qualified column reference like 'table.column' or 'schema.table.column' - /// - /// This function handles SQL's qualified column name resolution for expressions - /// that explicitly specify a table name, such as 'customers.id' or 'sales.price'. - /// The resolution process follows these steps: - /// - /// 1. Build the fully qualified name from the provided parts - /// 2. Look for an exact match in the table's column names - /// 3. If not found, attempt suffix matching for JOIN scenarios - /// (e.g., 'customers.id' might match 'orders_customers.id' in a join) - /// - /// This approach properly handles column name disambiguation in queries - /// involving multiple tables, particularly for JOIN operations where - /// columns from different tables may have the same names. - fn resolve_qualified_column_reference( - &self, - parts: &[sqlparser::ast::Ident], - row: &[Value], - table: &Table, - ) -> SqawkResult { - // Build the fully qualified column name from parts - let qualified_name = parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."); - - // Try to find an exact match for the qualified column - if let Some(idx) = table.column_index(&qualified_name) { - return self.get_row_value_at_index(idx, row); - } - - // If we didn't find an exact match, try a suffix match - // This helps with cases like "users.id" matching "users_orders_cross.users.id" - if parts.len() == 2 { - return self.try_suffix_match(parts, row, table); - } - - // If we got here, the qualified column wasn't found - Err(SqawkError::ColumnNotFound(qualified_name)) - } - - /// Try to match a column reference as a suffix - /// - /// This helps with joins where the full reference might be something like - /// 'users_orders_cross.users.id' but the user references 'users.id' - fn try_suffix_match( - &self, - parts: &[sqlparser::ast::Ident], - row: &[Value], - table: &Table, - ) -> SqawkResult { - let suffix = format!("{}.{}", parts[0].value, parts[1].value); - - for (i, col) in table.columns().iter().enumerate() { - if col.ends_with(&suffix) { - return self.get_row_value_at_index(i, row); - } - } - - // If no suffix match was found, report the column as not found - Err(SqawkError::ColumnNotFound( - parts - .iter() - .map(|ident| ident.value.clone()) - .collect::>() - .join("."), - )) - } - - /// Safely retrieves a value from a row at the specified column index with bounds checking - /// - /// This helper function is used throughout the SQL expression evaluation to access - /// row values while ensuring we don't cause out-of-bounds access errors. It - /// provides a consistent error message format for column index boundary issues. - /// - /// # Arguments - /// * `idx` - The column index to access - /// * `row` - The row vector containing values - /// - /// # Returns - /// * `Ok(Value)` - A cloned copy of the value at the specified index - /// * `Err` - If the index is out of bounds for the given row - fn get_row_value_at_index(&self, idx: usize, row: &[Value]) -> SqawkResult { - if idx < row.len() { - Ok(row[idx].clone()) - } else { - Err(SqawkError::InvalidSqlQuery(format!( - "Column index {} out of bounds for row with {} columns", - idx, - row.len() - ))) - } - } - - /// Execute a SQL UPDATE statement - /// - /// This function implements the SQL UPDATE operation by: - /// 1. Identifying the target table - /// 2. Applying WHERE clause filtering (if present) - /// 3. Modifying the specified columns on matching rows - /// 4. Tracking the table as modified for later write operations - /// 5. Tracking the number of affected rows for reporting - /// - /// If no WHERE condition is provided, all rows in the table will be updated. - /// The operation maintains SQL semantics for type conversions during assignment. - /// - /// # Arguments - /// * `table` - The table reference to update - /// * `assignments` - Column assignments to apply (column-value pairs) - /// * `selection` - Optional WHERE clause to filter which rows to update - /// - /// # Returns - /// * The number of rows that were updated - /// - /// Execute a CREATE TABLE statement - /// - /// Creates a new table with the specified schema. If a LOCATION is specified, - /// the table will be associated with that file for future saving. Supports - /// specifying a custom delimiter and file format through WITH options. - /// - /// # Arguments - /// * `name` - The name of the table to create - /// * `columns` - Column definitions with names and data types - /// * `file_format` - Optional file format specification - /// * `location` - Optional file path for the table - /// * `with_options` - Additional options like delimiter - /// - /// # Returns - /// * `Ok(())` if the table was created successfully - /// * `Err` if there was an error creating the table - /// - /// Execute a CREATE TABLE statement - /// - /// This function handles the creation of tables from SQL CREATE TABLE statements. - /// It parses the schema, location, file format, and options to create a new table. - /// The location is crucial for being able to save the table later. - /// - /// # Arguments - /// * `name` - Table name from SQL - /// * `columns` - Column definitions from SQL - /// * `file_format` - File format (TEXTFILE, etc.) from SQL - /// * `location` - File path location from SQL LOCATION clause - /// * `with_options` - Additional options from SQL WITH clause - /// - /// # Returns - /// * `SqawkResult<()>` - Success or error - fn execute_create_table( - &mut self, - name: ObjectName, - columns: Vec, - file_format: Option, - location: Option, - with_options: Vec, - ) -> SqawkResult<()> { - // Validate file format (must be TEXTFILE if specified) - if let Some(format) = &file_format { - match format { - SqlFileFormat::TEXTFILE => { - // This is the only supported format for now - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported file format: {:?}. Only TEXTFILE is supported.", - format - ))); - } - } - } - - // Extract table name - let table_name = name - .0 - .iter() - .map(|i| i.value.clone()) - .collect::>() - .join("."); - - // Check if table already exists - if self.file_handler.has_table(&table_name) { - return Err(SqawkError::TableAlreadyExists(table_name)); - } - - // Convert SQL column definitions to our internal ColumnDefinition type - let schema: Vec = columns - .into_iter() - .map(|col| { - let name = col.name.value; - - // Convert SQL data type to our internal DataType - let data_type = match col.data_type.to_string().to_uppercase().as_str() { - "INTEGER" | "INT" => DataType::Integer, - "REAL" | "FLOAT" | "DOUBLE" => DataType::Float, - "TEXT" | "VARCHAR" | "CHAR" | "STRING" => DataType::Text, - "BOOLEAN" | "BOOL" => DataType::Boolean, - other => { - // Default to TEXT for unsupported types - eprintln!( - "Warning: Unsupported data type '{}', using TEXT instead", - other - ); - DataType::Text - } - }; - - ColumnDefinition { name, data_type } - }) - .collect(); - - // Extract custom delimiter from WITH options if specified - let delimiter = with_options - .iter() - .find(|opt| opt.name.value.to_lowercase() == "delimiter") - .and_then(|opt| { - if let SqlValue::SingleQuotedString(s) = &opt.value { - Some(s.clone()) - } else { - // Only string literals are supported for delimiter - None - } - }); - - // Get the delimiter from options or default to comma - let delimiter_str = delimiter.unwrap_or_else(|| { - // Default to comma as separator if not specified - ",".to_string() - }); - - // Check if the LOCATION clause was provided - if self.config.verbose() { - if let Some(loc) = &location { - println!("LOCATION clause found: '{}'", loc); - } else { - eprintln!( - "Warning: CREATE TABLE without LOCATION clause - table cannot be saved to disk" - ); - } - } - - // Process the file path from the LOCATION clause - let file_path = location.map(|loc| { - // Remove any quotes that might be in the location string - let loc = loc.trim_matches('\'').trim_matches('"'); - - if self.config.verbose() { - println!("Setting file path for table '{}' to: {}", table_name, loc); - } - - // Convert to absolute path if needed - let path = if loc.starts_with('/') { - // Already absolute - std::path::PathBuf::from(loc) - } else { - // Convert relative path to absolute - match std::env::current_dir() { - Ok(mut cur_dir) => { - cur_dir.push(loc); - if self.config.verbose() { - println!("Resolved relative path to absolute: {:?}", cur_dir); - } - cur_dir - } - Err(_) => { - // Fall back to relative path if current dir can't be determined - if self.config.verbose() { - println!("Warning: Could not resolve absolute path, using relative"); - } - std::path::PathBuf::from(loc) - } - } - }; - - if self.config.verbose() { - println!("Final file path for table '{}': {:?}", table_name, path); - } - - path - }); - - // Create the table with schema and file information - let mut table = - Table::new_with_schema(&table_name, schema, file_path.clone(), Some(delimiter_str)); - - // Double-check file path is set and display it for debug purposes - if let Some(path) = file_path { - // Ensure the file path is set in the table - table.set_file_path(path.clone()); - - if self.config.verbose() { - println!("Table '{}' created with file path: {:?}", table_name, path); - } - } else if self.config.verbose() { - eprintln!( - "Warning: Table '{}' created without a file path", - table_name - ); + println!("Executing SQL: {}", sql); } - // Add the table to the file handler - self.file_handler.add_table(table_name.clone(), table)?; + // Execute via VM engine + let result = crate::vm::execute_vm(sql, self.database, self.config.verbose())?; - // Verify the table has a file path in the database - if let Ok(added_table) = self.file_handler.get_table(&table_name) { - if let Some(table_path) = added_table.file_path() { - if self.config.verbose() { - println!( - "Confirmed table '{}' has file path: {:?}", - table_name, table_path - ); - } - } else if self.config.verbose() { - eprintln!( - "Warning: Table '{}' lost its file path during creation", - table_name - ); - } + // Track modified tables + for table_name in result.modified_tables { + self.modified_tables.insert(table_name); } - // Mark the table as modified (for potential saving) - self.modified_tables.insert(table_name); - - Ok(()) - } - - fn execute_update( - &mut self, - table: TableWithJoins, - assignments: Vec, - selection: Option, - ) -> SqawkResult { - // Get the target table name as a string - let table_name = self.get_table_name(&table)?; - - // Verify the table exists and get necessary info - let table_ref = self.file_handler.get_table(&table_name)?; - - // Process assignments to get column indices and their new values - let column_assignments = self.process_update_assignments(&assignments, table_ref)?; - - // Find rows to update based on WHERE clause - let rows_to_update = self.find_rows_to_update(selection.as_ref(), table_ref)?; - - // Compute all values for each assignment before getting a mutable reference - let updates = self.compute_update_values(&rows_to_update, &column_assignments)?; - - // Apply updates with a mutable reference, now that all expressions have been evaluated - self.apply_updates(&table_name, updates) - } - - /// Process assignment expressions for an UPDATE statement - /// - /// Converts SQL assignments to column indices and expressions - fn process_update_assignments( - &self, - assignments: &[Assignment], - table: &Table, - ) -> SqawkResult> { - assignments - .iter() - .map(|assignment| { - // The id is a Vec but we only support simple column references - if assignment.id.len() != 1 { - return Err(SqawkError::UnsupportedSqlFeature( - "Compound column identifiers not supported".to_string(), - )); - } - - let column_name = assignment.id[0].value.clone(); + // Track affected rows from the last statement + self.affected_rows = result.affected_rows; - let column_idx = table - .column_index(&column_name) - .ok_or(SqawkError::ColumnNotFound(column_name))?; - - // Clone the Expr value since we can't take ownership of it - Ok((column_idx, assignment.value.clone())) - }) - .collect::>>() - } - - /// Find rows to update based on an optional WHERE clause - /// - /// If no WHERE clause is provided, all rows will be updated - fn find_rows_to_update( - &self, - where_expr: Option<&Expr>, - table: &Table, - ) -> SqawkResult> { - let mut rows_to_update = Vec::new(); - - if let Some(expr) = where_expr { - // Filter rows that match the WHERE condition - for (idx, row) in table.rows().iter().enumerate() { - if self.evaluate_condition(expr, row, table).unwrap_or(false) { - rows_to_update.push(idx); + // Set delimiter on result table to match config (for consistent output format) + let table = match result.table { + Some(mut t) => { + if let Some(delim) = self.config.field_separator() { + t.set_delimiter(delim); } + Some(t) } - } else { - // If no WHERE clause, update all rows - rows_to_update = (0..table.row_count()).collect(); - } - - Ok(rows_to_update) - } - - /// Compute all values for an UPDATE operation - /// - /// This avoids the borrow checker conflict between evaluate_expr and table_mut - /// by pre-computing all values before applying them - fn compute_update_values( - &self, - rows: &[usize], - column_assignments: &[(usize, Expr)], - ) -> SqawkResult> { - let mut updates = Vec::new(); - - // Pre-compute all values to be updated - for &row_idx in rows { - for &(col_idx, ref expr) in column_assignments { - let value = self.evaluate_expr(expr)?; - updates.push((row_idx, col_idx, value)); - } - } - - Ok(updates) - } - - /// Apply a set of pre-computed updates to a table - /// - /// Returns the number of rows that were affected - fn apply_updates( - &mut self, - table_name: &str, - updates: Vec<(usize, usize, Value)>, - ) -> SqawkResult { - // Calculate how many rows were affected (distinct row indices) - let row_indices: std::collections::HashSet = - updates.iter().map(|(row_idx, _, _)| *row_idx).collect(); - - let row_count = row_indices.len(); - - if row_count > 0 { - let table = self.file_handler.get_table_mut(table_name)?; - - // Apply all the pre-computed updates - for (row_idx, col_idx, value) in updates { - table.update_value(row_idx, col_idx, value)?; - } - - // Mark the table as modified - self.modified_tables.insert(table_name.to_string()); - } + None => None, + }; - Ok(row_count) + Ok(table) } /// Save all modified tables back to their source files @@ -3417,7 +108,7 @@ impl<'a> SqlExecutor<'a> { /// files if no changes were made. /// /// # Returns - /// * `Ok(())` if all modified tables were saved successfully + /// * `Ok(usize)` - Number of tables saved /// * `Err` if any error occurs during saving pub fn save_modified_tables(&self) -> Result { let mut count = 0; @@ -3446,7 +137,6 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `Vec` - List of table names pub fn table_names(&self) -> Vec { - // Get the table names from the database self.database.table_names() } @@ -3458,9 +148,7 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `SqawkResult>` - List of column names pub fn get_table_columns(&self, table_name: &str) -> SqawkResult> { - // Get the table from the database let table = self.database.get_table(table_name)?; - // Return the column names Ok(table.columns().to_vec()) } @@ -3472,16 +160,12 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `SqawkResult>` - List of column names with their data types pub fn get_table_column_types(&self, table_name: &str) -> SqawkResult> { - // Get the table from the database let table = self.database.get_table(table_name)?; - - // Get column metadata and return as name/type pairs let column_types = table .column_metadata() .iter() .map(|col| (col.name.clone(), col.data_type)) .collect(); - Ok(column_types) } @@ -3501,8 +185,6 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `SqawkResult>` - Tuple of (table_name, file_path) if successful pub fn load_file(&mut self, file_spec: &str) -> SqawkResult> { - // Use None for field_separator, as Tables already have their delimiter - // This method is typically used for tables created via CREATE TABLE self.file_handler.load_file(file_spec) } @@ -3533,7 +215,7 @@ impl<'a> SqlExecutor<'a> { self.file_handler.has_table(table_name) } - /// Check if a table is modified + /// Check if a table is modified (alias for is_table_modified) /// /// # Arguments /// * `table_name` - Name of the table to check @@ -3561,10 +243,15 @@ impl<'a> SqlExecutor<'a> { return Ok(()); } - // Use our enhanced file_handler.save_table method which ensures parent directories exist - // This ensures consistency between command-line loaded tables and CREATE TABLE tables self.file_handler.save_table(table_name) } + + /// Get the number of rows affected by the last executed statement + /// + /// Returns the count of rows affected by the most recent INSERT, UPDATE, or DELETE. + pub fn get_affected_row_count(&self) -> SqawkResult { + Ok(self.affected_rows) + } } /// Result set structure for REPL output diff --git a/src/storage/memory.rs b/src/storage/memory.rs new file mode 100644 index 0000000..d1adaf2 --- /dev/null +++ b/src/storage/memory.rs @@ -0,0 +1,60 @@ +//! In-memory storage backend +//! +//! This module provides traditional in-memory storage for table rows. +//! All data is heap-allocated and owned by the storage. + +use crate::table::Row; + +/// In-memory storage backend +/// +/// Stores table rows in a heap-allocated vector. This is the traditional +/// storage mode where all data is copied into memory during loading. +#[derive(Debug, Clone)] +pub struct MemoryStorage { + /// Row data + rows: Vec, +} + +impl MemoryStorage { + /// Create a new empty memory storage + pub fn new() -> Self { + Self { rows: Vec::new() } + } + + /// Create memory storage with existing rows + #[allow(dead_code)] + pub fn with_rows(rows: Vec) -> Self { + Self { rows } + } + + /// Get the number of rows + pub fn row_count(&self) -> usize { + self.rows.len() + } + + /// Get all rows as a slice + pub fn rows(&self) -> &[Row] { + &self.rows + } + + /// Get mutable reference to rows + pub fn rows_mut(&mut self) -> &mut Vec { + &mut self.rows + } + + /// Add a row to the storage + pub fn push_row(&mut self, row: Row) { + self.rows.push(row); + } + + /// Replace all rows + pub fn replace_rows(&mut self, rows: Vec) { + self.rows = rows; + } +} + +impl Default for MemoryStorage { + fn default() -> Self { + Self::new() + } +} diff --git a/src/storage/mmap.rs b/src/storage/mmap.rs new file mode 100644 index 0000000..779e15f --- /dev/null +++ b/src/storage/mmap.rs @@ -0,0 +1,366 @@ +//! Memory-mapped storage backend +//! +//! This module provides zero-copy access to CSV files by memory-mapping them. +//! String values reference data directly in the mmap'd region instead of +//! being copied to the heap. + +use std::borrow::Cow; +use std::fs::File; +use std::path::Path; + +use memmap2::Mmap; + +use crate::capacity::{ + estimate_row_count, generate_alpha_columns, is_comment_line, is_likely_data_row, + DEFAULT_ROW_CAPACITY, +}; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::{Row, Value}; + +/// Memory-mapped storage backend +/// +/// This storage backend maps a CSV file directly into memory and parses +/// rows lazily. String values borrow directly from the mmap'd region, +/// avoiding heap allocation. +/// +/// # Safety +/// +/// This struct uses unsafe code to create `'static` references from the +/// mmap'd region. This is safe because: +/// 1. The Mmap is owned by this struct +/// 2. The rows reference data within the Mmap +/// 3. The struct cannot be mutated after creation +/// 4. When the struct is dropped, both the rows and Mmap are dropped together +#[derive(Debug)] +pub struct MmapStorage { + /// The memory-mapped file data + #[allow(dead_code)] + mmap: Mmap, + /// Column names from the header row + columns: Vec, + /// Parsed row data (borrows from mmap) + rows: Vec, +} + +#[allow(dead_code)] +impl MmapStorage { + /// Open a CSV file with memory-mapping + /// + /// # Arguments + /// * `path` - Path to the CSV file + /// * `delimiter` - Field delimiter (e.g., ',' for CSV, '\t' for TSV) + /// + /// # Returns + /// * `Ok(MmapStorage)` if the file was successfully opened and parsed + /// * `Err` if the file couldn't be opened or parsed + pub fn open>(path: P, delimiter: u8) -> SqawkResult { + Self::open_with_columns(path, delimiter, None) + } + + /// Open with predefined column names (for --tabledef support) + /// + /// When predefined columns are provided, the first row is treated as data, + /// not headers. + pub fn open_with_columns>( + path: P, + delimiter: u8, + predefined_columns: Option>, + ) -> SqawkResult { + let file = File::open(path.as_ref()).map_err(SqawkError::IoError)?; + + // Safety: We're only reading the file, and the Mmap will be kept alive + // for as long as the MmapStorage exists + let mmap = unsafe { Mmap::map(&file).map_err(SqawkError::IoError)? }; + + // Hint to the kernel for better read-ahead + #[cfg(unix)] + { + let ptr = mmap.as_ptr() as *mut libc::c_void; + let len = mmap.len(); + unsafe { + // MADV_SEQUENTIAL: Expect sequential page references + libc::madvise(ptr, len, libc::MADV_SEQUENTIAL); + // MADV_WILLNEED: Will need these pages, initiate read-ahead + libc::madvise(ptr, len, libc::MADV_WILLNEED); + } + } + + // Parse the CSV structure + let (columns, rows) = Self::parse_csv(&mmap, delimiter, predefined_columns)?; + + Ok(Self { + mmap, + columns, + rows, + }) + } + + /// Parse CSV data from memory-mapped region + /// + /// # Arguments + /// * `data` - Memory-mapped file data + /// * `delimiter` - Field delimiter byte + /// * `predefined_columns` - If Some, use these as column names and treat first row as data + /// + /// # Safety + /// This function creates Value::String with Cow::Borrowed references that + /// point into the mmap'd memory. The caller must ensure the returned rows + /// do not outlive the mmap. + fn parse_csv( + data: &Mmap, + delimiter: u8, + predefined_columns: Option>, + ) -> SqawkResult<(Vec, Vec)> { + if data.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + + // Find line boundaries - estimate capacity from file size + let estimated_lines = estimate_row_count(data.len()); + let mut lines: Vec<&[u8]> = Vec::with_capacity(estimated_lines); + let mut start = 0; + + for (i, &byte) in data.iter().enumerate() { + if byte == b'\n' { + // Handle \r\n line endings + let end = if i > 0 && data[i - 1] == b'\r' { + i - 1 + } else { + i + }; + if end > start { + lines.push(&data[start..end]); + } + start = i + 1; + } + } + // Handle last line without newline + if start < data.len() { + let end = if data[data.len() - 1] == b'\r' { + data.len() - 1 + } else { + data.len() + }; + if end > start { + lines.push(&data[start..end]); + } + } + + // Filter out comment lines (starting with #) + lines.retain(|line| !is_comment_line(line)); + + if lines.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + + // Determine columns and where data rows start + let (columns, data_start) = if let Some(cols) = predefined_columns { + // Use predefined columns, all lines are data + (cols, 0) + } else { + // Parse first row and check if it looks like data or headers + let header_line = lines[0]; + let fields: Vec = Self::split_fields(header_line, delimiter) + .iter() + .map(|field| String::from_utf8_lossy(field).into_owned()) + .collect(); + + if is_likely_data_row(&fields) { + // First row is data, generate a,b,c column names + (generate_alpha_columns(fields.len()), 0) + } else { + // First row is headers + (fields, 1) + } + }; + + // Parse data rows + let mut rows = Vec::with_capacity(lines.len() - data_start); + + for line in lines.iter().skip(data_start) { + let fields = Self::split_fields(line, delimiter); + let mut row = Vec::with_capacity(columns.len()); + + for field in fields { + // Convert field bytes to a Value + // Safety: We use unsafe to extend the lifetime to 'static + // This is safe because the mmap outlives the rows (both are in the same struct) + let value = unsafe { Self::parse_field_borrowed(field) }; + row.push(value); + } + + // Pad row with nulls if it has fewer fields than headers + while row.len() < columns.len() { + row.push(Value::Null); + } + + rows.push(row); + } + + Ok((columns, rows)) + } + + /// Split a line into fields by delimiter + fn split_fields(line: &[u8], delimiter: u8) -> Vec<&[u8]> { + let mut fields = Vec::with_capacity(DEFAULT_ROW_CAPACITY); + let mut start = 0; + let mut in_quotes = false; + + for (i, &byte) in line.iter().enumerate() { + if byte == b'"' { + in_quotes = !in_quotes; + } else if byte == delimiter && !in_quotes { + fields.push(&line[start..i]); + start = i + 1; + } + } + + // Add the last field + fields.push(&line[start..]); + + fields + } + + /// Parse a field as a Value with borrowed string data + /// + /// # Safety + /// The returned Value contains a Cow::Borrowed that references the input bytes. + /// The caller must ensure the bytes outlive the Value. + unsafe fn parse_field_borrowed(field: &[u8]) -> Value { + // Convert to str first + let s = match std::str::from_utf8(field) { + Ok(s) => s, + Err(_) => { + // If not valid UTF-8, convert lossily and return owned + return Value::String(Cow::Owned(String::from_utf8_lossy(field).into_owned())); + } + }; + + // Trim whitespace and quotes + let trimmed = s.trim(); + let trimmed = if trimmed.starts_with('"') && trimmed.ends_with('"') && trimmed.len() >= 2 { + &trimmed[1..trimmed.len() - 1] + } else { + trimmed + }; + + // Try to parse as integer + if let Ok(i) = trimmed.parse::() { + return Value::Integer(i); + } + + // Try to parse as float + if let Ok(f) = trimmed.parse::() { + return Value::Float(f); + } + + // Try to parse as boolean (case-insensitive without allocation) + if trimmed.is_empty() { + return Value::Null; + } + if trimmed.eq_ignore_ascii_case("true") || trimmed.eq_ignore_ascii_case("yes") { + return Value::Boolean(true); + } + if trimmed.eq_ignore_ascii_case("false") || trimmed.eq_ignore_ascii_case("no") { + return Value::Boolean(false); + } + + // Return as borrowed string + // Safety: We extend the lifetime to 'static. The caller must ensure the + // data lives long enough. + let static_str: &'static str = std::mem::transmute(trimmed); + Value::String(Cow::Borrowed(static_str)) + } + + /// Get the column names + pub fn columns(&self) -> &[String] { + &self.columns + } + + /// Get the number of rows + pub fn row_count(&self) -> usize { + self.rows.len() + } + + /// Get all rows as a slice + pub fn rows(&self) -> &[Row] { + &self.rows + } +} + +// MmapStorage is not Clone because the mmap'd references would become invalid +// if the Mmap were dropped independently + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_mmap_storage_basic() { + // Create a temporary CSV file + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "name,age,city").unwrap(); + writeln!(file, "Alice,30,NYC").unwrap(); + writeln!(file, "Bob,25,LA").unwrap(); + + // Open with mmap + let storage = MmapStorage::open(file.path(), b',').unwrap(); + + // Check columns + assert_eq!(storage.columns(), &["name", "age", "city"]); + + // Check row count + assert_eq!(storage.row_count(), 2); + + // Check first row values + let rows = storage.rows(); + assert_eq!(rows[0].len(), 3); + + // String values should be borrowed + match &rows[0][0] { + Value::String(cow) => { + assert!(matches!(cow, Cow::Borrowed(_))); + assert_eq!(cow.as_ref(), "Alice"); + } + _ => panic!("Expected string value"), + } + + // Integer values should be parsed + assert_eq!(rows[0][1], Value::Integer(30)); + + // Another string value + match &rows[0][2] { + Value::String(cow) => { + assert!(matches!(cow, Cow::Borrowed(_))); + assert_eq!(cow.as_ref(), "NYC"); + } + _ => panic!("Expected string value"), + } + } + + #[test] + fn test_mmap_storage_empty_file() { + let file = NamedTempFile::new().unwrap(); + let storage = MmapStorage::open(file.path(), b',').unwrap(); + + assert!(storage.columns().is_empty()); + assert_eq!(storage.row_count(), 0); + } + + #[test] + fn test_mmap_storage_tsv() { + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "name\tvalue").unwrap(); + writeln!(file, "test\t42").unwrap(); + + let storage = MmapStorage::open(file.path(), b'\t').unwrap(); + + assert_eq!(storage.columns(), &["name", "value"]); + assert_eq!(storage.row_count(), 1); + + let rows = storage.rows(); + assert_eq!(rows[0][1], Value::Integer(42)); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs new file mode 100644 index 0000000..ca5c70f --- /dev/null +++ b/src/storage/mod.rs @@ -0,0 +1,140 @@ +//! Storage backends for table row data +//! +//! This module provides different storage backends for table rows: +//! - `MemoryStorage`: Traditional in-memory storage with owned data +//! - `MmapStorage`: Memory-mapped file storage for zero-copy access + +pub mod memory; +pub mod mmap; + +use crate::table::Row; + +/// Storage backend for table row data +/// +/// This enum wraps different storage implementations, allowing tables +/// to use either in-memory storage or memory-mapped file storage. +/// Column metadata is managed by the Table struct, not the storage. +#[derive(Debug)] +pub enum Storage { + /// In-memory storage with owned data + Memory(memory::MemoryStorage), + /// Memory-mapped file storage (read-only, zero-copy) + #[allow(dead_code)] + Mmap(mmap::MmapStorage), +} + +impl Storage { + /// Create a new empty in-memory storage + pub fn new_memory() -> Self { + Storage::Memory(memory::MemoryStorage::new()) + } + + /// Create in-memory storage with existing rows + #[allow(dead_code)] + pub fn with_rows(rows: Vec) -> Self { + Storage::Memory(memory::MemoryStorage::with_rows(rows)) + } + + /// Get the number of rows + pub fn row_count(&self) -> usize { + match self { + Storage::Memory(m) => m.row_count(), + Storage::Mmap(m) => m.row_count(), + } + } + + /// Get all rows as a slice + pub fn rows(&self) -> &[Row] { + match self { + Storage::Memory(m) => m.rows(), + Storage::Mmap(m) => m.rows(), + } + } + + /// Get a mutable reference to rows (only for mutable backends) + /// + /// Returns None for read-only storage backends like Mmap. + pub fn rows_mut(&mut self) -> Option<&mut Vec> { + match self { + Storage::Memory(m) => Some(m.rows_mut()), + Storage::Mmap(_) => None, // Mmap is read-only + } + } + + /// Add a row to the storage + /// + /// Note: For mmap storage, this will panic. Use is_mutable() to check first. + pub fn push_row(&mut self, row: Row) { + match self { + Storage::Memory(m) => m.push_row(row), + Storage::Mmap(_) => panic!("Cannot add rows to read-only mmap storage"), + } + } + + /// Check if this storage is mutable + #[allow(dead_code)] + pub fn is_mutable(&self) -> bool { + match self { + Storage::Memory(_) => true, + Storage::Mmap(_) => false, + } + } + + /// Replace all rows + /// + /// Note: For mmap storage, this will panic. Use is_mutable() to check first, + /// or call ensure_mutable() before calling this method. + pub fn replace_rows(&mut self, rows: Vec) { + match self { + Storage::Memory(m) => m.replace_rows(rows), + Storage::Mmap(_) => panic!("Cannot replace rows in read-only mmap storage"), + } + } + + /// Convert this storage to a mutable in-memory storage if needed + /// + /// If the storage is already mutable (Memory), this is a no-op. + /// If the storage is read-only (Mmap), it copies all rows to memory, + /// converting borrowed strings to owned strings. + pub fn ensure_mutable(&mut self) { + use crate::table::Value; + use std::borrow::Cow; + + match self { + Storage::Memory(_) => { + // Already mutable, nothing to do + } + Storage::Mmap(mmap) => { + // Copy all rows to memory, converting borrowed strings to owned + let rows: Vec = mmap + .rows() + .iter() + .map(|row| { + row.iter() + .map(|value| match value { + Value::String(cow) => { + // Convert borrowed to owned + Value::String(Cow::Owned(cow.to_string())) + } + // Other value types are Copy or already owned + v => v.clone(), + }) + .collect() + }) + .collect(); + *self = Storage::Memory(memory::MemoryStorage::with_rows(rows)); + } + } + } + + /// Get column names from the storage (for mmap only) + /// + /// Returns None for memory storage (columns are stored in Table). + #[allow(dead_code)] + pub fn columns(&self) -> Option<&[String]> { + match self { + Storage::Memory(_) => None, + Storage::Mmap(m) => Some(m.columns()), + } + } +} diff --git a/src/string_functions.rs b/src/string_functions.rs deleted file mode 100644 index bc47ca1..0000000 --- a/src/string_functions.rs +++ /dev/null @@ -1,562 +0,0 @@ -//! String function implementation module for sqawk -//! -//! This module implements standard SQL string functions including: -//! - UPPER(): Convert string to uppercase -//! - LOWER(): Convert string to lowercase -//! - TRIM(): Remove leading and trailing whitespace -//! - SUBSTR(): Extract a substring -//! - REPLACE(): Replace occurrences of a substring - -use crate::error::{SqawkError, SqawkResult}; -use crate::table::Value; - -/// Enum of supported string functions -#[derive(Debug, Clone, PartialEq)] -pub enum StringFunction { - /// Convert string to uppercase - UPPER(str) - Upper, - /// Convert string to lowercase - LOWER(str) - Lower, - /// Remove leading/trailing whitespace - TRIM(str) - Trim, - /// Extract substring - SUBSTR(str, start_pos[, length]) - Substr, - /// Replace occurrences of a substring - REPLACE(str, search, replace) - Replace, -} - -impl StringFunction { - /// Create a StringFunction from its name - pub fn from_name(name: &str) -> Option { - match name.to_uppercase().as_str() { - "UPPER" => Some(StringFunction::Upper), - "LOWER" => Some(StringFunction::Lower), - "TRIM" => Some(StringFunction::Trim), - "SUBSTR" => Some(StringFunction::Substr), - "REPLACE" => Some(StringFunction::Replace), - _ => None, - } - } - - /// Apply the string function to its arguments - pub fn apply(&self, args: &[Value]) -> SqawkResult { - match self { - StringFunction::Lower => self.apply_lower(args), - StringFunction::Upper => self.apply_upper(args), - StringFunction::Trim => self.apply_trim(args), - StringFunction::Substr => self.apply_substr(args), - StringFunction::Replace => self.apply_replace(args), - } - } - - /// Apply LOWER function - convert to lowercase - fn apply_lower(&self, args: &[Value]) -> SqawkResult { - // Validate argument count - if args.len() != 1 { - return Err(SqawkError::InvalidFunctionArguments( - "LOWER requires exactly one argument".to_string(), - )); - } - - // Handle input value - match &args[0] { - // Pass NULL through - Value::Null => Ok(Value::Null), - - // Convert string to lowercase - Value::String(s) => Ok(Value::String(s.to_lowercase())), - - // Error for non-string inputs - _ => Err(SqawkError::TypeError(format!( - "LOWER function requires a string argument, got {:?}", - args[0] - ))), - } - } - - /// Apply UPPER function - convert to uppercase - fn apply_upper(&self, args: &[Value]) -> SqawkResult { - // Validate argument count - if args.len() != 1 { - return Err(SqawkError::InvalidFunctionArguments( - "UPPER requires exactly one argument".to_string(), - )); - } - - // Handle input value - match &args[0] { - // Pass NULL through - Value::Null => Ok(Value::Null), - - // Convert string to uppercase - Value::String(s) => Ok(Value::String(s.to_uppercase())), - - // Error for non-string inputs - _ => Err(SqawkError::TypeError(format!( - "UPPER function requires a string argument, got {:?}", - args[0] - ))), - } - } - - /// Apply TRIM function - remove leading/trailing whitespace - fn apply_trim(&self, args: &[Value]) -> SqawkResult { - // Validate argument count - if args.len() != 1 { - return Err(SqawkError::InvalidFunctionArguments( - "TRIM requires exactly one argument".to_string(), - )); - } - - // Handle input value - match &args[0] { - // Pass NULL through - Value::Null => Ok(Value::Null), - - // Trim string - Value::String(s) => Ok(Value::String(s.trim().to_string())), - - // Error for non-string inputs - _ => Err(SqawkError::TypeError(format!( - "TRIM function requires a string argument, got {:?}", - args[0] - ))), - } - } - - /// Apply SUBSTR function - extract substring - fn apply_substr(&self, args: &[Value]) -> SqawkResult { - // Validate argument count - if args.len() < 2 || args.len() > 3 { - return Err(SqawkError::InvalidFunctionArguments( - "SUBSTR requires two or three arguments: (string, start_pos[, length])".to_string(), - )); - } - - // Handle NULL input - if let Value::Null = args[0] { - return Ok(Value::Null); - } - - // Get the string - let string = match &args[0] { - Value::String(s) => s, - _ => { - return Err(SqawkError::TypeError(format!( - "First argument to SUBSTR must be a string, got {:?}", - args[0] - ))) - } - }; - - // Get the start position (1-indexed in SQL) - let start_pos = match &args[1] { - Value::Integer(n) => { - if *n < 1 { - return Err(SqawkError::InvalidFunctionArguments( - "Start position in SUBSTR must be at least 1".to_string(), - )); - } - *n as usize - } - _ => { - return Err(SqawkError::TypeError(format!( - "Second argument to SUBSTR must be an integer, got {:?}", - args[1] - ))) - } - }; - - // Convert to 0-indexed for Rust - let start_index = start_pos - 1; - - // If start_index is beyond the string length, return empty string - if start_index >= string.len() { - return Ok(Value::String("".to_string())); - } - - // Get the optional length argument - let result = if args.len() == 3 { - match &args[2] { - Value::Integer(n) => { - if *n < 0 { - return Err(SqawkError::InvalidFunctionArguments( - "Length in SUBSTR must be non-negative".to_string(), - )); - } - - // Get the substring of specified length - let end_index = string.len().min(start_index + *n as usize); - string[start_index..end_index].to_string() - } - _ => { - return Err(SqawkError::TypeError(format!( - "Third argument to SUBSTR must be an integer, got {:?}", - args[2] - ))) - } - } - } else { - // No length argument, take everything from start_index to the end - string[start_index..].to_string() - }; - - Ok(Value::String(result)) - } - - /// Apply REPLACE function - replace substring occurrences - fn apply_replace(&self, args: &[Value]) -> SqawkResult { - // Validate argument count - if args.len() != 3 { - return Err(SqawkError::InvalidFunctionArguments( - "REPLACE requires exactly three arguments: (string, search, replace)".to_string(), - )); - } - - // Handle NULL input - if let Value::Null = args[0] { - return Ok(Value::Null); - } - - // Get the source string - let string = match &args[0] { - Value::String(s) => s, - _ => { - return Err(SqawkError::TypeError(format!( - "First argument to REPLACE must be a string, got {:?}", - args[0] - ))) - } - }; - - // Get the search pattern - let pattern = match &args[1] { - Value::String(s) => s, - _ => { - return Err(SqawkError::TypeError(format!( - "Second argument to REPLACE must be a string, got {:?}", - args[1] - ))) - } - }; - - // Get the replacement string - let replacement = match &args[2] { - Value::String(s) => s, - _ => { - return Err(SqawkError::TypeError(format!( - "Third argument to REPLACE must be a string, got {:?}", - args[2] - ))) - } - }; - - // Perform the replacement - let result = string.replace(pattern, replacement); - Ok(Value::String(result)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_upper_function() { - let func = StringFunction::Upper; - - // Test with valid string - let result = func.apply(&[Value::String("hello".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "HELLO", - _ => false, - }); - - // Test with NULL - let result = func.apply(&[Value::Null]); - assert!(match &result { - Ok(Value::Null) => true, - _ => false, - }); - - // Test with wrong argument type - let result = func.apply(&[Value::Integer(42)]); - assert!(result.is_err()); - - // Test with wrong argument count - let result = func.apply(&[]); - assert!(result.is_err()); - let result = func.apply(&[ - Value::String("a".to_string()), - Value::String("b".to_string()), - ]); - assert!(result.is_err()); - } - - #[test] - fn test_lower_function() { - let func = StringFunction::Lower; - - // Test with valid string - let result = func.apply(&[Value::String("HELLO".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with mixed case - let result = func.apply(&[Value::String("HeLLo".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with NULL - let result = func.apply(&[Value::Null]); - assert!(match &result { - Ok(Value::Null) => true, - _ => false, - }); - - // Test with wrong argument type - let result = func.apply(&[Value::Integer(42)]); - assert!(result.is_err()); - - // Test with wrong argument count - let result = func.apply(&[]); - assert!(result.is_err()); - } - - #[test] - fn test_trim_function() { - let func = StringFunction::Trim; - - // Test with spaces on both sides - let result = func.apply(&[Value::String(" hello ".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with spaces on left side only - let result = func.apply(&[Value::String(" hello".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with spaces on right side only - let result = func.apply(&[Value::String("hello ".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with no spaces - let result = func.apply(&[Value::String("hello".to_string())]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with NULL - let result = func.apply(&[Value::Null]); - assert!(match &result { - Ok(Value::Null) => true, - _ => false, - }); - - // Test with wrong argument type - let result = func.apply(&[Value::Integer(42)]); - assert!(result.is_err()); - - // Test with wrong argument count - let result = func.apply(&[]); - assert!(result.is_err()); - } - - #[test] - fn test_substr_function() { - let func = StringFunction::Substr; - - // Test basic substring (start, no length) - let result = func.apply(&[Value::String("hello world".to_string()), Value::Integer(7)]); - assert!(match &result { - Ok(Value::String(s)) => s == "world", - _ => false, - }); - - // Test with start and length - let result = func.apply(&[ - Value::String("hello world".to_string()), - Value::Integer(1), - Value::Integer(5), - ]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with start beyond string length - let result = func.apply(&[Value::String("hello".to_string()), Value::Integer(10)]); - assert!(match &result { - Ok(Value::String(s)) => s.is_empty(), - _ => false, - }); - - // Test with length beyond string end - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::Integer(1), - Value::Integer(100), - ]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello", - _ => false, - }); - - // Test with NULL - let result = func.apply(&[Value::Null, Value::Integer(1)]); - assert!(match &result { - Ok(Value::Null) => true, - _ => false, - }); - - // Test with invalid start position - let result = func.apply(&[Value::String("hello".to_string()), Value::Integer(0)]); - assert!(result.is_err()); - - // Test with negative length - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::Integer(1), - Value::Integer(-5), - ]); - assert!(result.is_err()); - - // Test with wrong argument types - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::String("world".to_string()), - ]); - assert!(result.is_err()); - - // Test with wrong argument count - let result = func.apply(&[Value::String("hello".to_string())]); - assert!(result.is_err()); - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::Integer(1), - Value::Integer(2), - Value::Integer(3), - ]); - assert!(result.is_err()); - } - - #[test] - fn test_replace_function() { - let func = StringFunction::Replace; - - // Test basic replacement - let result = func.apply(&[ - Value::String("hello world".to_string()), - Value::String("world".to_string()), - Value::String("Rust".to_string()), - ]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello Rust", - _ => false, - }); - - // Test replacement with empty string (deletion) - let result = func.apply(&[ - Value::String("hello world".to_string()), - Value::String("o".to_string()), - Value::String("".to_string()), - ]); - assert!(match &result { - Ok(Value::String(s)) => s == "hell wrld", - _ => false, - }); - - // Test with pattern not found - let result = func.apply(&[ - Value::String("hello world".to_string()), - Value::String("xyz".to_string()), - Value::String("abc".to_string()), - ]); - assert!(match &result { - Ok(Value::String(s)) => s == "hello world", - _ => false, - }); - - // Test with NULL - let result = func.apply(&[ - Value::Null, - Value::String("a".to_string()), - Value::String("b".to_string()), - ]); - assert!(match &result { - Ok(Value::Null) => true, - _ => false, - }); - - // Test with wrong argument types - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::Integer(123), - Value::String("world".to_string()), - ]); - assert!(result.is_err()); - - // Test with wrong argument count - let result = func.apply(&[ - Value::String("hello".to_string()), - Value::String("world".to_string()), - ]); - assert!(result.is_err()); - } - - #[test] - fn test_from_name() { - // Test exact matches - assert_eq!( - StringFunction::from_name("UPPER"), - Some(StringFunction::Upper) - ); - assert_eq!( - StringFunction::from_name("LOWER"), - Some(StringFunction::Lower) - ); - assert_eq!( - StringFunction::from_name("TRIM"), - Some(StringFunction::Trim) - ); - assert_eq!( - StringFunction::from_name("SUBSTR"), - Some(StringFunction::Substr) - ); - assert_eq!( - StringFunction::from_name("REPLACE"), - Some(StringFunction::Replace) - ); - - // Test case insensitivity - assert_eq!( - StringFunction::from_name("upper"), - Some(StringFunction::Upper) - ); - assert_eq!( - StringFunction::from_name("Lower"), - Some(StringFunction::Lower) - ); - assert_eq!( - StringFunction::from_name("trim"), - Some(StringFunction::Trim) - ); - - // Test non-existent function - assert_eq!(StringFunction::from_name("UNKNOWN"), None); - assert_eq!(StringFunction::from_name(""), None); - } -} diff --git a/src/table.rs b/src/table.rs index 4d8de52..880b733 100644 --- a/src/table.rs +++ b/src/table.rs @@ -9,58 +9,25 @@ //! - Table joins (cross joins and inner joins via WHERE conditions) //! - Column resolution with qualified names (table.column) +use std::borrow::Cow; use std::collections::HashMap; use std::fmt; use std::path::PathBuf; use anyhow::Result; -/// Represents a reference to a column, which can be qualified with a table name -/// -/// This structure is used for handling column references in SQL queries, -/// particularly for supporting table-qualified column names (e.g., "table.column") -/// which are essential for resolving column names in JOINs and multi-table queries. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ColumnRef { - /// Optional table name qualifier - /// When present, it specifies the table to which the column belongs - pub table_name: Option, - - /// Column name - /// The actual name of the column being referenced - pub column_name: String, -} - -impl ColumnRef { - // Removed unused methods -} - use crate::error::{SqawkError, SqawkResult}; - -/// A unique identifier for a row in a table -/// -/// This is used to track rows for transactions and versioning support. -/// Each row gets a unique ID when inserted into a table. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct RowId(pub u64); - -impl RowId { - /// Create a new RowId with the given ID value - pub fn new(id: u64) -> Self { - RowId(id) - } - - /// Get the underlying ID value - pub fn value(&self) -> u64 { - self.0 - } -} +use crate::storage::Storage; /// Represents a value in a table cell /// /// This enum provides the possible data types for a cell value in a table. /// It supports the common SQL data types and allows for type conversions /// between numeric types (Integer <-> Float) for comparison operations. +/// +/// String values use `Cow<'static, str>` to support both: +/// - Owned strings (heap-allocated, from in-memory tables) +/// - Borrowed strings (zero-copy, from memory-mapped files) #[derive(Debug, Clone)] pub enum Value { /// Represents a NULL or missing value @@ -69,8 +36,8 @@ pub enum Value { Integer(i64), /// 64-bit floating point number Float(f64), - /// UTF-8 string - String(String), + /// UTF-8 string (owned or borrowed via Cow) + String(Cow<'static, str>), /// Boolean value (true/false) Boolean(bool), } @@ -225,16 +192,437 @@ impl From<&str> for Value { _ => {} } - // Default to string - Value::String(s.to_string()) + // Default to owned string + Value::String(Cow::Owned(s.to_string())) + } +} + +impl From for Value { + fn from(s: String) -> Self { + // Try to parse as integer first + if let Ok(i) = s.parse::() { + return Value::Integer(i); + } + + // Try to parse as float + if let Ok(fl) = s.parse::() { + return Value::Float(fl); + } + + // Try to parse as boolean + match s.to_lowercase().as_str() { + "true" | "yes" | "1" => return Value::Boolean(true), + "false" | "no" | "0" => return Value::Boolean(false), + "" => return Value::Null, + _ => {} + } + + // Default to owned string (avoid extra allocation) + Value::String(Cow::Owned(s)) } } /// Represents a row in a table pub type Row = Vec; -/// Represents an in-memory table -#[derive(Debug, Clone)] +// ============================================================================= +// Index-Based Result Types (Phase 4A) +// ============================================================================= +// These types enable efficient query processing by storing row indices instead +// of cloning entire rows. Materialization only happens at output time. +// +// Note: These types are defined in Phase 4A and will be integrated in Phases 4B-4F. +// The #[allow(dead_code)] attributes will be removed as each type is adopted. + +/// Reference to a row within a specific table +/// +/// This lightweight struct (16 bytes) replaces cloning entire rows (~100+ bytes). +/// Used for tracking which rows match query conditions. +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RowRef { + /// Index of the table in the source table list (for JOINs) + pub table_idx: usize, + /// Index of the row within that table + pub row_idx: usize, +} + +#[allow(dead_code)] +impl RowRef { + /// Create a new row reference + pub fn new(table_idx: usize, row_idx: usize) -> Self { + Self { table_idx, row_idx } + } + + /// Create a row reference for single-table operations (table_idx = 0) + pub fn single(row_idx: usize) -> Self { + Self { + table_idx: 0, + row_idx, + } + } +} + +/// Set of row references for single-table operations +/// +/// Instead of `Vec>` (clones all row data), this stores only +/// indices into the original table. Memory usage: O(n * 8 bytes) vs O(n * row_size). +#[allow(dead_code)] +#[derive(Debug)] +pub struct RowSet<'a> { + /// Reference to the source table (borrowed, not owned) + source: &'a Table, + /// Indices of selected rows (into source.rows) + indices: Vec, +} + +#[allow(dead_code)] +impl<'a> RowSet<'a> { + /// Create a new empty RowSet referencing the given table + pub fn new(source: &'a Table) -> Self { + Self { + source, + indices: Vec::new(), + } + } + + /// Create a RowSet with pre-allocated capacity + pub fn with_capacity(source: &'a Table, capacity: usize) -> Self { + Self { + source, + indices: Vec::with_capacity(capacity), + } + } + + /// Add a row index to the set + pub fn push(&mut self, row_idx: usize) { + self.indices.push(row_idx); + } + + /// Get the number of rows in this set + pub fn len(&self) -> usize { + self.indices.len() + } + + /// Check if the set is empty + pub fn is_empty(&self) -> bool { + self.indices.is_empty() + } + + /// Get the source table + pub fn source(&self) -> &'a Table { + self.source + } + + /// Get the row indices + pub fn indices(&self) -> &[usize] { + &self.indices + } + + /// Get a value from a specific row and column (no clone) + pub fn get_value(&self, row_idx: usize, col_idx: usize) -> Option<&Value> { + let actual_row = *self.indices.get(row_idx)?; + self.source.rows().get(actual_row)?.get(col_idx) + } + + /// Materialize a single row (clones values - use sparingly) + pub fn materialize_row(&self, row_idx: usize) -> Option> { + let actual_row = *self.indices.get(row_idx)?; + self.source.rows().get(actual_row).cloned() + } + + /// Iterate over row indices + pub fn iter(&self) -> impl Iterator + '_ { + self.indices.iter().copied() + } +} + +/// Row references for multi-table operations (JOINs) +/// +/// For JOIN operations, we need to track which rows from each table +/// form a result row. Option handles NULL rows in outer joins. +#[allow(dead_code)] +#[derive(Debug)] +pub struct JoinedRowSet<'a> { + /// References to source tables (borrowed) + sources: Vec<&'a Table>, + /// Each entry is row indices from each source table + /// Option allows NULL rows for outer joins + row_pairs: Vec>>, +} + +#[allow(dead_code)] +impl<'a> JoinedRowSet<'a> { + /// Create a new JoinedRowSet for two tables + pub fn new(left: &'a Table, right: &'a Table) -> Self { + Self { + sources: vec![left, right], + row_pairs: Vec::new(), + } + } + + /// Create with pre-allocated capacity + pub fn with_capacity(left: &'a Table, right: &'a Table, capacity: usize) -> Self { + Self { + sources: vec![left, right], + row_pairs: Vec::with_capacity(capacity), + } + } + + /// Add a matched row pair (INNER JOIN) + pub fn add_match(&mut self, left_idx: usize, right_idx: usize) { + self.row_pairs.push(vec![Some(left_idx), Some(right_idx)]); + } + + /// Add unmatched left row with NULL right (LEFT JOIN) + pub fn add_left_only(&mut self, left_idx: usize) { + self.row_pairs.push(vec![Some(left_idx), None]); + } + + /// Add unmatched right row with NULL left (RIGHT JOIN) + pub fn add_right_only(&mut self, right_idx: usize) { + self.row_pairs.push(vec![None, Some(right_idx)]); + } + + /// Get the number of result rows + pub fn len(&self) -> usize { + self.row_pairs.len() + } + + /// Check if empty + pub fn is_empty(&self) -> bool { + self.row_pairs.is_empty() + } + + /// Get value from a joined row (returns &Value or NULL for outer join misses) + pub fn get_value(&self, row_idx: usize, table_idx: usize, col_idx: usize) -> &Value { + // Static NULL for outer join misses + static NULL_VALUE: Value = Value::Null; + + if let Some(row_pair) = self.row_pairs.get(row_idx) { + if let Some(Some(actual_row)) = row_pair.get(table_idx) { + if let Some(table) = self.sources.get(table_idx) { + if let Some(row) = table.rows().get(*actual_row) { + if let Some(value) = row.get(col_idx) { + return value; + } + } + } + } + } + &NULL_VALUE + } + + /// Get the source tables + pub fn sources(&self) -> &[&'a Table] { + &self.sources + } + + /// Materialize a single result row (clones values) + pub fn materialize_row( + &self, + row_idx: usize, + left_cols: usize, + right_cols: usize, + ) -> Option> { + // Verify row exists + let _ = self.row_pairs.get(row_idx)?; + let mut result = Vec::with_capacity(left_cols + right_cols); + + // Left table columns + for col_idx in 0..left_cols { + result.push(self.get_value(row_idx, 0, col_idx).clone()); + } + + // Right table columns + for col_idx in 0..right_cols { + result.push(self.get_value(row_idx, 1, col_idx).clone()); + } + + Some(result) + } +} + +/// Unified VM result type - either single-table indices, joined indices, or materialized +/// +/// This enum allows the VM to work with index-based results throughout query +/// processing, only materializing to actual values when necessary. +#[allow(dead_code)] +#[derive(Debug)] +pub enum IndexedResult<'a> { + /// Result from single table scan/filter + Single(RowSet<'a>), + /// Result from JOIN operations + Joined(JoinedRowSet<'a>), + /// Materialized result (for complex expressions, aggregates, or final output) + Materialized(Vec>), +} + +#[allow(dead_code)] +impl<'a> IndexedResult<'a> { + /// Get the number of result rows + pub fn len(&self) -> usize { + match self { + IndexedResult::Single(rs) => rs.len(), + IndexedResult::Joined(js) => js.len(), + IndexedResult::Materialized(rows) => rows.len(), + } + } + + /// Check if empty + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Check if this result is already materialized + pub fn is_materialized(&self) -> bool { + matches!(self, IndexedResult::Materialized(_)) + } +} + +/// Lazy row view - provides access to row values without immediate cloning +/// +/// This struct allows iteration over result rows while deferring value +/// cloning until actually needed (e.g., for output). +#[allow(dead_code)] +#[derive(Debug)] +pub struct LazyRow<'a, 'b> { + /// Reference to the indexed result + result: &'b IndexedResult<'a>, + /// Index within the result + row_idx: usize, +} + +#[allow(dead_code)] +impl<'a, 'b> LazyRow<'a, 'b> { + /// Create a new lazy row view + pub fn new(result: &'b IndexedResult<'a>, row_idx: usize) -> Self { + Self { result, row_idx } + } + + /// Get the row index + pub fn index(&self) -> usize { + self.row_idx + } + + /// Materialize this row to owned values (clones - use when needed for output) + pub fn materialize(&self) -> Option> { + match self.result { + IndexedResult::Single(rs) => rs.materialize_row(self.row_idx), + IndexedResult::Joined(js) => { + // For joined rows, we need column counts + let left_cols = js.sources().first().map(|t| t.column_count()).unwrap_or(0); + let right_cols = js.sources().get(1).map(|t| t.column_count()).unwrap_or(0); + js.materialize_row(self.row_idx, left_cols, right_cols) + } + IndexedResult::Materialized(rows) => rows.get(self.row_idx).cloned(), + } + } +} + +/// Builder for constructing IndexedResult during VM execution +/// +/// This builder accumulates row indices during query processing and +/// produces an IndexedResult at the end. +#[allow(dead_code)] +#[derive(Debug)] +pub struct IndexedResultBuilder<'a> { + /// Source tables being queried + sources: Vec<&'a Table>, + /// Accumulated row indices (single table: one index per row) + single_indices: Vec, + /// Accumulated row pairs (joins: multiple indices per row) + join_indices: Vec>>, + /// Whether this is a join operation + is_join: bool, +} + +#[allow(dead_code)] +impl<'a> IndexedResultBuilder<'a> { + /// Create a new builder + pub fn new() -> Self { + Self { + sources: Vec::new(), + single_indices: Vec::new(), + join_indices: Vec::new(), + is_join: false, + } + } + + /// Add a source table + pub fn add_source(&mut self, table: &'a Table) { + self.sources.push(table); + if self.sources.len() > 1 { + self.is_join = true; + } + } + + /// Add a single row index (for single-table operations) + pub fn add_row_single(&mut self, row_idx: usize) { + self.single_indices.push(row_idx); + } + + /// Add a joined row (for multi-table operations) + pub fn add_row_joined(&mut self, indices: Vec>) { + self.join_indices.push(indices); + } + + /// Check if any rows have been added + pub fn is_empty(&self) -> bool { + self.single_indices.is_empty() && self.join_indices.is_empty() + } + + /// Get the number of result rows + pub fn len(&self) -> usize { + if self.is_join { + self.join_indices.len() + } else { + self.single_indices.len() + } + } + + /// Clear accumulated results (for reuse) + pub fn clear(&mut self) { + self.single_indices.clear(); + self.join_indices.clear(); + } + + /// Build the final IndexedResult + pub fn build(self) -> Option> { + if self.sources.is_empty() { + return None; + } + + if self.is_join && self.sources.len() >= 2 { + let joined = JoinedRowSet { + sources: self.sources, + row_pairs: self.join_indices, + }; + Some(IndexedResult::Joined(joined)) + } else if let Some(source) = self.sources.into_iter().next() { + let row_set = RowSet { + source, + indices: self.single_indices, + }; + Some(IndexedResult::Single(row_set)) + } else { + None + } + } +} + +impl<'a> Default for IndexedResultBuilder<'a> { + fn default() -> Self { + Self::new() + } +} + +// ============================================================================= +// End Index-Based Result Types +// ============================================================================= + +/// Represents a table with a storage backend +#[derive(Debug)] pub struct Table { /// Name of the table name: String, @@ -245,14 +633,8 @@ pub struct Table { /// Map of column names to their indices column_map: HashMap, - /// Rows of data - rows: Vec, - - /// Row IDs to uniquely identify each row - row_ids: Vec, - - /// Next row ID to assign (increments with each row added) - next_row_id: u64, + /// Row data storage backend (memory or mmap) + storage: Storage, /// File path associated with this table (for loading or saving) file_path: Option, @@ -312,15 +694,6 @@ pub struct ColumnDefinition { pub data_type: DataType, } -/// Sort direction for a column in ORDER BY clause -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SortDirection { - /// Sort in ascending order (default) - Ascending, - /// Sort in descending order - Descending, -} - impl Table { /// Create a new table with the given name and column names /// @@ -346,9 +719,7 @@ impl Table { name: name.to_string(), cols, column_map, - rows: Vec::new(), - row_ids: Vec::new(), - next_row_id: 1, // Start with 1 as the first row ID + storage: Storage::new_memory(), file_path, modified: false, delimiter: ",".to_string(), // Default to comma delimiter @@ -367,6 +738,44 @@ impl Table { table } + /// Create a new table with a specific storage backend + /// + /// This constructor is used for memory-mapped file storage where the + /// storage already contains the parsed data. + pub fn with_storage( + name: &str, + column_names: Vec, + file_path: Option, + delimiter: String, + storage: Storage, + ) -> Self { + // Create column_map from column names + let column_map = column_names + .iter() + .enumerate() + .map(|(i, name)| (name.clone(), i)) + .collect(); + + // Create full column metadata objects (default to Text type) + let cols = column_names + .iter() + .map(|name| Column { + name: name.clone(), + data_type: DataType::Text, + }) + .collect(); + + Table { + name: name.to_string(), + cols, + column_map, + storage, + file_path, + modified: false, + delimiter, + } + } + /// Create a new table with a schema pub fn new_with_schema( name: &str, @@ -397,9 +806,7 @@ impl Table { name: name.to_string(), cols, column_map, - rows: Vec::new(), - row_ids: Vec::new(), - next_row_id: 1, // Start with 1 as the first row ID + storage: Storage::new_memory(), file_path, modified: true, // Tables created with schema are considered modified delimiter: delimiter.unwrap_or_else(|| ",".to_string()), @@ -439,7 +846,7 @@ impl Table { /// of Value enums representing the cell values. This provides read-only /// access to the table data for processing or querying. pub fn rows(&self) -> &[Row] { - &self.rows + self.storage.rows() } /// Get the name of the table @@ -456,7 +863,7 @@ impl Table { /// Returns the number of rows in the table. This is useful for /// determining the size of the result set or for validation. pub fn row_count(&self) -> usize { - self.rows.len() + self.storage.row_count() } /// Add a row to the table @@ -482,106 +889,39 @@ impl Table { ))); } - // Add the row with a new unique row ID - let row_id = RowId::new(self.next_row_id); - self.next_row_id += 1; - - self.rows.push(row); - self.row_ids.push(row_id); + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); + self.storage.push_row(row); self.modified = true; Ok(()) } - /// Add a row without validation for recovery mode - /// - /// This method is used internally by the CSV handler in recovery mode - /// to add rows that may have had their structure altered to fix issues, - /// such as padding missing fields or truncating extra fields. + /// Add a row by cloning values from a slice /// - /// # Arguments - /// * `row` - Vector of values to add as a new row - /// - /// # Returns - /// * `Ok(())` always succeeds - pub fn add_row_recovery(&mut self, row: Row) -> SqawkResult<()> { - // No validation, assumes the row has been fixed already - // Still assign a unique row ID - let row_id = RowId::new(self.next_row_id); - self.next_row_id += 1; - - self.rows.push(row); - self.row_ids.push(row_id); - self.modified = true; - Ok(()) - } - - /// Get a row by its unique row ID - /// - /// # Arguments - /// * `row_id` - The unique identifier for the row - /// - /// # Returns - /// * `Some(&Row)` if a row with the given ID exists - /// * `None` if no row with that ID exists - pub fn get_row_by_id(&self, row_id: RowId) -> Option<&Row> { - // Find the index of the row with this ID - let index = self.row_ids.iter().position(|id| *id == row_id)?; - self.rows.get(index) - } - - /// Get a mutable reference to a row by its unique row ID + /// This method allows reusing a row buffer by cloning values from a slice. + /// It validates that the slice length matches the table column count. /// /// # Arguments - /// * `row_id` - The unique identifier for the row + /// * `row` - Slice of values to clone and add as a new row /// /// # Returns - /// * `Some(&mut Row)` if a row with the given ID exists - /// * `None` if no row with that ID exists - pub fn get_row_by_id_mut(&mut self, row_id: RowId) -> Option<&mut Row> { - // Find the index of the row with this ID - let index = self.row_ids.iter().position(|id| *id == row_id)?; - self.rows.get_mut(index) - } - - /// Get the row ID for a row at a specific index - /// - /// # Arguments - /// * `index` - The index of the row - /// - /// # Returns - /// * `Some(RowId)` if the index is valid - /// * `None` if the index is out of bounds - pub fn get_row_id_at_index(&self, index: usize) -> Option { - self.row_ids.get(index).copied() - } - - /// Remove a row by its unique row ID - /// - /// # Arguments - /// * `row_id` - The unique identifier for the row to remove - /// - /// # Returns - /// * `true` if a row with the given ID was found and removed - /// * `false` if no row with that ID exists - pub fn remove_row_by_id(&mut self, row_id: RowId) -> bool { - if let Some(index) = self.row_ids.iter().position(|id| *id == row_id) { - self.rows.remove(index); - self.row_ids.remove(index); - self.modified = true; - true - } else { - false + /// * `Ok(())` if the row was successfully added + /// * `Err` if the slice length doesn't match the table schema + pub fn add_row_from_slice(&mut self, row: &[Value]) -> SqawkResult<()> { + if row.len() != self.column_count() { + return Err(SqawkError::InvalidSqlQuery(format!( + "Row has {} columns, but table '{}' has {} columns", + row.len(), + self.name, + self.column_count() + ))); } - } - /// Get all row IDs in this table - /// - /// This is useful for iterating over all rows by ID. - /// - /// # Returns - /// * A slice containing all row IDs in the table - pub fn row_ids(&self) -> &[RowId] { - &self.row_ids + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); + self.storage.push_row(row.to_vec()); + self.modified = true; + Ok(()) } /// Get the file path associated with this table @@ -598,20 +938,6 @@ impl Table { self.file_path.as_ref() } - /// Set file path for this table - /// - /// Sets the path to the file associated with this table. - /// This is useful for tables created with CREATE TABLE. - /// - /// # Arguments - /// * `path` - The new file path - pub fn set_file_path(&mut self, path: PathBuf) { - // Note: Verbose logging now happens at the caller level via AppConfig.verbose() - self.file_path = Some(path); - } - - // The set_verbose method has been removed as part of cleanup - /// Get the delimiter for this table /// /// Returns the delimiter used for this table. @@ -623,6 +949,14 @@ impl Table { &self.delimiter } + /// Set the delimiter for this table + /// + /// # Arguments + /// * `delimiter` - The new delimiter string to use + pub fn set_delimiter(&mut self, delimiter: String) { + self.delimiter = delimiter; + } + /// Get the index of a column by name /// /// Looks up a column by name and returns its index in the table. @@ -648,21 +982,23 @@ impl Table { /// * `Ok(())` if the table was successfully printed /// * `Err` if there was an error writing to stdout pub fn print_to_stdout(&self) -> Result<()> { + let delim = &self.delimiter; + // Print header let column_names = self.columns(); for (i, col) in column_names.iter().enumerate() { if i > 0 { - print!(","); + print!("{}", delim); } print!("{}", col); } println!(); // Print rows - for row in &self.rows { + for row in self.rows() { for (i, val) in row.iter().enumerate() { if i > 0 { - print!(","); + print!("{}", delim); } print!("{}", val); } @@ -672,37 +1008,6 @@ impl Table { Ok(()) } - /// Create a new table with a subset of rows matching a predicate - /// - /// Filters the table rows based on a provided predicate function. - /// This is the core implementation of SQL WHERE clause functionality. - /// - /// # Arguments - /// * `predicate` - A function that takes a row reference and returns a boolean - /// indicating whether the row should be included in the result - /// - /// # Returns - /// * A new table containing only the rows that match the predicate - pub fn select(&self, predicate: F) -> Self - where - F: Fn(&Row) -> bool, - { - let mut result = Table::new_with_delimiter( - &self.name, - self.columns().to_vec(), - None, - self.delimiter.clone(), - ); - - for row in &self.rows { - if predicate(row) { - result.rows.push(row.clone()); - } - } - - result - } - /// Replace all rows with a new set /// /// This method is useful for operations like DELETE that need to replace @@ -711,80 +1016,67 @@ impl Table { /// # Arguments /// * `new_rows` - The new set of rows to replace the existing ones pub fn replace_rows(&mut self, new_rows: Vec) { - self.rows = new_rows; + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); + self.storage.replace_rows(new_rows); self.modified = true; } /// Add a column to the table with a specified data type - /// - /// This method adds a new column to the table with the given name and data type. - /// It's primarily used when creating tables with specific schema definitions. - /// - /// # Arguments - /// * `name` - Name of the column to add - /// * `data_type` - Data type for the column (as a string, e.g., "INT", "TEXT") - /// - /// # Returns - /// * `()` - This method doesn't return a result as it cannot fail + #[cfg(test)] pub fn add_column(&mut self, name: String, data_type_str: String) { - // Map the string data type to our internal DataType enum let data_type = match data_type_str.to_uppercase().as_str() { "INT" | "INTEGER" => DataType::Integer, "FLOAT" | "REAL" | "DOUBLE" => DataType::Float, "BOOL" | "BOOLEAN" => DataType::Boolean, - _ => DataType::Text, // Default to Text for unknown types + _ => DataType::Text, }; - - // Create a new Column instance let column = Column { name: name.clone(), data_type, }; - - // Add the column to the table's column list self.cols.push(column); - - // Update the column map with the new column's index let new_index = self.cols.len() - 1; self.column_map.insert(name, new_index); - - // Mark the table as modified + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); self.modified = true; } - /// Update a single value in a specific row and column - /// - /// # Arguments - /// * `row_idx` - The index of the row to update - /// * `col_idx` - The index of the column to update - /// * `value` - The new value to set - /// - /// # Returns - /// * `Ok(())` if the update was successful - /// * `Err` if the row or column index is out of bounds - pub fn update_value( + /// Add a new column with a default value for all existing rows + pub fn add_column_with_default( &mut self, - row_idx: usize, - col_idx: usize, - value: Value, - ) -> SqawkResult<()> { - if row_idx >= self.rows.len() { - return Err(SqawkError::InvalidSqlQuery(format!( - "Row index {} is out of bounds (table has {} rows)", - row_idx, - self.rows.len() - ))); - } + name: String, + data_type: DataType, + default_value: Value, + ) -> crate::error::SqawkResult<()> { + let column = Column { + name: name.clone(), + data_type, + }; + self.cols.push(column); + let new_index = self.cols.len() - 1; + self.column_map.insert(name, new_index); - if col_idx >= self.column_count() { - return Err(SqawkError::ColumnNotFound(format!( - "Column index {} is out of bounds (table has {} columns)", - col_idx, - self.column_count() - ))); + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); + + // Add default value to all existing rows + if let Some(rows) = self.storage.rows_mut() { + for row in rows { + row.push(default_value.clone()); + } } - self.rows[row_idx][col_idx] = value; + self.modified = true; + Ok(()) + } + + /// Clear all rows from the table (TRUNCATE TABLE) + pub fn clear_rows(&mut self) -> crate::error::SqawkResult<()> { + // Ensure storage is mutable (converts mmap to memory if needed) + self.storage.ensure_mutable(); + self.storage.replace_rows(Vec::new()); self.modified = true; Ok(()) } @@ -797,401 +1089,9 @@ impl Table { /// # Returns /// * Vec> - All rows converted to strings pub fn rows_as_strings(&self) -> Vec> { - self.rows + self.rows() .iter() .map(|row| row.iter().map(|value| value.to_string()).collect()) .collect() } - - /// Create a new table with only specified columns - /// - /// Projects the table to include only the columns specified by their indices. - /// This is the core implementation of the SQL SELECT column list functionality, - /// allowing queries to specify which columns should be included in the result. - /// - /// # Arguments - /// * `column_indices` - Array of column indices to include in the result table - /// - /// # Returns - /// * `Ok(Table)` containing only the specified columns from the original table - /// * `Err` if any column index is out of bounds - /// Create a new table with only specified columns and optional aliases - /// - /// Projects the table to include only the columns specified by their indices, - /// applying any aliases provided. - /// - /// # Arguments - /// * `column_specs` - Array of column indices and optional aliases to include in the result table - /// - /// # Returns - /// * `Ok(Table)` containing only the specified columns from the original table with aliases applied - /// * `Err` if any column index is out of bounds - pub fn project_with_aliases( - &self, - column_specs: &[(usize, Option)], - ) -> SqawkResult { - // Validate column indices - for &(idx, _) in column_specs { - if idx >= self.column_count() { - return Err(SqawkError::ColumnNotFound(format!( - "Column index {} out of bounds", - idx - ))); - } - } - - // Create new column list with aliases where specified - let column_names = self.columns(); - let columns: Vec = column_specs - .iter() - .map(|&(idx, ref alias)| { - if let Some(alias_name) = alias { - alias_name.clone() - } else { - column_names[idx].clone() - } - }) - .collect(); - - let mut result = Table::new(&self.name, columns, self.file_path.clone()); - result.delimiter = self.delimiter.clone(); - - // Project rows - for row in &self.rows { - let projected_row: Vec = column_specs - .iter() - .map(|&(idx, _)| row[idx].clone()) - .collect(); - - result.add_row(projected_row)?; - } - - Ok(result) - } - - /// Execute a CROSS JOIN with another table - /// - /// This creates a cartesian product of the two tables. - /// - /// # Arguments - /// * `right` - The right table to join with - /// - /// # Returns - /// * A new table containing the joined data - /// - /// Perform a cross join between two tables - /// - /// This method implements the Cartesian product of two tables, combining every row from - /// the left table with every row from the right table. - /// - /// # Arguments - /// * `right` - The right-hand table to join with - /// - /// # Returns - /// * A new table containing the cross join result - pub fn cross_join(&self, right: &Self) -> SqawkResult { - // Create result columns with proper prefixes - let columns = self.create_joined_columns(right); - - // Create a new table to hold the join result - let mut result = Table::new("join_result", columns, None); - result.delimiter = self.delimiter.clone(); - - // Fill with cross-joined rows - self.fill_cross_joined_rows(right, &mut result)?; - - Ok(result) - } - - /// Perform an INNER JOIN between two tables with a custom condition - /// - /// This method implements the SQL INNER JOIN operation, which combines rows from - /// two tables that satisfy a join condition. The implementation follows a - /// two-step approach: first creating a cross join (Cartesian product), then - /// filtering the combined rows based on the provided condition. - /// - /// # Arguments - /// * `right` - The right-hand table to join with - /// * `join_condition` - A closure that evaluates whether a combined row should be included. - /// The closure receives: - /// - A combined row from both tables - /// - A reference to the combined table (for column lookups) - /// - /// The closure returns a boolean indicating whether the row satisfies the join condition - /// - /// # Returns - /// * A new table containing only the rows that satisfy the join condition - /// * `Err` if there was an error evaluating the condition or adding rows - /// - /// # Usage Example - /// - /// This method is typically used to implement SQL's INNER JOIN operation - /// with an ON condition. For example, implementing: - /// - /// SELECT * FROM employees INNER JOIN departments - /// ON employees.dept_id = departments.id - /// - /// The implementation first finds the column indexes for the join keys, - /// then compares the values in those columns for each row combination. - pub fn inner_join(&self, right: &Self, join_condition: F) -> SqawkResult - where - F: Fn(&[Value], &Self) -> SqawkResult, - { - // Step 1: Create the output columns structure - this must be done before creating - // the result table to ensure columns from both tables are properly qualified - let columns = self.create_joined_columns(right); - - // Step 2: Create a new table to hold the join result - let name = format!("{}_inner_join", self.name()); - let mut result = Table::new(&name, columns, None); - result.delimiter = self.delimiter.clone(); - - // Step 3: First create the cross join (Cartesian product) to evaluate conditions against - // This creates every possible combination of rows from both tables - let combined_tables = self.cross_join(right)?; - - // Step 4: Filter the cross join result based on the join condition - // This is effectively the ON clause in SQL's "INNER JOIN ... ON" syntax - for row in combined_tables.rows().iter() { - // Evaluate the condition for this row, which comes from the closure - // provided by the SQL executor based on the ON condition - if join_condition(row, &combined_tables)? { - // Add matching rows to the result table - result.add_row(row.clone())?; - } - } - - Ok(result) - } - - /// Create column names for a joined table - /// - /// This function creates a list of qualified column names by prefixing - /// each column name with its table name if it doesn't already have a prefix. - /// - /// # Arguments - /// * `right` - The right-hand table for the join - /// - /// # Returns - /// * A vector of qualified column names - fn create_joined_columns(&self, right: &Self) -> Vec { - let mut columns = Vec::new(); - - // Add columns from left table (self) with prefixes - self.add_prefixed_columns(&mut columns); - - // Add columns from right table with prefixes - right.add_prefixed_columns(&mut columns); - - columns - } - - /// Add columns with table name prefixes to a column list - /// - /// This function adds column names to a list, prefixing them with - /// the table name if they don't already have a prefix. - /// - /// # Arguments - /// * `columns` - The column list to add to - fn add_prefixed_columns(&self, columns: &mut Vec) { - for col in self.columns() { - // If the column already has a table prefix, keep it as is - // Otherwise, add the table name prefix - if col.contains('.') { - columns.push(col.clone()); - } else { - columns.push(format!("{}.{}", self.name, col)); - } - } - } - - /// Fill a table with cross-joined rows - /// - /// This function creates rows for a cross join by combining each row - /// from the left table with each row from the right table. - /// - /// # Arguments - /// * `right` - The right-hand table for the join - /// * `result` - The result table to fill with rows - /// - /// # Returns - /// * Ok(()) if all rows were successfully added - /// * Err if there was an error adding a row - fn fill_cross_joined_rows(&self, right: &Self, result: &mut Self) -> SqawkResult<()> { - // For CROSS JOIN, we include every combination of rows - for left_row in self.rows() { - for right_row in right.rows() { - // Create combined row - let new_row = self.combine_rows(left_row, right_row, right.column_count()); - - // Add the combined row to the result table - result.add_row(new_row)?; - } - } - - Ok(()) - } - - /// Combine two rows into a single row - /// - /// This function combines a row from the left table with a row - /// from the right table to create a joined row. - /// - /// # Arguments - /// * `left_row` - A row from the left table - /// * `right_row` - A row from the right table - /// * `right_column_count` - The number of columns in the right table - /// - /// # Returns - /// * A new combined row - fn combine_rows( - &self, - left_row: &[Value], - right_row: &[Value], - right_column_count: usize, - ) -> Vec { - let mut new_row = Vec::with_capacity(self.column_count() + right_column_count); - - // Add values from left row - for i in 0..self.column_count() { - new_row.push(left_row.get(i).unwrap_or(&Value::Null).clone()); - } - - // Add values from right row - for i in 0..right_column_count { - new_row.push(right_row.get(i).unwrap_or(&Value::Null).clone()); - } - - new_row - } - - /// Remove duplicate rows from the table - /// - /// This method implements the DISTINCT functionality for SQL queries. - /// It creates a new table with the same structure but with duplicate rows removed. - /// Two rows are considered identical if all of their column values match exactly. - /// - /// # Returns - /// * A new table with duplicate rows removed - pub fn distinct(&self) -> SqawkResult { - // Create a new table with the same structure - let column_names = self.columns().to_vec(); - let mut result = Table::new(&self.name, column_names, self.file_path.clone()); - - // Use a vector to track rows we've already seen - // We can't use a HashSet directly because Row is Vec which may not implement Hash - let mut seen_rows: Vec> = Vec::new(); - - for row in &self.rows { - // Check if this row is already in our seen rows - let is_duplicate = seen_rows.iter().any(|seen_row| { - // Two rows are identical if they have the same length and all values match - row.len() == seen_row.len() && row.iter().zip(seen_row.iter()).all(|(a, b)| a == b) - }); - - // If it's not a duplicate, add it to result and to seen rows - if !is_duplicate { - seen_rows.push(row.clone()); - result.add_row(row.clone())?; - } - } - - Ok(result) - } - - /// Sort the table by one or more columns - /// - /// This method implements the ORDER BY functionality for SQL queries. - /// It takes a list of column indices and their respective sort directions, - /// then sorts the table rows accordingly. Sort direction can be either - /// ascending (the default) or descending. - /// - /// # Arguments - /// * `sort_columns` - A vector of tuples containing (column_index, sort_direction) - /// - /// # Returns - /// * A new sorted table if successful - /// * Error if any column index is invalid - pub fn sort(&self, sort_columns: Vec<(usize, SortDirection)>) -> SqawkResult { - // Validate column indices - for (col_idx, _) in &sort_columns { - if *col_idx >= self.column_count() { - return Err(SqawkError::ColumnNotFound(format!( - "Column index {} out of bounds for ORDER BY (table has {} columns)", - col_idx, - self.column_count() - ))); - } - } - - // Create a new table with the same structure - let mut result = Table::new(&self.name, self.columns().to_vec(), self.file_path.clone()); - - // Clone the rows for sorting - let mut sorted_rows = self.rows.clone(); - - // Sort the rows based on the specified columns and directions - sorted_rows.sort_by(|row_a, row_b| { - // Compare each sort column in order until a difference is found - for &(col_idx, direction) in &sort_columns { - // Compare values using our PartialOrd implementation - match row_a[col_idx].partial_cmp(&row_b[col_idx]) { - Some(ordering) => { - // If not equal, return the ordering (possibly reversed for DESC) - if ordering != std::cmp::Ordering::Equal { - return match direction { - SortDirection::Ascending => ordering, - SortDirection::Descending => ordering.reverse(), - }; - } - } - // If values can't be compared (which shouldn't happen with our implementation), - // continue to the next column - None => continue, - } - } - - // If all specified columns are equal, maintain stable sort - std::cmp::Ordering::Equal - }); - - // Add the sorted rows to the result table - for row in sorted_rows { - result.add_row(row)?; - } - - Ok(result) - } - - /// Limits the number of rows in the table - /// - /// This method applies LIMIT and OFFSET to the table, returning a new table - /// containing at most `limit` rows, starting from the `offset` position. - /// This is used to implement the SQL LIMIT and OFFSET clauses. - /// - /// # Arguments - /// * `limit` - The maximum number of rows to include - /// * `offset` - The number of rows to skip before starting to include rows (default: 0) - /// - /// # Returns - /// * A new table with the specified limit and offset applied - pub fn limit(&self, limit: usize, offset: usize) -> SqawkResult { - // Create a new table with the same structure - let mut result = Table::new(&self.name, self.columns().to_vec(), self.file_path.clone()); - - // If offset is greater than or equal to the number of rows, return an empty table - if offset >= self.rows.len() { - return Ok(result); - } - - // Calculate the end index, capped at the table size - let end = std::cmp::min(offset + limit, self.rows.len()); - - // Add rows from offset to end - for row in &self.rows[offset..end] { - result.add_row(row.clone())?; - } - - Ok(result) - } } diff --git a/src/tsq.rs b/src/tsq.rs new file mode 100644 index 0000000..e192af2 --- /dev/null +++ b/src/tsq.rs @@ -0,0 +1,2177 @@ +//! TSQ - Test SQL Query Generator for sqawk +//! +//! Generates deterministic test data and SQL queries for comprehensive sqawk testing. + +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::process; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use clap::Parser; + +// Simple RNG using libc srand/rand +struct Rng; + +impl Rng { + fn seed(seed: u32) { + unsafe { + libc::srand(seed); + } + } + + fn next_u32() -> u32 { + unsafe { libc::rand() as u32 } + } + + fn gen_range_usize(min: usize, max: usize) -> usize { + if min >= max { + return min; + } + min + (Self::next_u32() as usize % (max - min + 1)) + } + + fn gen_range_i32(min: i32, max: i32) -> i32 { + if min >= max { + return min; + } + min + (Self::next_u32() as i32).abs() % (max - min + 1) + } + + fn gen_range_f64(min: f64, max: f64) -> f64 { + let r = Self::next_u32() as f64 / u32::MAX as f64; + min + r * (max - min) + } + + fn gen_bool(probability: f64) -> bool { + Self::gen_range_f64(0.0, 1.0) < probability + } +} + +/// Generate default seed from time XOR pid +fn default_seed() -> u64 { + let time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let pid = process::id() as u64; + time ^ pid +} + +// ============================================================================ +// CLI Arguments +// ============================================================================ + +#[derive(Parser, Debug)] +#[clap(name = "tsq", about = "Test SQL Query Generator for sqawk")] +struct Args { + /// Random seed for reproducible generation (prints seed if not specified) + #[clap(long, short = 's')] + seed: Option, + + /// Number of customer rows (base count, other tables scale proportionally) + #[clap(long, short = 'r', default_value = "100000")] + rows: usize, + + /// Output directory for generated files + #[clap(long, short = 'o')] + output_dir: String, + + /// Verbose output showing generation progress + #[clap(long, short = 'v')] + verbose: bool, +} + +// ============================================================================ +// Data Pools - Static data for random generation +// ============================================================================ + +const FIRST_NAMES: &[&str] = &[ + "James", + "Mary", + "John", + "Patricia", + "Robert", + "Jennifer", + "Michael", + "Linda", + "William", + "Elizabeth", + "David", + "Barbara", + "Richard", + "Susan", + "Joseph", + "Jessica", + "Thomas", + "Sarah", + "Charles", + "Karen", + "Christopher", + "Nancy", + "Daniel", + "Lisa", + "Matthew", + "Betty", + "Anthony", + "Margaret", + "Mark", + "Sandra", + "Donald", + "Ashley", + "Steven", + "Kimberly", + "Paul", + "Emily", + "Andrew", + "Donna", + "Joshua", + "Michelle", + "Kenneth", + "Dorothy", + "Kevin", + "Carol", + "Brian", + "Amanda", + "George", + "Melissa", + "Timothy", + "Deborah", + "Ronald", + "Stephanie", + "Edward", + "Rebecca", + "Jason", + "Sharon", + "Jeffrey", + "Laura", + "Ryan", + "Cynthia", + "Jacob", + "Kathleen", + "Gary", + "Amy", +]; + +const LAST_NAMES: &[&str] = &[ + "Smith", + "Johnson", + "Williams", + "Brown", + "Jones", + "Garcia", + "Miller", + "Davis", + "Rodriguez", + "Martinez", + "Hernandez", + "Lopez", + "Gonzalez", + "Wilson", + "Anderson", + "Thomas", + "Taylor", + "Moore", + "Jackson", + "Martin", + "Lee", + "Perez", + "Thompson", + "White", + "Harris", + "Sanchez", + "Clark", + "Ramirez", + "Lewis", + "Robinson", + "Walker", + "Young", + "Allen", + "King", + "Wright", + "Scott", + "Torres", + "Nguyen", + "Hill", + "Flores", + "Green", + "Adams", + "Nelson", + "Baker", + "Hall", + "Rivera", + "Campbell", + "Mitchell", + "Carter", + "Roberts", + "Gomez", + "Phillips", + "Evans", + "Turner", + "Diaz", + "Parker", +]; + +const CITIES: &[(&str, &str)] = &[ + ("New York", "NY"), + ("Los Angeles", "CA"), + ("Chicago", "IL"), + ("Houston", "TX"), + ("Phoenix", "AZ"), + ("Philadelphia", "PA"), + ("San Antonio", "TX"), + ("San Diego", "CA"), + ("Dallas", "TX"), + ("San Jose", "CA"), + ("Austin", "TX"), + ("Jacksonville", "FL"), + ("Fort Worth", "TX"), + ("Columbus", "OH"), + ("Charlotte", "NC"), + ("San Francisco", "CA"), + ("Indianapolis", "IN"), + ("Seattle", "WA"), + ("Denver", "CO"), + ("Boston", "MA"), +]; + +const COUNTRIES: &[&str] = &["USA", "Canada", "Mexico", "UK", "Germany"]; + +const CATEGORIES: &[&str] = &[ + "Electronics", + "Clothing", + "Home & Garden", + "Sports", + "Books", + "Toys", + "Automotive", + "Health", + "Beauty", + "Food", +]; + +const SUBCATEGORIES: &[(&str, &[&str])] = &[ + ( + "Electronics", + &["Phones", "Laptops", "Tablets", "Cameras", "Audio"], + ), + ( + "Clothing", + &["Shirts", "Pants", "Dresses", "Shoes", "Accessories"], + ), + ( + "Home & Garden", + &["Furniture", "Kitchen", "Bedding", "Decor", "Tools"], + ), + ( + "Sports", + &[ + "Fitness", + "Outdoor", + "Team Sports", + "Water Sports", + "Winter", + ], + ), + ( + "Books", + &["Fiction", "Non-Fiction", "Science", "History", "Children"], + ), + ( + "Toys", + &[ + "Action Figures", + "Board Games", + "Puzzles", + "Dolls", + "Building", + ], + ), + ( + "Automotive", + &["Parts", "Accessories", "Tools", "Care", "Electronics"], + ), + ( + "Health", + &[ + "Vitamins", + "First Aid", + "Personal Care", + "Fitness", + "Medical", + ], + ), + ( + "Beauty", + &["Skincare", "Makeup", "Hair Care", "Fragrance", "Bath"], + ), + ( + "Food", + &["Snacks", "Beverages", "Organic", "Frozen", "Pantry"], + ), +]; + +const ORDER_STATUSES: &[&str] = &["pending", "shipped", "delivered", "cancelled", "returned"]; + +const SHIPPING_METHODS: &[&str] = &["standard", "express", "overnight", "pickup"]; + +const EMAIL_DOMAINS: &[&str] = &[ + "gmail.com", + "yahoo.com", + "hotmail.com", + "outlook.com", + "example.com", + "mail.com", + "proton.me", + "icloud.com", +]; + +const PRODUCT_ADJECTIVES: &[&str] = &[ + "Premium", + "Pro", + "Ultra", + "Basic", + "Advanced", + "Classic", + "Modern", + "Compact", + "Deluxe", + "Essential", + "Elite", + "Standard", + "Professional", + "Portable", + "Smart", +]; + +const PRODUCT_NOUNS: &[&str] = &[ + "Widget", + "Gadget", + "Device", + "Tool", + "Kit", + "Set", + "Pack", + "Bundle", + "System", + "Unit", + "Module", + "Component", + "Accessory", + "Item", + "Product", +]; + +// ============================================================================ +// Verification Data - Tracks counts for verification +// ============================================================================ + +#[derive(Default)] +struct VerificationData { + // Row counts + customer_count: usize, + product_count: usize, + order_count: usize, + order_item_count: usize, + review_count: usize, + + // Distribution counts + city_counts: HashMap, + state_counts: HashMap, + category_counts: HashMap, + status_counts: HashMap, + shipping_counts: HashMap, + rating_counts: HashMap, + + // NULL counts + customers_with_notes: usize, + reviews_with_body: usize, + + // Active/inactive counts + active_customers: usize, + discontinued_products: usize, + + // Aggregate values + total_order_amount: f64, + total_product_price: f64, + + // Credit score distribution + credit_below_400: usize, + credit_600_to_750: usize, + credit_above_700: usize, + credit_above_800: usize, + min_credit_score: i32, + max_credit_score: i32, + + // Price distribution + price_below_100: usize, + + // Date ranges + min_order_date: String, + max_order_date: String, + + // Customer orders distribution + customers_with_orders: usize, +} + +// ============================================================================ +// Data Generator +// ============================================================================ + +struct DataGenerator { + row_count: usize, + verbose: bool, + verification: VerificationData, +} + +impl DataGenerator { + fn new(seed: u64, row_count: usize, verbose: bool) -> Self { + // Seed the global RNG + Rng::seed(seed as u32); + Self { + row_count, + verbose, + verification: VerificationData { + min_credit_score: i32::MAX, + max_credit_score: i32::MIN, + min_order_date: "9999-12-31".to_string(), + max_order_date: "0000-01-01".to_string(), + ..Default::default() + }, + } + } + + fn log(&self, msg: &str) { + if self.verbose { + eprintln!("[tsq] {}", msg); + } + } + + fn escape_csv(value: &str) -> String { + if value.contains(',') + || value.contains('"') + || value.contains('\n') + || value.contains('\r') + { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } + } + + fn random_date(&mut self, year_start: i32, year_end: i32) -> String { + let year = Rng::gen_range_i32(year_start, year_end); + let month = Rng::gen_range_i32(1, 12); + let day = match month { + 2 => Rng::gen_range_i32(1, 28), + 4 | 6 | 9 | 11 => Rng::gen_range_i32(1, 30), + _ => Rng::gen_range_i32(1, 31), + }; + format!("{:04}-{:02}-{:02}", year, month, day) + } + + fn random_text(&mut self, with_special_chars: bool) -> String { + let words: Vec<&str> = vec![ + "lorem", + "ipsum", + "dolor", + "sit", + "amet", + "consectetur", + "adipiscing", + "elit", + "sed", + "do", + "eiusmod", + "tempor", + "incididunt", + "ut", + "labore", + ]; + let word_count = Rng::gen_range_usize(3, 10); + let mut text: Vec = (0..word_count) + .map(|_| words[Rng::gen_range_usize(0, words.len() - 1)].to_string()) + .collect(); + + if with_special_chars && Rng::gen_bool(0.1) { + // Add some special characters occasionally + let specials = ["can't", "won't", "it's", "O'Brien", "Smith & Co"]; + text.push(specials[Rng::gen_range_usize(0, specials.len() - 1)].to_string()); + } + + text.join(" ") + } + + fn generate_customers(&mut self, path: &Path) -> Result<()> { + self.log(&format!("Generating {} customers...", self.row_count)); + + let file = File::create(path).context("Failed to create customers.csv")?; + let mut writer = BufWriter::new(file); + + // Header + writeln!( + writer, + "customer_id,name,email,city,state,country,signup_date,is_active,credit_score,notes" + )?; + + for i in 1..=self.row_count { + let first = FIRST_NAMES[Rng::gen_range_usize(0, FIRST_NAMES.len() - 1)]; + let last = LAST_NAMES[Rng::gen_range_usize(0, LAST_NAMES.len() - 1)]; + let name = format!("{} {}", first, last); + + let email_domain = EMAIL_DOMAINS[Rng::gen_range_usize(0, EMAIL_DOMAINS.len() - 1)]; + let email = format!( + "{}.{}{}@{}", + first.to_lowercase(), + last.to_lowercase(), + i % 1000, + email_domain + ); + + let (city, state) = CITIES[Rng::gen_range_usize(0, CITIES.len() - 1)]; + let country = COUNTRIES[Rng::gen_range_usize(0, COUNTRIES.len() - 1)]; + let signup_date = self.random_date(2020, 2025); + let is_active = if Rng::gen_bool(0.85) { 1 } else { 0 }; + let credit_score = Rng::gen_range_i32(300, 850); + + // ~20% have notes + let notes = if Rng::gen_bool(0.2) { + self.verification.customers_with_notes += 1; + Self::escape_csv(&self.random_text(true)) + } else { + String::new() + }; + + // Track verification data + *self + .verification + .city_counts + .entry(city.to_string()) + .or_insert(0) += 1; + *self + .verification + .state_counts + .entry(state.to_string()) + .or_insert(0) += 1; + + if is_active == 1 { + self.verification.active_customers += 1; + } + + if credit_score < 400 { + self.verification.credit_below_400 += 1; + } + if (600..=750).contains(&credit_score) { + self.verification.credit_600_to_750 += 1; + } + if credit_score > 700 { + self.verification.credit_above_700 += 1; + } + if credit_score > 800 { + self.verification.credit_above_800 += 1; + } + self.verification.min_credit_score = + self.verification.min_credit_score.min(credit_score); + self.verification.max_credit_score = + self.verification.max_credit_score.max(credit_score); + + writeln!( + writer, + "{},{},{},{},{},{},{},{},{},{}", + i, + Self::escape_csv(&name), + email, + Self::escape_csv(city), + state, + country, + signup_date, + is_active, + credit_score, + notes + )?; + } + + self.verification.customer_count = self.row_count; + self.log(&format!(" Created {} customers", self.row_count)); + Ok(()) + } + + fn generate_products(&mut self, path: &Path) -> Result<()> { + let count = (self.row_count / 100).max(100); + self.log(&format!("Generating {} products...", count)); + + let file = File::create(path).context("Failed to create products.csv")?; + let mut writer = BufWriter::new(file); + + // Header + writeln!( + writer, + "product_id,name,category,subcategory,price,cost,quantity_in_stock,is_discontinued,created_date,description" + )?; + + // Build subcategory lookup + let subcategory_map: HashMap<&str, &[&str]> = SUBCATEGORIES.iter().cloned().collect(); + + for i in 1..=count { + let adj = PRODUCT_ADJECTIVES[Rng::gen_range_usize(0, PRODUCT_ADJECTIVES.len() - 1)]; + let noun = PRODUCT_NOUNS[Rng::gen_range_usize(0, PRODUCT_NOUNS.len() - 1)]; + let name = format!("{} {} {}", adj, noun, i); + + let category = CATEGORIES[Rng::gen_range_usize(0, CATEGORIES.len() - 1)]; + let subcats = subcategory_map.get(category).unwrap(); + let subcategory = subcats[Rng::gen_range_usize(0, subcats.len() - 1)]; + + let price: f64 = Rng::gen_range_f64(0.99, 9999.99); + let price = (price * 100.0).round() / 100.0; + let cost = (price * Rng::gen_range_f64(0.5, 0.9) * 100.0).round() / 100.0; + let quantity_in_stock = Rng::gen_range_usize(0, 10000); + let is_discontinued = if Rng::gen_bool(0.05) { 1 } else { 0 }; + let created_date = self.random_date(2018, 2025); + + // ~10% have description + let description = if Rng::gen_bool(0.9) { + Self::escape_csv(&self.random_text(false)) + } else { + String::new() + }; + + // Track verification data + *self + .verification + .category_counts + .entry(category.to_string()) + .or_insert(0) += 1; + self.verification.total_product_price += price; + + if price < 100.0 { + self.verification.price_below_100 += 1; + } + if is_discontinued == 1 { + self.verification.discontinued_products += 1; + } + + writeln!( + writer, + "{},{},{},{},{:.2},{:.2},{},{},{},{}", + i, + Self::escape_csv(&name), + Self::escape_csv(category), + Self::escape_csv(subcategory), + price, + cost, + quantity_in_stock, + is_discontinued, + created_date, + description + )?; + } + + self.verification.product_count = count; + self.log(&format!(" Created {} products", count)); + Ok(()) + } + + fn generate_orders(&mut self, path: &Path) -> Result> { + let count = self.row_count * 3; + self.log(&format!("Generating {} orders...", count)); + + let file = File::create(path).context("Failed to create orders.csv")?; + let mut writer = BufWriter::new(file); + + // Header + writeln!( + writer, + "order_id,customer_id,order_date,status,total_amount,discount_percent,shipping_method,notes" + )?; + + // Track which customers have orders + let mut customer_order_counts: HashMap = HashMap::new(); + + // Store order info for order_items generation + let mut order_info: Vec<(usize, usize, f64)> = Vec::with_capacity(count); + + for i in 1..=count { + // Pareto distribution: 80% of orders from 20% of customers + let customer_id = if Rng::gen_bool(0.8) { + // Top 20% of customers + Rng::gen_range_usize(1, (self.row_count / 5).max(1)) + } else { + Rng::gen_range_usize(1, self.row_count) + }; + + *customer_order_counts.entry(customer_id).or_insert(0) += 1; + + let order_date = self.random_date(2023, 2025); + let status = ORDER_STATUSES[Rng::gen_range_usize(0, ORDER_STATUSES.len() - 1)]; + let total_amount: f64 = Rng::gen_range_f64(10.0, 5000.0); + let total_amount = (total_amount * 100.0).round() / 100.0; + let discount_percent = [0, 5, 10, 15, 20, 25][Rng::gen_range_usize(0, 5)]; + let shipping_method = + SHIPPING_METHODS[Rng::gen_range_usize(0, SHIPPING_METHODS.len() - 1)]; + + // ~50% have notes + let notes = if Rng::gen_bool(0.5) { + Self::escape_csv(&self.random_text(false)) + } else { + String::new() + }; + + // Track verification data + *self + .verification + .status_counts + .entry(status.to_string()) + .or_insert(0) += 1; + *self + .verification + .shipping_counts + .entry(shipping_method.to_string()) + .or_insert(0) += 1; + self.verification.total_order_amount += total_amount; + + if order_date < self.verification.min_order_date { + self.verification.min_order_date = order_date.clone(); + } + if order_date > self.verification.max_order_date { + self.verification.max_order_date = order_date.clone(); + } + + order_info.push((i, customer_id, total_amount)); + + writeln!( + writer, + "{},{},{},{},{:.2},{},{},{}", + i, + customer_id, + order_date, + status, + total_amount, + discount_percent, + shipping_method, + notes + )?; + } + + self.verification.order_count = count; + self.verification.customers_with_orders = customer_order_counts.len(); + self.log(&format!(" Created {} orders", count)); + Ok(order_info) + } + + fn generate_order_items( + &mut self, + path: &Path, + order_info: &[(usize, usize, f64)], + ) -> Result<()> { + let count = self.row_count * 10; + self.log(&format!("Generating {} order items...", count)); + + let file = File::create(path).context("Failed to create order_items.csv")?; + let mut writer = BufWriter::new(file); + + // Header + writeln!( + writer, + "item_id,order_id,product_id,quantity,unit_price,line_total" + )?; + + let product_count = self.verification.product_count; + let order_count = order_info.len(); + + for i in 1..=count { + let order_id = Rng::gen_range_usize(1, order_count); + let product_id = Rng::gen_range_usize(1, product_count); + let quantity = Rng::gen_range_usize(1, 10); + let unit_price: f64 = Rng::gen_range_f64(5.0, 500.0); + let unit_price = (unit_price * 100.0).round() / 100.0; + let line_total = (quantity as f64 * unit_price * 100.0).round() / 100.0; + + writeln!( + writer, + "{},{},{},{},{:.2},{:.2}", + i, order_id, product_id, quantity, unit_price, line_total + )?; + } + + self.verification.order_item_count = count; + self.log(&format!(" Created {} order items", count)); + Ok(()) + } + + fn generate_reviews(&mut self, path: &Path) -> Result<()> { + let count = self.row_count / 2; + self.log(&format!("Generating {} reviews...", count)); + + let file = File::create(path).context("Failed to create reviews.csv")?; + let mut writer = BufWriter::new(file); + + // Header + writeln!( + writer, + "review_id,customer_id,product_id,rating,review_date,title,body,helpful_votes" + )?; + + let product_count = self.verification.product_count; + + for i in 1..=count { + let customer_id = Rng::gen_range_usize(1, self.row_count); + let product_id = Rng::gen_range_usize(1, product_count); + let rating = Rng::gen_range_i32(1, 5); + let review_date = self.random_date(2023, 2025); + let title = Self::escape_csv(&self.random_text(false)); + + // ~85% have body + let body = if Rng::gen_bool(0.85) { + self.verification.reviews_with_body += 1; + Self::escape_csv(&self.random_text(true)) + } else { + String::new() + }; + + let helpful_votes = Rng::gen_range_usize(0, 1000); + + // Track verification data + *self.verification.rating_counts.entry(rating).or_insert(0) += 1; + + writeln!( + writer, + "{},{},{},{},{},{},{},{}", + i, customer_id, product_id, rating, review_date, title, body, helpful_votes + )?; + } + + self.verification.review_count = count; + self.log(&format!(" Created {} reviews", count)); + Ok(()) + } + + fn generate_all(&mut self, base_path: &Path) -> Result<()> { + let data_path = base_path.join("data"); + + self.generate_customers(&data_path.join("customers.csv"))?; + self.generate_products(&data_path.join("products.csv"))?; + let order_info = self.generate_orders(&data_path.join("orders.csv"))?; + self.generate_order_items(&data_path.join("order_items.csv"), &order_info)?; + self.generate_reviews(&data_path.join("reviews.csv"))?; + + Ok(()) + } +} + +// ============================================================================ +// Query Generator +// ============================================================================ + +struct QueryGenerator<'a> { + verification: &'a VerificationData, +} + +impl<'a> QueryGenerator<'a> { + fn new(verification: &'a VerificationData) -> Self { + Self { verification } + } + + fn write_query_file(&self, path: &Path, filename: &str, content: &str) -> Result<()> { + let file_path = path.join(filename); + let mut file = File::create(&file_path) + .with_context(|| format!("Failed to create {}", file_path.display()))?; + file.write_all(content.as_bytes())?; + Ok(()) + } + + fn generate_all(&self, base_path: &Path) -> Result<()> { + let queries_path = base_path.join("queries"); + + self.write_query_file( + &queries_path, + "01_select_basic.sql", + &self.gen_select_basic(), + )?; + self.write_query_file( + &queries_path, + "02_where_comparison.sql", + &self.gen_where_comparison(), + )?; + self.write_query_file( + &queries_path, + "03_where_logical.sql", + &self.gen_where_logical(), + )?; + self.write_query_file( + &queries_path, + "04_where_pattern.sql", + &self.gen_where_pattern(), + )?; + self.write_query_file(&queries_path, "05_join_inner.sql", &self.gen_join_inner())?; + self.write_query_file(&queries_path, "06_join_multi.sql", &self.gen_join_multi())?; + self.write_query_file( + &queries_path, + "07_aggregate_basic.sql", + &self.gen_aggregate_basic(), + )?; + self.write_query_file( + &queries_path, + "08_groupby_having.sql", + &self.gen_groupby_having(), + )?; + self.write_query_file(&queries_path, "09_orderby.sql", &self.gen_orderby())?; + self.write_query_file( + &queries_path, + "10_limit_offset.sql", + &self.gen_limit_offset(), + )?; + self.write_query_file(&queries_path, "11_distinct.sql", &self.gen_distinct())?; + self.write_query_file(&queries_path, "12_window.sql", &self.gen_window())?; + self.write_query_file( + &queries_path, + "13_subquery_scalar.sql", + &self.gen_subquery_scalar(), + )?; + self.write_query_file(&queries_path, "14_subquery_in.sql", &self.gen_subquery_in())?; + self.write_query_file( + &queries_path, + "15_subquery_exists.sql", + &self.gen_subquery_exists(), + )?; + self.write_query_file( + &queries_path, + "16_subquery_correlated.sql", + &self.gen_subquery_correlated(), + )?; + self.write_query_file(&queries_path, "17_setop_union.sql", &self.gen_setop_union())?; + self.write_query_file( + &queries_path, + "18_setop_intersect_except.sql", + &self.gen_setop_intersect_except(), + )?; + self.write_query_file( + &queries_path, + "19_string_functions.sql", + &self.gen_string_functions(), + )?; + self.write_query_file( + &queries_path, + "20_math_functions.sql", + &self.gen_math_functions(), + )?; + self.write_query_file( + &queries_path, + "21_case_coalesce.sql", + &self.gen_case_coalesce(), + )?; + self.write_query_file( + &queries_path, + "22_mutation_insert.sql", + &self.gen_mutation_insert(), + )?; + self.write_query_file( + &queries_path, + "23_mutation_update.sql", + &self.gen_mutation_update(), + )?; + self.write_query_file( + &queries_path, + "24_mutation_delete.sql", + &self.gen_mutation_delete(), + )?; + self.write_query_file( + &queries_path, + "25_complex_combined.sql", + &self.gen_complex_combined(), + )?; + + Ok(()) + } + + fn gen_select_basic(&self) -> String { + r#"-- 01_select_basic.sql - Basic SELECT queries +-- Generated by tsq + +-- Q001: Select all columns with LIMIT +SELECT * FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q002: Select specific columns +SELECT customer_id, name, email FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q003: Select with column alias +SELECT customer_id AS id, name AS customer_name FROM customers LIMIT 5; +-- EXPECTED_COUNT: 5 + +-- Q004: Select all from products +SELECT * FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q005: Select from orders +SELECT order_id, customer_id, total_amount FROM orders LIMIT 15; +-- EXPECTED_COUNT: 15 +"# + .to_string() + } + + fn gen_where_comparison(&self) -> String { + let v = &self.verification; + // Pick a city that exists + let city = v + .city_counts + .keys() + .next() + .map(|s| s.as_str()) + .unwrap_or("New York"); + let city_count = v.city_counts.get(city).copied().unwrap_or(0); + + format!( + r#"-- 02_where_comparison.sql - WHERE clause comparison operators +-- Generated by tsq + +-- Q010: Equals comparison (city) +SELECT * FROM customers WHERE city = '{city}'; +-- EXPECTED_COUNT: {city_count} + +-- Q011: Not equals +SELECT COUNT(*) FROM customers WHERE country != 'USA'; +-- EXPECTED_COUNT: 1 + +-- Q012: Less than +SELECT COUNT(*) FROM products WHERE price < 100.00; +-- EXPECTED_COUNT: 1 + +-- Q013: Greater than +SELECT COUNT(*) FROM customers WHERE credit_score > 700; +-- EXPECTED_COUNT: 1 + +-- Q014: Greater than or equal +SELECT COUNT(*) FROM customers WHERE credit_score >= 800; +-- EXPECTED_COUNT: 1 + +-- Q015: Less than or equal +SELECT COUNT(*) FROM customers WHERE credit_score <= 400; +-- EXPECTED_COUNT: 1 + +-- Q016: IS NULL +SELECT COUNT(*) FROM customers WHERE notes IS NULL; +-- EXPECTED_COUNT: 1 + +-- Q017: IS NOT NULL +SELECT COUNT(*) FROM customers WHERE notes IS NOT NULL; +-- EXPECTED_COUNT: 1 + +-- Q018: Combined comparison +SELECT * FROM products WHERE price >= 50.00 AND price <= 200.00 LIMIT 20; +-- EXPECTED_COUNT: 20 +"#, + city = city, + city_count = city_count + ) + } + + fn gen_where_logical(&self) -> String { + r#"-- 03_where_logical.sql - WHERE clause logical operators +-- Generated by tsq + +-- Q020: AND condition +SELECT COUNT(*) FROM customers WHERE is_active = 1 AND credit_score > 700; +-- EXPECTED_COUNT: 1 + +-- Q021: OR condition +SELECT COUNT(*) FROM orders WHERE status = 'cancelled' OR status = 'returned'; +-- EXPECTED_COUNT: 1 + +-- Q022: NOT condition +SELECT COUNT(*) FROM products WHERE NOT is_discontinued = 1; +-- EXPECTED_COUNT: 1 + +-- Q023: Complex AND/OR with parentheses +SELECT COUNT(*) FROM customers WHERE (city = 'New York' OR city = 'Los Angeles') AND is_active = 1; +-- EXPECTED_COUNT: 1 + +-- Q024: Multiple AND +SELECT COUNT(*) FROM orders WHERE status = 'delivered' AND discount_percent > 0 AND total_amount > 100; +-- EXPECTED_COUNT: 1 + +-- Q025: NOT with comparison +SELECT COUNT(*) FROM customers WHERE NOT credit_score < 600; +-- EXPECTED_COUNT: 1 +"#.to_string() + } + + fn gen_where_pattern(&self) -> String { + r#"-- 04_where_pattern.sql - Pattern matching and IN/BETWEEN +-- Generated by tsq + +-- Q030: LIKE with prefix +SELECT COUNT(*) FROM customers WHERE email LIKE 'john%'; +-- EXPECTED_COUNT: 1 + +-- Q031: LIKE with suffix +SELECT COUNT(*) FROM customers WHERE email LIKE '%@gmail.com'; +-- EXPECTED_COUNT: 1 + +-- Q032: LIKE with contains +SELECT COUNT(*) FROM products WHERE name LIKE '%Pro%'; +-- EXPECTED_COUNT: 1 + +-- Q033: IN list integers +SELECT COUNT(*) FROM reviews WHERE rating IN (4, 5); +-- EXPECTED_COUNT: 1 + +-- Q034: IN list strings +SELECT COUNT(*) FROM orders WHERE status IN ('pending', 'shipped'); +-- EXPECTED_COUNT: 1 + +-- Q035: NOT IN +SELECT COUNT(*) FROM orders WHERE shipping_method NOT IN ('overnight', 'express'); +-- EXPECTED_COUNT: 1 + +-- Q036: BETWEEN numeric +SELECT COUNT(*) FROM customers WHERE credit_score BETWEEN 600 AND 750; +-- EXPECTED_COUNT: 1 + +-- Q037: NOT BETWEEN +SELECT COUNT(*) FROM products WHERE price NOT BETWEEN 10.00 AND 100.00; +-- EXPECTED_COUNT: 1 +"# + .to_string() + } + + fn gen_join_inner(&self) -> String { + r#"-- 05_join_inner.sql - Two-table INNER JOIN queries +-- Generated by tsq + +-- Q040: Basic two-table join (implicit) +SELECT c.name, o.order_id, o.total_amount +FROM customers c, orders o +WHERE c.customer_id = o.customer_id +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q041: Join with additional filter +SELECT c.name, o.order_id, o.status +FROM customers c, orders o +WHERE c.customer_id = o.customer_id AND o.status = 'delivered' +LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q042: Join with aggregate +SELECT c.customer_id, c.name, COUNT(*) AS order_count +FROM customers c, orders o +WHERE c.customer_id = o.customer_id +GROUP BY c.customer_id, c.name +LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q043: Products and order items join +SELECT p.name, oi.quantity, oi.unit_price +FROM products p, order_items oi +WHERE p.product_id = oi.product_id +LIMIT 50; +-- EXPECTED_COUNT: 50 +"# + .to_string() + } + + fn gen_join_multi(&self) -> String { + r#"-- 06_join_multi.sql - Multi-table JOIN queries +-- Generated by tsq + +-- Q050: Three-table join +SELECT c.name, o.order_id, oi.quantity +FROM customers c, orders o, order_items oi +WHERE c.customer_id = o.customer_id + AND o.order_id = oi.order_id +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q051: Four-table join +SELECT c.name, o.order_date, p.name AS product_name, oi.quantity +FROM customers c, orders o, order_items oi, products p +WHERE c.customer_id = o.customer_id + AND o.order_id = oi.order_id + AND oi.product_id = p.product_id +LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q052: Three-table join with aggregate +SELECT c.city, COUNT(DISTINCT o.order_id) AS order_count, SUM(oi.line_total) AS total_value +FROM customers c, orders o, order_items oi +WHERE c.customer_id = o.customer_id + AND o.order_id = oi.order_id +GROUP BY c.city +LIMIT 20; +-- EXPECTED_COUNT: 20 +"# + .to_string() + } + + fn gen_aggregate_basic(&self) -> String { + let v = &self.verification; + format!( + r#"-- 07_aggregate_basic.sql - Basic aggregate functions +-- Generated by tsq + +-- Q060: COUNT(*) +SELECT COUNT(*) AS total_customers FROM customers; +-- EXPECTED_COUNT: 1 +-- EXPECTED_VALUE: {customer_count} + +-- Q061: COUNT(column) - excludes NULL +SELECT COUNT(notes) AS customers_with_notes FROM customers; +-- EXPECTED_COUNT: 1 +-- EXPECTED_VALUE: {customers_with_notes} + +-- Q062: SUM +SELECT SUM(total_amount) AS total_revenue FROM orders; +-- EXPECTED_COUNT: 1 + +-- Q063: AVG +SELECT AVG(credit_score) AS avg_credit FROM customers; +-- EXPECTED_COUNT: 1 + +-- Q064: MIN +SELECT MIN(price) AS min_price FROM products; +-- EXPECTED_COUNT: 1 + +-- Q065: MAX +SELECT MAX(price) AS max_price FROM products; +-- EXPECTED_COUNT: 1 + +-- Q066: Multiple aggregates +SELECT COUNT(*) AS cnt, SUM(quantity) AS total_qty, AVG(unit_price) AS avg_price +FROM order_items; +-- EXPECTED_COUNT: 1 + +-- Q067: MIN/MAX together +SELECT MIN(credit_score) AS min_credit, MAX(credit_score) AS max_credit FROM customers; +-- EXPECTED_COUNT: 1 +"#, + customer_count = v.customer_count, + customers_with_notes = v.customers_with_notes + ) + } + + fn gen_groupby_having(&self) -> String { + let v = &self.verification; + let num_cities = v.city_counts.len(); + let num_categories = v.category_counts.len(); + + format!( + r#"-- 08_groupby_having.sql - GROUP BY and HAVING +-- Generated by tsq + +-- Q070: Simple GROUP BY +SELECT city, COUNT(*) AS customer_count +FROM customers +GROUP BY city; +-- EXPECTED_COUNT: {num_cities} + +-- Q071: GROUP BY with multiple aggregates +SELECT category, COUNT(*) AS cnt, AVG(price) AS avg_price, SUM(quantity_in_stock) AS total_stock +FROM products +GROUP BY category; +-- EXPECTED_COUNT: {num_categories} + +-- Q072: GROUP BY with HAVING +SELECT city, COUNT(*) AS cnt +FROM customers +GROUP BY city +HAVING COUNT(*) > 100; +-- EXPECTED_COUNT varies + +-- Q073: GROUP BY with HAVING on SUM +SELECT customer_id, SUM(total_amount) AS total_spent +FROM orders +GROUP BY customer_id +HAVING SUM(total_amount) > 1000 +LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q074: GROUP BY multiple columns +SELECT city, state, COUNT(*) AS cnt +FROM customers +GROUP BY city, state; +-- EXPECTED_COUNT varies + +-- Q075: GROUP BY with ORDER BY +SELECT status, COUNT(*) AS cnt +FROM orders +GROUP BY status +ORDER BY cnt DESC; +-- EXPECTED_COUNT: 5 +"#, + num_cities = num_cities, + num_categories = num_categories + ) + } + + fn gen_orderby(&self) -> String { + r#"-- 09_orderby.sql - ORDER BY queries +-- Generated by tsq + +-- Q080: ORDER BY single column ASC +SELECT * FROM products ORDER BY price ASC LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q081: ORDER BY single column DESC +SELECT * FROM customers ORDER BY credit_score DESC LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q082: ORDER BY multiple columns +SELECT * FROM orders ORDER BY status ASC, total_amount DESC LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q083: ORDER BY with WHERE +SELECT * FROM customers WHERE is_active = 1 ORDER BY credit_score DESC LIMIT 15; +-- EXPECTED_COUNT: 15 + +-- Q084: ORDER BY with GROUP BY +SELECT category, AVG(price) AS avg_price +FROM products +GROUP BY category +ORDER BY avg_price DESC; +-- EXPECTED_COUNT varies +"# + .to_string() + } + + fn gen_limit_offset(&self) -> String { + r#"-- 10_limit_offset.sql - LIMIT and OFFSET +-- Generated by tsq + +-- Q090: LIMIT only +SELECT * FROM customers LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q091: LIMIT with OFFSET +SELECT * FROM customers LIMIT 25 OFFSET 100; +-- EXPECTED_COUNT: 25 + +-- Q092: Large OFFSET +SELECT * FROM orders LIMIT 10 OFFSET 1000; +-- EXPECTED_COUNT: 10 + +-- Q093: LIMIT 1 +SELECT * FROM products ORDER BY price DESC LIMIT 1; +-- EXPECTED_COUNT: 1 + +-- Q094: LIMIT with ORDER BY and WHERE +SELECT * FROM customers WHERE is_active = 1 ORDER BY credit_score DESC LIMIT 20 OFFSET 10; +-- EXPECTED_COUNT: 20 +"# + .to_string() + } + + fn gen_distinct(&self) -> String { + let v = &self.verification; + let num_cities = v.city_counts.len(); + let num_categories = v.category_counts.len(); + + format!( + r#"-- 11_distinct.sql - DISTINCT queries +-- Generated by tsq + +-- Q100: DISTINCT single column +SELECT DISTINCT city FROM customers; +-- EXPECTED_COUNT: {num_cities} + +-- Q101: DISTINCT multiple columns +SELECT DISTINCT city, state FROM customers; +-- EXPECTED_COUNT varies + +-- Q102: DISTINCT with ORDER BY +SELECT DISTINCT category FROM products ORDER BY category ASC; +-- EXPECTED_COUNT: {num_categories} + +-- Q103: DISTINCT with WHERE +SELECT DISTINCT status FROM orders WHERE total_amount > 500; +-- EXPECTED_COUNT varies + +-- Q104: DISTINCT on joined tables +SELECT DISTINCT c.city +FROM customers c, orders o +WHERE c.customer_id = o.customer_id AND o.status = 'delivered'; +-- EXPECTED_COUNT varies +"#, + num_cities = num_cities, + num_categories = num_categories + ) + } + + fn gen_window(&self) -> String { + r#"-- 12_window.sql - Window functions +-- Generated by tsq + +-- Q110: ROW_NUMBER without partition +SELECT customer_id, name, credit_score, + ROW_NUMBER() OVER (ORDER BY credit_score DESC) AS rank +FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q111: ROW_NUMBER with PARTITION BY +SELECT product_id, category, price, + ROW_NUMBER() OVER (PARTITION BY category ORDER BY price DESC) AS cat_rank +FROM products LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q112: RANK +SELECT review_id, rating, + RANK() OVER (ORDER BY rating DESC) AS rating_rank +FROM reviews LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q113: DENSE_RANK +SELECT customer_id, credit_score, + DENSE_RANK() OVER (ORDER BY credit_score DESC) AS dense_rank +FROM customers LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q114: SUM OVER (running total) +SELECT order_id, total_amount, + SUM(total_amount) OVER (ORDER BY order_id) AS running_total +FROM orders LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q115: AVG OVER with PARTITION +SELECT product_id, category, price, + AVG(price) OVER (PARTITION BY category) AS category_avg +FROM products LIMIT 50; +-- EXPECTED_COUNT: 50 +"# + .to_string() + } + + fn gen_subquery_scalar(&self) -> String { + r#"-- 13_subquery_scalar.sql - Scalar subqueries +-- Generated by tsq + +-- Q120: Scalar subquery with MAX +SELECT * FROM customers +WHERE credit_score = (SELECT MAX(credit_score) FROM customers); +-- EXPECTED_COUNT varies (ties possible) + +-- Q121: Scalar subquery with AVG comparison +SELECT COUNT(*) FROM products +WHERE price > (SELECT AVG(price) FROM products); +-- EXPECTED_COUNT: 1 + +-- Q122: Scalar subquery with MIN +SELECT * FROM products +WHERE price = (SELECT MIN(price) FROM products); +-- EXPECTED_COUNT varies (ties possible) + +-- Q123: Nested scalar in SELECT (if supported) +SELECT customer_id, name, + (SELECT COUNT(*) FROM customers) AS total_customers +FROM customers LIMIT 5; +-- EXPECTED_COUNT: 5 +"# + .to_string() + } + + fn gen_subquery_in(&self) -> String { + r#"-- 14_subquery_in.sql - IN subqueries +-- Generated by tsq + +-- Q130: IN subquery +SELECT * FROM customers +WHERE customer_id IN (SELECT DISTINCT customer_id FROM orders WHERE status = 'delivered') +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q131: NOT IN subquery +SELECT COUNT(*) FROM products +WHERE product_id NOT IN (SELECT DISTINCT product_id FROM order_items); +-- EXPECTED_COUNT: 1 + +-- Q132: IN subquery with aggregate filter +SELECT * FROM customers +WHERE customer_id IN ( + SELECT customer_id FROM orders + GROUP BY customer_id + HAVING COUNT(*) > 5 +) +LIMIT 50; +-- EXPECTED_COUNT: 50 +"# + .to_string() + } + + fn gen_subquery_exists(&self) -> String { + r#"-- 15_subquery_exists.sql - EXISTS subqueries +-- Generated by tsq + +-- Q140: EXISTS +SELECT * FROM customers c +WHERE EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id) +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q141: NOT EXISTS +SELECT COUNT(*) FROM products p +WHERE NOT EXISTS (SELECT 1 FROM order_items oi WHERE oi.product_id = p.product_id); +-- EXPECTED_COUNT: 1 + +-- Q142: EXISTS with additional condition +SELECT * FROM customers c +WHERE EXISTS ( + SELECT 1 FROM orders o + WHERE o.customer_id = c.customer_id AND o.status = 'delivered' +) +LIMIT 50; +-- EXPECTED_COUNT: 50 +"# + .to_string() + } + + fn gen_subquery_correlated(&self) -> String { + r#"-- 16_subquery_correlated.sql - Correlated subqueries +-- Generated by tsq + +-- Q150: Correlated subquery in WHERE +SELECT * FROM orders o +WHERE o.total_amount > ( + SELECT AVG(o2.total_amount) FROM orders o2 WHERE o2.customer_id = o.customer_id +) +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q151: Correlated subquery with COUNT +SELECT c.customer_id, c.name +FROM customers c +WHERE (SELECT COUNT(*) FROM orders o WHERE o.customer_id = c.customer_id) > 3 +LIMIT 50; +-- EXPECTED_COUNT: 50 +"# + .to_string() + } + + fn gen_setop_union(&self) -> String { + r#"-- 17_setop_union.sql - UNION operations +-- Generated by tsq + +-- Q160: UNION ALL +SELECT customer_id, name FROM customers WHERE city = 'New York' +UNION ALL +SELECT customer_id, name FROM customers WHERE city = 'Los Angeles' +LIMIT 100; +-- EXPECTED_COUNT varies + +-- Q161: UNION (removes duplicates) +SELECT city FROM customers WHERE state = 'CA' +UNION +SELECT city FROM customers WHERE state = 'NY'; +-- EXPECTED_COUNT varies + +-- Q162: UNION ALL with different filters +SELECT order_id, total_amount FROM orders WHERE status = 'pending' +UNION ALL +SELECT order_id, total_amount FROM orders WHERE status = 'shipped' +LIMIT 200; +-- EXPECTED_COUNT varies +"# + .to_string() + } + + fn gen_setop_intersect_except(&self) -> String { + r#"-- 18_setop_intersect_except.sql - INTERSECT and EXCEPT +-- Generated by tsq + +-- Q170: INTERSECT +SELECT customer_id FROM orders WHERE status = 'delivered' +INTERSECT +SELECT customer_id FROM reviews WHERE rating >= 4 +LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q171: EXCEPT +SELECT customer_id FROM customers WHERE is_active = 1 +EXCEPT +SELECT customer_id FROM orders WHERE status = 'cancelled' +LIMIT 100; +-- EXPECTED_COUNT varies + +-- Q172: EXCEPT to find customers without orders +SELECT customer_id FROM customers +EXCEPT +SELECT DISTINCT customer_id FROM orders +LIMIT 50; +-- EXPECTED_COUNT varies +"# + .to_string() + } + + fn gen_string_functions(&self) -> String { + r#"-- 19_string_functions.sql - String functions +-- Generated by tsq + +-- Q180: UPPER +SELECT * FROM customers WHERE UPPER(city) = 'NEW YORK' LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q181: LOWER +SELECT * FROM products WHERE LOWER(category) = 'electronics' LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q182: SUBSTR +SELECT * FROM customers WHERE SUBSTR(email, 1, 4) = 'john' LIMIT 20; +-- EXPECTED_COUNT varies + +-- Q183: CONCAT +SELECT customer_id, CONCAT(city, ', ', state) AS location FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q184: LEFT +SELECT * FROM customers WHERE LEFT(name, 1) = 'J' LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q185: RIGHT +SELECT * FROM customers WHERE RIGHT(email, 10) = '@gmail.com' LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q186: TRIM +SELECT customer_id, TRIM(city) AS trimmed_city FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 + +-- Q187: REPLACE +SELECT customer_id, REPLACE(email, '@', ' at ') AS safe_email FROM customers LIMIT 10; +-- EXPECTED_COUNT: 10 +"# + .to_string() + } + + fn gen_math_functions(&self) -> String { + r#"-- 20_math_functions.sql - Math functions +-- Generated by tsq + +-- Q190: ABS +SELECT * FROM orders WHERE ABS(discount_percent - 15) <= 5 LIMIT 50; +-- EXPECTED_COUNT varies + +-- Q191: ROUND +SELECT product_id, price, ROUND(price) AS rounded_price FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q192: CEIL +SELECT product_id, price, CEIL(price) AS ceiling FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q193: FLOOR +SELECT product_id, price, FLOOR(price) AS floor FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q194: Arithmetic expressions +SELECT product_id, price, cost, (price - cost) AS profit FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q195: Percentage calculation +SELECT product_id, price, cost, (price - cost) / price * 100 AS margin_pct FROM products WHERE price > 0 LIMIT 20; +-- EXPECTED_COUNT: 20 +"#.to_string() + } + + fn gen_case_coalesce(&self) -> String { + r#"-- 21_case_coalesce.sql - CASE, COALESCE, NULLIF +-- Generated by tsq + +-- Q200: Simple CASE +SELECT customer_id, credit_score, + CASE + WHEN credit_score >= 800 THEN 'Excellent' + WHEN credit_score >= 700 THEN 'Good' + WHEN credit_score >= 600 THEN 'Fair' + ELSE 'Poor' + END AS credit_tier +FROM customers LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q201: CASE in WHERE +SELECT * FROM customers +WHERE CASE WHEN credit_score > 700 THEN 1 ELSE 0 END = 1 +LIMIT 50; +-- EXPECTED_COUNT: 50 + +-- Q202: COALESCE +SELECT customer_id, COALESCE(notes, 'No notes') AS notes_display +FROM customers LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q203: NULLIF +SELECT product_id, NULLIF(quantity_in_stock, 0) AS stock_or_null +FROM products LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q204: Nested CASE +SELECT order_id, total_amount, + CASE + WHEN total_amount > 1000 THEN 'Premium' + WHEN total_amount > 500 THEN 'Standard' + WHEN total_amount > 100 THEN 'Basic' + ELSE 'Micro' + END AS order_tier +FROM orders LIMIT 20; +-- EXPECTED_COUNT: 20 +"# + .to_string() + } + + fn gen_mutation_insert(&self) -> String { + r#"-- 22_mutation_insert.sql - INSERT statements +-- Generated by tsq +-- NOTE: Run with --write flag to persist changes + +-- Q210: INSERT single row +INSERT INTO customers (customer_id, name, email, city, state, country, signup_date, is_active, credit_score) +VALUES (999999, 'Test User', 'test@example.com', 'Test City', 'TS', 'USA', '2026-01-01', 1, 750); + +-- Q211: Verify insert +SELECT * FROM customers WHERE customer_id = 999999; +-- EXPECTED_COUNT: 1 + +-- Q212: INSERT with expression values +INSERT INTO products (product_id, name, category, subcategory, price, cost, quantity_in_stock, is_discontinued, created_date) +VALUES (999999, 'Test Product', 'Electronics', 'Phones', 99.99, 49.99, 100, 0, '2026-01-01'); + +-- Q213: Verify product insert +SELECT * FROM products WHERE product_id = 999999; +-- EXPECTED_COUNT: 1 +"#.to_string() + } + + fn gen_mutation_update(&self) -> String { + r#"-- 23_mutation_update.sql - UPDATE statements +-- Generated by tsq +-- NOTE: Run with --write flag to persist changes + +-- Q220: Count before update +SELECT COUNT(*) AS before_count FROM customers WHERE is_active = 0 AND credit_score < 400; +-- Record this count + +-- Q221: UPDATE with WHERE +UPDATE customers SET is_active = 0 WHERE credit_score < 400; + +-- Q222: Verify update +SELECT COUNT(*) AS after_count FROM customers WHERE is_active = 0 AND credit_score < 400; +-- EXPECTED: count should match credit_below_400 + +-- Q223: UPDATE products +UPDATE products SET quantity_in_stock = quantity_in_stock + 10 WHERE is_discontinued = 0; + +-- Q224: Verify product update +SELECT COUNT(*) FROM products WHERE is_discontinued = 0; +-- EXPECTED_COUNT: 1 +"# + .to_string() + } + + fn gen_mutation_delete(&self) -> String { + let v = &self.verification; + let cancelled_count = v.status_counts.get("cancelled").copied().unwrap_or(0); + + format!( + r#"-- 24_mutation_delete.sql - DELETE statements +-- Generated by tsq +-- NOTE: Run with --write flag to persist changes + +-- Q230: Count before delete +SELECT COUNT(*) AS before_delete FROM orders WHERE status = 'cancelled'; +-- EXPECTED_VALUE: approximately {cancelled_count} + +-- Q231: DELETE with WHERE +DELETE FROM orders WHERE status = 'cancelled'; + +-- Q232: Verify delete +SELECT COUNT(*) AS after_delete FROM orders WHERE status = 'cancelled'; +-- EXPECTED_VALUE: 0 + +-- Q233: DELETE from reviews (low rating) +SELECT COUNT(*) FROM reviews WHERE rating = 1; +-- Record count before + +-- Q234: Execute delete +DELETE FROM reviews WHERE rating = 1; + +-- Q235: Verify +SELECT COUNT(*) FROM reviews WHERE rating = 1; +-- EXPECTED_VALUE: 0 +"#, + cancelled_count = cancelled_count + ) + } + + fn gen_complex_combined(&self) -> String { + r#"-- 25_complex_combined.sql - Complex combined queries +-- Generated by tsq + +-- Q240: Multi-table aggregate with GROUP BY and ORDER BY +SELECT c.city, c.state, + COUNT(DISTINCT o.order_id) AS order_count, + SUM(o.total_amount) AS total_revenue, + AVG(o.total_amount) AS avg_order +FROM customers c, orders o +WHERE c.customer_id = o.customer_id AND o.status = 'delivered' +GROUP BY c.city, c.state +ORDER BY total_revenue DESC +LIMIT 20; +-- EXPECTED_COUNT: 20 + +-- Q241: Subquery with aggregate +SELECT category, AVG(price) AS avg_price +FROM products +WHERE price > (SELECT AVG(price) FROM products) +GROUP BY category +ORDER BY avg_price DESC; +-- EXPECTED_COUNT varies + +-- Q242: Window function with JOIN +SELECT c.name, o.order_id, o.total_amount, + ROW_NUMBER() OVER (PARTITION BY c.customer_id ORDER BY o.total_amount DESC) AS order_rank +FROM customers c, orders o +WHERE c.customer_id = o.customer_id +LIMIT 100; +-- EXPECTED_COUNT: 100 + +-- Q243: Complex filter with multiple conditions +SELECT c.customer_id, c.name, c.credit_score, COUNT(o.order_id) AS orders +FROM customers c, orders o +WHERE c.customer_id = o.customer_id + AND c.is_active = 1 + AND c.credit_score > 650 + AND o.status IN ('delivered', 'shipped') + AND o.total_amount > 100 +GROUP BY c.customer_id, c.name, c.credit_score +HAVING COUNT(o.order_id) >= 2 +ORDER BY orders DESC +LIMIT 30; +-- EXPECTED_COUNT: 30 + +-- Q244: UNION with aggregates +SELECT 'High Value' AS segment, COUNT(*) AS cnt FROM orders WHERE total_amount > 1000 +UNION ALL +SELECT 'Medium Value' AS segment, COUNT(*) AS cnt FROM orders WHERE total_amount BETWEEN 100 AND 1000 +UNION ALL +SELECT 'Low Value' AS segment, COUNT(*) AS cnt FROM orders WHERE total_amount < 100; +-- EXPECTED_COUNT: 3 + +-- Q245: Four-table join with aggregates +SELECT p.category, + COUNT(DISTINCT c.customer_id) AS unique_customers, + COUNT(DISTINCT o.order_id) AS order_count, + SUM(oi.line_total) AS total_sales +FROM customers c, orders o, order_items oi, products p +WHERE c.customer_id = o.customer_id + AND o.order_id = oi.order_id + AND oi.product_id = p.product_id +GROUP BY p.category +ORDER BY total_sales DESC; +-- EXPECTED_COUNT varies by categories +"#.to_string() + } +} + +// ============================================================================ +// Verification Generator +// ============================================================================ + +struct VerificationGenerator<'a> { + verification: &'a VerificationData, + seed: u64, +} + +impl<'a> VerificationGenerator<'a> { + fn new(verification: &'a VerificationData, seed: u64) -> Self { + Self { verification, seed } + } + + fn generate_all(&self, base_path: &Path) -> Result<()> { + self.write_expected_counts(&base_path.join("verify/expected_counts.txt"))?; + self.write_verification_script(&base_path.join("verify/run_verification.sh"))?; + Ok(()) + } + + fn write_expected_counts(&self, path: &Path) -> Result<()> { + let v = &self.verification; + let content = format!( + r#"# Expected counts for verification +# Generated by tsq with seed: {} + +# Table row counts +customers: {} +products: {} +orders: {} +order_items: {} +reviews: {} + +# Distribution counts +customers_with_notes: {} +reviews_with_body: {} +active_customers: {} +discontinued_products: {} +customers_with_orders: {} + +# Credit score distribution +credit_below_400: {} +credit_600_to_750: {} +credit_above_700: {} +credit_above_800: {} +min_credit_score: {} +max_credit_score: {} + +# Price distribution +price_below_100: {} + +# Status distribution +{} + +# City distribution +{} +"#, + self.seed, + v.customer_count, + v.product_count, + v.order_count, + v.order_item_count, + v.review_count, + v.customers_with_notes, + v.reviews_with_body, + v.active_customers, + v.discontinued_products, + v.customers_with_orders, + v.credit_below_400, + v.credit_600_to_750, + v.credit_above_700, + v.credit_above_800, + v.min_credit_score, + v.max_credit_score, + v.price_below_100, + v.status_counts + .iter() + .map(|(k, c)| format!("status_{}: {}", k, c)) + .collect::>() + .join("\n"), + v.city_counts + .iter() + .take(5) + .map(|(k, c)| format!("city_{}: {}", k.replace(' ', "_"), c)) + .collect::>() + .join("\n"), + ); + + let mut file = File::create(path)?; + file.write_all(content.as_bytes())?; + Ok(()) + } + + fn write_verification_script(&self, path: &Path) -> Result<()> { + let v = &self.verification; + let content = format!( + r#"#!/bin/bash +# Verification script for tsq-generated data +# Seed: {} +# Run this script from the output directory +# +# Usage: SQAWK=/path/to/sqawk ./run_verification.sh +# or: SQAWK="cargo run --bin sqawk --" ./run_verification.sh + +SQAWK="${{SQAWK:-sqawk}}" +DATA_DIR="./data" +PASS=0 +FAIL=0 + +echo "=== TSQ Verification Script ===" +echo "Seed: {}" +echo "Data directory: $DATA_DIR" +echo "Using sqawk: $SQAWK" +echo "" + +# Test if sqawk is available +if ! $SQAWK -s "SELECT 1" /dev/null 2>/dev/null; then + echo "ERROR: sqawk not found or not working" + echo "Set SQAWK environment variable to the path of sqawk binary" + echo " e.g., SQAWK=/path/to/sqawk ./run_verification.sh" + echo " or: SQAWK='cargo run --bin sqawk --' ./run_verification.sh" + exit 1 +fi + +check_count() {{ + local desc="$1" + local expected="$2" + local sql="$3" + local files="$4" + + # Run sqawk and get the last line (data row, skipping header) + result=$($SQAWK -s "$sql" $files 2>/dev/null | tail -n 1) + + if [ "$result" = "$expected" ]; then + echo "[PASS] $desc: $result" + ((PASS++)) || true + else + echo "[FAIL] $desc: expected $expected, got '$result'" + ((FAIL++)) || true + fi +}} + +echo "--- Row Count Verification ---" +check_count "Customer count" "{}" "SELECT COUNT(*) FROM customers" "$DATA_DIR/customers.csv" +check_count "Product count" "{}" "SELECT COUNT(*) FROM products" "$DATA_DIR/products.csv" +check_count "Order count" "{}" "SELECT COUNT(*) FROM orders" "$DATA_DIR/orders.csv" +check_count "Order item count" "{}" "SELECT COUNT(*) FROM order_items" "$DATA_DIR/order_items.csv" +check_count "Review count" "{}" "SELECT COUNT(*) FROM reviews" "$DATA_DIR/reviews.csv" + +echo "" +echo "--- Distribution Verification ---" +check_count "Customers with notes" "{}" "SELECT COUNT(notes) FROM customers" "$DATA_DIR/customers.csv" +check_count "Active customers" "{}" "SELECT COUNT(*) FROM customers WHERE is_active = 1" "$DATA_DIR/customers.csv" +check_count "Credit > 700" "{}" "SELECT COUNT(*) FROM customers WHERE credit_score > 700" "$DATA_DIR/customers.csv" + +echo "" +echo "=== Results ===" +echo "Passed: $PASS" +echo "Failed: $FAIL" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi +echo "All tests passed!" +"#, + self.seed, + self.seed, + v.customer_count, + v.product_count, + v.order_count, + v.order_item_count, + v.review_count, + v.customers_with_notes, + v.active_customers, + v.credit_above_700, + ); + + let mut file = File::create(path)?; + file.write_all(content.as_bytes())?; + + // Make executable on Unix + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(path)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(path, perms)?; + } + + Ok(()) + } +} + +// ============================================================================ +// Metadata Writer +// ============================================================================ + +fn write_metadata(base_path: &Path, seed: u64, rows: usize, v: &VerificationData) -> Result<()> { + let content = format!( + r#"{{ + "seed": {}, + "base_rows": {}, + "row_counts": {{ + "customers": {}, + "products": {}, + "orders": {}, + "order_items": {}, + "reviews": {} + }}, + "tsq_version": "0.1.0" +}} +"#, + seed, + rows, + v.customer_count, + v.product_count, + v.order_count, + v.order_item_count, + v.review_count + ); + + let mut file = File::create(base_path.join("metadata.json"))?; + file.write_all(content.as_bytes())?; + Ok(()) +} + +// ============================================================================ +// Main Entry Point +// ============================================================================ + +fn main() -> Result<()> { + let args = Args::parse(); + + // Determine seed: CLI --seed or (time ^ pid) + let seed = args.seed.unwrap_or_else(default_seed); + + println!("TSQ - Test SQL Query Generator for sqawk"); + println!("========================================="); + println!("Seed: {}", seed); + println!("Rows: {} (base customer count)", args.rows); + println!("Output: {}", args.output_dir); + println!(); + + // Create output directories + let base_path = Path::new(&args.output_dir); + fs::create_dir_all(base_path.join("data")).context("Failed to create data directory")?; + fs::create_dir_all(base_path.join("queries")).context("Failed to create queries directory")?; + fs::create_dir_all(base_path.join("verify")).context("Failed to create verify directory")?; + + // Generate data + println!("Generating data..."); + let mut generator = DataGenerator::new(seed, args.rows, args.verbose); + generator.generate_all(base_path)?; + println!(); + + // Generate queries + println!("Generating queries..."); + let query_gen = QueryGenerator::new(&generator.verification); + query_gen.generate_all(base_path)?; + println!(" Created 25 query files"); + println!(); + + // Generate verification + println!("Generating verification scripts..."); + let verify_gen = VerificationGenerator::new(&generator.verification, seed); + verify_gen.generate_all(base_path)?; + println!(" Created expected_counts.txt"); + println!(" Created run_verification.sh"); + println!(); + + // Write metadata + write_metadata(base_path, seed, args.rows, &generator.verification)?; + println!(" Created metadata.json"); + println!(); + + // Summary + let v = &generator.verification; + println!("Generation complete!"); + println!("-----------------------------------------"); + println!("Tables generated:"); + println!(" customers: {:>10} rows", v.customer_count); + println!(" products: {:>10} rows", v.product_count); + println!(" orders: {:>10} rows", v.order_count); + println!(" order_items: {:>10} rows", v.order_item_count); + println!(" reviews: {:>10} rows", v.review_count); + println!("-----------------------------------------"); + println!( + "Total rows: {:>10}", + v.customer_count + v.product_count + v.order_count + v.order_item_count + v.review_count + ); + println!(); + println!("To run sqawk on generated data:"); + println!( + " sqawk -s \"SELECT * FROM customers LIMIT 10\" {}/data/customers.csv", + args.output_dir + ); + println!(); + println!("To run verification:"); + println!( + " cd {} && bash verify/run_verification.sh", + args.output_dir + ); + + Ok(()) +} diff --git a/src/vm/bytecode.rs b/src/vm/bytecode.rs index eded512..e0cc97b 100644 --- a/src/vm/bytecode.rs +++ b/src/vm/bytecode.rs @@ -7,11 +7,72 @@ //! Each instruction has a specific semantics that controls how data is loaded, //! manipulated, and stored during SQL query execution. -use crate::table::Value; +use crate::capacity::{DEFAULT_COLUMN_CAPACITY, DEFAULT_INSTRUCTION_CAPACITY}; +use crate::table::{DataType, Value}; use std::fmt; +use std::rc::Rc; + +// Aggregate function type constants for AggStep/AggFinal opcodes +/// COUNT aggregate function type +pub const AGG_COUNT: i64 = 0; +/// SUM aggregate function type +pub const AGG_SUM: i64 = 1; +/// AVG aggregate function type +pub const AGG_AVG: i64 = 2; +/// MIN aggregate function type +pub const AGG_MIN: i64 = 3; +/// MAX aggregate function type +pub const AGG_MAX: i64 = 4; + +/// A column in a result schema +#[derive(Debug, Clone)] +pub struct ResultColumn { + /// Name of the column (may include table prefix like "users.name") + pub name: String, + /// Data type of the column + pub data_type: DataType, +} + +impl ResultColumn { + pub fn new(name: String, data_type: DataType) -> Self { + Self { name, data_type } + } +} + +/// Schema definition for query results +/// +/// This is built at compile time and carries both column names and types, +/// allowing the VM to construct properly-typed result tables. +#[derive(Debug, Clone, Default)] +pub struct ResultSchema { + /// Columns in the result set + pub columns: Vec, +} + +impl ResultSchema { + pub fn new() -> Self { + Self { + columns: Vec::with_capacity(DEFAULT_COLUMN_CAPACITY), + } + } + + /// Add a column to the schema + pub fn add_column(&mut self, name: String, data_type: DataType) { + self.columns.push(ResultColumn::new(name, data_type)); + } + + /// Check if schema is empty + pub fn is_empty(&self) -> bool { + self.columns.is_empty() + } +} /// Opcodes for VM instructions +/// +/// Note: Some opcodes are defined but not yet used by the compiler. +/// They are kept for planned features in the SQL expansion roadmap. #[derive(Debug, Clone, Copy, PartialEq)] +#[allow(dead_code)] pub enum OpCode { // Program flow control Init, // Initialize VM @@ -24,38 +85,119 @@ pub enum OpCode { Close, // Close a cursor // Cursor operations - Rewind, // Move cursor to first row - Next, // Move cursor to next row - Column, // Read column value into register + Rewind, // Move cursor to first row + Next, // Move cursor to next row + Column, // Read column value into register + InsertRow, // Insert row from registers P2..P2+P3 into cursor P1 + DeleteRow, // Delete current row at cursor P1 // Data manipulation Integer, // Load integer constant String, // Load string constant Null, // Load NULL value ResultRow, // Return result row to client + Copy, // Copy register P1 to register P2 (SCopy in SQLite) // Transaction operations - Begin, // Begin a transaction - marks the start of a set of changes that can be committed or rolled back - Commit, // Commit a transaction - permanently applies all changes made since the Begin operation - Rollback, // Rollback a transaction - discards all changes made since the Begin operation + Begin, // Begin a transaction - marks the start of a set of changes that can be committed or rolled back + Commit, // Commit a transaction - permanently applies all changes made since the Begin operation + Rollback, // Rollback a transaction - discards all changes made since the Begin operation SavePoint, // Create a savepoint in the transaction - establishes a point to which a transaction can be partially rolled back Release, // Release a savepoint - confirms changes up to the specified savepoint // Comparison operations - Lt, // Less than comparison (P1 < P2, result in P3) - Le, // Less than or equal comparison (P1 <= P2, result in P3) - Eq, // Equal comparison (P1 == P2, result in P3) - Ne, // Not equal comparison (P1 != P2, result in P3) - Gt, // Greater than comparison (P1 > P2, result in P3) - Ge, // Greater than or equal comparison (P1 >= P2, result in P3) - + Lt, // Less than comparison (P1 < P2, result in P3) + Le, // Less than or equal comparison (P1 <= P2, result in P3) + Eq, // Equal comparison (P1 == P2, result in P3) + Ne, // Not equal comparison (P1 != P2, result in P3) + Gt, // Greater than comparison (P1 > P2, result in P3) + Ge, // Greater than or equal comparison (P1 >= P2, result in P3) + + // Pattern matching operations + Like, // LIKE pattern match (P1 LIKE P4 pattern, result in P3). Case-sensitive. + Glob, // GLOB pattern match (P1 GLOB P4 pattern, result in P3). Unix-style wildcards. + + // Type conversion + Cast, // Cast value in P1 to type in P4, result in P2. Types: "INTEGER", "TEXT", "REAL" + + // Null operations + IsNull, // Set P2 to 1 if P1 is NULL, 0 otherwise + // Conditional jumps - IfZ, // Jump to P2 if register P1 contains 0 - IfPos, // Jump to P2 if register P1 is positive (> 0) - IfNeg, // Jump to P2 if register P1 is negative (< 0) - + IfZ, // Jump to P2 if register P1 contains 0 + IfPos, // Jump to P2 if register P1 is positive (> 0) + IfNeg, // Jump to P2 if register P1 is negative (< 0) + // Utility opcodes Noop, // No operation + + // Set operations + Distinct, // Remove duplicate rows from results + Limit, // Limit results to P1 rows. P2 = offset (rows to skip, already applied). Post-processing opcode. + Intersect, // Keep only rows that exist in both left and right result sets + Except, // Keep only rows from left that don't exist in right result set + + // Join operations + NullRow, // Load NULL values into registers P1 through P1+P2-1 (P2 = count) + RewindInner, // Rewind cursor P1, used for inner loop of nested join + MarkMatch, // Set register P1 to 1 to indicate a match was found + CheckMatch, // If register P1 is 0 (no match), jump to P2; reset register to 0 + + // Subquery/Coroutine operations (inspired by SQLite) + InitCoroutine, // P1 = register to store return address, P2 = jump over coroutine, P3 = coroutine entry + Yield, // Swap program counter with register P1 (coroutine yield) + EndCoroutine, // Jump back to Yield that invoked this coroutine, set P1 for next Yield return + Once, // First time: continue. Subsequently: jump to P2. Used for one-time init. + + // Subquery result operations + AggStep, // P1 = function type, P2 = value reg, P3 = accumulator reg. Step aggregate. + AggFinal, // P1 = accumulator reg, P2 = result reg. Finalize aggregate. + AggReset, // P1 = accumulator reg. Reset accumulator to initial state. + Exists, // P1 = flag reg (set to 1 if cursor P2 has rows, else 0) + NotExists, // P1 = flag reg (set to 1 if cursor P2 has no rows, else 0) + + // Counter operations (LIMIT/OFFSET) + DecrJumpZero, // Decrement register P1. If result is zero, jump to P2. + + // Sorter operations (ORDER BY) + SorterOpen, // Open a sorter. P1 = sorter ID, P2 = number of columns, P4 = sort key spec + SorterInsert, // Insert row into sorter P1. P2 = start register, P3 = column count + SorterSort, // Sort the sorter P1. Must be called before SorterNext. + SorterData, // Copy current sorter P1 row to registers starting at P2. P3 = column count. + SorterNext, // Advance sorter P1. Jump to P2 if more rows, else continue. + + // Ephemeral table operations (for ORDER BY results, temp storage) + // Ephemeral tables use cursor IDs and work with Column/Next/Rewind opcodes + OpenEphemeral, // Open ephemeral cursor P1 with P2 columns. P4 = sort key spec for ordering. + IdxInsert, // Insert into ephemeral cursor P1. P2 = start register, P3 = column count. + Sort, // Sort ephemeral cursor P1. Jump to P2 when empty, else continue. + Sequence, // P1 = ephemeral cursor, P2 = dest register. Generate next sequence number. + + // DDL operations + CreateTable, // Create a table. P4 = "table_name:col1:type1,col2:type2,..." + DropTable, // Drop a table. P4 = table name + AlterTableAdd, // Add column to table. P4 = "table_name:col_name:col_type" + Truncate, // Remove all rows from table. P4 = table name + + // String functions + StringFunc, // P1 = src reg, P2 = dest reg, P3 = arg2 reg (for SUBSTR/REPLACE), P4 = function name + + // Math functions + MathFunc, // P1 = src reg, P2 = dest reg, P4 = function name (ABS, ROUND, CEIL, FLOOR) + + // Date/Time functions + DateFunc, // P1 = src reg (optional), P2 = dest reg, P4 = function name (DATE, TIME, NOW) + + // Arithmetic operations + Add, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left + right + Subtract, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left - right + Multiply, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left * right + Divide, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left / right + Remainder, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left % right + + // Window function operations + WindowAggStep, // P1 = func type, P2 = value reg, P3 = accum reg, P4 = window spec. Step window aggregate. + WindowValue, // P1 = accum reg, P2 = dest reg, P3 = func type. Get current window value. } /// A SQL VM instruction with opcode and parameters @@ -74,15 +216,17 @@ pub struct Instruction { pub p3: i64, /// P4 parameter (typically a string parameter) - pub p4: Option, + /// Uses Rc for O(1) clone in execution loop + pub p4: Option>, - // Removed unused p5 parameter /// Comment describing the instruction - pub comment: Option, + /// Uses Rc for O(1) clone in execution loop + pub comment: Option>, } impl Instruction { /// Create a new instruction with the given opcode and parameters + /// Converts String to Rc for O(1) clone in execution loop pub fn new( opcode: OpCode, p1: i64, @@ -97,8 +241,8 @@ impl Instruction { p1, p2, p3, - p4, - comment, + p4: p4.map(Rc::from), + comment: comment.map(Rc::from), } } } @@ -129,8 +273,9 @@ impl fmt::Display for Instruction { /// A program of bytecode instructions #[derive(Debug, Clone)] pub struct Program { - /// The list of instructions pub instructions: Vec, + /// Schema for the result set (built at compile time) + pub result_schema: ResultSchema, } impl Default for Program { @@ -140,13 +285,18 @@ impl Default for Program { } impl Program { - /// Create a new empty program pub fn new() -> Self { Self { - instructions: Vec::new(), + instructions: Vec::with_capacity(DEFAULT_INSTRUCTION_CAPACITY), + result_schema: ResultSchema::new(), } } + /// Set the result schema for this program + pub fn set_result_schema(&mut self, schema: ResultSchema) { + self.result_schema = schema; + } + /// Add an instruction to the program pub fn add_instruction(&mut self, instruction: Instruction) { self.instructions.push(instruction); @@ -198,7 +348,7 @@ impl From for Register { match value { Value::Integer(i) => Register::Integer(i), Value::Float(f) => Register::Float(f), - Value::String(s) => Register::String(s), + Value::String(s) => Register::String(s.into_owned()), Value::Boolean(b) => Register::Boolean(b), Value::Null => Register::Null, } @@ -207,10 +357,11 @@ impl From for Register { impl From for Value { fn from(register: Register) -> Self { + use std::borrow::Cow; match register { Register::Integer(i) => Value::Integer(i), Register::Float(f) => Value::Float(f), - Register::String(s) => Value::String(s), + Register::String(s) => Value::String(Cow::Owned(s)), Register::Boolean(b) => Value::Boolean(b), Register::Null => Value::Null, } diff --git a/src/vm/compiler.rs b/src/vm/compiler.rs index e89a040..3db2cc1 100644 --- a/src/vm/compiler.rs +++ b/src/vm/compiler.rs @@ -4,32 +4,55 @@ //! into bytecode instructions that can be executed by the SQL VM. //! It implements a visitor pattern to walk the AST generated by sqlparser. -use sqlparser::ast::{BinaryOperator, Expr, ObjectName, Query, Select, SelectItem, SetExpr, Statement, TableWithJoins, Value}; +use sqlparser::ast::{ + BinaryOperator, DataType as SqlDataType, Expr, Function, FunctionArg, FunctionArgExpr, + ObjectName, Query, Select, SelectItem, SetExpr, SetOperator, SetQuantifier, Statement, + TableWithJoins, UnaryOperator, Value, +}; use sqlparser::dialect::HiveDialect; use sqlparser::parser::Parser; use std::collections::HashMap; -use super::bytecode::{Instruction, OpCode, Program}; +use super::bytecode::{ + Instruction, OpCode, Program, ResultSchema, AGG_AVG, AGG_COUNT, AGG_MAX, AGG_MIN, AGG_SUM, +}; +use crate::aggregate::AggregateFunction; +use crate::capacity::DEFAULT_CURSOR_CAPACITY; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; -use crate::table::Table; +use crate::table::{DataType, Table, Value as TableValue}; + +/// Result type for multi-table projection resolution +/// Contains: (columns_per_table, output_column_refs, schema, is_wildcard) +pub(crate) type MultiTableProjection = (Vec>, Vec<(usize, usize)>, ResultSchema, bool); +/// Outer column reference found in a correlated subquery (Phase 4B) +#[derive(Debug, Clone)] +struct OuterColumnRef { + /// The table qualifier (alias or table name) + qualifier: String, + /// The column name + column: String, +} /// SQL statement compiler that generates bytecode for the VM engine pub struct SqlCompiler<'a> { /// The database containing tables - database: &'a Database, + pub(crate) database: &'a Database, /// Name mapping for tables referenced in the query - table_map: HashMap, + pub(crate) table_map: HashMap, /// Current bytecode program being generated - program: Program, + pub(crate) program: Program, /// Counter for register allocation - register_counter: i64, + pub(crate) register_counter: i64, /// Whether to generate verbose bytecode with detailed comments - verbose: bool, + pub(crate) verbose: bool, + + /// Current outer table alias for correlated subquery detection + pub(crate) current_outer_alias: Option, } impl<'a> SqlCompiler<'a> { @@ -37,32 +60,285 @@ impl<'a> SqlCompiler<'a> { pub fn new(database: &'a Database, verbose: bool) -> Self { SqlCompiler { database, - table_map: HashMap::new(), + table_map: HashMap::with_capacity(DEFAULT_CURSOR_CAPACITY), program: Program::new(), register_counter: 0, verbose, + current_outer_alias: None, } } /// Allocate a new register and return its index - fn allocate_register(&mut self) -> i64 { + pub(crate) fn allocate_register(&mut self) -> i64 { let reg = self.register_counter; self.register_counter += 1; reg } + /// Allocate multiple consecutive registers and return the starting index + pub(crate) fn allocate_registers(&mut self, count: usize) -> i64 { + let start = self.allocate_register(); + for _ in 1..count { + self.allocate_register(); + } + start + } + /// Reset the register counter (used between statements) fn reset_registers(&mut self) { self.register_counter = 0; } + /// Helper method to emit an instruction + pub(crate) fn emit( + &mut self, + opcode: OpCode, + p1: i64, + p2: i64, + p3: i64, + p4: Option, + comment: &str, + ) { + self.program.add_instruction(Instruction::new( + opcode, + p1, + p2, + p3, + p4, + 0, + Some(comment.to_string()), + )); + } + + /// Helper to emit Column opcodes for multiple columns + pub(crate) fn emit_column_loads( + &mut self, + cursor: i64, + cols: &[usize], + start_reg: i64, + label: &str, + ) { + for (i, col_idx) in cols.iter().enumerate() { + self.emit( + OpCode::Column, + cursor, + *col_idx as i64, + start_reg + i as i64, + None, + &format!("r[{}] = {}.col[{}]", start_reg + i as i64, label, col_idx), + ); + } + } + + /// Convert aggregate function name to type code + pub(crate) fn agg_func_type(name: &str) -> i64 { + match name { + "COUNT" => AGG_COUNT, + "SUM" => AGG_SUM, + "AVG" => AGG_AVG, + "MIN" => AGG_MIN, + "MAX" => AGG_MAX, + _ => AGG_COUNT, // Default to COUNT + } + } + + /// Patch a jump instruction's target address (p2) + pub(crate) fn patch_jump(&mut self, addr: usize, target: usize) { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = target as i64; + } + } + + /// Emit AND logic: result = left_reg && right_reg + /// Returns the register containing the result (1 if both true, 0 otherwise) + pub(crate) fn emit_and(&mut self, left_reg: i64, right_reg: i64) -> i64 { + let result_reg = self.allocate_register(); + + // Start with 0 (false) + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (AND init)", result_reg), + ); + + // If left is 0, skip to end (result stays 0) + let skip_addr1 = self.program.len(); + self.emit(OpCode::IfZ, left_reg, 0, 0, None, "Skip if left is false"); + + // If right is 0, skip to end (result stays 0) + let skip_addr2 = self.program.len(); + self.emit(OpCode::IfZ, right_reg, 0, 0, None, "Skip if right is false"); + + // Both true, set result to 1 + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (AND true)", result_reg), + ); + + // Patch skip addresses to end + let end_addr = self.program.len(); + self.patch_jump(skip_addr1, end_addr); + self.patch_jump(skip_addr2, end_addr); + + result_reg + } + + /// Emit OR logic: result = left_reg || right_reg + /// Returns the register containing the result (1 if either true, 0 otherwise) + pub(crate) fn emit_or(&mut self, left_reg: i64, right_reg: i64) -> i64 { + let result_reg = self.allocate_register(); + + // Start with 1 (optimistic - assume true) + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (OR init)", result_reg), + ); + + // If left is non-zero, skip to end (result stays 1) + let skip_addr1 = self.program.len(); + self.emit(OpCode::IfPos, left_reg, 0, 0, None, "Skip if left is true"); + + // If right is non-zero, skip to end (result stays 1) + let skip_addr2 = self.program.len(); + self.emit( + OpCode::IfPos, + right_reg, + 0, + 0, + None, + "Skip if right is true", + ); + + // Both false, set result to 0 + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (OR false)", result_reg), + ); + + // Patch skip addresses to end + let end_addr = self.program.len(); + self.patch_jump(skip_addr1, end_addr); + self.patch_jump(skip_addr2, end_addr); + + result_reg + } + + /// Emit a comparison operation and return the result register + pub(crate) fn emit_comparison( + &mut self, + op: &BinaryOperator, + left_reg: i64, + right_reg: i64, + ) -> SqawkResult { + let result_reg = self.allocate_register(); + let opcode = Self::binary_op_to_comparison_opcode(op)?; + + self.emit( + opcode, + left_reg, + right_reg, + result_reg, + None, + &format!( + "r[{}] = (r[{}] {:?} r[{}])", + result_reg, left_reg, op, right_reg + ), + ); + + Ok(result_reg) + } + + /// Convert a comparison binary operator to the corresponding OpCode + pub(crate) fn binary_op_to_comparison_opcode(op: &BinaryOperator) -> SqawkResult { + match op { + BinaryOperator::Eq => Ok(OpCode::Eq), + BinaryOperator::NotEq => Ok(OpCode::Ne), + BinaryOperator::Lt => Ok(OpCode::Lt), + BinaryOperator::LtEq => Ok(OpCode::Le), + BinaryOperator::Gt => Ok(OpCode::Gt), + BinaryOperator::GtEq => Ok(OpCode::Ge), + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported comparison operator: {:?}", + op + ))), + } + } + + /// Convert SQL type to internal type string + pub(crate) fn sql_type_to_internal(data_type: &str) -> &'static str { + match data_type.to_uppercase().as_str() { + "INTEGER" | "INT" => "INTEGER", + "REAL" | "FLOAT" | "DOUBLE" => "REAL", + "BOOLEAN" | "BOOL" => "BOOLEAN", + _ => "TEXT", + } + } + + /// Find a column by table name and column name in a list of tables. + /// Returns (table_idx, col_idx) if found. + pub(crate) fn find_column_in_tables( + table_name: &str, + col_name: &str, + tables: &[&Table], + table_names: &[&str], + ) -> SqawkResult<(usize, usize)> { + for (i, name) in table_names.iter().enumerate() { + if table_name.eq_ignore_ascii_case(name) { + let columns = tables[i].columns(); + if let Some(col_idx) = columns + .iter() + .position(|c| c.eq_ignore_ascii_case(col_name)) + { + return Ok((i, col_idx)); + } else { + return Err(SqawkError::ColumnNotFound(col_name.to_string())); + } + } + } + Err(SqawkError::TableNotFound(table_name.to_string())) + } + + /// Build a wildcard schema (SELECT *) from multiple tables. + /// Returns column indices per table and the result schema. + pub(crate) fn build_wildcard_schema( + tables: &[&Table], + table_names: &[&str], + ) -> (Vec>, ResultSchema) { + let mut table_cols = Vec::with_capacity(tables.len()); + let mut schema = ResultSchema::new(); + + for (i, table) in tables.iter().enumerate() { + let cols: Vec = (0..table.column_count()).collect(); + for meta in table.column_metadata() { + schema.add_column(format!("{}.{}", table_names[i], meta.name), meta.data_type); + } + table_cols.push(cols); + } + + (table_cols, schema) + } + // Removed unused add_halt method /// Add a comment to the program if verbose mode is enabled - fn add_comment(&mut self, comment: &str) { + pub(crate) fn add_comment(&mut self, comment: &str) { if self.verbose { - let noop = Instruction::new(OpCode::Noop, 0, 0, 0, None, 0, Some(comment.to_string())); - self.program.add_instruction(noop); + self.emit(OpCode::Noop, 0, 0, 0, None, comment); } } @@ -89,15 +365,14 @@ impl<'a> SqlCompiler<'a> { // Add initial "Init" instruction that will be filled in later // with the address of the main program body let init_addr = self.program.len(); - self.program.add_instruction(Instruction::new( + self.emit( OpCode::Init, 0, 0, 0, None, - 0, - Some("Start address will be filled in later".into()), - )); + "Start address will be filled in later", + ); // Compile each statement for statement in statements { @@ -105,22 +380,14 @@ impl<'a> SqlCompiler<'a> { } // Add Halt instruction at the end - self.program.add_instruction(Instruction::new( - OpCode::Halt, - 0, - 0, - 0, - None, - 0, - Some("End execution".to_string()), - )); + self.emit(OpCode::Halt, 0, 0, 0, None, "End execution"); // Go back and update the Init instruction with the correct start address // (usually this would point to setup code like transaction start) let transaction_addr = init_addr + 1; if let Some(instruction) = self.program.instructions.get_mut(init_addr) { instruction.p2 = transaction_addr as i64; - instruction.comment = Some(format!("Start at {}", transaction_addr)); + instruction.comment = Some(format!("Start at {}", transaction_addr).into()); } Ok(self.program.clone()) @@ -132,6 +399,47 @@ impl<'a> SqlCompiler<'a> { fn compile_statement(&mut self, statement: &Statement) -> SqawkResult<()> { match statement { Statement::Query(query) => self.compile_query(query), + Statement::Insert { + table_name, + columns, + source, + .. + } => self.compile_insert(table_name, columns, source), + Statement::Delete { + from, selection, .. + } => self.compile_delete(from, selection.as_ref()), + Statement::Update { + table, + assignments, + selection, + .. + } => self.compile_update(table, assignments, selection.as_ref()), + Statement::CreateTable { + name, + columns, + hive_formats, + location, + with_options, + query, + .. + } => { + if let Some(q) = query { + // CREATE TABLE ... AS SELECT + self.compile_create_table_as_select(name, q) + } else { + self.compile_create_table(name, columns, hive_formats, location, with_options) + } + } + Statement::Drop { + object_type, + names, + if_exists, + .. + } => self.compile_drop(object_type, names, *if_exists), + Statement::AlterTable { + name, operation, .. + } => self.compile_alter_table(name, operation), + Statement::Truncate { table_name, .. } => self.compile_truncate(table_name), _ => Err(SqawkError::UnsupportedSqlFeature(format!( "Unsupported SQL statement type: {:?}", statement @@ -141,10 +449,142 @@ impl<'a> SqlCompiler<'a> { /// Compile a SQL query fn compile_query(&mut self, query: &Query) -> SqawkResult<()> { + // Check if we have ORDER BY, LIMIT, or OFFSET + let has_order_by = !query.order_by.is_empty(); + let has_limit = query.limit.is_some() || query.offset.is_some(); + + // Compile the body (SELECT or set operation) match &*query.body { + SetExpr::Select(select) => { + if has_order_by || has_limit { + self.compile_select_with_post_processing(select, query)?; + } else { + self.compile_select(select)?; + } + } + SetExpr::SetOperation { + op, + left, + right, + set_quantifier, + .. + } => { + let all = *set_quantifier == SetQuantifier::All; + self.compile_set_operation(op, left, right, all)?; + // TODO: Handle ORDER BY/LIMIT for set operations + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT statements are supported".to_string(), + )); + } + } + + Ok(()) + } + + /// Compile a set operation (UNION, INTERSECT, EXCEPT) + fn compile_set_operation( + &mut self, + op: &SetOperator, + left: &SetExpr, + right: &SetExpr, + all: bool, + ) -> SqawkResult<()> { + self.add_comment(&format!( + "Set operation: {:?}{}", + op, + if all { " ALL" } else { "" } + )); + + // For INTERSECT and EXCEPT, we need to track where the left results end + let left_results_marker = match op { + SetOperator::Intersect | SetOperator::Except => { + // Use a register to store the count of left-side results + let marker_reg = self.allocate_register(); + Some(marker_reg) + } + _ => None, + }; + + // Compile the left side + self.compile_set_expr(left)?; + + // For INTERSECT and EXCEPT, record the boundary + if let Some(marker_reg) = left_results_marker { + // Add a marker instruction that the VM will use to know where left results end + // p1=-1 is a special value meaning "use current result count" + // p2 is the destination register + self.emit( + OpCode::Integer, + -1, // Special marker: -1 means "current result count" + marker_reg, + 0, + None, + "Store current result count", + ); + } + + // Compile the right side + self.compile_set_expr(right)?; + + // Add the appropriate set operation opcode + match op { + SetOperator::Union => { + if !all { + // UNION (without ALL) needs to remove duplicates + self.emit( + OpCode::Distinct, + 0, + 0, + 0, + None, + "Remove duplicate rows for UNION", + ); + } + // UNION ALL: no additional processing needed, results are already combined + } + SetOperator::Intersect => { + self.emit( + OpCode::Intersect, + left_results_marker.unwrap_or(0), + 0, + 0, + None, + "Keep only rows in both result sets", + ); + } + SetOperator::Except => { + self.emit( + OpCode::Except, + left_results_marker.unwrap_or(0), + 0, + 0, + None, + "Keep only rows in left but not in right", + ); + } + } + + Ok(()) + } + + /// Compile a set expression (can be a SELECT or nested set operation) + fn compile_set_expr(&mut self, expr: &SetExpr) -> SqawkResult<()> { + match expr { SetExpr::Select(select) => self.compile_select(select), + SetExpr::SetOperation { + op, + left, + right, + set_quantifier, + .. + } => { + let all = *set_quantifier == SetQuantifier::All; + self.compile_set_operation(op, left, right, all) + } _ => Err(SqawkError::UnsupportedSqlFeature( - "Only simple SELECT statements are supported".to_string(), + "Unsupported set expression type".to_string(), )), } } @@ -152,116 +592,900 @@ impl<'a> SqlCompiler<'a> { /// Compile a SELECT statement fn compile_select(&mut self, select: &Select) -> SqawkResult<()> { if select.from.is_empty() { - // This is a SELECT without a FROM clause (e.g., SELECT 1) self.compile_select_literal(&select.projection)?; } else { - // Regular table scan with optional WHERE clause + // Check for implicit join (comma-separated tables in FROM) + if select.from.len() > 1 { + // Multiple tables in FROM clause - treat as implicit cross join with WHERE filter + if self.verbose { + eprintln!("Processing multiple tables in FROM clause as CROSS JOINs"); + } + + // Check for GROUP BY or aggregates - requires specialized handling + let has_group_by = !select.group_by.is_empty(); + let has_aggregates = self.has_aggregates(&select.projection); + + if has_group_by || has_aggregates { + // Create a temporary query to pass to compile_implicit_join_with_group_by + let query = Query { + with: None, + body: Box::new(SetExpr::Select(Box::new(select.clone()))), + order_by: vec![], + limit: None, + offset: None, + fetch: None, + locks: vec![], + }; + return self.compile_implicit_join_with_group_by(select, &query); + } + + return self.compile_implicit_join(select); + } + let table_with_joins = &select.from[0]; - self.compile_table_scan_with_where(table_with_joins, &select.projection, &select.selection)?; + if !table_with_joins.joins.is_empty() { + self.compile_join(table_with_joins, &select.projection, &select.selection)?; + } else { + // Check for GROUP BY, window functions, or aggregates + let has_group_by = !select.group_by.is_empty(); + let has_window_functions = self.has_window_functions(&select.projection); + let has_aggregates = self.has_aggregates(&select.projection); + + if has_window_functions { + // Window function query (ROW_NUMBER, RANK, etc. with OVER clause) + let table_name = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, .. } => { + self.get_table_name(name)? + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported".into(), + )) + } + }; + let table = self.database.get_table(&table_name)?; + self.compile_select_with_window(select, table, &table_name)?; + } else if has_group_by { + // GROUP BY query + let table_name = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, .. } => { + self.get_table_name(name)? + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported".into(), + )) + } + }; + let table = self.database.get_table(&table_name)?; + self.compile_select_with_group_by(select, table, &table_name)?; + } else if has_aggregates { + // Aggregate without GROUP BY - entire table is one group + let table_name = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, .. } => { + self.get_table_name(name)? + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported".into(), + )) + } + }; + let table = self.database.get_table(&table_name)?; + self.compile_select_with_aggregate(select, table, &table_name)?; + } else { + self.compile_table_scan( + table_with_joins, + &select.projection, + &select.selection, + )?; + } + } + } + + // Handle DISTINCT - emit Distinct opcode to deduplicate results + if select.distinct.is_some() { + self.emit(OpCode::Distinct, 0, 0, 0, None, "Remove duplicate rows"); } Ok(()) } - /// Compile a SELECT statement with literals (e.g., SELECT 1, SELECT 'text') - fn compile_select_literal(&mut self, projection: &[SelectItem]) -> SqawkResult<()> { - self.add_comment("Literal value query (no table)"); + /// Compile a SELECT with ORDER BY and/or LIMIT/OFFSET + fn compile_select_with_post_processing( + &mut self, + select: &Select, + query: &Query, + ) -> SqawkResult<()> { + let has_order_by = !query.order_by.is_empty(); - // For each selected literal, add an instruction to load it into a register - let mut result_regs = Vec::new(); + if select.from.is_empty() { + self.compile_select_literal(&select.projection)?; + return Ok(()); + } - for item in projection.iter() { - match item { - SelectItem::UnnamedExpr(expr) => { - // Compile a simple expression and store the value in a register - let reg = self.compile_expr(expr)?; - result_regs.push(reg); - } - SelectItem::ExprWithAlias { expr, alias } => { - // Same as UnnamedExpr but with an alias (column name) - let reg = self.compile_expr(expr)?; - result_regs.push(reg); + // Check for implicit join (comma-separated tables in FROM) + if select.from.len() > 1 { + // Multiple tables in FROM clause + if self.verbose { + eprintln!("Processing multiple tables in FROM clause as CROSS JOINs"); + } - // We don't need to do anything special with the alias for now - // But we could store it for the result table column names - if self.verbose { - self.add_comment(&format!("Column alias: {}", alias)); - } - } - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple expressions supported in SELECT without FROM".to_string(), - )); - } + // Check for GROUP BY or aggregates - requires specialized handling + let has_group_by = !select.group_by.is_empty(); + let has_aggregates = self.has_aggregates(&select.projection); + + if has_group_by || has_aggregates { + // Multi-table query with GROUP BY or aggregates + return self.compile_implicit_join_with_group_by(select, query); } + + return self.compile_implicit_join_with_query(select, query); } - // Generate the result row - if !result_regs.is_empty() { - // Output the result row - self.program.add_instruction(Instruction::new( - OpCode::ResultRow, - result_regs[0], // First register - result_regs.len() as i64, // Number of columns - 0, - None, - 0, - Some("Output literal result row".to_string()), - )); + let table_with_joins = &select.from[0]; + + // Check for explicit JOINs (FROM table1 JOIN table2 ON ...) + if !table_with_joins.joins.is_empty() { + // For multi-table JOINs with ORDER BY, compile the JOIN normally + // ORDER BY support for JOINs requires post-processing which isn't fully implemented + // For now, compile the JOIN and skip ORDER BY + if self.verbose && has_order_by { + eprintln!("Note: ORDER BY with explicit JOINs - compiling join without sort"); + } + return self.compile_join(table_with_joins, &select.projection, &select.selection); + } + + // Get table and columns for ORDER BY resolution + let table_name = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported".into(), + )) + } + }; + + if !self.database.has_table(&table_name) { + return Err(SqawkError::TableNotFound(table_name)); + } + + let table = self.database.get_table(&table_name).unwrap(); + + // Check for GROUP BY - delegate to specialized handler + // GROUP BY with ORDER BY/LIMIT handles these internally + let has_group_by = !select.group_by.is_empty(); + let has_aggregates = self.has_aggregates(&select.projection); + + if has_group_by || has_aggregates { + if has_group_by { + return self.compile_select_with_group_by(select, table, &table_name); + } else { + return self.compile_select_with_aggregate(select, table, &table_name); + } + } + + // Check if projection contains function expressions or computed expressions + let has_computed_expr = select.projection.iter().any(|item| { + let expr = match item { + SelectItem::UnnamedExpr(e) => Some(e), + SelectItem::ExprWithAlias { expr: e, .. } => Some(e), + _ => None, + }; + if let Some(e) = expr { + matches!( + e, + Expr::Function(_) + | Expr::Trim { .. } + | Expr::Substring { .. } + | Expr::BinaryOp { .. } + | Expr::UnaryOp { .. } + | Expr::Ceil { .. } + | Expr::Floor { .. } + ) + } else { + false + } + }); + + // If we have functions/computed expressions and only LIMIT (no ORDER BY), use compile_table_scan + // which handles function expressions properly + if has_computed_expr && !has_order_by { + return self.compile_table_scan_with_limit(select, query, &table_name); + } + + let columns = self.resolve_projection(&select.projection, table)?; + + // Build result schema with column names and types + let schema = self.build_result_schema(&select.projection, table); + self.program.set_result_schema(schema); + + if has_order_by { + self.compile_select_with_sorter(select, query, table, &columns, &table_name)?; + } else { + self.compile_select_with_limit(select, query, table, &columns, &table_name)?; } Ok(()) } - /// Compile an expression into bytecode and return the register containing the result - fn compile_expr(&mut self, expr: &sqlparser::ast::Expr) -> SqawkResult { - let result_reg = self.allocate_register(); + /// Compile SELECT with ORDER BY using a sorter + fn compile_select_with_sorter( + &mut self, + select: &Select, + query: &Query, + table: &Table, + columns: &[usize], + table_name: &str, + ) -> SqawkResult<()> { + let cursor_idx = 0i64; + let sorter_id = 0i64; + let col_count = columns.len(); - match expr { - sqlparser::ast::Expr::Value(value) => { - // Load the appropriate value based on type - match value { - sqlparser::ast::Value::Number(num, _) => { - // Try to parse as integer first - if let Ok(int_val) = num.parse::() { - self.program.add_instruction(Instruction::new( - OpCode::Integer, - int_val, // Value + // Build sort spec from ORDER BY + let sort_spec = self.build_sort_spec(&query.order_by, table, columns)?; + + // Open sorter + self.emit( + OpCode::SorterOpen, + sorter_id, + col_count as i64, + 0, + Some(sort_spec), + "", + ); + + // Open table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + "", + ); + + // Rewind - jump past loop if empty + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + cursor_idx, + 0, // Will be patched + 0, + None, + "", + ); + + let loop_start = self.program.len(); + + // Load columns into registers + let start_reg = self.allocate_registers(col_count); + + for (i, col_idx) in columns.iter().enumerate() { + self.emit( + OpCode::Column, + cursor_idx, + *col_idx as i64, + start_reg + i as i64, + None, + "", + ); + } + + // Compile WHERE clause if present + if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + // Skip SorterInsert, go to Next + self.emit( + OpCode::IfZ, + cond_reg, + (self.program.len() + 2) as i64, + 0, + None, + "", + ); + } + + // Insert into sorter + self.emit( + OpCode::SorterInsert, + sorter_id, + start_reg, + col_count as i64, + None, + "", + ); + + // Next row + self.emit(OpCode::Next, cursor_idx, loop_start as i64, 0, None, ""); + + let after_scan = self.program.len(); + + // Patch rewind jump + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = after_scan as i64; + } + + // Sort the sorter + self.emit(OpCode::SorterSort, sorter_id, 0, 0, None, ""); + + // Handle LIMIT/OFFSET + // When DISTINCT is present, we apply LIMIT after DISTINCT (as a separate opcode) + // because DISTINCT may reduce the row count + let has_distinct = select.distinct.is_some(); + let (limit_val, offset_val) = self.extract_limit_offset(query)?; + + // Only apply LIMIT during sorter output if DISTINCT is not present + let limit_reg = if !has_distinct && (limit_val.is_some() || offset_val > 0) { + let reg = self.allocate_register(); + // Initialize limit counter (add offset to limit since we'll skip offset rows) + let total = limit_val.unwrap_or(i64::MAX); + self.emit(OpCode::Integer, total, reg, 0, None, ""); + Some(reg) + } else { + None + }; + + // Only apply offset during sorter output if DISTINCT is NOT present + // When DISTINCT is present, offset is applied after DISTINCT via the Limit opcode + let offset_reg = if offset_val > 0 && !has_distinct { + let reg = self.allocate_register(); + self.emit(OpCode::Integer, offset_val, reg, 0, None, ""); + Some(reg) + } else { + None + }; + + // Emit sorted results + let sorter_loop_start = self.program.len(); + + // Get row from sorter + self.emit( + OpCode::SorterData, + sorter_id, + start_reg, + col_count as i64, + None, + "", + ); + + // Handle OFFSET - skip first N rows + if let Some(off_reg) = offset_reg { + // If offset counter > 0, decrement and skip this row + self.emit( + OpCode::IfPos, + off_reg, + (self.program.len() + 2) as i64, // Jump to decrement and next (DecrJumpZero) + 0, + None, + "", + ); + // Offset exhausted, continue to output + let output_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to ResultRow + 0, + None, + "", + ); + // Decrement offset and go to next + self.emit( + OpCode::DecrJumpZero, + off_reg, + (self.program.len() + 1) as i64, // Continue to SorterNext + 0, + None, + "", + ); + // Skip offset depends on whether there's a LIMIT instruction after ResultRow + let skip_offset = if limit_reg.is_some() { 3 } else { 2 }; + self.emit( + OpCode::Goto, + 0, + (self.program.len() + skip_offset) as i64, // Skip to SorterNext + 0, + None, + "", + ); + // Patch the Goto to ResultRow + let result_row_addr = self.program.len() as i64; + if let Some(inst) = self.program.instructions.get_mut(output_addr) { + inst.p2 = result_row_addr; + } + } + + // Output result row + self.emit(OpCode::ResultRow, start_reg, col_count as i64, 0, None, ""); + + // Handle LIMIT + if let Some(lim_reg) = limit_reg { + // DecrJumpZero exits when limit reached + self.emit( + OpCode::DecrJumpZero, + lim_reg, + (self.program.len() + 2) as i64, // Jump past SorterNext to end + 0, + None, + "", + ); + } + + // Next sorted row + self.emit( + OpCode::SorterNext, + sorter_id, + sorter_loop_start as i64, + 0, + None, + "", + ); + + // Close cursor + self.emit(OpCode::Close, cursor_idx, 0, 0, None, ""); + + // Handle DISTINCT - emit Distinct opcode to deduplicate results + if select.distinct.is_some() { + self.emit(OpCode::Distinct, 0, 0, 0, None, "Remove duplicate rows"); + + // For DISTINCT queries, apply LIMIT/OFFSET after DISTINCT + // OFFSET was NOT applied during sorter output when DISTINCT is present + if limit_val.is_some() || offset_val > 0 { + self.emit( + OpCode::Limit, + limit_val.unwrap_or(i64::MAX), + offset_val, + 0, + None, + &format!( + "Limit {} offset {}", + limit_val.unwrap_or(i64::MAX), + offset_val + ), + ); + } + } + + Ok(()) + } + + /// Compile SELECT with LIMIT/OFFSET (no ORDER BY) + fn compile_select_with_limit( + &mut self, + select: &Select, + query: &Query, + table: &Table, + columns: &[usize], + table_name: &str, + ) -> SqawkResult<()> { + let cursor_idx = 0i64; + let col_count = columns.len(); + + let (limit_val, offset_val) = self.extract_limit_offset(query)?; + + // Open table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + "", + ); + + // Initialize limit counter + let limit_reg = if let Some(limit) = limit_val { + let reg = self.allocate_register(); + self.emit(OpCode::Integer, limit, reg, 0, None, ""); + Some(reg) + } else { + None + }; + + // LIMIT 0 optimization: skip the entire loop if limit is 0 at compile time + let limit_zero_jump = if limit_val == Some(0) { + let addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to after loop + 0, + None, + "LIMIT 0: skip loop", + ); + Some(addr) + } else { + None + }; + + // Initialize offset counter + let offset_reg = if offset_val > 0 { + let reg = self.allocate_register(); + self.emit(OpCode::Integer, offset_val, reg, 0, None, ""); + Some(reg) + } else { + None + }; + + // Rewind + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + cursor_idx, + 0, // Will be patched + 0, + None, + "", + ); + + let loop_start = self.program.len(); + + // Load columns + let start_reg = self.allocate_registers(col_count); + + for (i, col_idx) in columns.iter().enumerate() { + self.emit( + OpCode::Column, + cursor_idx, + *col_idx as i64, + start_reg + i as i64, + None, + "", + ); + } + + // WHERE clause + if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + self.emit( + OpCode::IfZ, + cond_reg, + (self.program.len() + 2) as i64, + 0, + None, + "", + ); + } + + // Handle OFFSET - skip first N matching rows + // Pattern: IfPos offset, skip -> output -> skip: DecrJumpZero -> Next + let offset_ifpos_addr = if let Some(off_reg) = offset_reg { + let addr = self.program.len(); + self.emit( + OpCode::IfPos, + off_reg, + 0, // Will patch to DecrJumpZero (skip block) + 0, + None, + "", + ); + Some(addr) + } else { + None + }; + + // Output row + self.emit(OpCode::ResultRow, start_reg, col_count as i64, 0, None, ""); + + // Handle LIMIT + let after_limit = if let Some(lim_reg) = limit_reg { + self.emit( + OpCode::DecrJumpZero, + lim_reg, + 0, // Will be patched to after loop + 0, + None, + "", + ); + Some(self.program.len() - 1) + } else { + None + }; + + // Goto past offset decrement (skip block) to Next + let goto_next_addr = if offset_reg.is_some() { + let addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will patch to Next + 0, + None, + "", + ); + Some(addr) + } else { + None + }; + + // Offset skip block: decrement offset counter then fall through to Next + let offset_decr_addr = if let Some(off_reg) = offset_reg { + let addr = self.program.len(); + self.emit( + OpCode::DecrJumpZero, + off_reg, + (self.program.len() + 1) as i64, // Always fall through to Next + 0, + None, + "", + ); + Some(addr) + } else { + None + }; + + // Next row + let next_addr = self.program.len(); + self.emit(OpCode::Next, cursor_idx, loop_start as i64, 0, None, ""); + + let after_loop = self.program.len(); + + // Patch jumps + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = after_loop as i64; + } + if let Some(addr) = after_limit { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = after_loop as i64; + } + } + // Patch offset IfPos to jump to decrement block + if let Some(ifpos_addr) = offset_ifpos_addr { + if let Some(decr_addr) = offset_decr_addr { + if let Some(inst) = self.program.instructions.get_mut(ifpos_addr) { + inst.p2 = decr_addr as i64; + } + } + } + // Patch Goto to jump to Next + if let Some(goto_addr) = goto_next_addr { + if let Some(inst) = self.program.instructions.get_mut(goto_addr) { + inst.p2 = next_addr as i64; + } + } + // Patch LIMIT 0 optimization Goto to jump past loop + if let Some(goto_addr) = limit_zero_jump { + if let Some(inst) = self.program.instructions.get_mut(goto_addr) { + inst.p2 = after_loop as i64; + } + } + + // Close cursor + self.emit(OpCode::Close, cursor_idx, 0, 0, None, ""); + + Ok(()) + } + + /// Build sort spec string from ORDER BY clause + fn build_sort_spec( + &self, + order_by: &[sqlparser::ast::OrderByExpr], + table: &Table, + projected_columns: &[usize], + ) -> SqawkResult { + let mut specs = Vec::new(); + for expr in order_by { + // Find column index in original table + let table_col_idx = match &expr.expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))? + } + Expr::CompoundIdentifier(parts) => { + let col_name = parts + .last() + .map(|p| p.value.to_lowercase()) + .unwrap_or_default(); + table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))? + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Complex ORDER BY expressions not supported".into(), + )) + } + }; + // Map to position in projected columns + let col_idx = projected_columns + .iter() + .position(|&c| c == table_col_idx) + .ok_or_else(|| { + SqawkError::InvalidSqlQuery( + "ORDER BY column must be in SELECT list".to_string(), + ) + })?; + let asc = expr.asc.unwrap_or(true); + specs.push(format!("{}:{}", col_idx, if asc { "asc" } else { "desc" })); + } + Ok(specs.join(",")) + } + + /// Extract LIMIT and OFFSET values from query + pub(crate) fn extract_limit_offset(&self, query: &Query) -> SqawkResult<(Option, i64)> { + let limit = if let Some(limit_expr) = &query.limit { + match limit_expr { + Expr::Value(Value::Number(n, _)) => Some(n.parse::().map_err(|_| { + SqawkError::InvalidSqlQuery(format!("Invalid LIMIT value: {}", n)) + })?), + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only constant LIMIT supported".into(), + )) + } + } + } else { + None + }; + + let offset = if let Some(offset_clause) = &query.offset { + match &offset_clause.value { + Expr::Value(Value::Number(n, _)) => n.parse::().map_err(|_| { + SqawkError::InvalidSqlQuery(format!("Invalid OFFSET value: {}", n)) + })?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only constant OFFSET supported".into(), + )) + } + } + } else { + 0 + }; + + Ok((limit, offset)) + } + + /// Compile a SELECT statement with literals (e.g., SELECT 1, SELECT 'text') + fn compile_select_literal(&mut self, projection: &[SelectItem]) -> SqawkResult<()> { + self.add_comment("Literal value query (no table)"); + + // For each selected literal, add an instruction to load it into a register + let mut result_regs = Vec::new(); + + for item in projection.iter() { + match item { + SelectItem::UnnamedExpr(expr) => { + // Compile a simple expression and store the value in a register + let reg = self.compile_expr(expr)?; + result_regs.push(reg); + } + SelectItem::ExprWithAlias { expr, alias } => { + // Same as UnnamedExpr but with an alias (column name) + let reg = self.compile_expr(expr)?; + result_regs.push(reg); + + // We don't need to do anything special with the alias for now + // But we could store it for the result table column names + if self.verbose { + self.add_comment(&format!("Column alias: {}", alias)); + } + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple expressions supported in SELECT without FROM".to_string(), + )); + } + } + } + + // Generate the result row + if !result_regs.is_empty() { + // ResultRow requires contiguous registers, but compile_expr may have + // allocated non-contiguous registers. Copy results to contiguous block. + let start_reg = self.allocate_registers(result_regs.len()); + + // Copy each result to the contiguous block + for (i, &src_reg) in result_regs.iter().enumerate() { + let dest_reg = start_reg + i as i64; + if src_reg != dest_reg { + self.emit( + OpCode::Copy, + src_reg, + dest_reg, + 0, + None, + &format!("Copy r[{}] -> r[{}]", src_reg, dest_reg), + ); + } + } + + // Output the result row from contiguous registers + self.emit( + OpCode::ResultRow, + start_reg, // First register + result_regs.len() as i64, // Number of columns + 0, + None, + "Output literal result row", + ); + } + + Ok(()) + } + + /// Compile an expression into a specific register + pub(crate) fn compile_expr_into_register( + &mut self, + expr: &sqlparser::ast::Expr, + target_reg: usize, + ) -> SqawkResult<()> { + // Compile the expression + let src_reg = self.compile_expr(expr)?; + + // If not already in target register, copy + if src_reg != target_reg as i64 { + self.emit( + OpCode::Copy, + src_reg, + target_reg as i64, + 0, + None, + "Copy expr result to target", + ); + } + + Ok(()) + } + + /// Compile an expression into bytecode and return the register containing the result + fn compile_expr(&mut self, expr: &sqlparser::ast::Expr) -> SqawkResult { + let result_reg = self.allocate_register(); + + match expr { + sqlparser::ast::Expr::Value(value) => { + // Load the appropriate value based on type + match value { + sqlparser::ast::Value::Number(num, _) => { + // Try to parse as integer first + if let Ok(int_val) = num.parse::() { + self.emit( + OpCode::Integer, + int_val, // Value result_reg, // Target register 0, None, + &format!("r[{}] = {}", result_reg, int_val), + ); + } else if let Ok(float_val) = num.parse::() { + // For floats, use String opcode and rely on runtime conversion + self.emit( + OpCode::String, 0, - Some(format!("r[{}] = {}", result_reg, int_val)), - )); + result_reg, + 0, + Some(float_val.to_string()), + &format!("r[{}] = {} (float)", result_reg, float_val), + ); } else { - // Return an error for now - we could add float support later return Err(SqawkError::UnsupportedSqlFeature(format!( - "Non-integer literals not yet supported: {}", + "Invalid numeric literal: {}", num ))); } } sqlparser::ast::Value::SingleQuotedString(s) => { - self.program.add_instruction(Instruction::new( + self.emit( OpCode::String, 0, result_reg, 0, Some(s.clone()), - 0, - Some(format!("r[{}] = '{}'", result_reg, s)), - )); + &format!("r[{}] = '{}'", result_reg, s), + ); } sqlparser::ast::Value::Null => { - self.program.add_instruction(Instruction::new( + self.emit( OpCode::Null, 0, result_reg, 0, None, - 0, - Some(format!("r[{}] = NULL", result_reg)), - )); + &format!("r[{}] = NULL", result_reg), + ); } _ => { return Err(SqawkError::UnsupportedSqlFeature(format!( @@ -271,165 +1495,3935 @@ impl<'a> SqlCompiler<'a> { } } } - // Could add support for other expression types here - _ => { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported expression type: {:?}", - expr - ))); + sqlparser::ast::Expr::UnaryOp { op, expr } => { + // Handle unary operators (e.g., -5, +3) + match op { + sqlparser::ast::UnaryOperator::Minus => { + // Compile the inner expression + let inner_reg = self.compile_expr(expr)?; + // Negate using 0 - value (avoids Integer -1 special case) + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0", zero_reg), + ); + self.emit( + OpCode::Subtract, + zero_reg, + inner_reg, + result_reg, + None, + &format!("r[{}] = -r[{}]", result_reg, inner_reg), + ); + } + sqlparser::ast::UnaryOperator::Plus => { + // Unary plus is a no-op, just compile the inner expression + let inner_reg = self.compile_expr(expr)?; + self.emit( + OpCode::Copy, + inner_reg, + result_reg, + 0, + None, + &format!("r[{}] = +r[{}]", result_reg, inner_reg), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported unary operator: {:?}", + op + ))); + } + } + } + sqlparser::ast::Expr::BinaryOp { left, op, right } => { + // Handle arithmetic binary operators + let left_reg = self.compile_expr(left)?; + let right_reg = self.compile_expr(right)?; + + let opcode = match op { + sqlparser::ast::BinaryOperator::Plus => OpCode::Add, + sqlparser::ast::BinaryOperator::Minus => OpCode::Subtract, + sqlparser::ast::BinaryOperator::Multiply => OpCode::Multiply, + sqlparser::ast::BinaryOperator::Divide => OpCode::Divide, + sqlparser::ast::BinaryOperator::Modulo => OpCode::Remainder, + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported binary operator: {:?}", + op + ))); + } + }; + + self.emit( + opcode, + left_reg, + right_reg, + result_reg, + None, + &format!( + "r[{}] = r[{}] {:?} r[{}]", + result_reg, left_reg, op, right_reg + ), + ); + } + sqlparser::ast::Expr::Function(func) => { + // Handle function calls (ABS, ROUND, etc.) + let func_name = func + .name + .0 + .iter() + .map(|id| id.value.as_str()) + .collect::>() + .join(".") + .to_uppercase(); + + match func_name.as_str() { + "ABS" | "ROUND" | "CEIL" | "CEILING" | "FLOOR" => { + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires one argument", + func_name + ))); + } + let arg_expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.compile_expr(&arg_expr)?; + self.emit( + OpCode::MathFunc, + src_reg, + result_reg, + 0, + Some(func_name.clone()), + &format!("r[{}] = {}(r[{}])", result_reg, func_name, src_reg), + ); + } + "NOW" | "CURRENT_TIMESTAMP" | "CURRENT_DATE" | "CURRENT_TIME" => { + // No-argument date/time functions + self.emit( + OpCode::DateFunc, + 0, // No source register needed + result_reg, + 0, + Some(func_name.clone()), + &format!("r[{}] = {}()", result_reg, func_name), + ); + } + "DATE" | "TIME" => { + // Single-argument date/time functions + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires one argument", + func_name + ))); + } + let arg_expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.compile_expr(&arg_expr)?; + self.emit( + OpCode::DateFunc, + src_reg, + result_reg, + 0, + Some(func_name.clone()), + &format!("r[{}] = {}(r[{}])", result_reg, func_name, src_reg), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported function in expression: {}", + func_name + ))); + } + } + } + sqlparser::ast::Expr::Ceil { expr, .. } => { + // CEIL is a special expression type in sqlparser + let src_reg = self.compile_expr(expr)?; + self.emit( + OpCode::MathFunc, + src_reg, + result_reg, + 0, + Some("CEIL".to_string()), + &format!("r[{}] = CEIL(r[{}])", result_reg, src_reg), + ); + } + sqlparser::ast::Expr::Floor { expr, .. } => { + // FLOOR is a special expression type in sqlparser + let src_reg = self.compile_expr(expr)?; + self.emit( + OpCode::MathFunc, + src_reg, + result_reg, + 0, + Some("FLOOR".to_string()), + &format!("r[{}] = FLOOR(r[{}])", result_reg, src_reg), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression type: {:?}", + expr + ))); + } + } + + Ok(result_reg) + } + + /// Compile a table scan with optional WHERE clause + fn compile_table_scan( + &mut self, + table_with_joins: &TableWithJoins, + projection: &[SelectItem], + where_clause: &Option, + ) -> SqawkResult<()> { + // Get the table name and alias + let (table_name, table_alias) = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let talias = alias.as_ref().map(|a| a.name.value.clone()); + (tname, talias) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans are supported".into(), + )) + } + }; + + // Set the current outer alias for correlated subquery detection + self.current_outer_alias = table_alias; + + // Check if the table exists by trying to get it + if !self.database.has_table(&table_name) { + return Err(SqawkError::TableNotFound(table_name)); + } + + let table = self.database.get_table(&table_name).unwrap(); + + // Check if projection contains function expressions or computed expressions + let has_function_expr = projection.iter().any(|item| { + let expr = match item { + SelectItem::UnnamedExpr(e) => Some(e), + SelectItem::ExprWithAlias { expr: e, .. } => Some(e), + _ => None, + }; + if let Some(e) = expr { + matches!( + e, + Expr::Function(_) + | Expr::Trim { .. } + | Expr::Substring { .. } + | Expr::Position { .. } + | Expr::Overlay { .. } + | Expr::BinaryOp { .. } + | Expr::UnaryOp { .. } + | Expr::Ceil { .. } + | Expr::Floor { .. } + ) + } else { + false + } + }); + + // Build result schema with column names and types + let schema = self.build_result_schema(projection, table); + self.program.set_result_schema(schema); + + self.add_comment(&format!("Scanning table: {}", table_name)); + + // Open the table for reading (cursor 0) + let cursor_idx = 0; + // Use a simple counter for now, since we don't have get_table_id + let table_id = 1i64; // Just assign a default ID + + self.emit( + OpCode::OpenRead, + cursor_idx as i64, + table_id, + 0, + Some(table_name.clone()), + &format!("Open table {} for reading", table_name), + ); + + // Set up the loop to scan the table + // Store the rewind address to patch later with correct exit point + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + cursor_idx as i64, + 0, // Placeholder - will be patched with actual exit address + 0, + None, + "Position cursor at first row", + ); + + // Loop body start address + let loop_addr = self.program.len(); + + // Load projection values into registers + let mut result_regs = Vec::new(); + if has_function_expr { + // Pre-allocate all result registers to ensure they are contiguous + let num_items = projection.len(); + for _ in 0..num_items { + let value_reg = self.allocate_register(); + result_regs.push(value_reg); + } + + // Handle projection with function expressions + for (idx, item) in projection.iter().enumerate() { + let value_reg = result_regs[idx]; + + match item { + SelectItem::Wildcard(_) => { + // For wildcard, load all columns + for col_idx in 0..table.column_count() { + if col_idx > 0 { + let extra_reg = self.allocate_register(); + result_regs.push(extra_reg); + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + extra_reg, + None, + &format!("r[{}] = column {}", extra_reg, col_idx), + ); + } else { + self.emit( + OpCode::Column, + cursor_idx as i64, + 0, + value_reg, + None, + &format!("r[{}] = column 0", value_reg), + ); + } + } + } + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + match expr { + Expr::Function(func) => { + self.compile_function(func, table, cursor_idx, value_reg)?; + } + Expr::Trim { + expr: trim_expr, .. + } => { + // Compile TRIM expression + let src_reg = self.allocate_register(); + self.compile_where_operand(trim_expr, table, cursor_idx, src_reg)?; + self.emit( + OpCode::StringFunc, + src_reg, + value_reg, + 0, + Some("TRIM".to_string()), + &format!("r[{}] = TRIM(r[{}])", value_reg, src_reg), + ); + } + Expr::Substring { + expr: sub_expr, + substring_from, + substring_for, + .. + } => { + // Compile SUBSTRING expression + let src_reg = self.allocate_register(); + self.compile_where_operand(sub_expr, table, cursor_idx, src_reg)?; + + // Compile start position + let start_reg = self.allocate_register(); + if let Some(from_expr) = substring_from { + self.compile_where_operand( + from_expr, table, cursor_idx, start_reg, + )?; + } else { + self.emit( + OpCode::Integer, + 1, + start_reg, + 0, + None, + &format!("r[{}] = 1 (default start)", start_reg), + ); + } + + // Check if we have a length + let func_spec = if let Some(for_expr) = substring_for { + let len_reg = self.allocate_register(); + self.compile_where_operand( + for_expr, table, cursor_idx, len_reg, + )?; + format!("SUBSTR:{}", len_reg) + } else { + "SUBSTR".to_string() + }; + + self.emit( + OpCode::StringFunc, + src_reg, + value_reg, + start_reg, + Some(func_spec), + &format!("r[{}] = SUBSTR(...)", value_reg), + ); + } + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + let col_idx = table + .column_index(&col_name) + .ok_or_else(|| SqawkError::ColumnNotFound(col_name.clone()))?; + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + value_reg, + None, + &format!("r[{}] = column {}", value_reg, col_idx), + ); + } + Expr::CompoundIdentifier(parts) => { + let col_name = parts + .last() + .map(|p| p.value.to_lowercase()) + .unwrap_or_default(); + let col_idx = table + .column_index(&col_name) + .ok_or_else(|| SqawkError::ColumnNotFound(col_name.clone()))?; + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + value_reg, + None, + &format!("r[{}] = column {}", value_reg, col_idx), + ); + } + Expr::BinaryOp { left, op, right } => { + // Handle arithmetic binary operators + let left_reg = self.allocate_register(); + let right_reg = self.allocate_register(); + + self.compile_where_operand(left, table, cursor_idx, left_reg)?; + self.compile_where_operand(right, table, cursor_idx, right_reg)?; + + let opcode = match op { + BinaryOperator::Plus => OpCode::Add, + BinaryOperator::Minus => OpCode::Subtract, + BinaryOperator::Multiply => OpCode::Multiply, + BinaryOperator::Divide => OpCode::Divide, + BinaryOperator::Modulo => OpCode::Remainder, + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported binary operator in SELECT: {:?}", + op + ))); + } + }; + + self.emit( + opcode, + left_reg, + right_reg, + value_reg, + None, + &format!( + "r[{}] = r[{}] {:?} r[{}]", + value_reg, left_reg, op, right_reg + ), + ); + } + Expr::UnaryOp { op, expr: inner } => { + // Handle unary operators (e.g., -value) + match op { + sqlparser::ast::UnaryOperator::Minus => { + let inner_reg = self.allocate_register(); + self.compile_where_operand( + inner, table, cursor_idx, inner_reg, + )?; + // Negate using 0 - value (avoids Integer -1 special case) + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0", zero_reg), + ); + self.emit( + OpCode::Subtract, + zero_reg, + inner_reg, + value_reg, + None, + &format!("r[{}] = -r[{}]", value_reg, inner_reg), + ); + } + sqlparser::ast::UnaryOperator::Plus => { + self.compile_where_operand( + inner, table, cursor_idx, value_reg, + )?; + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported unary operator: {:?}", + op + ))); + } + } + } + Expr::Ceil { expr: inner, .. } => { + let src_reg = self.allocate_register(); + self.compile_where_operand(inner, table, cursor_idx, src_reg)?; + self.emit( + OpCode::MathFunc, + src_reg, + value_reg, + 0, + Some("CEIL".to_string()), + &format!("r[{}] = CEIL(r[{}])", value_reg, src_reg), + ); + } + Expr::Floor { expr: inner, .. } => { + let src_reg = self.allocate_register(); + self.compile_where_operand(inner, table, cursor_idx, src_reg)?; + self.emit( + OpCode::MathFunc, + src_reg, + value_reg, + 0, + Some("FLOOR".to_string()), + &format!("r[{}] = FLOOR(r[{}])", value_reg, src_reg), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression in SELECT: {:?}", + expr + ))); + } + } + } + SelectItem::QualifiedWildcard(_, _) => { + for col_idx in 0..table.column_count() { + if col_idx > 0 { + let extra_reg = self.allocate_register(); + result_regs.push(extra_reg); + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + extra_reg, + None, + &format!("r[{}] = column {}", extra_reg, col_idx), + ); + } else { + self.emit( + OpCode::Column, + cursor_idx as i64, + 0, + value_reg, + None, + &format!("r[{}] = column 0", value_reg), + ); + } + } + } + } + } + } else { + // Original path for simple column references + let columns = self.resolve_projection(projection, table)?; + for col_idx in columns.iter() { + let value_reg = self.allocate_register(); + result_regs.push(value_reg); + + self.emit( + OpCode::Column, + cursor_idx as i64, + *col_idx as i64, + value_reg, + None, + &format!("r[{}] = column {} value", value_reg, col_idx), + ); + } + } + + // Compile WHERE clause filtering if present + if let Some(where_expr) = where_clause { + self.add_comment("WHERE clause filtering"); + if self.verbose { + eprintln!("WHERE comparison: {:?}", where_expr); + } + let skip_addr = self.compile_where_condition(where_expr, table, cursor_idx)?; + + // If WHERE condition failed, jump to Next instruction to skip this row + self.emit( + OpCode::IfZ, + skip_addr, // Register containing condition result + (self.program.len() + 2) as i64, // Jump to Next instruction + 0, + None, + "Skip row if WHERE condition is false", + ); + } + + // Output the result row (only reached if WHERE condition passes) + self.emit( + OpCode::ResultRow, + result_regs[0], // First register in result + result_regs.len() as i64, // Number of columns + 0, + None, + "Output result row", + ); + + // Move to next row and continue loop + self.emit( + OpCode::Next, + cursor_idx as i64, + loop_addr as i64, // Jump back to start of loop for next row + 0, + None, + "Move to next row or exit loop", + ); + + // Close the table cursor + // First, get the close instruction address (this is where Rewind should jump on empty table) + let close_addr = self.program.len(); + self.emit(OpCode::Close, cursor_idx as i64, 0, 0, None, "Close cursor"); + + // Patch the Rewind instruction to jump to Close on empty table + if let Some(rewind_inst) = self.program.instructions.get_mut(rewind_addr) { + rewind_inst.p2 = close_addr as i64; + } + + Ok(()) + } + + /// Compile a table scan with LIMIT/OFFSET and function expressions + fn compile_table_scan_with_limit( + &mut self, + select: &Select, + query: &Query, + table_name: &str, + ) -> SqawkResult<()> { + let table = self.database.get_table(table_name).unwrap(); + let projection = &select.projection; + + // Build result schema with column names and types + let schema = self.build_result_schema(projection, table); + self.program.set_result_schema(schema); + + let (limit_val, offset_val) = self.extract_limit_offset(query)?; + + let cursor_idx = 0i64; + + // Open table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + &format!("Open table {} for reading", table_name), + ); + + // Initialize limit counter + let limit_reg = if let Some(limit) = limit_val { + let reg = self.allocate_register(); + self.emit( + OpCode::Integer, + limit, + reg, + 0, + None, + "Initialize limit counter", + ); + Some(reg) + } else { + None + }; + + // Initialize offset counter + let offset_reg = if offset_val > 0 { + let reg = self.allocate_register(); + self.emit( + OpCode::Integer, + offset_val, + reg, + 0, + None, + "Initialize offset counter", + ); + Some(reg) + } else { + None + }; + + // Rewind + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, cursor_idx, 0, 0, None, "Rewind cursor"); + + let loop_start = self.program.len(); + + // Pre-allocate result registers to ensure they are contiguous + let mut result_regs = Vec::new(); + for _ in projection { + let value_reg = self.allocate_register(); + result_regs.push(value_reg); + } + + // Load projection values into registers with function support + for (idx, item) in projection.iter().enumerate() { + let value_reg = result_regs[idx]; + + match item { + SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _) => { + for col_idx in 0..table.column_count() { + if col_idx > 0 { + let extra_reg = self.allocate_register(); + result_regs.push(extra_reg); + self.emit( + OpCode::Column, + cursor_idx, + col_idx as i64, + extra_reg, + None, + &format!("r[{}] = column {}", extra_reg, col_idx), + ); + } else { + self.emit( + OpCode::Column, + cursor_idx, + 0, + value_reg, + None, + &format!("r[{}] = column 0", value_reg), + ); + } + } + } + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + self.compile_projection_expr(expr, table, cursor_idx as usize, value_reg)?; + } + } + } + + // WHERE clause + if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + // Jump to Next if condition fails + let next_addr = self.program.len() + + 1 + + if offset_reg.is_some() { 2 } else { 0 } + + if limit_reg.is_some() { 2 } else { 0 } + + 1; // ResultRow + self.emit( + OpCode::IfZ, + cond_reg, + next_addr as i64, + 0, + None, + "Skip if WHERE false", + ); + } + + // Handle offset + if let Some(offset_r) = offset_reg { + // If offset counter > 0, decrement and skip this row + let after_skip = self.program.len() + 2; + self.emit( + OpCode::DecrJumpZero, + offset_r, + after_skip as i64, + 0, + None, + "Decrement offset, skip if done", + ); + // Jump to next row (skip output) + let next_addr = self.program.len() + if limit_reg.is_some() { 3 } else { 1 } + 1; + self.emit( + OpCode::Goto, + 0, + next_addr as i64, + 0, + None, + "Skip row for offset", + ); + } + + // Output result row + self.emit( + OpCode::ResultRow, + result_regs[0], + result_regs.len() as i64, + 0, + None, + "Output result row", + ); + + // Handle limit + if let Some(limit_r) = limit_reg { + // Decrement limit counter, stop if zero + let loop_end = self.program.len() + 2; + self.emit( + OpCode::DecrJumpZero, + limit_r, + loop_end as i64, + 0, + None, + "Decrement limit, stop if zero", + ); + } + + // Next row + self.emit( + OpCode::Next, + cursor_idx, + loop_start as i64, + 0, + None, + "Next row", + ); + + let loop_end = self.program.len(); + + // Close cursor + self.emit(OpCode::Close, cursor_idx, 0, 0, None, "Close cursor"); + + // Patch rewind jump + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = loop_end as i64; + } + + Ok(()) + } + + /// Compile a projection expression (column or function) into a register + fn compile_projection_expr( + &mut self, + expr: &Expr, + table: &Table, + cursor_idx: usize, + dest_reg: i64, + ) -> SqawkResult<()> { + match expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + if let Some(col_idx) = table.column_index(&col_name) { + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + dest_reg, + None, + &format!("r[{}] = {}", dest_reg, col_name), + ); + } + } + Expr::Function(func) => { + let func_name = func.name.to_string().to_uppercase(); + if let Some(sqlparser::ast::FunctionArg::Unnamed( + sqlparser::ast::FunctionArgExpr::Expr(Expr::Identifier(ident)), + )) = func.args.first() + { + let col_name = ident.value.to_lowercase(); + if let Some(col_idx) = table.column_index(&col_name) { + let src_reg = self.allocate_register(); + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + src_reg, + None, + &format!("Load {} for {}", col_name, func_name), + ); + + // Handle different string functions + match func_name.as_str() { + "UPPER" | "LOWER" | "TRIM" | "LTRIM" | "RTRIM" | "LENGTH" => { + self.emit( + OpCode::StringFunc, + src_reg, + dest_reg, + 0, + Some(func_name.clone()), + &format!("{}()", func_name), + ); + } + "SUBSTR" | "SUBSTRING" => { + // Handle SUBSTR with start and length + let (start, len) = self.extract_substr_args(func)?; + let start_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + start, + start_reg, + 0, + None, + "SUBSTR start", + ); + let len_reg = self.allocate_register(); + self.emit(OpCode::Integer, len, len_reg, 0, None, "SUBSTR length"); + // P3 = start register, P4 = "SUBSTR:len_reg" + self.emit( + OpCode::StringFunc, + src_reg, + dest_reg, + start_reg, + Some(format!("SUBSTR:{}", len_reg)), + "SUBSTR()", + ); + } + _ => { + // Copy source to dest for unknown functions + self.emit(OpCode::Copy, src_reg, dest_reg, 0, None, "Copy value"); + } + }; + } + } + } + _ => { + // For other expressions, try to resolve as column + if let Ok(col_idx) = self.resolve_column_expr(expr, table) { + self.emit( + OpCode::Column, + cursor_idx as i64, + col_idx as i64, + dest_reg, + None, + &format!("r[{}] = column {}", dest_reg, col_idx), + ); + } + } + } + Ok(()) + } + + /// Extract SUBSTR arguments (start, length) from function + fn extract_substr_args(&self, func: &sqlparser::ast::Function) -> SqawkResult<(i64, i64)> { + let mut start = 1i64; + let mut length = 100i64; // Default length + + let args: Vec<_> = func.args.iter().collect(); + if args.len() >= 2 { + if let sqlparser::ast::FunctionArg::Unnamed(sqlparser::ast::FunctionArgExpr::Expr( + Expr::Value(sqlparser::ast::Value::Number(n, _)), + )) = &args[1] + { + start = n.parse().unwrap_or(1); + } + } + if args.len() >= 3 { + if let sqlparser::ast::FunctionArg::Unnamed(sqlparser::ast::FunctionArgExpr::Expr( + Expr::Value(sqlparser::ast::Value::Number(n, _)), + )) = &args[2] + { + length = n.parse().unwrap_or(100); + } + } + Ok((start, length)) + } + + /// Compile a WHERE condition and return the register containing the result + pub(crate) fn compile_where_condition( + &mut self, + where_expr: &Expr, + table: &Table, + cursor_idx: usize, + ) -> SqawkResult { + match where_expr { + Expr::BinaryOp { left, op, right } => { + self.compile_binary_comparison(left, op, right, table, cursor_idx) + } + Expr::Like { + negated, + expr, + pattern, + .. + } => self.compile_like_pattern(expr, pattern, *negated, table, cursor_idx, false), + Expr::ILike { + negated, + expr, + pattern, + .. + } => self.compile_like_pattern(expr, pattern, *negated, table, cursor_idx, true), + Expr::Between { + expr, + negated, + low, + high, + } => self.compile_between(expr, low, high, *negated, table, cursor_idx), + Expr::InList { + expr, + list, + negated, + } => self.compile_in_list(expr, list, *negated, table, cursor_idx), + Expr::Case { + operand, + conditions, + results, + else_result, + } => self.compile_case( + operand.as_deref(), + conditions, + results, + else_result.as_deref(), + table, + cursor_idx, + None, + ), + Expr::IsNull(expr) => { + // Compile IS NULL as a condition + let expr_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + + self.compile_where_operand(expr, table, cursor_idx, expr_reg)?; + + self.emit( + OpCode::IsNull, + expr_reg, + result_reg, + 0, + None, + &format!("r[{}] = (r[{}] IS NULL)", result_reg, expr_reg), + ); + + Ok(result_reg) + } + Expr::IsNotNull(expr) => { + // Compile IS NOT NULL as a condition + let expr_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + + self.compile_where_operand(expr, table, cursor_idx, expr_reg)?; + + // First check IS NULL + self.emit( + OpCode::IsNull, + expr_reg, + result_reg, + 0, + None, + &format!("r[{}] = (r[{}] IS NULL)", result_reg, expr_reg), + ); + + // Then negate the result (if is_null is 1, set to 0; if 0, set to 1) + // We can do this with: result = 1 - result + let one_reg = self.allocate_register(); + let final_reg = self.allocate_register(); + + self.emit( + OpCode::Integer, + 1, + one_reg, + 0, + None, + &format!("r[{}] = 1", one_reg), + ); + + // Subtract using: if result is 0, final = 1; if result is 1, final = 0 + // We need a subtraction operation, but we don't have one yet + // Alternative: use IfZ to branch + let is_null_label = self.program.len(); + self.emit( + OpCode::IfZ, + result_reg, + 0, // Will be patched + 0, + None, + "if IS NULL result is 0 (not null), jump", + ); + + // result_reg was 1 (is null), so set final to 0 + self.emit( + OpCode::Integer, + 0, + final_reg, + 0, + None, + &format!("r[{}] = 0 (was null)", final_reg), + ); + + let skip_else_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched + 0, + None, + "skip else branch", + ); + + // Patch the IfZ to jump here when result was 0 (not null) + let not_null_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(is_null_label) { + inst.p2 = not_null_label as i64; + } + + // result_reg was 0 (not null), so set final to 1 + self.emit( + OpCode::Integer, + 1, + final_reg, + 0, + None, + &format!("r[{}] = 1 (was not null)", final_reg), + ); + + // Patch skip_else to jump here + let end_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_else_addr) { + inst.p2 = end_label as i64; + } + + Ok(final_reg) + } + // Phase 4F: Subquery support + Expr::Subquery(subquery) => { + // Scalar subquery - returns a single value + self.compile_scalar_subquery(subquery, table, cursor_idx) + } + Expr::InSubquery { + expr, + subquery, + negated, + } => { + // IN (SELECT ...) - check if value exists in subquery results + self.compile_in_subquery(expr, subquery, *negated, table, cursor_idx) + } + Expr::Exists { subquery, negated } => { + // EXISTS (SELECT ...) - check if subquery returns any rows + self.compile_exists_subquery(subquery, *negated, table, cursor_idx) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported WHERE clause expression: {:?}", + where_expr + ))), + } + } + + /// Compile a scalar subquery (Phase 4F/4B) + /// + /// Scalar subqueries return a single value that can be used in expressions. + /// - Non-correlated: uses compile-time execution + /// - Correlated: uses runtime execution with coroutines + /// + /// Example: SELECT * FROM users WHERE age = (SELECT MAX(age) FROM users) + fn compile_scalar_subquery( + &mut self, + subquery: &Query, + outer_table: &Table, + outer_cursor_idx: usize, + ) -> SqawkResult { + let result_reg = self.allocate_register(); + + // Detect if this is a correlated subquery + let outer_refs = self.detect_outer_references(subquery, outer_table.name(), None); + + if outer_refs.is_empty() { + // Non-correlated: evaluate at compile time + let result_value = self.evaluate_scalar_subquery(subquery)?; + self.emit_value_literal(result_reg, &result_value)?; + } else { + // Correlated: emit runtime execution bytecode + self.compile_correlated_scalar_subquery( + subquery, + outer_table, + outer_cursor_idx, + &outer_refs, + result_reg, + )?; + } + + Ok(result_reg) + } + + /// Extract the column index from an aggregate function argument + fn extract_agg_column_index(&self, func: &Function, table: &Table) -> SqawkResult { + // Handle COUNT(*) specially + if !func.args.is_empty() { + match &func.args[0] { + FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => { + // COUNT(*) - use first column (doesn't matter which for count) + return Ok(0); + } + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident))) => { + let col_name = ident.value.to_lowercase(); + return table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name)); + } + _ => {} + } + } + // Default to first column + Ok(0) + } + + /// Emit bytecode to load a value as a literal + fn emit_value_literal(&mut self, reg: i64, value: &TableValue) -> SqawkResult<()> { + match value { + TableValue::Integer(i) => { + self.emit( + OpCode::Integer, + *i, + reg, + 0, + None, + &format!("r[{}] = {} (subquery result)", reg, i), + ); + } + TableValue::Float(f) => { + // Store float as string and use String opcode, then cast + self.emit( + OpCode::String, + 0, + reg, + 0, + Some(f.to_string()), + &format!("r[{}] = {} (subquery result)", reg, f), + ); + // Cast to float + self.emit( + OpCode::Cast, + reg, + reg, + 0, + Some("REAL".to_string()), + &format!("Cast r[{}] to REAL", reg), + ); + } + TableValue::String(s) => { + self.emit( + OpCode::String, + 0, + reg, + 0, + Some(s.clone().into_owned()), + &format!("r[{}] = '{}' (subquery result)", reg, s), + ); + } + TableValue::Boolean(b) => { + self.emit( + OpCode::Integer, + if *b { 1 } else { 0 }, + reg, + 0, + None, + &format!("r[{}] = {} (subquery result)", reg, b), + ); + } + TableValue::Null => { + self.emit( + OpCode::Null, + 0, + reg, + 0, + None, + &format!("r[{}] = NULL (subquery result)", reg), + ); + } + } + Ok(()) + } + + /// Evaluate a scalar subquery at compile time and return the result value + /// This is used when a subquery appears as an operand in a comparison + fn evaluate_scalar_subquery(&self, subquery: &Query) -> SqawkResult { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "Subquery must have a FROM clause".to_string(), + )); + } + + // Get the table name + let table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in subqueries".to_string(), + )) + } + }; + + // Get the table + if !self.database.has_table(&table_name) { + return Err(SqawkError::TableNotFound(table_name.clone())); + } + let table = self.database.get_table(&table_name).unwrap(); + + // Filter rows if there's a WHERE clause in the subquery + let filtered_rows: Vec<&Vec> = if let Some(where_expr) = &select.selection { + table + .rows() + .iter() + .filter(|row| self.evaluate_where_at_compile_time(where_expr, row, table)) + .collect() + } else { + table.rows().iter().collect() + }; + + // Check the projection to determine what value to compute + if select.projection.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "Subquery must have a projection".to_string(), + )); + } + + let result_value = match &select.projection[0] { + SelectItem::UnnamedExpr(expr) => match expr { + Expr::Function(func) => { + // Handle aggregate functions (MAX, MIN, COUNT, SUM, AVG) + let func_name = func + .name + .0 + .first() + .map(|id| id.value.to_uppercase()) + .unwrap_or_default(); + + // Get the aggregate function + let agg_func = AggregateFunction::from_name(&func_name).ok_or_else(|| { + SqawkError::UnsupportedSqlFeature(format!( + "Unsupported aggregate function in subquery: {}", + func_name + )) + })?; + + // Extract the column index from the function argument + let col_idx = self.extract_agg_column_index(func, table)?; + + // Collect all values from that column + let values: Vec = filtered_rows + .iter() + .map(|row| row[col_idx].clone()) + .collect(); + + // Execute the aggregate function + agg_func.execute(&values)? + } + Expr::Identifier(ident) => { + // Simple column reference - get first row's value + let col_name = ident.value.to_lowercase(); + let col_idx = table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name.clone()))?; + + if filtered_rows.is_empty() { + TableValue::Null + } else { + filtered_rows[0][col_idx].clone() + } + } + _ => { + // Default: get first column of first row + if filtered_rows.is_empty() { + TableValue::Null + } else { + filtered_rows[0][0].clone() + } + } + }, + SelectItem::Wildcard(_) => { + // SELECT * - just get first column of first row + if filtered_rows.is_empty() { + TableValue::Null + } else { + filtered_rows[0][0].clone() + } + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Unsupported projection in scalar subquery".to_string(), + )) + } + }; + + Ok(result_value) + } + + /// Evaluate an EXISTS subquery at compile time and return true/false + fn evaluate_exists_subquery(&self, subquery: &Query, negated: bool) -> SqawkResult { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in EXISTS clause".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "EXISTS subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in EXISTS subqueries".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Check if any rows match the subquery's WHERE clause + let has_rows = if let Some(where_expr) = &select.selection { + subquery_table + .rows() + .iter() + .any(|row| self.evaluate_where_at_compile_time(where_expr, row, subquery_table)) + } else { + !subquery_table.rows().is_empty() + }; + + Ok(if negated { !has_rows } else { has_rows }) + } + + /// Evaluate an IN subquery at compile time and return the list of values + fn evaluate_in_subquery_values(&self, subquery: &Query) -> SqawkResult> { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in IN clause".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "IN subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in IN subqueries".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Filter rows if there's a WHERE clause in the subquery + let filtered_rows: Vec<&Vec> = if let Some(where_expr) = &select.selection { + subquery_table + .rows() + .iter() + .filter(|row| self.evaluate_where_at_compile_time(where_expr, row, subquery_table)) + .collect() + } else { + subquery_table.rows().iter().collect() + }; + + // Get the column index from the subquery projection + let col_idx = if select.projection.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "IN subquery must have a projection".to_string(), + )); + } else { + match &select.projection[0] { + SelectItem::UnnamedExpr(Expr::Identifier(ident)) => { + let col_name = ident.value.to_lowercase(); + subquery_table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))? + } + SelectItem::Wildcard(_) => 0, + _ => 0, // Default to first column + } + }; + + // Collect values from the subquery result + let values: Vec = filtered_rows + .iter() + .map(|row| row[col_idx].clone()) + .collect(); + + Ok(values) + } + + /// Evaluate a WHERE expression at compile time for subquery filtering + fn evaluate_where_at_compile_time( + &self, + expr: &Expr, + row: &[TableValue], + table: &Table, + ) -> bool { + match expr { + Expr::BinaryOp { left, op, right } => { + match op { + BinaryOperator::And => { + self.evaluate_where_at_compile_time(left, row, table) + && self.evaluate_where_at_compile_time(right, row, table) + } + BinaryOperator::Or => { + self.evaluate_where_at_compile_time(left, row, table) + || self.evaluate_where_at_compile_time(right, row, table) + } + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq => { + let left_val = self.evaluate_expr_at_compile_time(left, row, table); + let right_val = self.evaluate_expr_at_compile_time(right, row, table); + self.compare_values(&left_val, &right_val, op) + } + _ => true, // Unknown op, assume true + } + } + Expr::IsNull(inner) => { + let val = self.evaluate_expr_at_compile_time(inner, row, table); + matches!(val, TableValue::Null) + } + Expr::IsNotNull(inner) => { + let val = self.evaluate_expr_at_compile_time(inner, row, table); + !matches!(val, TableValue::Null) + } + _ => true, // Unknown expression, assume true + } + } + + /// Evaluate an expression at compile time to get a value + fn evaluate_expr_at_compile_time( + &self, + expr: &Expr, + row: &[TableValue], + table: &Table, + ) -> TableValue { + match expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + if let Some(idx) = table.column_index(&col_name) { + row[idx].clone() + } else { + TableValue::Null + } + } + Expr::Value(val) => match val { + Value::Number(n, _) => { + if let Ok(i) = n.parse::() { + TableValue::Integer(i) + } else if let Ok(f) = n.parse::() { + TableValue::Float(f) + } else { + TableValue::String(n.clone().into()) + } + } + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { + TableValue::String(s.clone().into()) + } + Value::Boolean(b) => TableValue::Boolean(*b), + Value::Null => TableValue::Null, + _ => TableValue::Null, + }, + _ => TableValue::Null, + } + } + + /// Compare two values using the given operator + fn compare_values(&self, left: &TableValue, right: &TableValue, op: &BinaryOperator) -> bool { + match (left, right) { + (TableValue::Integer(l), TableValue::Integer(r)) => match op { + BinaryOperator::Eq => l == r, + BinaryOperator::NotEq => l != r, + BinaryOperator::Lt => l < r, + BinaryOperator::LtEq => l <= r, + BinaryOperator::Gt => l > r, + BinaryOperator::GtEq => l >= r, + _ => false, + }, + (TableValue::Float(l), TableValue::Float(r)) => match op { + BinaryOperator::Eq => (l - r).abs() < f64::EPSILON, + BinaryOperator::NotEq => (l - r).abs() >= f64::EPSILON, + BinaryOperator::Lt => l < r, + BinaryOperator::LtEq => l <= r, + BinaryOperator::Gt => l > r, + BinaryOperator::GtEq => l >= r, + _ => false, + }, + (TableValue::Integer(l), TableValue::Float(r)) => { + let l = *l as f64; + match op { + BinaryOperator::Eq => (l - r).abs() < f64::EPSILON, + BinaryOperator::NotEq => (l - r).abs() >= f64::EPSILON, + BinaryOperator::Lt => l < *r, + BinaryOperator::LtEq => l <= *r, + BinaryOperator::Gt => l > *r, + BinaryOperator::GtEq => l >= *r, + _ => false, + } + } + (TableValue::Float(l), TableValue::Integer(r)) => { + let r = *r as f64; + match op { + BinaryOperator::Eq => (l - r).abs() < f64::EPSILON, + BinaryOperator::NotEq => (l - r).abs() >= f64::EPSILON, + BinaryOperator::Lt => *l < r, + BinaryOperator::LtEq => *l <= r, + BinaryOperator::Gt => *l > r, + BinaryOperator::GtEq => *l >= r, + _ => false, + } + } + (TableValue::String(l), TableValue::String(r)) => match op { + BinaryOperator::Eq => l == r, + BinaryOperator::NotEq => l != r, + BinaryOperator::Lt => l < r, + BinaryOperator::LtEq => l <= r, + BinaryOperator::Gt => l > r, + BinaryOperator::GtEq => l >= r, + _ => false, + }, + (TableValue::Boolean(l), TableValue::Boolean(r)) => match op { + BinaryOperator::Eq => l == r, + BinaryOperator::NotEq => l != r, + _ => false, + }, + // Handle string to number comparisons + (TableValue::String(s), TableValue::Integer(i)) => { + if let Ok(n) = s.parse::() { + match op { + BinaryOperator::Eq => n == *i, + BinaryOperator::NotEq => n != *i, + BinaryOperator::Lt => n < *i, + BinaryOperator::LtEq => n <= *i, + BinaryOperator::Gt => n > *i, + BinaryOperator::GtEq => n >= *i, + _ => false, + } + } else { + false + } + } + (TableValue::Integer(i), TableValue::String(s)) => { + if let Ok(n) = s.parse::() { + match op { + BinaryOperator::Eq => *i == n, + BinaryOperator::NotEq => *i != n, + BinaryOperator::Lt => *i < n, + BinaryOperator::LtEq => *i <= n, + BinaryOperator::Gt => *i > n, + BinaryOperator::GtEq => *i >= n, + _ => false, + } + } else { + false + } + } + _ => false, // NULL comparisons return false + } + } + + /// Compile an IN (SELECT ...) subquery (Phase 4F) + /// + /// Uses compile-time execution: executes the subquery during compilation + /// and generates comparison code similar to IN (list). + /// + /// Example: SELECT * FROM orders WHERE user_id IN (SELECT id FROM users WHERE active = 1) + fn compile_in_subquery( + &mut self, + expr: &Expr, + subquery: &Query, + negated: bool, + table: &Table, + cursor_idx: usize, + ) -> SqawkResult { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in IN clause".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "IN subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in IN subqueries".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Filter rows if there's a WHERE clause in the subquery + let filtered_rows: Vec<&Vec> = if let Some(where_expr) = &select.selection { + subquery_table + .rows() + .iter() + .filter(|row| self.evaluate_where_at_compile_time(where_expr, row, subquery_table)) + .collect() + } else { + subquery_table.rows().iter().collect() + }; + + // Get the column index from the subquery projection + let col_idx = if select.projection.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "IN subquery must have a projection".to_string(), + )); + } else { + match &select.projection[0] { + SelectItem::UnnamedExpr(Expr::Identifier(ident)) => { + let col_name = ident.value.to_lowercase(); + subquery_table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))? + } + SelectItem::Wildcard(_) => 0, + _ => 0, // Default to first column + } + }; + + // Collect values from the subquery result + let subquery_values: Vec = filtered_rows + .iter() + .map(|row| row[col_idx].clone()) + .collect(); + + // If subquery returns no values, handle specially + if subquery_values.is_empty() { + let result_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + if negated { 1 } else { 0 }, + result_reg, + 0, + None, + &format!( + "r[{}] = {} (empty IN subquery)", + result_reg, + if negated { 1 } else { 0 } + ), + ); + return Ok(result_reg); + } + + // Allocate registers + let expr_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + let temp_reg = self.allocate_register(); + + // Compile the expression value (from the outer query) + self.compile_where_operand(expr, table, cursor_idx, expr_reg)?; + + // Initialize result to 0 (false, not found) + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (IN subquery initial)", result_reg), + ); + + // Track jump addresses that need to be patched to skip to end when match found + let mut jump_to_end_addrs = Vec::with_capacity(subquery_values.len()); + + // Compare against each value from the subquery + for value in &subquery_values { + let item_reg = self.allocate_register(); + + // Emit the subquery value as a literal + self.emit_value_literal(item_reg, value)?; + + // Compare expr == item + self.emit( + OpCode::Eq, + expr_reg, + item_reg, + temp_reg, + None, + &format!("r[{}] = (r[{}] == r[{}])", temp_reg, expr_reg, item_reg), + ); + + // If match found (temp_reg != 0), set result to 1 and jump to end + let skip_set_addr = self.program.len(); + self.emit( + OpCode::IfZ, + temp_reg, + 0, // Will be patched to skip the next instructions + 0, + None, + "if no match, skip to next item", + ); + + // Set result to 1 (found) + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (found in subquery)", result_reg), + ); + + // Jump to end (skip remaining comparisons) + jump_to_end_addrs.push(self.program.len()); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to end + 0, + None, + "skip to end", + ); + + // Patch the skip address to continue to next comparison + let continue_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_set_addr) { + inst.p2 = continue_addr as i64; + } + } + + // End label - all "found" jumps come here + let end_label = self.program.len(); + + // Patch all jump-to-end addresses + for addr in jump_to_end_addrs { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = end_label as i64; + } + } + + // Handle negation: if NOT IN, invert the result + if negated { + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0 for NOT IN", zero_reg), + ); + self.emit( + OpCode::Eq, + result_reg, + zero_reg, + result_reg, + None, + &format!("r[{}] = (r[{}] == 0) for NOT IN", result_reg, result_reg), + ); + } + + Ok(result_reg) + } + + /// Compile an EXISTS (SELECT ...) subquery (Phase 4F/4B) + /// + /// - Non-correlated: uses compile-time execution + /// - Correlated: uses runtime execution with inline subquery iteration + /// + /// Example: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id) + fn compile_exists_subquery( + &mut self, + subquery: &Query, + negated: bool, + outer_table: &Table, + outer_cursor_idx: usize, + ) -> SqawkResult { + let result_reg = self.allocate_register(); + + // Detect if this is a correlated subquery + let outer_refs = self.detect_outer_references(subquery, outer_table.name(), None); + + if !outer_refs.is_empty() { + // Correlated: emit runtime execution bytecode + return self.compile_correlated_exists_subquery( + subquery, + negated, + outer_table, + outer_cursor_idx, + &outer_refs, + result_reg, + ); + } + + // Non-correlated: use compile-time execution + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in EXISTS clause".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "EXISTS subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in EXISTS subqueries".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Check if any rows match the subquery's WHERE clause + let has_rows = if let Some(where_expr) = &select.selection { + subquery_table + .rows() + .iter() + .any(|row| self.evaluate_where_at_compile_time(where_expr, row, subquery_table)) + } else { + !subquery_table.rows().is_empty() + }; + + // Emit the result + let exists = if negated { !has_rows } else { has_rows }; + self.emit( + OpCode::Integer, + if exists { 1 } else { 0 }, + result_reg, + 0, + None, + &format!( + "r[{}] = {} ({}EXISTS result)", + result_reg, + if exists { 1 } else { 0 }, + if negated { "NOT " } else { "" } + ), + ); + + Ok(result_reg) + } + + // ========================================================================= + // Phase 4B: Correlated Subquery Support + // ========================================================================= + + /// Detect if a subquery references outer query columns (making it correlated) + /// + /// A correlated subquery contains references to columns from the outer query, + /// like: SELECT * FROM orders o WHERE amount > (SELECT AVG(amount) FROM orders WHERE user_id = o.user_id) + /// Here, `o.user_id` is an outer reference. + fn detect_outer_references( + &self, + subquery: &Query, + outer_table_name: &str, + outer_alias: Option<&str>, + ) -> Vec { + let mut outer_refs = Vec::new(); + + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => return outer_refs, + }; + + // Get the subquery's table name to distinguish from outer refs + let subquery_table_name = select.from.first().and_then(|f| match &f.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let table_name = name + .0 + .last() + .map(|p| p.value.to_lowercase()) + .unwrap_or_default(); + let alias_name = alias.as_ref().map(|a| a.name.value.to_lowercase()); + Some((table_name, alias_name)) + } + _ => None, + }); + + // Use the passed outer_alias, or fall back to self.current_outer_alias + let effective_outer_alias = outer_alias.or(self.current_outer_alias.as_deref()); + + // Walk the subquery's WHERE clause looking for outer references + if let Some(where_expr) = &select.selection { + self.collect_outer_refs_from_expr( + where_expr, + outer_table_name, + effective_outer_alias, + subquery_table_name.as_ref(), + &mut outer_refs, + ); + } + + outer_refs + } + + /// Recursively collect outer column references from an expression + fn collect_outer_refs_from_expr( + &self, + expr: &Expr, + outer_table_name: &str, + outer_alias: Option<&str>, + subquery_table: Option<&(String, Option)>, + refs: &mut Vec, + ) { + match expr { + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let qualifier = parts[0].value.to_lowercase(); + let column = parts[1].value.to_lowercase(); + + // Check if this references the outer table (not the subquery's own table) + let is_outer_ref = { + let matches_outer_alias = outer_alias + .map(|a| a.to_lowercase() == qualifier) + .unwrap_or(false); + let matches_outer_table = outer_table_name.to_lowercase() == qualifier; + + // Make sure it's not referencing the subquery's own table + let matches_subquery = subquery_table + .map(|(name, alias)| { + name == &qualifier + || alias.as_ref().map(|a| a == &qualifier).unwrap_or(false) + }) + .unwrap_or(false); + + (matches_outer_alias || matches_outer_table) && !matches_subquery + }; + + if is_outer_ref { + refs.push(OuterColumnRef { qualifier, column }); + } + } + Expr::BinaryOp { left, right, .. } => { + self.collect_outer_refs_from_expr( + left, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + self.collect_outer_refs_from_expr( + right, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + Expr::Nested(inner) => { + self.collect_outer_refs_from_expr( + inner, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + Expr::UnaryOp { expr, .. } => { + self.collect_outer_refs_from_expr( + expr, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + Expr::IsNull(inner) | Expr::IsNotNull(inner) => { + self.collect_outer_refs_from_expr( + inner, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + Expr::Between { + expr, + low, + high, + negated: _, + } => { + self.collect_outer_refs_from_expr( + expr, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + self.collect_outer_refs_from_expr( + low, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + self.collect_outer_refs_from_expr( + high, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + Expr::InList { expr, list, .. } => { + self.collect_outer_refs_from_expr( + expr, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + for item in list { + self.collect_outer_refs_from_expr( + item, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + } + Expr::Case { + operand, + conditions, + results, + else_result, + } => { + if let Some(op) = operand { + self.collect_outer_refs_from_expr( + op, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + for cond in conditions { + self.collect_outer_refs_from_expr( + cond, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + for result in results { + self.collect_outer_refs_from_expr( + result, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + if let Some(else_r) = else_result { + self.collect_outer_refs_from_expr( + else_r, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + } + Expr::Function(func) => { + for arg in &func.args { + if let FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) = arg { + self.collect_outer_refs_from_expr( + e, + outer_table_name, + outer_alias, + subquery_table, + refs, + ); + } + } + } + // Other expression types don't contain column references + _ => {} + } + } + + /// Compile a correlated EXISTS subquery (Phase 4B) + /// + /// For each outer row, iterates through the subquery table and checks + /// if any row matches the correlated condition. + fn compile_correlated_exists_subquery( + &mut self, + subquery: &Query, + negated: bool, + outer_table: &Table, + outer_cursor_idx: usize, + outer_refs: &[OuterColumnRef], + result_reg: i64, + ) -> SqawkResult { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in correlated EXISTS".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "Correlated EXISTS subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in correlated EXISTS".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Use a unique cursor ID for the subquery (offset from outer cursor) + let sub_cursor_idx = outer_cursor_idx + 100; + + // Initialize result to 0 (not found) + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (EXISTS init)", result_reg), + ); + + // Open the subquery table + self.emit( + OpCode::OpenRead, + sub_cursor_idx as i64, + 0, + 0, + Some(subquery_table_name.clone()), + &format!("Open {} for correlated EXISTS", subquery_table_name), + ); + + // Rewind to first row (jump to done if empty) + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + sub_cursor_idx as i64, + 0, // Will be patched to done_addr + 0, + None, + "Rewind subquery cursor", + ); + + let loop_start = self.program.len(); + + // For the WHERE clause, we need to evaluate the correlated condition + // This requires comparing subquery columns with outer columns + if let Some(where_expr) = &select.selection { + let cmp_reg = self.allocate_register(); + + // Compile the WHERE condition with both tables accessible + // For correlated subqueries, we need to handle outer references specially + self.compile_correlated_where( + where_expr, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + cmp_reg, + )?; + + // If condition is false (0), jump to next row + let skip_match_addr = self.program.len(); + self.emit( + OpCode::IfZ, + cmp_reg, + 0, // Will be patched to next_row + 0, + None, + "Skip if WHERE condition false", + ); + + // Match found! Set result to 1 + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (EXISTS match found)", result_reg), + ); + + // Jump to done (exit early) + let goto_done_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to done_addr + 0, + None, + "Exit early on match", + ); + + // Patch skip_match to jump here + let next_row_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_match_addr) { + inst.p2 = next_row_addr as i64; + } + + // Next row + self.emit( + OpCode::Next, + sub_cursor_idx as i64, + loop_start as i64, + 0, + None, + "Next subquery row", + ); + + // Done label + let done_addr = self.program.len(); + + // Patch rewind to jump to done if empty + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = done_addr as i64; + } + + // Patch goto_done to jump here + if let Some(inst) = self.program.instructions.get_mut(goto_done_addr) { + inst.p2 = done_addr as i64; + } + } else { + // No WHERE clause - if table has any rows, EXISTS is true + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (EXISTS: table has rows)", result_reg), + ); + + // Done + let done_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = done_addr as i64; + } + } + + // Close the subquery cursor + self.emit( + OpCode::Close, + sub_cursor_idx as i64, + 0, + 0, + None, + "Close subquery cursor", + ); + + // Handle NOT EXISTS: flip the result + if negated { + // result = (result == 0) ? 1 : 0 + // Use branching: if result is 0, set to 1; else set to 0 + let skip_zero_addr = self.program.len(); + self.emit( + OpCode::IfZ, + result_reg, + 0, // Will be patched + 0, + None, + &format!("if r[{}] == 0, jump to set 1", result_reg), + ); + + // result was non-zero (EXISTS found rows), so NOT EXISTS = 0 + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (NOT EXISTS false)", result_reg), + ); + + let goto_end_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched + 0, + None, + "skip to end", + ); + + // result was zero (EXISTS found no rows), so NOT EXISTS = 1 + let set_one_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_zero_addr) { + inst.p2 = set_one_addr as i64; + } + + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (NOT EXISTS true)", result_reg), + ); + + // Patch goto to end + let end_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(goto_end_addr) { + inst.p2 = end_addr as i64; + } + } + + Ok(result_reg) + } + + /// Compile a WHERE clause with correlated column references + #[allow(clippy::too_many_arguments)] + fn compile_correlated_where( + &mut self, + where_expr: &Expr, + subquery_table: &Table, + sub_cursor_idx: usize, + outer_table: &Table, + outer_cursor_idx: usize, + outer_refs: &[OuterColumnRef], + result_reg: i64, + ) -> SqawkResult<()> { + match where_expr { + Expr::BinaryOp { left, op, right } => { + match op { + BinaryOperator::And => { + // Compile both sides and AND them + let left_reg = self.allocate_register(); + let right_reg = self.allocate_register(); + self.compile_correlated_where( + left, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + left_reg, + )?; + self.compile_correlated_where( + right, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + right_reg, + )?; + // AND: result = left AND right + // If left is 0, result is 0; else result is right + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + left_reg, + 0, // Will patch to set result=0 + 0, + None, + "AND: skip if left is false", + ); + // Left is true, result = right + self.emit( + OpCode::Copy, + right_reg, + result_reg, + 0, + None, + &format!("r[{}] = r[{}] (AND right)", result_reg, right_reg), + ); + let goto_end_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will patch to end + 0, + None, + "", + ); + // Left is false, result = 0 + let false_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = false_addr as i64; + } + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (AND false)", result_reg), + ); + let end_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(goto_end_addr) { + inst.p2 = end_addr as i64; + } + Ok(()) + } + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq => { + // Compile left and right operands + let left_reg = self.allocate_register(); + let right_reg = self.allocate_register(); + + self.compile_correlated_operand( + left, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + left_reg, + )?; + self.compile_correlated_operand( + right, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + right_reg, + )?; + + // Emit comparison + let cmp_opcode = Self::binary_op_to_comparison_opcode(op)?; + + self.emit( + cmp_opcode, + left_reg, + right_reg, + result_reg, + None, + &format!( + "r[{}] = r[{}] {:?} r[{}]", + result_reg, left_reg, op, right_reg + ), + ); + Ok(()) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported operator in correlated subquery WHERE: {:?}", + op + ))), + } + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression in correlated subquery WHERE: {:?}", + where_expr + ))), + } + } + + /// Compile an operand in a correlated WHERE clause + #[allow(clippy::too_many_arguments)] + fn compile_correlated_operand( + &mut self, + expr: &Expr, + subquery_table: &Table, + sub_cursor_idx: usize, + outer_table: &Table, + outer_cursor_idx: usize, + outer_refs: &[OuterColumnRef], + dest_reg: i64, + ) -> SqawkResult<()> { + match expr { + Expr::Identifier(ident) => { + // Unqualified column - must be from subquery table + let col_name = ident.value.to_lowercase(); + let col_idx = subquery_table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name.clone()))?; + self.emit( + OpCode::Column, + sub_cursor_idx as i64, + col_idx as i64, + dest_reg, + None, + &format!("r[{}] = subquery.{}", dest_reg, col_name), + ); + Ok(()) + } + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let qualifier = parts[0].value.to_lowercase(); + let col_name = parts[1].value.to_lowercase(); + + // Check if this is an outer reference + let is_outer = outer_refs + .iter() + .any(|r| r.qualifier == qualifier && r.column == col_name) + || qualifier == outer_table.name().to_lowercase(); + + if is_outer { + // Load from outer table + let col_idx = outer_table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name.clone()))?; + self.emit( + OpCode::Column, + outer_cursor_idx as i64, + col_idx as i64, + dest_reg, + None, + &format!("r[{}] = outer.{}", dest_reg, col_name), + ); + } else { + // Load from subquery table + let col_idx = subquery_table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name.clone()))?; + self.emit( + OpCode::Column, + sub_cursor_idx as i64, + col_idx as i64, + dest_reg, + None, + &format!("r[{}] = subquery.{}", dest_reg, col_name), + ); + } + Ok(()) + } + Expr::Value(v) => { + match v { + Value::Number(n, _) => { + if let Ok(i) = n.parse::() { + self.emit( + OpCode::Integer, + i, + dest_reg, + 0, + None, + &format!("r[{}] = {}", dest_reg, i), + ); + } else if let Ok(f) = n.parse::() { + self.emit( + OpCode::String, + 0, + dest_reg, + 0, + Some(f.to_string()), + &format!("r[{}] = {}", dest_reg, f), + ); + self.emit( + OpCode::Cast, + dest_reg, + dest_reg, + 0, + Some("REAL".to_string()), + "", + ); + } + } + Value::SingleQuotedString(s) => { + self.emit( + OpCode::String, + 0, + dest_reg, + 0, + Some(s.clone()), + &format!("r[{}] = '{}'", dest_reg, s), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported value type in correlated subquery: {:?}", + v + ))) + } + } + Ok(()) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported operand in correlated subquery: {:?}", + expr + ))), + } + } + + /// Compile a correlated scalar subquery (Phase 4B) + /// + /// For each outer row, iterates through the subquery table and computes + /// an aggregate over matching rows. + fn compile_correlated_scalar_subquery( + &mut self, + subquery: &Query, + outer_table: &Table, + outer_cursor_idx: usize, + outer_refs: &[OuterColumnRef], + result_reg: i64, + ) -> SqawkResult<()> { + // Get the subquery's SELECT body + let select = match &*subquery.body { + SetExpr::Select(select) => select, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple SELECT subqueries are supported in correlated scalar".to_string(), + )) + } + }; + + // Verify it's a single-table query + if select.from.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "Correlated scalar subquery must have a FROM clause".to_string(), + )); + } + + // Get the subquery table name + let subquery_table_name = match &select.from[0].relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in correlated scalar".to_string(), + )) + } + }; + + // Get the subquery table + if !self.database.has_table(&subquery_table_name) { + return Err(SqawkError::TableNotFound(subquery_table_name.clone())); + } + let subquery_table = self.database.get_table(&subquery_table_name).unwrap(); + + // Get the aggregate function from the SELECT list + if select.projection.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "Correlated scalar subquery must have a SELECT expression".to_string(), + )); + } + + let (agg_func_name, agg_col_idx) = match &select.projection[0] { + SelectItem::UnnamedExpr(Expr::Function(func)) => { + let func_name = func.name.to_string().to_uppercase(); + let col_idx = self.extract_agg_column_index(func, subquery_table)?; + (func_name, col_idx) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Correlated scalar subquery must select an aggregate function".to_string(), + )) + } + }; + + // Use a unique cursor ID for the subquery + let sub_cursor_idx = outer_cursor_idx + 100; + + // Allocate registers for aggregate computation + let acc_reg = self.allocate_register(); // Accumulator + let _count_reg = self.allocate_register(); // Count for AVG (unused for now) + let val_reg = self.allocate_register(); // Current value + + // Initialize accumulator with Null (like SQLite does) + // AggStep/AggFinal will treat Null as "no data yet" + self.emit( + OpCode::Null, + 0, + acc_reg, + acc_reg, // P3=P2 means just set one register + None, + &format!("r[{}]=NULL; Init {} accumulator", acc_reg, agg_func_name), + ); + + // Validate aggregate function is supported + match agg_func_name.as_str() { + "COUNT" | "SUM" | "AVG" | "MIN" | "MAX" => {} + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported aggregate in correlated subquery: {}", + agg_func_name + ))) + } + } + + // Open the subquery table + self.emit( + OpCode::OpenRead, + sub_cursor_idx as i64, + 0, + 0, + Some(subquery_table_name.clone()), + &format!("Open {} for correlated scalar", subquery_table_name), + ); + + // Rewind to first row + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + sub_cursor_idx as i64, + 0, // Will be patched to done_addr + 0, + None, + "Rewind subquery cursor", + ); + + let loop_start = self.program.len(); + + // Compile WHERE condition if present + let cmp_reg = self.allocate_register(); + let has_where = select.selection.is_some(); + + if let Some(where_expr) = &select.selection { + self.compile_correlated_where( + where_expr, + subquery_table, + sub_cursor_idx, + outer_table, + outer_cursor_idx, + outer_refs, + cmp_reg, + )?; + } + + // If WHERE fails, skip to next row + let skip_agg_addr = if has_where { + let addr = self.program.len(); + self.emit( + OpCode::IfZ, + cmp_reg, + 0, // Will be patched + 0, + None, + "Skip if WHERE false", + ); + Some(addr) + } else { + None + }; + + // Load the value for aggregation + self.emit( + OpCode::Column, + sub_cursor_idx as i64, + agg_col_idx as i64, + val_reg, + None, + &format!("r[{}] = subquery column for agg", val_reg), + ); + + // Update accumulator based on aggregate type + self.emit( + OpCode::AggStep, + Self::agg_func_type(agg_func_name.as_str()), + val_reg, + acc_reg, + None, + &format!( + "Aggregate step: {} r[{}] -> r[{}]", + agg_func_name, val_reg, acc_reg + ), + ); + + // Next row + let next_row_addr = self.program.len(); + if let Some(skip_addr) = skip_agg_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = next_row_addr as i64; + } + } + + self.emit( + OpCode::Next, + sub_cursor_idx as i64, + loop_start as i64, + 0, + None, + "Next subquery row", + ); + + // Done - finalize aggregate + let done_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = done_addr as i64; + } + + // Finalize aggregate into result_reg + // P3 = function type hint for when no AggStep was called (register still Null) + self.emit( + OpCode::AggFinal, + acc_reg, + result_reg, + Self::agg_func_type(agg_func_name.as_str()), + None, + &format!("r[{}] = final {} result", result_reg, agg_func_name), + ); + + // Close the subquery cursor + self.emit( + OpCode::Close, + sub_cursor_idx as i64, + 0, + 0, + None, + "Close subquery cursor", + ); + + Ok(()) + } + + /// Compile a LIKE/ILIKE pattern match operation + fn compile_like_pattern( + &mut self, + expr: &Expr, + pattern: &Expr, + negated: bool, + table: &Table, + cursor_idx: usize, + case_insensitive: bool, + ) -> SqawkResult { + // Allocate registers for the value and result + let value_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + + // Compile the expression (usually a column reference) + self.compile_where_operand(expr, table, cursor_idx, value_reg)?; + + // Get the pattern string + let pattern_str = match pattern { + Expr::Value(Value::SingleQuotedString(s)) => s.clone(), + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "LIKE pattern must be a string literal".to_string(), + )) + } + }; + + // Generate the Like opcode + // P1 = value register, P3 = result register, P4 = pattern string + // P2 encodes: bit 0 = negated, bit 1 = case_insensitive + let flags = (if negated { 1 } else { 0 }) | (if case_insensitive { 2 } else { 0 }); + + self.emit( + OpCode::Like, + value_reg, + flags, + result_reg, + Some(pattern_str.clone()), + &format!( + "r[{}] = r[{}] {}LIKE '{}'", + result_reg, + value_reg, + if negated { "NOT " } else { "" }, + pattern_str + ), + ); + + Ok(result_reg) + } + + /// Compile a BETWEEN expression: expr BETWEEN low AND high + /// + /// This compiles to: (expr >= low) AND (expr <= high) + /// Uses conditional jumps for short-circuit evaluation. + fn compile_between( + &mut self, + expr: &Expr, + low: &Expr, + high: &Expr, + negated: bool, + table: &Table, + cursor_idx: usize, + ) -> SqawkResult { + // Allocate registers + let expr_reg = self.allocate_register(); + let low_reg = self.allocate_register(); + let high_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + let temp_reg = self.allocate_register(); + + // Compile the expression value + self.compile_where_operand(expr, table, cursor_idx, expr_reg)?; + + // Compile the low bound + self.compile_where_operand(low, table, cursor_idx, low_reg)?; + + // Compile the high bound + self.compile_where_operand(high, table, cursor_idx, high_reg)?; + + // Compare expr >= low (result in temp_reg) + self.emit( + OpCode::Ge, + expr_reg, + low_reg, + temp_reg, + None, + &format!("r[{}] = r[{}] >= r[{}]", temp_reg, expr_reg, low_reg), + ); + + // If expr < low, result is false (0) + // Jump past the second comparison if first is false + let jump_to_false = self.program.len(); + self.emit( + OpCode::IfZ, + temp_reg, + 0, // Will be patched + 0, + None, + "if first comparison false, skip to result", + ); + + // Compare expr <= high (result in result_reg) + self.emit( + OpCode::Le, + expr_reg, + high_reg, + result_reg, + None, + &format!("r[{}] = r[{}] <= r[{}]", result_reg, expr_reg, high_reg), + ); + + // Jump past the "set to false" instruction + let jump_to_end = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched + 0, + None, + "skip false assignment", + ); + + // Set result to 0 (false) - this is where we jump if first comparison fails + let false_label = self.program.len(); + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (BETWEEN false)", result_reg), + ); + + let end_label = self.program.len(); + + // Patch the jump addresses + if let Some(inst) = self.program.instructions.get_mut(jump_to_false) { + inst.p2 = false_label as i64; + } + if let Some(inst) = self.program.instructions.get_mut(jump_to_end) { + inst.p2 = end_label as i64; + } + + // Handle negation: if NOT BETWEEN, invert the result + // NOT BETWEEN is true when BETWEEN is false (result == 0) + if negated { + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0 for NOT BETWEEN", zero_reg), + ); + self.emit( + OpCode::Eq, + result_reg, + zero_reg, + result_reg, + None, + &format!( + "r[{}] = (r[{}] == 0) for NOT BETWEEN", + result_reg, result_reg + ), + ); + } + + Ok(result_reg) + } + + /// Compile an IN list expression: expr IN (val1, val2, ...) + /// + /// This compiles to a series of equality checks with short-circuit OR logic. + fn compile_in_list( + &mut self, + expr: &Expr, + list: &[Expr], + negated: bool, + table: &Table, + cursor_idx: usize, + ) -> SqawkResult { + if list.is_empty() { + // Empty list: IN () is always false, NOT IN () is always true + let result_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + if negated { 1 } else { 0 }, + result_reg, + 0, + None, + &format!( + "r[{}] = {} (empty IN list)", + result_reg, + if negated { 1 } else { 0 } + ), + ); + return Ok(result_reg); + } + + // Allocate registers + let expr_reg = self.allocate_register(); + let result_reg = self.allocate_register(); + let temp_reg = self.allocate_register(); + + // Compile the expression value + self.compile_where_operand(expr, table, cursor_idx, expr_reg)?; + + // Initialize result to 0 (false, not found in list) + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (IN list initial)", result_reg), + ); + + // Track jump addresses that need to be patched to skip to end when match found + let mut jump_to_end_addrs = Vec::new(); + + // Compare against each value in the list + for item in list { + let item_reg = self.allocate_register(); + + // Compile the list item value + self.compile_where_operand(item, table, cursor_idx, item_reg)?; + + // Compare expr == item + self.emit( + OpCode::Eq, + expr_reg, + item_reg, + temp_reg, + None, + &format!("r[{}] = (r[{}] == r[{}])", temp_reg, expr_reg, item_reg), + ); + + // If match found (temp_reg != 0), set result to 1 and jump to end + // We use IfZ to skip setting result if no match + let skip_set_addr = self.program.len(); + self.emit( + OpCode::IfZ, + temp_reg, + 0, // Will be patched to skip the next instructions + 0, + None, + "if no match, skip to next item", + ); + + // Set result to 1 (found) + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (found in list)", result_reg), + ); + + // Jump to end (skip remaining comparisons) + jump_to_end_addrs.push(self.program.len()); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to end + 0, + None, + "skip to end", + ); + + // Patch the skip address to continue to next comparison + let continue_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_set_addr) { + inst.p2 = continue_addr as i64; + } + } + + // End label - all "found" jumps come here + let end_label = self.program.len(); + + // Patch all jump-to-end addresses + for addr in jump_to_end_addrs { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = end_label as i64; + } + } + + // Handle negation: if NOT IN, invert the result + if negated { + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0 for NOT IN", zero_reg), + ); + self.emit( + OpCode::Eq, + result_reg, + zero_reg, + result_reg, + None, + &format!("r[{}] = (r[{}] == 0) for NOT IN", result_reg, result_reg), + ); + } + + Ok(result_reg) + } + + /// Compile a CASE expression + /// + /// Supports both: + /// - Simple CASE: CASE expr WHEN val1 THEN res1 ... ELSE default END + /// - Searched CASE: CASE WHEN cond1 THEN res1 ... ELSE default END + #[allow(clippy::too_many_arguments)] + fn compile_case( + &mut self, + operand: Option<&Expr>, + conditions: &[Expr], + results: &[Expr], + else_result: Option<&Expr>, + table: &Table, + cursor_idx: usize, + target_reg: Option, + ) -> SqawkResult { + if conditions.len() != results.len() { + return Err(SqawkError::InvalidSqlQuery( + "CASE: conditions and results must have same length".to_string(), + )); + } + + let result_reg = target_reg.unwrap_or_else(|| self.allocate_register()); + let temp_reg = self.allocate_register(); + + // For simple CASE, compile the operand once + let operand_reg = if let Some(op_expr) = operand { + let reg = self.allocate_register(); + self.compile_where_operand(op_expr, table, cursor_idx, reg)?; + Some(reg) + } else { + None + }; + + // Track jump addresses that need to go to the end + let mut jump_to_end_addrs = Vec::new(); + + // Process each WHEN clause + for (condition, result_expr) in conditions.iter().zip(results.iter()) { + let cond_reg = self.allocate_register(); + + if let Some(op_reg) = operand_reg { + // Simple CASE: compare operand with condition value + self.compile_where_operand(condition, table, cursor_idx, cond_reg)?; + self.emit( + OpCode::Eq, + op_reg, + cond_reg, + temp_reg, + None, + &format!( + "r[{}] = (r[{}] == r[{}]) for CASE", + temp_reg, op_reg, cond_reg + ), + ); + } else { + // Searched CASE: evaluate condition directly + // For now, we only support simple boolean conditions + // The condition should evaluate to a boolean/integer result + self.compile_where_operand(condition, table, cursor_idx, temp_reg)?; + } + + // If condition is false (temp_reg == 0), skip to next branch + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + temp_reg, + 0, // Will be patched + 0, + None, + "if WHEN condition false, try next", + ); + + // Condition is true - compile the result value into result_reg + self.compile_where_operand(result_expr, table, cursor_idx, result_reg)?; + + // Jump to end + jump_to_end_addrs.push(self.program.len()); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to end + 0, + None, + "jump to CASE end", + ); + + // Patch skip address to continue to next WHEN + let next_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = next_addr as i64; + } + } + + // ELSE clause (or NULL if no ELSE) + if let Some(else_expr) = else_result { + self.compile_where_operand(else_expr, table, cursor_idx, result_reg)?; + } else { + // No ELSE - result is NULL + self.emit( + OpCode::Null, + 0, + result_reg, + 0, + None, + &format!("r[{}] = NULL (no ELSE in CASE)", result_reg), + ); + } + + // End label + let end_label = self.program.len(); + + // Patch all jump-to-end addresses + for addr in jump_to_end_addrs { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = end_label as i64; } } Ok(result_reg) } - /// Compile a table scan with optional WHERE clause - fn compile_table_scan_with_where( + /// Extract function name from a Function AST node + fn get_function_name(func: &Function) -> String { + func.name + .0 + .iter() + .map(|id| id.value.as_str()) + .collect::>() + .join(".") + .to_uppercase() + } + + /// Compile COALESCE function - returns first non-NULL argument + fn compile_func_coalesce( &mut self, - table_with_joins: &TableWithJoins, - projection: &[SelectItem], - where_clause: &Option, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, ) -> SqawkResult<()> { - // Get the table name - let table_name = match &table_with_joins.relation { - sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, - _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple table scans are supported".into(), - )) + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "COALESCE requires at least one argument".to_string(), + )); + } + + let mut jump_to_end_addrs = Vec::new(); + + for arg in &func.args { + let expr = self.extract_function_arg_expr(arg)?; + let arg_reg = self.allocate_register(); + let null_check_reg = self.allocate_register(); + + // Compile the argument + self.compile_where_operand(&expr, table, cursor_idx, arg_reg)?; + + // Check if it's NULL + self.emit( + OpCode::IsNull, + arg_reg, + null_check_reg, + 0, + None, + &format!("r[{}] = (r[{}] IS NULL)", null_check_reg, arg_reg), + ); + + // If NULL (null_check_reg == 1), skip to next argument + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + null_check_reg, + 0, // Will be patched to "use this value" + 0, + None, + "if NOT NULL, use this value", + ); + + // Patch to jump to next iteration (will be set at end of this iteration) + let continue_to_next = self.program.len(); + + // Add a goto to skip to next arg check (this arg was NULL) + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to next arg check + 0, + None, + "skip to next arg (this was NULL)", + ); + + // Patch the IfZ to come here when not NULL + let use_value_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = use_value_addr as i64; } - }; - // Check if the table exists by trying to get it - if !self.database.has_table(&table_name) { - return Err(SqawkError::TableNotFound(table_name)); + // Copy value to target register (value is not NULL) + // Cast to same type as workaround for copy + self.emit( + OpCode::Cast, + arg_reg, + target_reg, + 0, + Some("TEXT".to_string()), + &format!("r[{}] = r[{}] (COALESCE result)", target_reg, arg_reg), + ); + + // Jump to end (we found a non-NULL value) + jump_to_end_addrs.push(self.program.len()); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched to end + 0, + None, + "jump to COALESCE end", + ); + + // Patch the continue_to_next goto to jump here + let next_arg_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(continue_to_next) { + inst.p2 = next_arg_addr as i64; + } } - // We know the table exists, so unwrap is safe - let table = self.database.get_table(&table_name).unwrap(); + // If all args were NULL, result is NULL + self.emit( + OpCode::Null, + 0, + target_reg, + 0, + None, + &format!("r[{}] = NULL (all COALESCE args were NULL)", target_reg), + ); - // Get column indices for result row based on projection - let columns = self.resolve_projection(projection, table)?; + // Patch all jump-to-end addresses + let end_label = self.program.len(); + for addr in jump_to_end_addrs { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = end_label as i64; + } + } - self.add_comment(&format!("Scanning table: {}", table_name)); + Ok(()) + } - // Open the table for reading (cursor 0) - let cursor_idx = 0; - // Use a simple counter for now, since we don't have get_table_id - let table_id = 1i64; // Just assign a default ID + /// Compile NULLIF function - returns NULL if both args are equal + fn compile_func_nullif( + &mut self, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.len() != 2 { + return Err(SqawkError::InvalidSqlQuery( + "NULLIF requires exactly two arguments".to_string(), + )); + } - self.program.add_instruction(Instruction::new( - OpCode::OpenRead, - cursor_idx as i64, - table_id, + let expr1 = self.extract_function_arg_expr(&func.args[0])?; + let expr2 = self.extract_function_arg_expr(&func.args[1])?; + + let reg1 = self.allocate_register(); + let reg2 = self.allocate_register(); + let cmp_reg = self.allocate_register(); + + // Compile both arguments + self.compile_where_operand(&expr1, table, cursor_idx, reg1)?; + self.compile_where_operand(&expr2, table, cursor_idx, reg2)?; + + // Compare them + self.emit( + OpCode::Eq, + reg1, + reg2, + cmp_reg, + None, + &format!("r[{}] = (r[{}] == r[{}])", cmp_reg, reg1, reg2), + ); + + // If equal (cmp_reg == 1), return NULL; else return reg1 + let not_equal_addr = self.program.len(); + self.emit( + OpCode::IfZ, + cmp_reg, + 0, // Will be patched 0, - Some(table_name.clone()), + None, + "if NOT equal, jump", + ); + + // Values are equal, set result to NULL + self.emit( + OpCode::Null, 0, - Some(format!("Open table {} for reading", table_name)), - )); + target_reg, + 0, + None, + &format!("r[{}] = NULL (values equal)", target_reg), + ); - // Set up the loop to scan the table - self.program.add_instruction(Instruction::new( - OpCode::Rewind, - cursor_idx as i64, - (self.program.len() + 4) as i64, // Jump past the loop if empty + let skip_else_addr = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched 0, None, + "skip else branch", + ); + + // Patch IfZ to jump here (values not equal) + let else_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(not_equal_addr) { + inst.p2 = else_addr as i64; + } + + // Values not equal, copy reg1 to target + self.emit( + OpCode::Cast, + reg1, + target_reg, 0, - Some("Position cursor at first row".to_string()), - )); + Some("TEXT".to_string()), + &format!("r[{}] = r[{}] (values not equal)", target_reg, reg1), + ); - // Loop body start address - let loop_addr = self.program.len(); + // Patch skip_else to jump here + let end_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_else_addr) { + inst.p2 = end_addr as i64; + } - // Load each column value into registers for both WHERE clause and SELECT - let mut result_regs = Vec::new(); - for col_idx in columns.iter() { - let value_reg = self.allocate_register(); - result_regs.push(value_reg); + Ok(()) + } - self.program.add_instruction(Instruction::new( - OpCode::Column, - cursor_idx as i64, - *col_idx as i64, - value_reg, - None, - 0, - Some(format!("r[{}] = column {} value", value_reg, col_idx)), + /// Compile simple string functions (UPPER, LOWER, TRIM, LTRIM, RTRIM, LENGTH) + fn compile_func_string_simple( + &mut self, + func_name: &str, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires one argument", + func_name + ))); + } + + let expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.allocate_register(); + + // Compile the argument + self.compile_where_operand(&expr, table, cursor_idx, src_reg)?; + + // Emit StringFunc opcode + self.emit( + OpCode::StringFunc, + src_reg, + target_reg, + 0, + Some(func_name.to_string()), + &format!("r[{}] = {}(r[{}])", target_reg, func_name, src_reg), + ); + + Ok(()) + } + + /// Compile SUBSTR/SUBSTRING function + fn compile_func_substr( + &mut self, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.len() < 2 { + return Err(SqawkError::InvalidSqlQuery( + "SUBSTR requires at least two arguments".to_string(), )); } - // Compile WHERE clause filtering if present - if let Some(where_expr) = where_clause { - self.add_comment("WHERE clause filtering"); - let skip_addr = self.compile_where_condition(where_expr, table, cursor_idx)?; - - // If WHERE condition failed, jump to Next instruction to skip this row - self.program.add_instruction(Instruction::new( - OpCode::IfZ, - skip_addr, // Register containing condition result - (self.program.len() + 2) as i64, // Jump to Next instruction - 0, - None, - 0, - Some("Skip row if WHERE condition is false".to_string()), + let str_expr = self.extract_function_arg_expr(&func.args[0])?; + let start_expr = self.extract_function_arg_expr(&func.args[1])?; + + let str_reg = self.allocate_register(); + let start_reg = self.allocate_register(); + + // Compile string and start arguments + self.compile_where_operand(&str_expr, table, cursor_idx, str_reg)?; + self.compile_where_operand(&start_expr, table, cursor_idx, start_reg)?; + + // Check if we have a length argument + let func_spec = if func.args.len() >= 3 { + let len_expr = self.extract_function_arg_expr(&func.args[2])?; + let len_reg = self.allocate_register(); + self.compile_where_operand(&len_expr, table, cursor_idx, len_reg)?; + format!("SUBSTR:{}", len_reg) + } else { + "SUBSTR".to_string() + }; + + // Emit StringFunc opcode + self.emit( + OpCode::StringFunc, + str_reg, + target_reg, + start_reg, + Some(func_spec), + &format!("r[{}] = SUBSTR(...)", target_reg), + ); + + Ok(()) + } + + /// Compile REPLACE function + fn compile_func_replace( + &mut self, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.len() != 3 { + return Err(SqawkError::InvalidSqlQuery( + "REPLACE requires three arguments".to_string(), )); } - // Output the result row (only reached if WHERE condition passes) - self.program.add_instruction(Instruction::new( - OpCode::ResultRow, - result_regs[0], // First register in result - result_regs.len() as i64, // Number of columns - 0, - None, - 0, - Some("Output result row".to_string()), - )); + let str_expr = self.extract_function_arg_expr(&func.args[0])?; + let from_expr = self.extract_function_arg_expr(&func.args[1])?; + let to_expr = self.extract_function_arg_expr(&func.args[2])?; - // Move to next row and continue loop - self.program.add_instruction(Instruction::new( - OpCode::Next, - cursor_idx as i64, - loop_addr as i64, // Jump back to start of loop for next row - 0, - None, + let str_reg = self.allocate_register(); + let from_reg = self.allocate_register(); + let to_reg = self.allocate_register(); + + // Compile all three arguments + self.compile_where_operand(&str_expr, table, cursor_idx, str_reg)?; + self.compile_where_operand(&from_expr, table, cursor_idx, from_reg)?; + self.compile_where_operand(&to_expr, table, cursor_idx, to_reg)?; + + // Emit StringFunc opcode with from_reg and to_reg in P4 + self.emit( + OpCode::StringFunc, + str_reg, + target_reg, 0, - Some("Move to next row or exit loop".to_string()), - )); + Some(format!("REPLACE:{}:{}", from_reg, to_reg)), + &format!("r[{}] = REPLACE(...)", target_reg), + ); - // Close the table cursor - self.program.add_instruction(Instruction::new( - OpCode::Close, - cursor_idx as i64, + Ok(()) + } + + /// Compile CONCAT function + fn compile_func_concat( + &mut self, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.len() < 2 { + return Err(SqawkError::InvalidSqlQuery( + "CONCAT requires at least two arguments".to_string(), + )); + } + + // Compile first argument to src_reg + let first_expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.allocate_register(); + self.compile_where_operand(&first_expr, table, cursor_idx, src_reg)?; + + // Compile remaining arguments and build P4 spec + let mut arg_regs = Vec::with_capacity(func.args.len().saturating_sub(1)); + for arg in func.args.iter().skip(1) { + let expr = self.extract_function_arg_expr(arg)?; + let reg = self.allocate_register(); + self.compile_where_operand(&expr, table, cursor_idx, reg)?; + arg_regs.push(reg.to_string()); + } + + // P4 = "CONCAT:reg1:reg2:..." + let func_spec = format!("CONCAT:{}", arg_regs.join(":")); + + self.emit( + OpCode::StringFunc, + src_reg, + target_reg, 0, + Some(func_spec), + &format!("r[{}] = CONCAT(...)", target_reg), + ); + + Ok(()) + } + + /// Compile LEFT and RIGHT string functions + fn compile_func_left_right( + &mut self, + func_name: &str, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.len() != 2 { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires exactly two arguments", + func_name + ))); + } + + let str_expr = self.extract_function_arg_expr(&func.args[0])?; + let len_expr = self.extract_function_arg_expr(&func.args[1])?; + + let str_reg = self.allocate_register(); + let len_reg = self.allocate_register(); + + self.compile_where_operand(&str_expr, table, cursor_idx, str_reg)?; + self.compile_where_operand(&len_expr, table, cursor_idx, len_reg)?; + + self.emit( + OpCode::StringFunc, + str_reg, + target_reg, + len_reg, + Some(func_name.to_string()), + &format!( + "r[{}] = {}(r[{}], r[{}])", + target_reg, func_name, str_reg, len_reg + ), + ); + + Ok(()) + } + + /// Compile math functions (ABS, ROUND, CEIL, CEILING, FLOOR) + fn compile_func_math( + &mut self, + func_name: &str, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires one argument", + func_name + ))); + } + + let expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.allocate_register(); + + // Compile the argument + self.compile_where_operand(&expr, table, cursor_idx, src_reg)?; + + // Emit MathFunc opcode + self.emit( + OpCode::MathFunc, + src_reg, + target_reg, 0, - None, + Some(func_name.to_string()), + &format!("r[{}] = {}(r[{}])", target_reg, func_name, src_reg), + ); + + Ok(()) + } + + /// Compile date/time functions with no arguments (NOW, CURRENT_TIMESTAMP, etc.) + fn compile_func_datetime_noarg(&mut self, func_name: &str, target_reg: i64) -> SqawkResult<()> { + self.emit( + OpCode::DateFunc, + 0, // No source register needed + target_reg, 0, - Some("Close cursor".to_string()), - )); + Some(func_name.to_string()), + &format!("r[{}] = {}()", target_reg, func_name), + ); Ok(()) } - /// Compile a WHERE condition and return the register containing the result - fn compile_where_condition( + /// Compile date/time functions with one argument (DATE, TIME) + fn compile_func_datetime( &mut self, - where_expr: &Expr, + func_name: &str, + func: &Function, table: &Table, cursor_idx: usize, - ) -> SqawkResult { - match where_expr { - Expr::BinaryOp { left, op, right } => { - self.compile_binary_comparison(left, op, right, table, cursor_idx) + target_reg: i64, + ) -> SqawkResult<()> { + if func.args.is_empty() { + return Err(SqawkError::InvalidSqlQuery(format!( + "{} requires one argument", + func_name + ))); + } + + let expr = self.extract_function_arg_expr(&func.args[0])?; + let src_reg = self.allocate_register(); + + // Compile the argument + self.compile_where_operand(&expr, table, cursor_idx, src_reg)?; + + // Emit DateFunc opcode + self.emit( + OpCode::DateFunc, + src_reg, + target_reg, + 0, + Some(func_name.to_string()), + &format!("r[{}] = {}(r[{}])", target_reg, func_name, src_reg), + ); + + Ok(()) + } + + /// Compile a function call - dispatches to specialized handlers + fn compile_function( + &mut self, + func: &Function, + table: &Table, + cursor_idx: usize, + target_reg: i64, + ) -> SqawkResult<()> { + let func_name = Self::get_function_name(func); + + match func_name.as_str() { + "COALESCE" => self.compile_func_coalesce(func, table, cursor_idx, target_reg), + "NULLIF" => self.compile_func_nullif(func, table, cursor_idx, target_reg), + "UPPER" | "LOWER" | "TRIM" | "LTRIM" | "RTRIM" | "LENGTH" => { + self.compile_func_string_simple(&func_name, func, table, cursor_idx, target_reg) + } + "SUBSTR" | "SUBSTRING" => self.compile_func_substr(func, table, cursor_idx, target_reg), + "REPLACE" => self.compile_func_replace(func, table, cursor_idx, target_reg), + "CONCAT" => self.compile_func_concat(func, table, cursor_idx, target_reg), + "LEFT" | "RIGHT" => { + self.compile_func_left_right(&func_name, func, table, cursor_idx, target_reg) + } + "ABS" | "ROUND" | "CEIL" | "CEILING" | "FLOOR" => { + self.compile_func_math(&func_name, func, table, cursor_idx, target_reg) + } + "NOW" | "CURRENT_TIMESTAMP" | "CURRENT_DATE" | "CURRENT_TIME" => { + self.compile_func_datetime_noarg(&func_name, target_reg) + } + "DATE" | "TIME" => { + self.compile_func_datetime(&func_name, func, table, cursor_idx, target_reg) } _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported WHERE clause expression: {:?}", - where_expr + "Unsupported function: {}", + func_name ))), } } + /// Extract expression from a function argument + pub(crate) fn extract_function_arg_expr(&self, arg: &FunctionArg) -> SqawkResult { + match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Ok(expr.clone()), + FunctionArg::Named { + arg: FunctionArgExpr::Expr(expr), + .. + } => Ok(expr.clone()), + _ => Err(SqawkError::UnsupportedSqlFeature( + "Unsupported function argument type".to_string(), + )), + } + } + /// Compile a binary comparison operation (e.g., column > value) fn compile_binary_comparison( &mut self, @@ -439,6 +5433,183 @@ impl<'a> SqlCompiler<'a> { table: &Table, cursor_idx: usize, ) -> SqawkResult { + // Handle logical operators (AND, OR) specially + match op { + BinaryOperator::And => { + // For AND, both conditions must be true + // Compile left condition + let left_result = self.compile_where_condition(left, table, cursor_idx)?; + + // If left is false, skip right evaluation and return 0 + let skip_to_false = self.program.len(); + self.emit( + OpCode::IfZ, + left_result, + 0, // Will be patched + 0, + None, + "if left is false, skip to false", + ); + + // Compile right condition + let right_result = self.compile_where_condition(right, table, cursor_idx)?; + + // Result is right_result (if we got here, left was true) + let result_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (default AND result)", result_reg), + ); + + // If right is false, skip setting result to 1 + let skip_to_end = self.program.len(); + self.emit( + OpCode::IfZ, + right_result, + 0, // Will be patched to skip to end + 0, + None, + "if right is false, skip to end", + ); + + // Both are true, set result to 1 + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (AND true)", result_reg), + ); + + // Jump to end + let goto_end = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched + 0, + None, + "goto end", + ); + + // False label (left was false) + let false_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_to_false) { + inst.p2 = false_label as i64; + } + + // Result is already 0 (set above), so we just fall through + + // End label + let end_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_to_end) { + inst.p2 = end_label as i64; + } + if let Some(inst) = self.program.instructions.get_mut(goto_end) { + inst.p2 = end_label as i64; + } + + return Ok(result_reg); + } + BinaryOperator::Or => { + // For OR, at least one condition must be true + let result_reg = self.allocate_register(); + + // Initialize result to 0 + self.emit( + OpCode::Integer, + 0, + result_reg, + 0, + None, + &format!("r[{}] = 0 (default OR result)", result_reg), + ); + + // Compile left condition + let left_result = self.compile_where_condition(left, table, cursor_idx)?; + + // If left is true, set result to 1 and skip right + let skip_if_left_true = self.program.len(); + self.emit( + OpCode::IfZ, + left_result, + 0, // Will be patched to check right + 0, + None, + "if left is false, check right", + ); + + // Left is true, set result to 1 + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (OR left true)", result_reg), + ); + + // Jump to end + let goto_end = self.program.len(); + self.emit( + OpCode::Goto, + 0, + 0, // Will be patched + 0, + None, + "goto end", + ); + + // Check right label + let check_right_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_if_left_true) { + inst.p2 = check_right_label as i64; + } + + // Compile right condition + let right_result = self.compile_where_condition(right, table, cursor_idx)?; + + // If right is false, skip setting result to 1 + let skip_to_end = self.program.len(); + self.emit( + OpCode::IfZ, + right_result, + 0, // Will be patched to skip to end + 0, + None, + "if right is false, skip to end", + ); + + // Right is true, set result to 1 + self.emit( + OpCode::Integer, + 1, + result_reg, + 0, + None, + &format!("r[{}] = 1 (OR right true)", result_reg), + ); + + // End label + let end_label = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(goto_end) { + inst.p2 = end_label as i64; + } + if let Some(inst) = self.program.instructions.get_mut(skip_to_end) { + inst.p2 = end_label as i64; + } + + return Ok(result_reg); + } + _ => {} + } + + // Handle comparison operators // Allocate registers for operands and result let left_reg = self.allocate_register(); let right_reg = self.allocate_register(); @@ -451,36 +5622,25 @@ impl<'a> SqlCompiler<'a> { self.compile_where_operand(right, table, cursor_idx, right_reg)?; // Generate comparison instruction based on operator - let opcode = match op { - BinaryOperator::Gt => OpCode::Gt, - BinaryOperator::Lt => OpCode::Lt, - BinaryOperator::GtEq => OpCode::Ge, - BinaryOperator::LtEq => OpCode::Le, - BinaryOperator::Eq => OpCode::Eq, - BinaryOperator::NotEq => OpCode::Ne, - _ => { - return Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported comparison operator: {:?}", - op - ))); - } - }; + let opcode = Self::binary_op_to_comparison_opcode(op)?; - self.program.add_instruction(Instruction::new( + self.emit( opcode, left_reg, right_reg, result_reg, None, - 0, - Some(format!("r[{}] = r[{}] {:?} r[{}]", result_reg, left_reg, op, right_reg)), - )); + &format!( + "r[{}] = r[{}] {:?} r[{}]", + result_reg, left_reg, op, right_reg + ), + ); Ok(result_reg) } /// Compile a WHERE clause operand (column reference or literal) - fn compile_where_operand( + pub(crate) fn compile_where_operand( &mut self, expr: &Expr, table: &Table, @@ -493,58 +5653,67 @@ impl<'a> SqlCompiler<'a> { let column_name = &ident.value; let column_idx = self.find_column_index(table, column_name)?; - self.program.add_instruction(Instruction::new( + self.emit( OpCode::Column, cursor_idx as i64, column_idx as i64, target_reg, None, - 0, - Some(format!("r[{}] = column {} ({})", target_reg, column_idx, column_name)), - )); + &format!( + "r[{}] = column {} ({})", + target_reg, column_idx, column_name + ), + ); } Expr::Value(value) => { // This is a literal value match value { Value::Number(num, _) => { if let Ok(int_val) = num.parse::() { - self.program.add_instruction(Instruction::new( + self.emit( OpCode::Integer, int_val, target_reg, 0, None, + &format!("r[{}] = {}", target_reg, int_val), + ); + } else if let Ok(float_val) = num.parse::() { + // Store float as string for runtime conversion + self.emit( + OpCode::String, 0, - Some(format!("r[{}] = {}", target_reg, int_val)), - )); + target_reg, + 0, + Some(float_val.to_string()), + &format!("r[{}] = {} (float)", target_reg, float_val), + ); } else { return Err(SqawkError::UnsupportedSqlFeature(format!( - "Non-integer literals not supported in WHERE: {}", + "Invalid numeric literal in WHERE: {}", num ))); } } Value::SingleQuotedString(s) => { - self.program.add_instruction(Instruction::new( + self.emit( OpCode::String, 0, target_reg, 0, Some(s.clone()), - 0, - Some(format!("r[{}] = '{}'", target_reg, s)), - )); + &format!("r[{}] = '{}'", target_reg, s), + ); } Value::Null => { - self.program.add_instruction(Instruction::new( + self.emit( OpCode::Null, 0, target_reg, 0, None, - 0, - Some(format!("r[{}] = NULL", target_reg)), - )); + &format!("r[{}] = NULL", target_reg), + ); } _ => { return Err(SqawkError::UnsupportedSqlFeature(format!( @@ -554,6 +5723,278 @@ impl<'a> SqlCompiler<'a> { } } } + Expr::Case { + operand, + conditions, + results, + else_result, + } => { + // Compile CASE expression, storing result directly in target_reg + self.compile_case( + operand.as_deref(), + conditions, + results, + else_result.as_deref(), + table, + cursor_idx, + Some(target_reg), + )?; + } + Expr::BinaryOp { left, op, right } => { + // Handle binary operations as operands (e.g., in CASE WHEN conditions or arithmetic) + let left_reg = self.allocate_register(); + let right_reg = self.allocate_register(); + + self.compile_where_operand(left, table, cursor_idx, left_reg)?; + self.compile_where_operand(right, table, cursor_idx, right_reg)?; + + let opcode = match op { + // Comparison operators + BinaryOperator::Gt => OpCode::Gt, + BinaryOperator::Lt => OpCode::Lt, + BinaryOperator::GtEq => OpCode::Ge, + BinaryOperator::LtEq => OpCode::Le, + BinaryOperator::Eq => OpCode::Eq, + BinaryOperator::NotEq => OpCode::Ne, + // Arithmetic operators + BinaryOperator::Plus => OpCode::Add, + BinaryOperator::Minus => OpCode::Subtract, + BinaryOperator::Multiply => OpCode::Multiply, + BinaryOperator::Divide => OpCode::Divide, + BinaryOperator::Modulo => OpCode::Remainder, + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported binary operator in operand: {:?}", + op + ))); + } + }; + + self.emit( + opcode, + left_reg, + right_reg, + target_reg, + None, + &format!( + "r[{}] = (r[{}] {:?} r[{}])", + target_reg, left_reg, op, right_reg + ), + ); + } + Expr::Cast { + expr, data_type, .. + } => { + // Compile the expression to cast + let src_reg = self.allocate_register(); + self.compile_where_operand(expr, table, cursor_idx, src_reg)?; + + // Get the target type name + let type_name = self.sql_data_type_to_string(data_type); + + // Emit Cast instruction + self.emit( + OpCode::Cast, + src_reg, + target_reg, + 0, + Some(type_name.clone()), + &format!("r[{}] = CAST(r[{}] AS {})", target_reg, src_reg, type_name), + ); + } + Expr::Function(func) => { + // Handle COALESCE and NULLIF functions + self.compile_function(func, table, cursor_idx, target_reg)?; + } + Expr::Subquery(subquery) => { + // Handle scalar subquery as an operand + // Check for correlation first + let outer_refs = self.detect_outer_references(subquery, table.name(), None); + if outer_refs.is_empty() { + // Non-correlated: Execute subquery at compile time + let result_value = self.evaluate_scalar_subquery(subquery)?; + self.emit_value_literal(target_reg, &result_value)?; + } else { + // Correlated: emit runtime execution bytecode + self.compile_correlated_scalar_subquery( + subquery, + table, + cursor_idx, + &outer_refs, + target_reg, + )?; + } + } + Expr::Exists { subquery, negated } => { + // Handle EXISTS (SELECT ...) as an operand + // Check for correlation first + let outer_refs = self.detect_outer_references(subquery, table.name(), None); + if outer_refs.is_empty() { + // Non-correlated: Evaluate at compile time and emit 1 or 0 + let exists_result = self.evaluate_exists_subquery(subquery, *negated)?; + self.emit( + OpCode::Integer, + if exists_result { 1 } else { 0 }, + target_reg, + 0, + None, + &format!( + "r[{}] = {} ({}EXISTS result)", + target_reg, + if exists_result { 1 } else { 0 }, + if *negated { "NOT " } else { "" } + ), + ); + } else { + // Correlated: emit runtime execution bytecode + self.compile_correlated_exists_subquery( + subquery, + *negated, + table, + cursor_idx, + &outer_refs, + target_reg, + )?; + } + } + Expr::InSubquery { + expr: inner_expr, + subquery, + negated, + } => { + // Handle IN (SELECT ...) as an operand when combined with other conditions + // We need to evaluate this at compile time for non-correlated subqueries + let subquery_values = self.evaluate_in_subquery_values(subquery)?; + if subquery_values.is_empty() { + // Empty subquery: IN is false, NOT IN is true + self.emit( + OpCode::Integer, + if *negated { 1 } else { 0 }, + target_reg, + 0, + None, + &format!( + "r[{}] = {} (empty IN subquery)", + target_reg, + if *negated { 1 } else { 0 } + ), + ); + } else { + // Get the expression value + let expr_reg = self.allocate_register(); + self.compile_where_operand(inner_expr, table, cursor_idx, expr_reg)?; + + // Initialize result + self.emit( + OpCode::Integer, + 0, + target_reg, + 0, + None, + &format!("r[{}] = 0 (IN subquery initial)", target_reg), + ); + + let temp_reg = self.allocate_register(); + let mut jump_to_end_addrs = Vec::new(); + + for value in &subquery_values { + let item_reg = self.allocate_register(); + self.emit_value_literal(item_reg, value)?; + + self.emit( + OpCode::Eq, + expr_reg, + item_reg, + temp_reg, + None, + &format!("r[{}] = (r[{}] == r[{}])", temp_reg, expr_reg, item_reg), + ); + + let skip_set_addr = self.program.len(); + self.emit(OpCode::IfZ, temp_reg, 0, 0, None, "if no match, skip"); + + self.emit( + OpCode::Integer, + 1, + target_reg, + 0, + None, + &format!("r[{}] = 1 (found)", target_reg), + ); + + jump_to_end_addrs.push(self.program.len()); + self.emit(OpCode::Goto, 0, 0, 0, None, "skip to end"); + + let continue_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_set_addr) { + inst.p2 = continue_addr as i64; + } + } + + let end_label = self.program.len(); + for addr in jump_to_end_addrs { + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = end_label as i64; + } + } + + if *negated { + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0 for NOT IN", zero_reg), + ); + self.emit( + OpCode::Eq, + target_reg, + zero_reg, + target_reg, + None, + &format!("r[{}] = (r[{}] == 0) for NOT IN", target_reg, target_reg), + ); + } + } + } + Expr::UnaryOp { op, expr: inner } => { + match op { + UnaryOperator::Minus => { + // Handle unary minus using 0 - value (avoids Integer -1 special case) + let inner_reg = self.allocate_register(); + self.compile_where_operand(inner, table, cursor_idx, inner_reg)?; + let zero_reg = self.allocate_register(); + self.emit( + OpCode::Integer, + 0, + zero_reg, + 0, + None, + &format!("r[{}] = 0", zero_reg), + ); + self.emit( + OpCode::Subtract, + zero_reg, + inner_reg, + target_reg, + None, + &format!("r[{}] = -r[{}]", target_reg, inner_reg), + ); + } + UnaryOperator::Plus => { + // Unary plus is a no-op, just compile the inner expression + self.compile_where_operand(inner, table, cursor_idx, target_reg)?; + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported unary operator: {:?}", + op + ))); + } + } + } _ => { return Err(SqawkError::UnsupportedSqlFeature(format!( "Unsupported WHERE operand: {:?}", @@ -566,7 +6007,7 @@ impl<'a> SqlCompiler<'a> { } /// Find column index by name - fn find_column_index(&self, table: &Table, column_name: &str) -> SqawkResult { + pub(crate) fn find_column_index(&self, table: &Table, column_name: &str) -> SqawkResult { for (i, column) in table.column_metadata().iter().enumerate() { if column.name == column_name { return Ok(i); @@ -575,8 +6016,25 @@ impl<'a> SqlCompiler<'a> { Err(SqawkError::ColumnNotFound(column_name.to_string())) } + /// Convert a SQL DataType to a string representation for the VM + fn sql_data_type_to_string(&self, data_type: &SqlDataType) -> String { + match data_type { + SqlDataType::Int(_) + | SqlDataType::Integer(_) + | SqlDataType::BigInt(_) + | SqlDataType::SmallInt(_) => "INTEGER".to_string(), + SqlDataType::Real | SqlDataType::Float(_) | SqlDataType::Double => "REAL".to_string(), + SqlDataType::Text + | SqlDataType::Varchar(_) + | SqlDataType::Char(_) + | SqlDataType::String => "TEXT".to_string(), + SqlDataType::Boolean => "BOOLEAN".to_string(), + _ => format!("{:?}", data_type), + } + } + /// Get a normalized table name from an ObjectName - fn get_table_name(&self, name: &ObjectName) -> SqawkResult { + pub(crate) fn get_table_name(&self, name: &ObjectName) -> SqawkResult { if name.0.is_empty() { return Err(SqawkError::InvalidSqlQuery("Empty table name".to_string())); } @@ -592,197 +6050,324 @@ impl<'a> SqlCompiler<'a> { projection: &[SelectItem], table: &Table, ) -> SqawkResult> { - // Check if we have a SELECT * - let is_select_star = projection - .iter() - .any(|item| matches!(item, SelectItem::Wildcard(_))); + let mut columns = Vec::new(); + + for item in projection { + match item { + SelectItem::Wildcard(_) => { + // SELECT * - include all columns + for i in 0..table.column_count() { + columns.push(i); + } + } + SelectItem::UnnamedExpr(expr) => { + let col_idx = self.resolve_column_expr(expr, table)?; + columns.push(col_idx); + } + SelectItem::ExprWithAlias { expr, .. } => { + let col_idx = self.resolve_column_expr(expr, table)?; + columns.push(col_idx); + } + SelectItem::QualifiedWildcard(name, _) => { + // table.* - for now just include all columns + let _ = name; // Ignore table qualifier for single-table queries + for i in 0..table.column_count() { + columns.push(i); + } + } + } + } - if is_select_star { - // For SELECT *, include all columns in their original order - let column_count = table.column_count(); - Ok((0..column_count).collect()) + if columns.is_empty() { + // Default to all columns + Ok((0..table.column_count()).collect()) } else { - Err(SqawkError::UnsupportedSqlFeature( - "Only SELECT * is supported for now".into(), - )) + Ok(columns) } } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::database::Database; - use crate::table::Table; - - /// Create a test database with a simple table - fn create_test_database() -> Database { - let mut database = Database::new(); - let mut test_table = Table::new("users", vec![], None); - - // Add columns: id (INTEGER), age (INTEGER), name (TEXT) - test_table.add_column("id".to_string(), "INTEGER".to_string()); - test_table.add_column("age".to_string(), "INTEGER".to_string()); - test_table.add_column("name".to_string(), "TEXT".to_string()); - - // Add test data - test_table.add_row(vec![ - crate::table::Value::Integer(1), - crate::table::Value::Integer(25), - crate::table::Value::String("Alice".to_string()), - ]).expect("Failed to add test row"); - - test_table.add_row(vec![ - crate::table::Value::Integer(2), - crate::table::Value::Integer(35), - crate::table::Value::String("Bob".to_string()), - ]).expect("Failed to add test row"); - - database.add_table("users".to_string(), test_table).expect("Failed to add table"); - database - } - - #[test] - fn test_compile_select_with_where_gt() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, false); // Use non-verbose mode for predictable output - - // Test: SELECT * FROM users WHERE age > 30 - let sql = "SELECT * FROM users WHERE age > 30"; - let program = compiler.compile(sql).expect("Compilation failed"); - - // Verify the generated bytecode contains the expected instructions - let instructions = &program.instructions; - - // Should have: Init, OpenRead, Rewind, Column (for all columns), - // Column (for age), Integer (30), Gt, IfZ, ResultRow, Next, Close, Halt - assert!(instructions.len() >= 8, "Expected at least 8 instructions"); - - // Find key instructions - let gt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Gt); - let ifz_found = instructions.iter().any(|inst| inst.opcode == OpCode::IfZ); - let integer_found = instructions.iter().any(|inst| inst.opcode == OpCode::Integer && inst.p1 == 30); - let halt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Halt); - - assert!(gt_found, "Expected Gt comparison instruction"); - assert!(ifz_found, "Expected IfZ conditional jump instruction"); - assert!(integer_found, "Expected Integer instruction with value 30"); - assert!(halt_found, "Expected Halt instruction"); - } - - #[test] - fn test_compile_select_with_where_eq() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, true); - - // Test: SELECT * FROM users WHERE age = 25 - let sql = "SELECT * FROM users WHERE age = 25"; - let program = compiler.compile(sql).expect("Compilation failed"); - - let instructions = &program.instructions; - - // Verify we have Eq comparison and the correct literal value - let eq_found = instructions.iter().any(|inst| inst.opcode == OpCode::Eq); - let value_found = instructions.iter().any(|inst| inst.opcode == OpCode::Integer && inst.p1 == 25); - - assert!(eq_found, "Expected Eq comparison instruction"); - assert!(value_found, "Expected Integer instruction with value 25"); - } - - #[test] - fn test_compile_select_with_where_string() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, true); - - // Test: SELECT * FROM users WHERE name = 'Alice' - let sql = "SELECT * FROM users WHERE name = 'Alice'"; - let program = compiler.compile(sql).expect("Compilation failed"); - - let instructions = &program.instructions; - - // Verify we have Eq comparison and String instruction - let eq_found = instructions.iter().any(|inst| inst.opcode == OpCode::Eq); - let string_found = instructions.iter().any(|inst| { - inst.opcode == OpCode::String && - inst.p4.as_ref().map_or(false, |s| s == "Alice") - }); - - assert!(eq_found, "Expected Eq comparison instruction"); - assert!(string_found, "Expected String instruction with value 'Alice'"); - } - - #[test] - fn test_compile_select_with_where_lt() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, true); - - // Test: SELECT * FROM users WHERE age < 30 - let sql = "SELECT * FROM users WHERE age < 30"; - let program = compiler.compile(sql).expect("Compilation failed"); - - let instructions = &program.instructions; - - // Verify we have Lt comparison - let lt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Lt); - assert!(lt_found, "Expected Lt comparison instruction"); - } - - #[test] - fn test_compile_select_without_where() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, true); - - // Test: SELECT * FROM users (no WHERE clause) - let sql = "SELECT * FROM users"; - let program = compiler.compile(sql).expect("Compilation failed"); - - let instructions = &program.instructions; - - // Should NOT have comparison or conditional jump instructions - let comparison_found = instructions.iter().any(|inst| { - matches!(inst.opcode, OpCode::Gt | OpCode::Lt | OpCode::Eq | OpCode::Ne | OpCode::Ge | OpCode::Le) - }); - let ifz_found = instructions.iter().any(|inst| inst.opcode == OpCode::IfZ); - - assert!(!comparison_found, "Should not have comparison instructions without WHERE"); - assert!(!ifz_found, "Should not have conditional jump without WHERE"); - } - - #[test] - fn test_column_not_found_error() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, false); - - // Test: SELECT * FROM users WHERE invalid_column > 10 - let sql = "SELECT * FROM users WHERE invalid_column > 10"; - let result = compiler.compile(sql); - - // Should fail with column not found error - assert!(result.is_err(), "Expected compilation to fail"); - match result.unwrap_err() { - SqawkError::ColumnNotFound(col) => { - assert_eq!(col, "invalid_column"); - } - other => panic!("Expected ColumnNotFound error, got {:?}", other), - } - } - - #[test] - fn test_unsupported_operator_error() { - let database = create_test_database(); - let mut compiler = SqlCompiler::new(&database, false); - - // Test with unsupported operator (LIKE is not implemented) - let sql = "SELECT * FROM users WHERE name LIKE 'A%'"; - let result = compiler.compile(sql); - - // Should fail with unsupported feature error - assert!(result.is_err(), "Expected compilation to fail"); - match result.unwrap_err() { - SqawkError::UnsupportedSqlFeature(_) => { - // Expected - } - other => panic!("Expected UnsupportedSqlFeature error, got {:?}", other), + pub(crate) fn resolve_column_expr(&self, expr: &Expr, table: &Table) -> SqawkResult { + match expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name)) + } + Expr::CompoundIdentifier(parts) => { + let col_name = parts + .last() + .map(|p| p.value.to_lowercase()) + .unwrap_or_default(); + table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name)) + } + _ => Err(SqawkError::UnsupportedSqlFeature( + "Only column references supported in SELECT".into(), + )), + } + } + + pub(crate) fn get_column_name_from_expr(&self, expr: &Expr) -> String { + match expr { + Expr::Identifier(ident) => ident.value.clone(), + Expr::CompoundIdentifier(parts) => { + parts.last().map(|p| p.value.clone()).unwrap_or_default() + } + Expr::Function(func) => { + // For aggregate functions, return just the function name (e.g., "COUNT" not "COUNT(*)") + // For other functions, include arguments for clarity + let func_name = func.name.to_string().to_uppercase(); + if matches!(func_name.as_str(), "COUNT" | "SUM" | "AVG" | "MIN" | "MAX") { + func_name + } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Wildcard)) = + func.args.first() + { + format!("{}(*)", func_name) + } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg_expr))) = + func.args.first() + { + format!( + "{}({})", + func_name, + self.get_column_name_from_expr(arg_expr) + ) + } else { + format!("{}()", func_name) + } + } + _ => "expr".to_string(), + } + } + + /// Build a ResultSchema from a projection and table + /// + /// This creates a proper schema with column names and types for the result set. + pub(crate) fn build_result_schema( + &self, + projection: &[SelectItem], + table: &Table, + ) -> ResultSchema { + let mut schema = ResultSchema::new(); + let col_metadata = table.column_metadata(); + + for item in projection { + match item { + SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _) => { + // Add all columns from the table with their types + for meta in col_metadata { + schema.add_column(meta.name.clone(), meta.data_type); + } + } + SelectItem::UnnamedExpr(expr) => { + let name = self.get_column_name_from_expr(expr); + let data_type = self.infer_expr_type(expr, table); + schema.add_column(name, data_type); + } + SelectItem::ExprWithAlias { expr, alias } => { + let data_type = self.infer_expr_type(expr, table); + schema.add_column(alias.value.clone(), data_type); + } + } + } + schema + } + + /// Infer the data type of an expression + pub(crate) fn infer_expr_type(&self, expr: &Expr, table: &Table) -> DataType { + match expr { + Expr::Identifier(ident) => { + // Look up the column type in the table + let col_meta = table.column_metadata(); + for meta in col_meta { + if meta.name.eq_ignore_ascii_case(&ident.value) { + return meta.data_type; + } + } + DataType::Text // Default to text if not found + } + Expr::CompoundIdentifier(parts) => { + // For table.column references, use the column name + if let Some(col_name) = parts.last() { + let col_meta = table.column_metadata(); + for meta in col_meta { + if meta.name.eq_ignore_ascii_case(&col_name.value) { + return meta.data_type; + } + } + } + DataType::Text + } + Expr::Function(func) => { + let func_name = func.name.to_string().to_uppercase(); + match func_name.as_str() { + "COUNT" => DataType::Integer, + "SUM" | "AVG" => { + // SUM/AVG preserves numeric type, defaults to Float for AVG + if func_name == "AVG" { + DataType::Float + } else { + // Try to infer from argument + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))) = + func.args.first() + { + let arg_type = self.infer_expr_type(arg, table); + if matches!(arg_type, DataType::Integer) { + return DataType::Integer; + } + } + DataType::Float + } + } + "MIN" | "MAX" => { + // MIN/MAX preserves the type of the argument + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))) = + func.args.first() + { + return self.infer_expr_type(arg, table); + } + DataType::Text + } + "UPPER" | "LOWER" | "TRIM" | "LTRIM" | "RTRIM" | "SUBSTR" | "REPLACE" => { + DataType::Text + } + "COALESCE" => { + // COALESCE returns the type of its first non-null argument + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))) = + func.args.first() + { + return self.infer_expr_type(arg, table); + } + DataType::Text + } + _ => DataType::Text, + } + } + Expr::Value(val) => { + match val { + Value::Number(_, _) => DataType::Integer, // Could be Float, but Integer is common + Value::SingleQuotedString(_) | Value::DoubleQuotedString(_) => DataType::Text, + Value::Boolean(_) => DataType::Boolean, + Value::Null => DataType::Text, // NULL doesn't have a type, default to Text + _ => DataType::Text, + } + } + Expr::BinaryOp { left, op, right: _ } => { + // Arithmetic ops return numeric, comparisons return boolean + match op { + BinaryOperator::Plus + | BinaryOperator::Minus + | BinaryOperator::Multiply + | BinaryOperator::Divide + | BinaryOperator::Modulo => self.infer_expr_type(left, table), + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + | BinaryOperator::And + | BinaryOperator::Or => DataType::Boolean, + BinaryOperator::StringConcat => DataType::Text, + _ => DataType::Text, + } + } + Expr::Case { + results, + else_result, + .. + } => { + // CASE returns the type of its first result + if let Some(first_result) = results.first() { + return self.infer_expr_type(first_result, table); + } + if let Some(else_expr) = else_result { + return self.infer_expr_type(else_expr, table); + } + DataType::Text + } + Expr::Cast { data_type, .. } => { + // CAST returns the target type + match data_type { + SqlDataType::Int(_) + | SqlDataType::Integer(_) + | SqlDataType::BigInt(_) + | SqlDataType::SmallInt(_) => DataType::Integer, + SqlDataType::Real | SqlDataType::Float(_) | SqlDataType::Double => { + DataType::Float + } + SqlDataType::Boolean => DataType::Boolean, + _ => DataType::Text, + } + } + _ => DataType::Text, // Default + } + } + + /// Check if a SELECT projection contains aggregate functions + fn has_aggregates(&self, projection: &[SelectItem]) -> bool { + for item in projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + if self.is_aggregate_expr(expr) { + return true; + } + } + _ => {} + } + } + false + } + + /// Check if an expression is or contains an aggregate function + fn is_aggregate_expr(&self, expr: &Expr) -> bool { + match expr { + Expr::Function(func) => { + let name = func.name.to_string().to_uppercase(); + matches!(name.as_str(), "COUNT" | "SUM" | "AVG" | "MIN" | "MAX") + } + _ => false, + } + } + + /// Check if the projection contains any window functions + fn has_window_functions(&self, projection: &[SelectItem]) -> bool { + for item in projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + if self.is_window_expr(expr) { + return true; + } + } + _ => {} + } + } + false + } + + /// Check if an expression is a window function (has OVER clause) + fn is_window_expr(&self, expr: &Expr) -> bool { + match expr { + Expr::Function(func) => func.over.is_some(), + _ => false, + } + } + + /// Get the window function type code + pub(crate) fn get_window_func_type(&self, name: &str) -> i64 { + match name.to_uppercase().as_str() { + "ROW_NUMBER" => 0, + "RANK" => 1, + "DENSE_RANK" => 2, + "LAG" => 3, + "LEAD" => 4, + "SUM" => 5, + "AVG" => 6, + "COUNT" => 7, + "MIN" => 8, + "MAX" => 9, + _ => 0, } } } diff --git a/src/vm/compiler_aggregate.rs b/src/vm/compiler_aggregate.rs new file mode 100644 index 0000000..b70a990 --- /dev/null +++ b/src/vm/compiler_aggregate.rs @@ -0,0 +1,674 @@ +//! Aggregate function and GROUP BY compilation +//! +//! This module extends SqlCompiler with aggregate and GROUP BY compilation methods. + +use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, Select, SelectItem}; + +use super::bytecode::{OpCode, ResultSchema}; +use super::compiler::SqlCompiler; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::Table; + +impl<'a> SqlCompiler<'a> { + pub(crate) fn compile_select_with_aggregate( + &mut self, + select: &Select, + table: &Table, + table_name: &str, + ) -> SqawkResult<()> { + let cursor_idx = 0i64; + + // Build result schema with column names and types + let schema = self.build_aggregate_result_schema(&select.projection, table); + self.program.set_result_schema(schema); + + self.add_comment("Aggregate query (no GROUP BY)"); + + // Open table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + "", + ); + + // Rewind + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, cursor_idx, 0, 0, None, ""); + + let loop_start = self.program.len(); + + // Track where we'll need to jump to skip this row (for WHERE filtering) + let mut where_skip_addr: Option = None; + + // WHERE clause filtering - must be evaluated BEFORE aggregates + // If WHERE condition is false, skip to Next + if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + // If condition is false (0), jump past the AggStep calls + where_skip_addr = Some(self.program.len()); + self.emit( + OpCode::IfZ, + cond_reg, + 0, + 0, + None, + "Skip row if WHERE condition is false", + ); + } + + // For each aggregate in the projection, emit AggStep + let mut acc_regs = Vec::new(); + for item in &select.projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + if let Some((func_type, col_reg)) = + self.compile_aggregate_step(expr, table, cursor_idx)? + { + let acc_reg = self.allocate_register(); + acc_regs.push((acc_reg, func_type)); + self.emit(OpCode::AggStep, func_type, col_reg, acc_reg, None, ""); + } + } + _ => {} + } + } + + // Next row + let next_addr = self.program.len(); + self.emit(OpCode::Next, cursor_idx, loop_start as i64, 0, None, ""); + + // Patch the WHERE skip jump to point to Next + if let Some(skip_addr) = where_skip_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = next_addr as i64; + } + } + + let after_loop = self.program.len(); + + // Patch rewind jump + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = after_loop as i64; + } + + // Finalize aggregates and output result + let result_start_reg = self.allocate_registers(acc_regs.len()); + + for (i, (acc_reg, _)) in acc_regs.iter().enumerate() { + self.emit( + OpCode::AggFinal, + *acc_reg, + result_start_reg + i as i64, + 0, + None, + "", + ); + } + + // Output result row + self.emit( + OpCode::ResultRow, + result_start_reg, + acc_regs.len() as i64, + 0, + None, + "", + ); + + // Close cursor + self.emit(OpCode::Close, cursor_idx, 0, 0, None, ""); + + Ok(()) + } + + /// Compile an aggregate function step and return (func_type, value_reg) + pub(crate) fn compile_aggregate_step( + &mut self, + expr: &Expr, + table: &Table, + cursor_idx: i64, + ) -> SqawkResult> { + match expr { + Expr::Function(func) => { + let name = func.name.to_string().to_uppercase(); + let func_type = Self::agg_func_type(&name); + + // Check for COUNT(*) + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Wildcard)) = func.args.first() { + // COUNT(*) - use -1 to indicate no column + return Ok(Some((func_type, -1))); + } + + // Get the column argument + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg_expr))) = + func.args.first() + { + let col_idx = self.resolve_column_expr(arg_expr, table)?; + let col_reg = self.allocate_register(); + self.emit( + OpCode::Column, + cursor_idx, + col_idx as i64, + col_reg, + None, + "", + ); + return Ok(Some((func_type, col_reg))); + } + + Ok(None) + } + _ => Ok(None), + } + } + + /// Build result schema for aggregate queries + pub(crate) fn build_aggregate_result_schema( + &self, + projection: &[SelectItem], + table: &Table, + ) -> ResultSchema { + let mut schema = ResultSchema::new(); + for item in projection { + match item { + SelectItem::UnnamedExpr(expr) => { + let name = self.get_column_name_from_expr(expr); + let data_type = self.infer_expr_type(expr, table); + schema.add_column(name, data_type); + } + SelectItem::ExprWithAlias { expr, alias } => { + let data_type = self.infer_expr_type(expr, table); + schema.add_column(alias.value.clone(), data_type); + } + _ => {} + } + } + schema + } + + /// Compile a SELECT with GROUP BY + pub(crate) fn compile_select_with_group_by( + &mut self, + select: &Select, + table: &Table, + table_name: &str, + ) -> SqawkResult<()> { + let cursor_idx = 0i64; + let sorter_id = 0i64; + + // Determine GROUP BY column indices + let mut group_col_indices: Vec = Vec::new(); + for expr in select.group_by.iter() { + match expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + let col_idx = table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))?; + group_col_indices.push(col_idx); + } + Expr::CompoundIdentifier(parts) => { + let col_name = parts + .last() + .map(|p| p.value.to_lowercase()) + .unwrap_or_default(); + let col_idx = table + .column_index(&col_name) + .ok_or(SqawkError::ColumnNotFound(col_name))?; + group_col_indices.push(col_idx); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only column references supported in GROUP BY".into(), + )); + } + } + } + + // Determine aggregates in projection + let mut agg_info: Vec<(i64, Option)> = Vec::new(); // (func_type, col_idx or None for COUNT(*)) + for item in &select.projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + if let Expr::Function(func) = expr { + let name = func.name.to_string().to_uppercase(); + if matches!(name.as_str(), "COUNT" | "SUM" | "AVG" | "MIN" | "MAX") { + let func_type = Self::agg_func_type(&name); + if let Some(FunctionArg::Unnamed(FunctionArgExpr::Wildcard)) = + func.args.first() + { + agg_info.push((func_type, None)); + } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr( + arg_expr, + ))) = func.args.first() + { + let col_idx = self.resolve_column_expr(arg_expr, table)?; + agg_info.push((func_type, Some(col_idx))); + } + } + } + } + _ => {} + } + } + + // Build result schema with column names and types + let schema = self.build_aggregate_result_schema(&select.projection, table); + self.program.set_result_schema(schema); + + self.add_comment("GROUP BY query"); + + // Build sort spec for sorting by GROUP BY columns + // Use position within sorter (0, 1, 2, ...) not original table column indices + let sort_spec: String = (0..group_col_indices.len()) + .map(|i| format!("{}:asc", i)) + .collect::>() + .join(","); + + // Total columns to store: group columns + aggregate value columns + let total_cols = group_col_indices.len() + agg_info.len(); + + // Open sorter + self.emit( + OpCode::SorterOpen, + sorter_id, + total_cols as i64, + 0, + Some(sort_spec), + "", + ); + + // Open table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + "", + ); + + // Rewind + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, cursor_idx, 0, 0, None, ""); + + let loop_start = self.program.len(); + + // Allocate registers for row data + let row_start_reg = self.allocate_registers(total_cols); + + // Load GROUP BY columns + for (i, &col_idx) in group_col_indices.iter().enumerate() { + self.emit( + OpCode::Column, + cursor_idx, + col_idx as i64, + row_start_reg + i as i64, + None, + "", + ); + } + + // Load aggregate value columns + for (i, (_, col_idx_opt)) in agg_info.iter().enumerate() { + let dest_reg = row_start_reg + group_col_indices.len() as i64 + i as i64; + if let Some(col_idx) = col_idx_opt { + self.emit( + OpCode::Column, + cursor_idx, + *col_idx as i64, + dest_reg, + None, + "", + ); + } else { + // COUNT(*) - just use a constant 1 + self.emit(OpCode::Integer, 1, dest_reg, 0, None, ""); + } + } + + // Insert into sorter + self.emit( + OpCode::SorterInsert, + sorter_id, + row_start_reg, + total_cols as i64, + None, + "", + ); + + // Next row + self.emit(OpCode::Next, cursor_idx, loop_start as i64, 0, None, ""); + + let after_scan = self.program.len(); + + // Patch rewind + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = after_scan as i64; + } + + // Sort the sorter + self.emit(OpCode::SorterSort, sorter_id, 0, 0, None, ""); + + // Now iterate through sorted rows, grouping by GROUP BY columns + // For each group, accumulate aggregates and output when group changes + + // Allocate registers for current group key + let group_key_reg = self.allocate_registers(group_col_indices.len()); + + // Allocate accumulator registers + let acc_base_reg = self.allocate_registers(agg_info.len()); + + // Initialize group key with NULL (first group detection) + for i in 0..group_col_indices.len() { + self.emit(OpCode::Null, 0, group_key_reg + i as i64, 0, None, ""); + } + + // Flag to indicate we've seen at least one row + let first_row_reg = self.allocate_register(); + self.emit(OpCode::Integer, 1, first_row_reg, 0, None, ""); + + // Allocate result registers upfront + let result_start_reg = self.allocate_registers(group_col_indices.len() + agg_info.len()); + + // Start group iteration + let sorter_loop_start = self.program.len(); + + // Get row from sorter + self.emit( + OpCode::SorterData, + sorter_id, + row_start_reg, + total_cols as i64, + None, + "", + ); + + // Check if first row - if so, skip group change check + let first_row_jump_addr = self.program.len(); + self.emit(OpCode::IfPos, first_row_reg, 0, 0, None, ""); + + // Compare first group column with saved group key + let cmp_reg = self.allocate_register(); + self.emit(OpCode::Ne, row_start_reg, group_key_reg, cmp_reg, None, ""); + + // If group NOT changed (cmp_reg is 0), skip output and go to step aggregates + let skip_output_addr = self.program.len(); + self.emit(OpCode::IfZ, cmp_reg, 0, 0, None, ""); + + // Group changed - output previous group + // Copy group key to result + for i in 0..group_col_indices.len() { + self.emit( + OpCode::Copy, + group_key_reg + i as i64, + result_start_reg + i as i64, + 0, + None, + "", + ); + } + + // Finalize aggregates + for (i, _) in agg_info.iter().enumerate() { + let dest_reg = result_start_reg + group_col_indices.len() as i64 + i as i64; + self.emit( + OpCode::AggFinal, + acc_base_reg + i as i64, + dest_reg, + 0, + None, + "", + ); + } + + // Apply HAVING filter if present + let mut having_skip_addr: Option = None; + if let Some(having_expr) = &select.having { + let having_reg = self.compile_having_condition( + having_expr, + &group_col_indices, + &agg_info, + result_start_reg, + group_key_reg, + )?; + // If HAVING condition is false (0), skip ResultRow + having_skip_addr = Some(self.program.len()); + self.emit(OpCode::IfZ, having_reg, 0, 0, None, ""); + } + + // Output result row + self.emit( + OpCode::ResultRow, + result_start_reg, + (group_col_indices.len() + agg_info.len()) as i64, + 0, + None, + "", + ); + + // Patch HAVING skip jump if present + if let Some(addr) = having_skip_addr { + let after_result = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = after_result as i64; + } + } + + // Reset accumulators for new group + for (i, _) in agg_info.iter().enumerate() { + self.emit(OpCode::AggReset, acc_base_reg + i as i64, 0, 0, None, ""); + } + + // === Update group key section (first row jumps here) === + let update_group_key_addr = self.program.len(); + + // Patch first row jump + if let Some(inst) = self.program.instructions.get_mut(first_row_jump_addr) { + inst.p2 = update_group_key_addr as i64; + } + + // Initialize/update group key from current row + for i in 0..group_col_indices.len() { + self.emit( + OpCode::Copy, + row_start_reg + i as i64, + group_key_reg + i as i64, + 0, + None, + "", + ); + } + + // Clear first row flag + self.emit(OpCode::Integer, 0, first_row_reg, 0, None, ""); + + // === Step aggregates section (skip output jumps here) === + let step_agg_addr = self.program.len(); + + // Patch skip output jump + if let Some(inst) = self.program.instructions.get_mut(skip_output_addr) { + inst.p2 = step_agg_addr as i64; + } + + // Step aggregates for current row + for (i, (func_type, _)) in agg_info.iter().enumerate() { + let value_reg = row_start_reg + group_col_indices.len() as i64 + i as i64; + self.emit( + OpCode::AggStep, + *func_type, + value_reg, + acc_base_reg + i as i64, + None, + "", + ); + } + + // Next sorted row + self.emit( + OpCode::SorterNext, + sorter_id, + sorter_loop_start as i64, + 0, + None, + "", + ); + + // Output final group + // Copy group key to result + for i in 0..group_col_indices.len() { + self.emit( + OpCode::Copy, + group_key_reg + i as i64, + result_start_reg + i as i64, + 0, + None, + "", + ); + } + + // Finalize aggregates + for (i, _) in agg_info.iter().enumerate() { + let dest_reg = result_start_reg + group_col_indices.len() as i64 + i as i64; + self.emit( + OpCode::AggFinal, + acc_base_reg + i as i64, + dest_reg, + 0, + None, + "", + ); + } + + // Apply HAVING filter for final group if present + let mut final_having_skip_addr: Option = None; + if let Some(having_expr) = &select.having { + let having_reg = self.compile_having_condition( + having_expr, + &group_col_indices, + &agg_info, + result_start_reg, + group_key_reg, + )?; + // If HAVING condition is false (0), skip ResultRow + final_having_skip_addr = Some(self.program.len()); + self.emit(OpCode::IfZ, having_reg, 0, 0, None, ""); + } + + // Output final result row + self.emit( + OpCode::ResultRow, + result_start_reg, + (group_col_indices.len() + agg_info.len()) as i64, + 0, + None, + "", + ); + + // Patch final HAVING skip jump if present + if let Some(addr) = final_having_skip_addr { + let after_result = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p2 = after_result as i64; + } + } + + // Close cursor + self.emit(OpCode::Close, cursor_idx, 0, 0, None, ""); + + Ok(()) + } + + /// Compile a HAVING condition and return the register containing the result (1 or 0) + /// agg_result_regs maps aggregate function signatures to their result registers + fn compile_having_condition( + &mut self, + having_expr: &Expr, + group_col_indices: &[usize], + agg_info: &[(i64, Option)], + result_start_reg: i64, + group_key_reg: i64, + ) -> SqawkResult { + match having_expr { + Expr::BinaryOp { left, op, right } => { + let left_reg = self.compile_having_operand( + left, + group_col_indices, + agg_info, + result_start_reg, + group_key_reg, + )?; + let right_reg = self.compile_having_operand( + right, + group_col_indices, + agg_info, + result_start_reg, + group_key_reg, + )?; + self.emit_comparison(op, left_reg, right_reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported HAVING expression: {:?}", + having_expr + ))), + } + } + + /// Compile an operand in a HAVING condition + fn compile_having_operand( + &mut self, + expr: &Expr, + group_col_indices: &[usize], + agg_info: &[(i64, Option)], + result_start_reg: i64, + group_key_reg: i64, + ) -> SqawkResult { + match expr { + Expr::Function(func) => { + // Find the aggregate function in agg_info and return its result register + let name = func.name.to_string().to_uppercase(); + let func_type = Self::agg_func_type(&name); + + // Find matching aggregate in agg_info by function type + for (i, (agg_func_type, _)) in agg_info.iter().enumerate() { + if *agg_func_type == func_type { + // Found matching aggregate - copy its result to a new register + let agg_result_reg = + result_start_reg + group_col_indices.len() as i64 + i as i64; + let reg = self.allocate_register(); + self.emit(OpCode::Copy, agg_result_reg, reg, 0, None, ""); + return Ok(reg); + } + } + + Err(SqawkError::UnsupportedSqlFeature(format!( + "Aggregate function in HAVING not found in SELECT: {}", + name + ))) + } + Expr::Value(value) => { + let reg = self.allocate_register(); + if let sqlparser::ast::Value::Number(n, _) = value { + if let Ok(i) = n.parse::() { + self.emit(OpCode::Integer, i, reg, 0, None, ""); + } + } + Ok(reg) + } + Expr::Identifier(_) => { + // GROUP BY column reference - copy from first group key register + // (simplified - for multi-column GROUP BY would need column name matching) + let reg = self.allocate_register(); + self.emit(OpCode::Copy, group_key_reg, reg, 0, None, ""); + Ok(reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported HAVING operand: {:?}", + expr + ))), + } + } +} diff --git a/src/vm/compiler_ddl.rs b/src/vm/compiler_ddl.rs new file mode 100644 index 0000000..0c2e590 --- /dev/null +++ b/src/vm/compiler_ddl.rs @@ -0,0 +1,472 @@ +//! DDL statement compilation (CREATE, DROP, ALTER, TRUNCATE) +//! +//! This module extends SqlCompiler with DDL statement compilation methods. + +use sqlparser::ast::{ + AlterTableOperation, ObjectName, ObjectType, SelectItem, SetExpr, TableFactor, Value, +}; + +use super::bytecode::OpCode; +use super::compiler::SqlCompiler; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::DataType; + +impl<'a> SqlCompiler<'a> { + /// Compile a DROP statement (DROP TABLE) + pub(crate) fn compile_drop( + &mut self, + object_type: &ObjectType, + names: &[ObjectName], + if_exists: bool, + ) -> SqawkResult<()> { + // Only support DROP TABLE for now + if *object_type != ObjectType::Table { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "DROP {:?} is not supported, only DROP TABLE", + object_type + ))); + } + + if names.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "DROP TABLE requires a table name".to_string(), + )); + } + + let table_name = self.get_table_name(&names[0])?; + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start DROP TABLE"); + + // Emit DropTable instruction (p1=1 if IF EXISTS) + self.emit( + OpCode::DropTable, + if if_exists { 1 } else { 0 }, + 0, + 0, + Some(table_name.clone()), + &format!( + "Drop table {}{}", + table_name, + if if_exists { " IF EXISTS" } else { "" } + ), + ); + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, "End DROP TABLE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile an ALTER TABLE statement + pub(crate) fn compile_alter_table( + &mut self, + name: &ObjectName, + operation: &AlterTableOperation, + ) -> SqawkResult<()> { + let table_name = self.get_table_name(name)?; + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start ALTER TABLE"); + + // Process the operation + match operation { + AlterTableOperation::AddColumn { column_def, .. } => { + let col_name = column_def.name.value.clone(); + let col_type = Self::sql_type_to_internal(&column_def.data_type.to_string()); + + let spec = format!("{}:{}:{}", table_name, col_name, col_type); + + self.emit( + OpCode::AlterTableAdd, + 0, + 0, + 0, + Some(spec), + &format!("Add column {} to {}", col_name, table_name), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "ALTER TABLE operation {:?} is not supported", + operation + ))); + } + } + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, "End ALTER TABLE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile a TRUNCATE TABLE statement + pub(crate) fn compile_truncate(&mut self, name: &ObjectName) -> SqawkResult<()> { + let table_name = self.get_table_name(name)?; + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start TRUNCATE TABLE"); + + self.emit( + OpCode::Truncate, + 0, + 0, + 0, + Some(table_name.clone()), + &format!("Truncate table {}", table_name), + ); + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, "End TRUNCATE TABLE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile CREATE TABLE ... AS SELECT + pub(crate) fn compile_create_table_as_select( + &mut self, + name: &ObjectName, + query: &sqlparser::ast::Query, + ) -> SqawkResult<()> { + let table_name = self.get_table_name(name)?; + + // First, compile the SELECT query to get the result schema + // We need to figure out the column types from the SELECT + // For now, we'll use a simplified approach: compile the query, + // then create the table with columns inferred from the result schema + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start CREATE TABLE AS SELECT"); + + // Get the SELECT to analyze + let select = match &*query.body { + SetExpr::Select(s) => s, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "CREATE TABLE AS only supports simple SELECT".to_string(), + )); + } + }; + + // Extract source table info to get column types + let source_table_name = if let Some(from) = select.from.first() { + match &from.relation { + TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "CREATE TABLE AS requires a simple table source".to_string(), + )); + } + } + } else { + return Err(SqawkError::InvalidSqlQuery( + "CREATE TABLE AS SELECT requires a FROM clause".to_string(), + )); + }; + + let source_table = self.database.get_table(&source_table_name)?; + + // Build column definitions from the SELECT projection + let mut col_specs = Vec::new(); + for item in &select.projection { + match item { + SelectItem::UnnamedExpr(expr) => { + let (col_name, col_type) = self.infer_expr_schema(expr, source_table); + let type_str = match col_type { + DataType::Integer => "INTEGER", + DataType::Float => "REAL", + DataType::Boolean => "BOOLEAN", + _ => "TEXT", + }; + col_specs.push(format!("{}:{}", col_name, type_str)); + } + SelectItem::ExprWithAlias { expr, alias } => { + let (_, col_type) = self.infer_expr_schema(expr, source_table); + let type_str = match col_type { + DataType::Integer => "INTEGER", + DataType::Float => "REAL", + DataType::Boolean => "BOOLEAN", + _ => "TEXT", + }; + col_specs.push(format!("{}:{}", alias.value, type_str)); + } + SelectItem::Wildcard(_) => { + // Add all columns from source table + for col in source_table.column_metadata() { + let type_str = match col.data_type { + DataType::Integer => "INTEGER", + DataType::Float => "REAL", + DataType::Boolean => "BOOLEAN", + _ => "TEXT", + }; + col_specs.push(format!("{}:{}", col.name, type_str)); + } + } + _ => {} + } + } + + // Build table spec: "table_name:col1:type1:col2:type2:..." + let mut spec = table_name.clone(); + for col_spec in &col_specs { + spec.push(':'); + spec.push_str(col_spec); + } + // Add empty file path and delimiter + spec.push_str("||"); + + // Emit CreateTable instruction + self.emit( + OpCode::CreateTable, + 0, + 0, + 0, + Some(spec), + &format!("Create table {}", table_name), + ); + + // Now compile the SELECT and INSERT into the new table + // Open the new table for writing + let cursor_idx = 0i64; + self.emit( + OpCode::OpenWrite, + cursor_idx, + 0, + 0, + Some(table_name.clone()), + "Open new table for writing", + ); + + // Open source table for reading + let source_cursor = 1i64; + self.emit( + OpCode::OpenRead, + source_cursor, + 1, + 0, + Some(source_table_name.clone()), + "Open source table", + ); + + // Rewind source + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + source_cursor, + 0, + 0, + None, + "Start reading source", + ); + + let loop_start = self.program.len(); + + // Handle WHERE clause if present + let mut skip_insert_addr: Option = None; + if let Some(selection) = &select.selection { + // Compile the WHERE clause condition (returns the result register) + let cond_reg = + self.compile_where_condition(selection, source_table, source_cursor as usize)?; + + // If condition is false (zero), skip the insert + skip_insert_addr = Some(self.program.len()); + self.emit( + OpCode::IfZ, + cond_reg, + 0, // Will be patched to point to Next + 0, + None, + "Skip row if WHERE is false", + ); + } + + // Allocate registers for columns + let col_count = col_specs.len(); + let start_reg = self.allocate_registers(col_count); + + // Load columns from source based on projection + let mut reg_idx = 0; + for item in &select.projection { + match item { + SelectItem::UnnamedExpr(sqlparser::ast::Expr::Identifier(ident)) + | SelectItem::ExprWithAlias { + expr: sqlparser::ast::Expr::Identifier(ident), + .. + } => { + if let Some(col_idx) = source_table.column_index(&ident.value) { + self.emit( + OpCode::Column, + source_cursor, + col_idx as i64, + start_reg + reg_idx as i64, + None, + &format!("Load column {}", ident.value), + ); + reg_idx += 1; + } + } + SelectItem::Wildcard(_) => { + for col_idx in 0..source_table.column_count() { + self.emit( + OpCode::Column, + source_cursor, + col_idx as i64, + start_reg + reg_idx as i64, + None, + "Load column", + ); + reg_idx += 1; + } + } + _ => { + // For other expressions, try to compile them + self.compile_where_operand( + match item { + SelectItem::UnnamedExpr(e) => e, + SelectItem::ExprWithAlias { expr, .. } => expr, + _ => continue, + }, + source_table, + source_cursor as usize, + start_reg + reg_idx as i64, + )?; + reg_idx += 1; + } + } + } + + // Insert row into new table + self.emit( + OpCode::InsertRow, + cursor_idx, + start_reg, + col_count as i64, + None, + "Insert row into new table", + ); + + // Next row in source + let next_addr = self.program.len(); + self.emit( + OpCode::Next, + source_cursor, + loop_start as i64, + 0, + None, + "Next source row", + ); + + // Patch the skip-insert jump to point to Next + if let Some(addr) = skip_insert_addr { + self.patch_jump(addr, next_addr); + } + + let after_loop = self.program.len(); + + // Patch rewind + self.patch_jump(rewind_addr, after_loop); + + // Close cursors + self.emit(OpCode::Close, cursor_idx, 0, 0, None, "Close new table"); + self.emit( + OpCode::Close, + source_cursor, + 0, + 0, + None, + "Close source table", + ); + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, "End CREATE TABLE AS SELECT"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile a CREATE TABLE statement + pub(crate) fn compile_create_table( + &mut self, + name: &ObjectName, + columns: &[sqlparser::ast::ColumnDef], + hive_formats: &Option, + location: &Option, + with_options: &[sqlparser::ast::SqlOption], + ) -> SqawkResult<()> { + // Extract table name + let table_name = self.get_table_name(name)?; + + // Build column specification: "table_name:col1:type1:col2:type2:..." + let mut spec = table_name.clone(); + for col in columns { + spec.push(':'); + spec.push_str(&col.name.value); + spec.push(':'); + let type_str = Self::sql_type_to_internal(&col.data_type.to_string()); + spec.push_str(type_str); + } + + // Get location from hive_formats or direct location + let file_path = hive_formats + .as_ref() + .and_then(|hf| hf.location.clone()) + .or_else(|| location.clone()); + + // Extract delimiter from WITH options + let delimiter = with_options.iter().find_map(|opt| { + if opt.name.value.to_lowercase() == "delimiter" { + if let Value::SingleQuotedString(s) = &opt.value { + return Some(s.clone()); + } + } + None + }); + + // Add file path and delimiter to spec: "...|filepath|delimiter" + spec.push('|'); + if let Some(ref fp) = file_path { + spec.push_str(fp); + } + spec.push('|'); + if let Some(ref d) = delimiter { + spec.push_str(d); + } + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start CREATE TABLE"); + + // Emit CreateTable instruction + self.emit( + OpCode::CreateTable, + 0, + 0, + 0, + Some(spec), + &format!("Create table {}", table_name), + ); + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, "End CREATE TABLE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } +} diff --git a/src/vm/compiler_dml.rs b/src/vm/compiler_dml.rs new file mode 100644 index 0000000..508861f --- /dev/null +++ b/src/vm/compiler_dml.rs @@ -0,0 +1,590 @@ +//! DML statement compilation (INSERT, UPDATE, DELETE) +//! +//! This module extends SqlCompiler with DML statement compilation methods. + +use sqlparser::ast::{Expr, Select, SelectItem, SetExpr, TableFactor}; + +use super::bytecode::OpCode; +use super::compiler::SqlCompiler; +use crate::error::{SqawkError, SqawkResult}; + +impl<'a> SqlCompiler<'a> { + /// Compile an INSERT statement + pub(crate) fn compile_insert( + &mut self, + table_name: &sqlparser::ast::ObjectName, + columns: &[sqlparser::ast::Ident], + source: &sqlparser::ast::Query, + ) -> SqawkResult<()> { + // Get table name as string + let table_name_str = self.get_table_name(table_name)?; + + // Get table info + let table = self.database.get_table(&table_name_str)?; + let column_count = table.column_count(); + + // Determine column indices for insertion + let column_indices: Vec = if columns.is_empty() { + (0..column_count).collect() + } else { + columns + .iter() + .map(|ident| { + table + .column_index(&ident.value) + .ok_or_else(|| SqawkError::ColumnNotFound(ident.value.clone())) + }) + .collect::, _>>()? + }; + + // Handle different source types: VALUES or SELECT + match &*source.body { + SetExpr::Values(values) => { + // INSERT ... VALUES + // Generate Init and jump + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start INSERT"); + + // Open table for writing (cursor 0) + self.emit( + OpCode::OpenWrite, + 0, + 0, + 0, + Some(table_name_str.clone()), + &format!("Open {} for writing", table_name_str), + ); + + // Process each row of values + let base_reg = self.register_counter as usize; + for value_row in &values.rows { + if value_row.len() != column_indices.len() { + return Err(SqawkError::InvalidSqlQuery(format!( + "INSERT statement has {} values but {} columns were specified", + value_row.len(), + column_indices.len() + ))); + } + + // Initialize all columns to NULL + for i in 0..column_count { + self.emit( + OpCode::Null, + (base_reg + i) as i64, + 0, + 0, + None, + &format!("Init col {} to NULL", i), + ); + } + + // Compile each value expression into the appropriate register + for (i, expr) in value_row.iter().enumerate() { + let col_idx = column_indices[i]; + let value_reg = base_reg + col_idx; + self.compile_expr_into_register(expr, value_reg)?; + } + + // Insert the row + self.emit( + OpCode::InsertRow, + 0, // cursor + base_reg as i64, + column_count as i64, + Some(table_name_str.clone()), + "Insert row", + ); + } + + // Close cursor and halt + self.emit(OpCode::Close, 0, 0, 0, None, "Close cursor"); + self.emit(OpCode::Halt, 0, 0, 0, None, "End INSERT"); + + // Patch init to jump past itself + self.patch_jump(init_addr, init_addr + 1); + + // Update register counter + self.register_counter = (base_reg + column_count) as i64; + } + + SetExpr::Select(select) => { + // INSERT ... SELECT + self.compile_insert_select(&table_name_str, column_count, &column_indices, select)?; + } + + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only INSERT ... VALUES and INSERT ... SELECT are supported".to_string(), + )); + } + } + + Ok(()) + } + + /// Compile INSERT ... SELECT statement + pub(crate) fn compile_insert_select( + &mut self, + target_table_name: &str, + target_column_count: usize, + column_indices: &[usize], + select: &Select, + ) -> SqawkResult<()> { + // Get the source table from the SELECT + if select.from.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "INSERT ... SELECT requires a FROM clause".to_string(), + )); + } + + let source_table_name = match &select.from[0].relation { + TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in INSERT...SELECT".to_string(), + )) + } + }; + + let source_table = self.database.get_table(&source_table_name)?; + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start INSERT...SELECT"); + + // Open source table for reading (cursor 0) + self.emit( + OpCode::OpenRead, + 0, + 0, + 0, + Some(source_table_name.clone()), + &format!("Open {} for reading", source_table_name), + ); + + // Open target table for writing (cursor 1) + self.emit( + OpCode::OpenWrite, + 1, + 0, + 0, + Some(target_table_name.to_string()), + &format!("Open {} for writing", target_table_name), + ); + + // Rewind source cursor + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, 0, 0, 0, None, "Rewind source cursor"); + + // Loop body start + let loop_start = self.program.len(); + + // Allocate registers for the row data + let base_reg = self.register_counter as usize; + self.register_counter += target_column_count as i64; + + // Initialize all columns to NULL + for i in 0..target_column_count { + self.emit( + OpCode::Null, + (base_reg + i) as i64, + 0, + 0, + None, + &format!("Init col {} to NULL", i), + ); + } + + // Load columns from SELECT projection + for (i, proj_item) in select.projection.iter().enumerate() { + if i >= column_indices.len() { + break; + } + let target_col_idx = column_indices[i]; + let dest_reg = base_reg + target_col_idx; + + match proj_item { + SelectItem::Wildcard(_) => { + // SELECT * - load all columns from source + for col_idx in 0..source_table.column_count() { + if col_idx < target_column_count { + self.emit( + OpCode::Column, + 0, + col_idx as i64, + (base_reg + col_idx) as i64, + None, + &format!("Load source col {} to target col {}", col_idx, col_idx), + ); + } + } + break; // Wildcard handles all columns + } + SelectItem::UnnamedExpr(expr) => { + // Compile the expression + match expr { + Expr::Identifier(ident) => { + let col_name = ident.value.to_lowercase(); + if let Some(col_idx) = source_table.column_index(&col_name) { + self.emit( + OpCode::Column, + 0, + col_idx as i64, + dest_reg as i64, + None, + &format!("Load {} to reg {}", col_name, dest_reg), + ); + } + } + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let col_name = parts[1].value.to_lowercase(); + if let Some(col_idx) = source_table.column_index(&col_name) { + self.emit( + OpCode::Column, + 0, + col_idx as i64, + dest_reg as i64, + None, + &format!("Load {} to reg {}", col_name, dest_reg), + ); + } + } + _ => { + // For other expressions, try to compile them + self.compile_expr_into_register(expr, dest_reg)?; + } + } + } + SelectItem::ExprWithAlias { expr, .. } => { + self.compile_expr_into_register(expr, dest_reg)?; + } + _ => {} + } + } + + // Check WHERE clause if present + let skip_addr = if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, source_table, 0)?; + let addr = self.program.len(); + self.emit( + OpCode::IfZ, + cond_reg, + 0, // will patch + 0, + None, + "Skip if WHERE is false", + ); + Some(addr) + } else { + None + }; + + // Insert the row into target table + self.emit( + OpCode::InsertRow, + 1, // target cursor + base_reg as i64, + target_column_count as i64, + Some(target_table_name.to_string()), + "Insert row into target", + ); + + // Patch skip if present + if let Some(addr) = skip_addr { + self.patch_jump(addr, self.program.len()); + } + + // Next row + self.emit( + OpCode::Next, + 0, // source cursor + loop_start as i64, + 0, + None, + "Next source row", + ); + + // Patch rewind + let end_addr = self.program.len(); + self.patch_jump(rewind_addr, end_addr); + + // Close cursors and halt + self.emit(OpCode::Close, 0, 0, 0, None, "Close source cursor"); + self.emit(OpCode::Close, 1, 0, 0, None, "Close target cursor"); + self.emit(OpCode::Halt, 0, 0, 0, None, "End INSERT...SELECT"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile a DELETE statement + pub(crate) fn compile_delete( + &mut self, + from: &[sqlparser::ast::TableWithJoins], + selection: Option<&Expr>, + ) -> SqawkResult<()> { + if from.len() != 1 { + return Err(SqawkError::UnsupportedSqlFeature( + "DELETE with multiple tables is not supported".to_string(), + )); + } + + // Get table name + let table_with_joins = &from[0]; + let table_name = match &table_with_joins.relation { + TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in DELETE".to_string(), + )) + } + }; + + // Get table reference for WHERE condition compilation + let table = self.database.get_table(&table_name)?; + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start DELETE"); + + // Open table for reading (cursor 0) + let cursor_idx = 0; + self.emit( + OpCode::OpenRead, + cursor_idx, + 0, + 0, + Some(table_name.clone()), + &format!("Open {} for reading", table_name), + ); + + // Rewind to first row + let rewind_addr = self.program.len(); + self.emit( + OpCode::Rewind, + cursor_idx, + 0, + 0, + None, + "Rewind to first row", + ); + + // Main loop: iterate over rows + let loop_start = self.program.len(); + + // If there's a WHERE clause, check if row matches + if let Some(where_expr) = selection { + // Compile WHERE condition using table context + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + + // If condition is false (0), skip to next row + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + cond_reg, + 0, // will patch + 0, + None, + "Skip if WHERE is false", + ); + + // Delete the current row + self.emit( + OpCode::DeleteRow, + cursor_idx, + 0, + 0, + Some(table_name.clone()), + "Delete matching row", + ); + + // Patch skip to jump to Next + let next_addr = self.program.len(); + self.patch_jump(skip_addr, next_addr); + } else { + // No WHERE clause - delete all rows + self.emit( + OpCode::DeleteRow, + 0, + 0, + 0, + Some(table_name.clone()), + "Delete row", + ); + } + + // Next row + self.emit( + OpCode::Next, + 0, + loop_start as i64, + 0, + None, + "Move to next row", + ); + + // Patch Rewind to jump here if table is empty + let end_addr = self.program.len(); + self.patch_jump(rewind_addr, end_addr); + + // Close and halt + self.emit(OpCode::Close, 0, 0, 0, None, "Close cursor"); + self.emit(OpCode::Halt, 0, 0, 0, None, "End DELETE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } + + /// Compile an UPDATE statement + pub(crate) fn compile_update( + &mut self, + table: &sqlparser::ast::TableWithJoins, + assignments: &[sqlparser::ast::Assignment], + selection: Option<&Expr>, + ) -> SqawkResult<()> { + // Get table name + let table_name = match &table.relation { + TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references are supported in UPDATE".to_string(), + )) + } + }; + + // Get table info for column lookups + let table_ref = self.database.get_table(&table_name)?; + let column_count = table_ref.column_count(); + + // Parse assignments to get column indices and expressions + let mut assignment_map: Vec<(usize, &Expr)> = Vec::new(); + for assignment in assignments { + // Handle different column identifier formats + let col_name = if !assignment.id.is_empty() { + assignment.id[0].value.clone() + } else { + return Err(SqawkError::InvalidSqlQuery( + "Invalid assignment target".to_string(), + )); + }; + + let col_idx = table_ref + .column_index(&col_name) + .ok_or_else(|| SqawkError::ColumnNotFound(col_name.clone()))?; + + assignment_map.push((col_idx, &assignment.value)); + } + + // Generate Init + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start UPDATE"); + + // Open table for reading (cursor 0) + self.emit( + OpCode::OpenRead, + 0, + 0, + 0, + Some(table_name.clone()), + &format!("Open {} for reading", table_name), + ); + + // Rewind to first row + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, 0, 0, 0, None, "Rewind to first row"); + + // Main loop + let loop_start = self.program.len(); + + // Allocate registers for the row data + let base_reg = self.register_counter as usize; + self.register_counter += column_count as i64; + + // Load all columns into registers + for col_idx in 0..column_count { + self.emit( + OpCode::Column, + 0, // cursor + col_idx as i64, + (base_reg + col_idx) as i64, + None, + &format!("Load column {}", col_idx), + ); + } + + // If there's a WHERE clause, check if row matches + let skip_addr = if let Some(where_expr) = selection { + // Compile WHERE condition using table context + let cond_reg = self.compile_where_condition(where_expr, table_ref, 0)?; + + // If condition is false (0), skip to next row + let addr = self.program.len(); + self.emit( + OpCode::IfZ, + cond_reg, + 0, // will patch + 0, + None, + "Skip if WHERE is false", + ); + Some(addr) + } else { + None + }; + + // Apply assignments + for (col_idx, expr) in &assignment_map { + self.compile_expr_into_register(expr, base_reg + *col_idx)?; + } + + // Delete old row and insert new row + self.emit( + OpCode::DeleteRow, + 0, + 0, + 0, + Some(table_name.clone()), + "Delete old row", + ); + self.emit( + OpCode::InsertRow, + 0, + base_reg as i64, + column_count as i64, + Some(table_name.clone()), + "Insert updated row", + ); + + // Patch skip if present + if let Some(addr) = skip_addr { + self.patch_jump(addr, self.program.len()); + } + + // Next row + self.emit( + OpCode::Next, + 0, + loop_start as i64, + 0, + None, + "Move to next row", + ); + + // Patch Rewind to jump here if table is empty + let end_addr = self.program.len(); + self.patch_jump(rewind_addr, end_addr); + + // Close and halt + self.emit(OpCode::Close, 0, 0, 0, None, "Close cursor"); + self.emit(OpCode::Halt, 0, 0, 0, None, "End UPDATE"); + + // Patch init + self.patch_jump(init_addr, init_addr + 1); + + Ok(()) + } +} diff --git a/src/vm/compiler_join.rs b/src/vm/compiler_join.rs new file mode 100644 index 0000000..f6753af --- /dev/null +++ b/src/vm/compiler_join.rs @@ -0,0 +1,2873 @@ +//! JOIN compilation +//! +//! This module extends SqlCompiler with JOIN compilation methods. + +use sqlparser::ast::{ + BinaryOperator, Expr, FunctionArg, FunctionArgExpr, JoinConstraint, JoinOperator, Query, + Select, SelectItem, TableWithJoins, Value, +}; + +/// Represents an item in a multi-table aggregate projection +#[derive(Clone, Debug)] +enum MultiTableProjectionItem { + /// A column reference: (table_idx, col_idx) + Column(usize, usize), + /// An aggregate: (func_type, optional column ref for the aggregate argument) + Aggregate(i64, Option<(usize, usize)>), +} + +use super::bytecode::{OpCode, ResultSchema}; +use super::compiler::{MultiTableProjection, SqlCompiler}; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::Table; + +impl<'a> SqlCompiler<'a> { + pub(crate) fn compile_join( + &mut self, + table_with_joins: &TableWithJoins, + projection: &[SelectItem], + where_clause: &Option, + ) -> SqawkResult<()> { + // Get the left table name + let left_table_name = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans are supported in joins".into(), + )) + } + }; + + if !self.database.has_table(&left_table_name) { + return Err(SqawkError::TableNotFound(left_table_name)); + } + + // Handle multiple joins (3+ tables) + if table_with_joins.joins.len() > 1 { + return self.compile_multi_join(table_with_joins, projection, where_clause); + } + + let join = &table_with_joins.joins[0]; + + // Get the right table name + let right_table_name = match &join.relation { + sqlparser::ast::TableFactor::Table { name, .. } => self.get_table_name(name)?, + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references supported in joins".into(), + )) + } + }; + + if !self.database.has_table(&right_table_name) { + return Err(SqawkError::TableNotFound(right_table_name)); + } + + // Check for CROSS JOIN - handle separately + if matches!(&join.join_operator, JoinOperator::CrossJoin) { + return self.compile_cross_join(&left_table_name, &right_table_name, projection); + } + + // Determine join type and constraint + let (join_type, join_condition) = match &join.join_operator { + JoinOperator::LeftOuter(constraint) => ("LEFT", constraint), + JoinOperator::RightOuter(constraint) => ("RIGHT", constraint), + JoinOperator::FullOuter(constraint) => ("FULL", constraint), + JoinOperator::Inner(constraint) => ("INNER", constraint), + JoinOperator::CrossJoin => unreachable!("Handled above"), + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Unsupported join type".into(), + )) + } + }; + + // Extract ON condition + let on_expr = match join_condition { + JoinConstraint::On(expr) => expr, + JoinConstraint::None => { + return Err(SqawkError::UnsupportedSqlFeature( + "Join without ON condition not supported".into(), + )) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only ON join constraints supported".into(), + )) + } + }; + + if self.verbose { + eprintln!( + "Join type: {}", + if join_type == "INNER" { + "Inner" + } else { + join_type + } + ); + eprintln!("Processing {} JOIN with ON condition", join_type); + } + self.add_comment(&format!("{} JOIN {} ON ...", join_type, right_table_name)); + + let left_table = self.database.get_table(&left_table_name)?; + let right_table = self.database.get_table(&right_table_name)?; + + // Determine which columns to output based on projection + let (left_cols, right_cols, schema) = self.resolve_join_projection( + projection, + left_table, + right_table, + &left_table_name, + &right_table_name, + )?; + + // Set result schema + self.program.set_result_schema(schema); + + // For LEFT and INNER joins: outer loop on left, inner on right + // For RIGHT join: outer loop on right, inner on left + // We'll use "outer" and "inner" variables to abstract this + let (outer_cursor, inner_cursor) = if join_type == "RIGHT" { + (1usize, 0usize) // right is outer, left is inner + } else { + (0usize, 1usize) // left is outer, right is inner + }; + + let (outer_table_name, inner_table_name) = if join_type == "RIGHT" { + (right_table_name.clone(), left_table_name.clone()) + } else { + (left_table_name.clone(), right_table_name.clone()) + }; + + // Note: outer_table and inner_table are defined here for potential future use + // (e.g., for compile-time column lookup), but currently unused + let (_outer_table, _inner_table) = if join_type == "RIGHT" { + (right_table, left_table) + } else { + (left_table, right_table) + }; + + let (outer_cols, inner_cols) = if join_type == "RIGHT" { + (right_cols.clone(), left_cols.clone()) + } else { + (left_cols.clone(), right_cols.clone()) + }; + + let left_cursor = 0usize; + let right_cursor = 1usize; + + // Open both tables + self.emit( + OpCode::OpenRead, + left_cursor as i64, + 1, + 0, + Some(left_table_name.clone()), + &format!("Open left table {} for reading", left_table_name), + ); + + self.emit( + OpCode::OpenRead, + right_cursor as i64, + 2, + 0, + Some(right_table_name.clone()), + &format!("Open right table {} for reading", right_table_name), + ); + + // Allocate registers for columns - always in left, right order + let left_start_reg = self.allocate_registers(left_cols.len()); + let right_start_reg = self.allocate_registers(right_cols.len()); + + // For RIGHT JOIN, outer/inner start regs are swapped + let (outer_start_reg, inner_start_reg) = if join_type == "RIGHT" { + (right_start_reg, left_start_reg) + } else { + (left_start_reg, right_start_reg) + }; + + // Allocate match flag register + let match_reg = self.allocate_register(); + + // Initialize match flag to 0 + self.emit( + OpCode::Integer, + 0, + match_reg, + 0, + None, + "Initialize match flag to 0", + ); + + // Rewind outer cursor + let outer_loop_end_placeholder = self.program.len(); + self.emit( + OpCode::Rewind, + outer_cursor as i64, + 0, // Placeholder - will be patched + 0, + None, + &format!("Rewind outer cursor ({})", outer_table_name), + ); + + // OUTER LOOP START + let outer_loop_start = self.program.len(); + + // Load outer table columns into their registers + self.emit_column_loads(outer_cursor as i64, &outer_cols, outer_start_reg, "outer"); + + // Reset match flag for this outer iteration + self.emit(OpCode::Integer, 0, match_reg, 0, None, "Reset match flag"); + + // Rewind inner cursor + let inner_loop_end_placeholder = self.program.len(); + self.emit( + OpCode::RewindInner, + inner_cursor as i64, + 0, // Placeholder - will be patched + 0, + None, + &format!("Rewind inner cursor ({})", inner_table_name), + ); + + // INNER LOOP START + let inner_loop_start = self.program.len(); + + // Load inner table columns into their registers + self.emit_column_loads(inner_cursor as i64, &inner_cols, inner_start_reg, "inner"); + + // Compile join condition + let condition_reg = self.compile_join_condition( + on_expr, + left_table, + right_table, + left_cursor, + right_cursor, + )?; + + // If ON condition fails, skip to next inner iteration (placeholder - will be patched) + let skip_on_addr = self.program.len(); + self.emit( + OpCode::IfZ, + condition_reg, + 0, + 0, + None, + "Skip if join ON condition false", + ); + + // Compile WHERE clause filtering if present + let skip_where_addr = if let Some(where_expr) = where_clause { + // Use compile_join_condition to handle table.column references in WHERE + let where_cond_reg = self.compile_join_condition( + where_expr, + left_table, + right_table, + left_cursor, + right_cursor, + )?; + + // If WHERE condition fails, skip to next inner iteration + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + where_cond_reg, + 0, + 0, + None, + "Skip if WHERE condition false", + ); + Some(skip_addr) + } else { + None + }; + + // Mark match found + self.emit( + OpCode::MarkMatch, + match_reg, + 0, + 0, + None, + "Mark that a match was found", + ); + + // Output combined row (matched) + self.emit( + OpCode::ResultRow, + left_start_reg, + (left_cols.len() + right_cols.len()) as i64, + 0, + None, + "Output matched row", + ); + + // Next inner + let next_inner_addr = self.program.len(); + self.emit( + OpCode::Next, + inner_cursor as i64, + inner_loop_start as i64, + 0, + None, + "Next inner row", + ); + + // Patch skip addresses to jump to Next inner + if let Some(inst) = self.program.instructions.get_mut(skip_on_addr) { + inst.p2 = next_inner_addr as i64; + } + if let Some(skip_addr) = skip_where_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = next_inner_addr as i64; + } + } + + // INNER LOOP END - patch the RewindInner jump address + let inner_loop_end = self.program.len(); + + // For LEFT/RIGHT/FULL JOIN: Check if no match was found, output outer row with NULLs for inner + // LEFT JOIN: outer=left, inner=right, fill right with NULL + // RIGHT JOIN: outer=right, inner=left, fill left with NULL + if join_type == "LEFT" || join_type == "RIGHT" || join_type == "FULL" { + // Check match flag - if match found (non-zero), skip NULL row output + // If no match (zero), continue to NullRow + // Jump calculation: IfPos (current), NullRow (+1), ResultRow (+2), Next outer (+3) + // So if match found, skip to +3 (Next outer) + self.emit( + OpCode::IfPos, + match_reg, + (self.program.len() + 3) as i64, + 0, + None, + "Skip NULL row if match found", + ); + + // No match: output NULLs for inner columns + self.emit( + OpCode::NullRow, + inner_start_reg, + inner_cols.len() as i64, + 0, + None, + "Fill inner columns with NULL", + ); + + // Output row with NULLs + self.emit( + OpCode::ResultRow, + left_start_reg, + (left_cols.len() + right_cols.len()) as i64, + 0, + None, + "Output outer row with NULLs for inner", + ); + } + + // Next outer + self.emit( + OpCode::Next, + outer_cursor as i64, + outer_loop_start as i64, + 0, + None, + "Next outer row", + ); + + // OUTER LOOP END + let mut outer_loop_end = self.program.len(); + // For FULL JOIN, we need to track where second pass starts + let mut second_pass_start: Option = None; + + // For FULL JOIN: Second pass to output unmatched right rows + // The first pass (LEFT JOIN style) already output matched rows and unmatched left rows + // Now we need to output right rows that have no match in left table + if join_type == "FULL" { + self.add_comment("FULL JOIN: Second pass for unmatched right rows"); + + // Save where second pass starts (this is where we jump if left table is empty) + second_pass_start = Some(self.program.len()); + + // Rewind right cursor + let right_loop_end_placeholder = self.program.len(); + self.emit( + OpCode::Rewind, + right_cursor as i64, + 0, + 0, + None, + "Rewind right cursor for second pass", + ); + + let right_loop_start = self.program.len(); + + // Load right columns + self.emit_column_loads(right_cursor as i64, &right_cols, right_start_reg, "right"); + + // Reset match flag + self.emit( + OpCode::Integer, + 0, + match_reg, + 0, + None, + "Reset match flag for right row", + ); + + // Rewind left cursor to check for matches + let left_check_end_placeholder = self.program.len(); + self.emit( + OpCode::RewindInner, + left_cursor as i64, + 0, + 0, + None, + "Rewind left cursor to check for matches", + ); + + let left_check_start = self.program.len(); + + // Load left columns for condition check + self.emit_column_loads(left_cursor as i64, &left_cols, left_start_reg, "left"); + + // Check join condition + let condition_reg2 = self.compile_join_condition( + on_expr, + left_table, + right_table, + left_cursor, + right_cursor, + )?; + + // If condition matches, mark it and skip to next right row + // IfZ skips over MarkMatch if condition is false + self.emit( + OpCode::IfZ, + condition_reg2, + (self.program.len() + 2) as i64, + 0, + None, + "Skip if condition false", + ); + + // Mark that a match was found + self.emit( + OpCode::MarkMatch, + match_reg, + 0, + 0, + None, + "Mark that right row has a match", + ); + + // Next left (continue checking) + self.emit( + OpCode::Next, + left_cursor as i64, + left_check_start as i64, + 0, + None, + "Next left row", + ); + + let left_check_end = self.program.len(); + + // After checking all left rows: if no match, output right row with NULLs for left + // If match found (IfPos), skip to next right row + self.emit( + OpCode::IfPos, + match_reg, + (self.program.len() + 3) as i64, + 0, + None, + "Skip if right row had a match", + ); + + // No match: fill left columns with NULL + self.emit( + OpCode::NullRow, + left_start_reg, + left_cols.len() as i64, + 0, + None, + "Fill left columns with NULL", + ); + + // Output row with NULLs for left + self.emit( + OpCode::ResultRow, + left_start_reg, + (left_cols.len() + right_cols.len()) as i64, + 0, + None, + "Output unmatched right row with NULLs", + ); + + // Next right row + self.emit( + OpCode::Next, + right_cursor as i64, + right_loop_start as i64, + 0, + None, + "Next right row", + ); + + let right_loop_end = self.program.len(); + + // Patch jump addresses for second pass + if let Some(inst) = self + .program + .instructions + .get_mut(right_loop_end_placeholder) + { + inst.p2 = right_loop_end as i64; + } + if let Some(inst) = self + .program + .instructions + .get_mut(left_check_end_placeholder) + { + inst.p2 = left_check_end as i64; + } + + outer_loop_end = right_loop_end; + } + + // Close cursors + self.emit( + OpCode::Close, + left_cursor as i64, + 0, + 0, + None, + "Close left cursor", + ); + self.emit( + OpCode::Close, + right_cursor as i64, + 0, + 0, + None, + "Close right cursor", + ); + + // Patch jump addresses for main loop + if let Some(inst) = self + .program + .instructions + .get_mut(outer_loop_end_placeholder) + { + // For FULL JOIN, when left table is empty (Rewind returns false), + // we still need to run second pass to output unmatched right rows + if let Some(second_start) = second_pass_start { + inst.p2 = second_start as i64; + } else { + inst.p2 = outer_loop_end as i64; + } + } + if let Some(inst) = self + .program + .instructions + .get_mut(inner_loop_end_placeholder) + { + inst.p2 = inner_loop_end as i64; + } + + Ok(()) + } + + /// Compile a CROSS JOIN (Cartesian product) + pub(crate) fn compile_cross_join( + &mut self, + left_table_name: &str, + right_table_name: &str, + projection: &[SelectItem], + ) -> SqawkResult<()> { + if self.verbose { + eprintln!("Join type: Cross"); + eprintln!("Processing CROSS JOIN"); + } + self.add_comment(&format!( + "CROSS JOIN {} x {}", + left_table_name, right_table_name + )); + + let left_table = self.database.get_table(left_table_name)?; + let right_table = self.database.get_table(right_table_name)?; + + // Determine which columns to output based on projection + let (left_cols, right_cols, schema) = self.resolve_join_projection( + projection, + left_table, + right_table, + left_table_name, + right_table_name, + )?; + + // Set result schema + self.program.set_result_schema(schema); + + let left_cursor = 0usize; + let right_cursor = 1usize; + + // Open both tables + self.emit( + OpCode::OpenRead, + left_cursor as i64, + 1, + 0, + Some(left_table_name.to_string()), + &format!("Open left table {} for reading", left_table_name), + ); + + self.emit( + OpCode::OpenRead, + right_cursor as i64, + 2, + 0, + Some(right_table_name.to_string()), + &format!("Open right table {} for reading", right_table_name), + ); + + // Allocate registers for columns + let left_start_reg = self.allocate_registers(left_cols.len()); + let right_start_reg = self.allocate_registers(right_cols.len()); + + // Rewind left cursor (outer loop) + let left_loop_end_placeholder = self.program.len(); + self.emit( + OpCode::Rewind, + left_cursor as i64, + 0, // Placeholder - will be patched + 0, + None, + "Rewind left cursor (outer loop)", + ); + + // OUTER LOOP START + let left_loop_start = self.program.len(); + + // Load left table columns + self.emit_column_loads(left_cursor as i64, &left_cols, left_start_reg, "left"); + + // Rewind right cursor (inner loop) + let right_loop_end_placeholder = self.program.len(); + self.emit( + OpCode::RewindInner, + right_cursor as i64, + 0, // Placeholder - will be patched + 0, + None, + "Rewind right cursor (inner loop)", + ); + + // INNER LOOP START + let right_loop_start = self.program.len(); + + // Load right table columns + self.emit_column_loads(right_cursor as i64, &right_cols, right_start_reg, "right"); + + // Output combined row + self.emit( + OpCode::ResultRow, + left_start_reg, + (left_cols.len() + right_cols.len()) as i64, + 0, + None, + "Output cross join row", + ); + + // Next right (inner) + self.emit( + OpCode::Next, + right_cursor as i64, + right_loop_start as i64, + 0, + None, + "Next right row", + ); + + // INNER LOOP END + let right_loop_end = self.program.len(); + + // Next left (outer) + self.emit( + OpCode::Next, + left_cursor as i64, + left_loop_start as i64, + 0, + None, + "Next left row", + ); + + // OUTER LOOP END + let left_loop_end = self.program.len(); + + // Close cursors + self.emit( + OpCode::Close, + left_cursor as i64, + 0, + 0, + None, + "Close left cursor", + ); + + self.emit( + OpCode::Close, + right_cursor as i64, + 0, + 0, + None, + "Close right cursor", + ); + + // Patch jump addresses + if let Some(inst) = self.program.instructions.get_mut(left_loop_end_placeholder) { + inst.p2 = left_loop_end as i64; + } + if let Some(inst) = self + .program + .instructions + .get_mut(right_loop_end_placeholder) + { + inst.p2 = right_loop_end as i64; + } + + Ok(()) + } + + /// Compile an implicit join with aggregates only (no GROUP BY) + /// This is a simpler pattern that doesn't require sorting + fn compile_implicit_join_aggregate_only( + &mut self, + select: &Select, + table_names: &[String], + table_refs: &[String], + tables: &[&Table], + projection_items: &[MultiTableProjectionItem], + ) -> SqawkResult<()> { + let num_tables = tables.len(); + self.add_comment("Multi-table aggregate (no GROUP BY)"); + + // Open all tables + for (i, table_name) in table_names.iter().enumerate() { + self.emit( + OpCode::OpenRead, + i as i64, + (i + 1) as i64, + 0, + Some(table_name.to_string()), + &format!("Open table {}", table_name), + ); + } + + // Allocate accumulator registers for each aggregate + let num_aggs = projection_items + .iter() + .filter(|i| matches!(i, MultiTableProjectionItem::Aggregate(_, _))) + .count(); + let acc_base_reg = self.allocate_registers(num_aggs.max(1)); + + // Create nested loops for the join + let mut loop_end_placeholders: Vec = Vec::with_capacity(num_tables); + let mut loop_starts: Vec = Vec::with_capacity(num_tables); + + for i in 0..num_tables { + let rewind_opcode = if i == 0 { + OpCode::Rewind + } else { + OpCode::RewindInner + }; + + let loop_end_placeholder = self.program.len(); + loop_end_placeholders.push(loop_end_placeholder); + self.emit( + rewind_opcode, + i as i64, + 0, + 0, + None, + &format!("Rewind cursor {}", i), + ); + + let loop_start = self.program.len(); + loop_starts.push(loop_start); + } + + // Compile WHERE condition + let skip_output_addr = if let Some(where_expr) = &select.selection { + let condition_reg = + self.compile_multi_table_condition(where_expr, tables, table_refs)?; + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + condition_reg, + 0, + 0, + None, + "Skip if WHERE false", + ); + Some(skip_addr) + } else { + None + }; + + // Step aggregates for this matching row + let mut agg_idx = 0; + for item in projection_items { + if let MultiTableProjectionItem::Aggregate(func_type, col_ref) = item { + let val_reg = if let Some((tbl_idx, col_idx)) = col_ref { + let r = self.allocate_register(); + self.emit( + OpCode::Column, + *tbl_idx as i64, + *col_idx as i64, + r, + None, + "Load aggregate source", + ); + r + } else { + // COUNT(*) - use 1 + let r = self.allocate_register(); + self.emit(OpCode::Integer, 1, r, 0, None, "COUNT(*) = 1"); + r + }; + + self.emit( + OpCode::AggStep, + *func_type, + val_reg, + acc_base_reg + agg_idx as i64, + None, + "", + ); + agg_idx += 1; + } + } + + // Patch skip address for WHERE + let innermost_next_addr = self.program.len(); + if let Some(skip_addr) = skip_output_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = innermost_next_addr as i64; + } + } + + // Close loops (innermost first) + let mut loop_ends: Vec = Vec::with_capacity(num_tables); + for i in (0..num_tables).rev() { + self.emit( + OpCode::Next, + i as i64, + loop_starts[i] as i64, + 0, + None, + &format!("Next {}", i), + ); + loop_ends.push(self.program.len()); + } + loop_ends.reverse(); + + // Close tables + for i in 0..num_tables { + self.emit( + OpCode::Close, + i as i64, + 0, + 0, + None, + &format!("Close cursor {}", i), + ); + } + + // Patch loop end placeholders + for (i, placeholder) in loop_end_placeholders.iter().enumerate() { + if let Some(inst) = self.program.instructions.get_mut(*placeholder) { + inst.p2 = loop_ends[i] as i64; + } + } + + // Finalize aggregates and output single result row + let result_reg = self.allocate_registers(projection_items.len()); + agg_idx = 0; + for (i, item) in projection_items.iter().enumerate() { + match item { + MultiTableProjectionItem::Column(_, _) => { + // This shouldn't happen in aggregate-only queries + // but handle it anyway with NULL + self.emit(OpCode::Null, 0, result_reg + i as i64, 0, None, ""); + } + MultiTableProjectionItem::Aggregate(_, _) => { + self.emit( + OpCode::AggFinal, + acc_base_reg + agg_idx as i64, + result_reg + i as i64, + 0, + None, + "", + ); + agg_idx += 1; + } + } + } + + // Output the single result row + self.emit( + OpCode::ResultRow, + result_reg, + projection_items.len() as i64, + 0, + None, + "Output aggregate result", + ); + + Ok(()) + } + + /// Compile an implicit join with GROUP BY or aggregates + pub(crate) fn compile_implicit_join_with_group_by( + &mut self, + select: &Select, + query: &Query, + ) -> SqawkResult<()> { + let num_tables = select.from.len(); + + // Collect table names and aliases + let mut table_names: Vec = Vec::with_capacity(num_tables); + let mut table_refs: Vec = Vec::with_capacity(num_tables); + for from_item in &select.from { + let (table_name, table_alias) = match &from_item.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let talias = alias.as_ref().map(|a| a.name.value.clone()); + (tname, talias) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references supported".into(), + )) + } + }; + if !self.database.has_table(&table_name) { + return Err(SqawkError::TableNotFound(table_name)); + } + table_refs.push(table_alias.unwrap_or_else(|| table_name.clone())); + table_names.push(table_name); + } + + let tables: Vec<&Table> = table_names + .iter() + .map(|name| self.database.get_table(name)) + .collect::, _>>()?; + + // Determine GROUP BY column references + let mut group_col_refs: Vec<(usize, usize)> = Vec::new(); // (table_idx, col_idx) + for expr in select.group_by.iter() { + let (tbl_idx, col_idx) = self.resolve_multi_table_column(expr, &tables, &table_refs)?; + group_col_refs.push((tbl_idx, col_idx)); + } + + // Analyze projection for aggregates and column references + let mut projection_items: Vec = Vec::new(); + for item in &select.projection { + let expr = match item { + SelectItem::UnnamedExpr(e) => e, + SelectItem::ExprWithAlias { expr: e, .. } => e, + SelectItem::Wildcard(_) => { + return Err(SqawkError::UnsupportedSqlFeature( + "SELECT * not supported with GROUP BY in multi-table queries".into(), + )) + } + _ => continue, + }; + + match expr { + Expr::Function(func) => { + let name = func.name.to_string().to_uppercase(); + if matches!(name.as_str(), "COUNT" | "SUM" | "AVG" | "MIN" | "MAX") { + let func_type = Self::agg_func_type(&name); + let col_ref = if let Some(FunctionArg::Unnamed(FunctionArgExpr::Wildcard)) = + func.args.first() + { + None // COUNT(*) + } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg_expr))) = + func.args.first() + { + let (tbl_idx, col_idx) = + self.resolve_multi_table_column(arg_expr, &tables, &table_refs)?; + Some((tbl_idx, col_idx)) + } else { + None + }; + projection_items + .push(MultiTableProjectionItem::Aggregate(func_type, col_ref)); + } else { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Function {} not supported in multi-table GROUP BY", + name + ))); + } + } + Expr::CompoundIdentifier(_) | Expr::Identifier(_) => { + let (tbl_idx, col_idx) = + self.resolve_multi_table_column(expr, &tables, &table_refs)?; + projection_items.push(MultiTableProjectionItem::Column(tbl_idx, col_idx)); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression in GROUP BY projection: {:?}", + expr + ))); + } + } + } + + // Build result schema + let schema = self.build_multi_table_agg_schema(&select.projection, &tables, &table_refs); + self.program.set_result_schema(schema); + + // Handle the no-GROUP-BY case (aggregate only) - simpler pattern without sorter + if group_col_refs.is_empty() { + return self.compile_implicit_join_aggregate_only( + select, + &table_names, + &table_refs, + &tables, + &projection_items, + ); + } + + // Calculate columns needed in sorter: group columns + aggregate source columns + let agg_source_cols: Vec<(usize, usize)> = projection_items + .iter() + .filter_map(|item| { + if let MultiTableProjectionItem::Aggregate(_, Some(col_ref)) = item { + Some(*col_ref) + } else { + None + } + }) + .collect(); + + let total_sorter_cols = group_col_refs.len() + agg_source_cols.len() + 1; // +1 for row count + + // Open sorter + let sorter_id = 0i64; + let sort_spec: String = (0..group_col_refs.len()) + .map(|i| format!("{}:asc", i)) + .collect::>() + .join(","); + + self.emit( + OpCode::SorterOpen, + sorter_id, + total_sorter_cols as i64, + 0, + Some(sort_spec), + "Open sorter for GROUP BY", + ); + + // Open all tables + for (i, table_name) in table_names.iter().enumerate() { + self.emit( + OpCode::OpenRead, + i as i64, + (i + 1) as i64, + 0, + Some(table_name.to_string()), + &format!("Open table {}", table_name), + ); + } + + // Allocate registers for sorter row + let sorter_row_reg = self.allocate_registers(total_sorter_cols); + + // Create nested loops + let mut loop_end_placeholders: Vec = Vec::with_capacity(num_tables); + let mut loop_starts: Vec = Vec::with_capacity(num_tables); + + for i in 0..num_tables { + let rewind_opcode = if i == 0 { + OpCode::Rewind + } else { + OpCode::RewindInner + }; + + let loop_end_placeholder = self.program.len(); + loop_end_placeholders.push(loop_end_placeholder); + self.emit( + rewind_opcode, + i as i64, + 0, + 0, + None, + &format!("Rewind cursor {}", i), + ); + + let loop_start = self.program.len(); + loop_starts.push(loop_start); + } + + // Compile WHERE condition + let skip_output_addr = if let Some(where_expr) = &select.selection { + let condition_reg = + self.compile_multi_table_condition(where_expr, &tables, &table_refs)?; + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + condition_reg, + 0, + 0, + None, + "Skip if WHERE false", + ); + Some(skip_addr) + } else { + None + }; + + // Load group columns into sorter row registers + for (i, (tbl_idx, col_idx)) in group_col_refs.iter().enumerate() { + self.emit( + OpCode::Column, + *tbl_idx as i64, + *col_idx as i64, + sorter_row_reg + i as i64, + None, + &format!("Load GROUP BY col {} from table {}", col_idx, tbl_idx), + ); + } + + // Load aggregate source columns + for (i, (tbl_idx, col_idx)) in agg_source_cols.iter().enumerate() { + self.emit( + OpCode::Column, + *tbl_idx as i64, + *col_idx as i64, + sorter_row_reg + group_col_refs.len() as i64 + i as i64, + None, + &format!("Load agg source col {} from table {}", col_idx, tbl_idx), + ); + } + + // Set row count to 1 + self.emit( + OpCode::Integer, + 1, + sorter_row_reg + (total_sorter_cols - 1) as i64, + 0, + None, + "Row count = 1", + ); + + // Insert into sorter + self.emit( + OpCode::SorterInsert, + sorter_id, + sorter_row_reg, + total_sorter_cols as i64, + None, + "Insert into sorter", + ); + + // Patch skip address + let innermost_next_addr = self.program.len(); + if let Some(skip_addr) = skip_output_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = innermost_next_addr as i64; + } + } + + // Close loops (innermost first) + let mut loop_ends: Vec = Vec::with_capacity(num_tables); + for i in (0..num_tables).rev() { + self.emit( + OpCode::Next, + i as i64, + loop_starts[i] as i64, + 0, + None, + &format!("Next {}", i), + ); + loop_ends.push(self.program.len()); + } + loop_ends.reverse(); + + // Close tables + for i in 0..num_tables { + self.emit( + OpCode::Close, + i as i64, + 0, + 0, + None, + &format!("Close cursor {}", i), + ); + } + + // Patch loop end placeholders + for (i, placeholder) in loop_end_placeholders.iter().enumerate() { + if let Some(inst) = self.program.instructions.get_mut(*placeholder) { + inst.p2 = loop_ends[i] as i64; + } + } + + // Sort the sorter + self.emit(OpCode::SorterSort, sorter_id, 0, 0, None, "Sort"); + + // Allocate registers for reading sorter data + let row_data_reg = self.allocate_registers(total_sorter_cols); + + // Allocate accumulators for aggregates + let num_aggs = projection_items + .iter() + .filter(|i| matches!(i, MultiTableProjectionItem::Aggregate(_, _))) + .count(); + let acc_base_reg = self.allocate_registers(num_aggs.max(1)); + + // Allocate registers for current and previous group key + let group_key_reg = self.allocate_registers(group_col_refs.len().max(1)); + + // Initialize group key with NULL (first group detection) + for i in 0..group_col_refs.len() { + self.emit( + OpCode::Null, + 0, + group_key_reg + i as i64, + 0, + None, + "Init group key NULL", + ); + } + + // Flag to indicate we've seen at least one row + let first_row_reg = self.allocate_register(); + self.emit(OpCode::Integer, 1, first_row_reg, 0, None, "first_row = 1"); + + // Allocate result registers + let result_reg = self.allocate_registers(projection_items.len()); + + // Start sorter iteration + let sorter_loop_start = self.program.len(); + + // Get row from sorter using SorterData + self.emit( + OpCode::SorterData, + sorter_id, + row_data_reg, + total_sorter_cols as i64, + None, + "Load sorter row", + ); + + // Check if first row - if so, skip group change check + let first_row_jump_addr = self.program.len(); + self.emit( + OpCode::IfPos, + first_row_reg, + 0, + 0, + None, + "Skip compare if first row", + ); + + // Compare first group column with saved group key + let cmp_reg = self.allocate_register(); + self.emit( + OpCode::Ne, + row_data_reg, + group_key_reg, + cmp_reg, + None, + "Compare group key", + ); + + // If group NOT changed (cmp_reg is 0), skip output and go to step aggregates + let skip_output_addr = self.program.len(); + self.emit( + OpCode::IfZ, + cmp_reg, + 0, + 0, + None, + "Skip output if same group", + ); + + // Group changed - output previous group + // Build result from group key and aggregates + let mut agg_idx = 0; + for (i, item) in projection_items.iter().enumerate() { + match item { + MultiTableProjectionItem::Column(_, _) => { + // Find the position in group_col_refs + let group_idx = group_col_refs + .iter() + .position(|g| { + if let MultiTableProjectionItem::Column(t, c) = item { + g.0 == *t && g.1 == *c + } else { + false + } + }) + .unwrap_or(0); + self.emit( + OpCode::Copy, + group_key_reg + group_idx as i64, + result_reg + i as i64, + 0, + None, + "", + ); + } + MultiTableProjectionItem::Aggregate(_, _) => { + self.emit( + OpCode::AggFinal, + acc_base_reg + agg_idx as i64, + result_reg + i as i64, + 0, + None, + "", + ); + agg_idx += 1; + } + } + } + + // Output result row + self.emit( + OpCode::ResultRow, + result_reg, + projection_items.len() as i64, + 0, + None, + "Output group", + ); + + // Reset accumulators for new group + for i in 0..num_aggs { + self.emit( + OpCode::AggReset, + acc_base_reg + i as i64, + 0, + 0, + None, + "Reset accumulator", + ); + } + + // === Update group key section (first row jumps here) === + let update_group_key_addr = self.program.len(); + + // Patch first row jump + if let Some(inst) = self.program.instructions.get_mut(first_row_jump_addr) { + inst.p2 = update_group_key_addr as i64; + } + + // Initialize/update group key from current row + for i in 0..group_col_refs.len() { + self.emit( + OpCode::Copy, + row_data_reg + i as i64, + group_key_reg + i as i64, + 0, + None, + "", + ); + } + + // Clear first row flag + self.emit(OpCode::Integer, 0, first_row_reg, 0, None, "first_row = 0"); + + // === Step aggregates section (skip output jumps here) === + let step_agg_addr = self.program.len(); + + // Patch skip output jump + if let Some(inst) = self.program.instructions.get_mut(skip_output_addr) { + inst.p2 = step_agg_addr as i64; + } + + // Step aggregates for current row + agg_idx = 0; + for item in &projection_items { + if let MultiTableProjectionItem::Aggregate(func_type, col_ref) = item { + let val_reg = if let Some((agg_src_idx, _)) = agg_source_cols + .iter() + .enumerate() + .find(|(_, c)| Some(*c) == col_ref.as_ref()) + { + // Value is in row_data_reg at offset group_col_refs.len() + agg_src_idx + row_data_reg + group_col_refs.len() as i64 + agg_src_idx as i64 + } else { + // COUNT(*) - use 1 + let r = self.allocate_register(); + self.emit(OpCode::Integer, 1, r, 0, None, "COUNT(*) = 1"); + r + }; + + self.emit( + OpCode::AggStep, + *func_type, + val_reg, + acc_base_reg + agg_idx as i64, + None, + "", + ); + agg_idx += 1; + } + } + + // Next sorted row + self.emit( + OpCode::SorterNext, + sorter_id, + sorter_loop_start as i64, + 0, + None, + "Next sorter row", + ); + + // Output final group (if first_row_reg is 0, we have data) + // Skip if first_row_reg is still 1 (no data) + let skip_final_addr = self.program.len(); + self.emit( + OpCode::IfPos, + first_row_reg, + 0, + 0, + None, + "Skip final if no data", + ); + + // Build final result + agg_idx = 0; + for (i, item) in projection_items.iter().enumerate() { + match item { + MultiTableProjectionItem::Column(_, _) => { + let group_idx = group_col_refs + .iter() + .position(|g| { + if let MultiTableProjectionItem::Column(t, c) = item { + g.0 == *t && g.1 == *c + } else { + false + } + }) + .unwrap_or(0); + self.emit( + OpCode::Copy, + group_key_reg + group_idx as i64, + result_reg + i as i64, + 0, + None, + "", + ); + } + MultiTableProjectionItem::Aggregate(_, _) => { + self.emit( + OpCode::AggFinal, + acc_base_reg + agg_idx as i64, + result_reg + i as i64, + 0, + None, + "", + ); + agg_idx += 1; + } + } + } + + // Output final result row + self.emit( + OpCode::ResultRow, + result_reg, + projection_items.len() as i64, + 0, + None, + "Output final group", + ); + + // Patch skip final + let end_addr = self.program.len(); + if let Some(inst) = self.program.instructions.get_mut(skip_final_addr) { + inst.p2 = end_addr as i64; + } + + // Handle LIMIT/OFFSET from query (not implemented yet for GROUP BY) + let _ = query; + + Ok(()) + } + + /// Compile an implicit join (comma-separated tables in FROM with WHERE filter) + /// e.g., SELECT * FROM users, orders WHERE users.id = orders.user_id + /// Supports 2 or more tables with nested loop joins + /// Compile an implicit join with optional LIMIT/OFFSET support + pub(crate) fn compile_implicit_join_with_query( + &mut self, + select: &Select, + query: &Query, + ) -> SqawkResult<()> { + // Extract LIMIT/OFFSET + let (limit_val, offset_val) = self.extract_limit_offset(query)?; + + // Handle LIMIT 0 optimization + if limit_val == Some(0) { + // LIMIT 0: output empty result + if let Ok(schema) = self.build_implicit_join_schema(select) { + self.program.set_result_schema(schema); + } + return Ok(()); + } + + self.compile_implicit_join_core(select, limit_val, offset_val) + } + + pub(crate) fn compile_implicit_join(&mut self, select: &Select) -> SqawkResult<()> { + self.compile_implicit_join_core(select, None, 0) + } + + fn compile_implicit_join_core( + &mut self, + select: &Select, + limit_val: Option, + offset_val: i64, + ) -> SqawkResult<()> { + let num_tables = select.from.len(); + if num_tables < 2 { + return Err(SqawkError::UnsupportedSqlFeature( + "Implicit join requires at least 2 tables".into(), + )); + } + + // Collect all table names and aliases + // table_names: actual table names for database lookups + // table_refs: aliases (if provided) or table names - used for column resolution + let mut table_names: Vec = Vec::with_capacity(num_tables); + let mut table_refs: Vec = Vec::with_capacity(num_tables); + for from_item in &select.from { + let (table_name, table_alias) = match &from_item.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let talias = alias.as_ref().map(|a| a.name.value.clone()); + (tname, talias) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table references supported in implicit join".into(), + )) + } + }; + if !self.database.has_table(&table_name) { + return Err(SqawkError::TableNotFound(table_name)); + } + // Use alias for column resolution if provided, otherwise use table name + table_refs.push(table_alias.unwrap_or_else(|| table_name.clone())); + table_names.push(table_name); + } + + // Get references to all tables + let tables: Vec<&Table> = table_names + .iter() + .map(|name| self.database.get_table(name)) + .collect::, _>>()?; + + // Resolve projection for all tables - determine which columns to output + // Use table_refs (aliases) for column resolution + let (table_col_indices, output_col_refs, schema, is_wildcard) = + self.resolve_multi_table_projection(&select.projection, &tables, &table_refs)?; + + // Set result schema + self.program.set_result_schema(schema); + + // Open all tables + for (i, table_name) in table_names.iter().enumerate() { + self.emit( + OpCode::OpenRead, + i as i64, + (i + 1) as i64, + 0, + Some(table_name.to_string()), + &format!("Open table {} for reading", table_name), + ); + } + + // Set up LIMIT counter register if needed + let limit_reg = if let Some(limit) = limit_val { + let reg = self.allocate_register(); + self.emit( + OpCode::Integer, + limit, + reg, + 0, + None, + &format!("r[{}] = {} (LIMIT counter)", reg, limit), + ); + Some(reg) + } else { + None + }; + + // Set up OFFSET counter register if needed + let offset_reg = if offset_val > 0 { + let reg = self.allocate_register(); + self.emit( + OpCode::Integer, + offset_val, + reg, + 0, + None, + &format!("r[{}] = {} (OFFSET counter)", reg, offset_val), + ); + Some(reg) + } else { + None + }; + + // Track loop start/end placeholders for patching + let mut loop_end_placeholders: Vec = Vec::with_capacity(num_tables); + let mut loop_starts: Vec = Vec::with_capacity(num_tables); + + // For SELECT *, allocate registers for each table's columns + // For explicit projection, we'll allocate output registers later + let mut table_start_regs: Vec = Vec::with_capacity(num_tables); + if is_wildcard { + for cols in &table_col_indices { + let start_reg = self.allocate_registers(cols.len().max(1)); + table_start_regs.push(start_reg); + } + } + + // Create nested loops for each table + for i in 0..num_tables { + let rewind_opcode = if i == 0 { + OpCode::Rewind + } else { + OpCode::RewindInner + }; + + // Rewind cursor + let loop_end_placeholder = self.program.len(); + loop_end_placeholders.push(loop_end_placeholder); + self.emit( + rewind_opcode, + i as i64, + 0, // Placeholder - will be patched + 0, + None, + &format!("Rewind cursor {} ({})", i, table_names[i]), + ); + + // Loop start + let loop_start = self.program.len(); + loop_starts.push(loop_start); + + // For SELECT *, load columns for this table inside the loop + if is_wildcard { + for (col_i, col_idx) in table_col_indices[i].iter().enumerate() { + self.emit( + OpCode::Column, + i as i64, + *col_idx as i64, + table_start_regs[i] + col_i as i64, + None, + &format!( + "r[{}] = {}.col[{}]", + table_start_regs[i] + col_i as i64, + table_names[i], + col_idx + ), + ); + } + } + } + + // Compile WHERE condition to filter rows at the innermost loop + let skip_output_addr = if let Some(where_expr) = &select.selection { + if self.verbose { + eprintln!("WHERE comparison: {:?}", where_expr); + } + // Use compile_multi_table_condition to handle table.column references + // Use table_refs (aliases) for column resolution + let condition_reg = + self.compile_multi_table_condition(where_expr, &tables, &table_refs)?; + + // Jump to next iteration if condition is false (0) + let skip_addr = self.program.len(); + self.emit( + OpCode::IfZ, + condition_reg, + 0, // Placeholder - will be patched + 0, + None, + "Skip row if WHERE condition is false", + ); + Some(skip_addr) + } else { + None + }; + + // Handle OFFSET - skip rows until offset is exhausted + let offset_skip_addr = if let Some(off_reg) = offset_reg { + let addr = self.program.len(); + self.emit( + OpCode::DecrJumpZero, + off_reg, + 0, // Placeholder - will jump to Next + 0, + None, + "Decrement OFFSET counter, skip if not yet at offset", + ); + Some(addr) + } else { + None + }; + + // Output combined row (only if WHERE passed and OFFSET exhausted) + if is_wildcard { + // For SELECT *, columns are already in consecutive registers + let total_cols: usize = table_col_indices.iter().map(|c| c.len()).sum(); + self.emit( + OpCode::ResultRow, + table_start_regs[0], + total_cols as i64, + 0, + None, + "Output implicit join row", + ); + } else { + // For explicit projection, load columns in SELECT order into output registers + let output_start_reg = self.allocate_registers(output_col_refs.len()); + + // Load each column in SELECT order + for (out_i, (tbl_idx, col_idx)) in output_col_refs.iter().enumerate() { + self.emit( + OpCode::Column, + *tbl_idx as i64, + *col_idx as i64, + output_start_reg + out_i as i64, + None, + &format!( + "r[{}] = {}.col[{}]", + output_start_reg + out_i as i64, + table_names[*tbl_idx], + col_idx + ), + ); + } + + self.emit( + OpCode::ResultRow, + output_start_reg, + output_col_refs.len() as i64, + 0, + None, + "Output implicit join row", + ); + } + + // Handle LIMIT - decrement counter and jump to end if exhausted + let limit_done_addr = if let Some(lim_reg) = limit_reg { + let addr = self.program.len(); + self.emit( + OpCode::DecrJumpZero, + lim_reg, + 0, // Placeholder - will jump to Close cursors + 0, + None, + "Decrement LIMIT counter, jump to end if done", + ); + Some(addr) + } else { + None + }; + + // Patch skip address to jump to innermost Next instruction + let innermost_next_addr = self.program.len(); + + // Patch OFFSET skip address to jump to innermost Next + if let Some(off_addr) = offset_skip_addr { + if let Some(inst) = self.program.instructions.get_mut(off_addr) { + inst.p2 = innermost_next_addr as i64; + } + } + if let Some(skip_addr) = skip_output_addr { + if let Some(inst) = self.program.instructions.get_mut(skip_addr) { + inst.p2 = innermost_next_addr as i64; + } + } + + // Close loops in reverse order (innermost first) + let mut loop_ends: Vec = Vec::with_capacity(num_tables); + for i in (0..num_tables).rev() { + // Next for this cursor + self.emit( + OpCode::Next, + i as i64, + loop_starts[i] as i64, + 0, + None, + &format!("Next row from {}", table_names[i]), + ); + loop_ends.push(self.program.len()); + } + loop_ends.reverse(); // Restore to table order for patching + + // Close all cursors - this is where LIMIT jumps to when done + let close_cursors_addr = self.program.len(); + for i in 0..num_tables { + self.emit( + OpCode::Close, + i as i64, + 0, + 0, + None, + &format!("Close cursor {}", i), + ); + } + + // Patch jump addresses for loop ends + for (i, loop_end_placeholder) in loop_end_placeholders.iter().enumerate() { + if let Some(inst) = self.program.instructions.get_mut(*loop_end_placeholder) { + inst.p2 = loop_ends[i] as i64; + } + } + + // Patch LIMIT done address to jump to Close cursors + if let Some(lim_addr) = limit_done_addr { + if let Some(inst) = self.program.instructions.get_mut(lim_addr) { + inst.p2 = close_cursors_addr as i64; + } + } + + Ok(()) + } + + /// Build schema for implicit join without executing (for LIMIT 0 optimization) + fn build_implicit_join_schema(&self, select: &Select) -> SqawkResult { + let mut table_names: Vec = Vec::new(); + let mut table_refs: Vec = Vec::new(); + + for from_item in &select.from { + let (table_name, table_alias) = match &from_item.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let talias = alias.as_ref().map(|a| a.name.value.clone()); + (tname, talias) + } + _ => continue, + }; + table_refs.push(table_alias.unwrap_or_else(|| table_name.clone())); + table_names.push(table_name); + } + + let tables: Vec<&Table> = table_names + .iter() + .filter_map(|name| self.database.get_table(name).ok()) + .collect(); + + if tables.is_empty() { + return Ok(ResultSchema::new()); + } + + let (_, _, schema, _) = + self.resolve_multi_table_projection(&select.projection, &tables, &table_refs)?; + Ok(schema) + } + + /// Resolve projection for multiple tables + /// Returns (table_cols_for_loading, output_column_refs, schema, is_wildcard) + /// - table_cols_for_loading: columns to load per table (for SELECT *) + /// - output_column_refs: (table_idx, col_idx) pairs in SELECT order (for explicit projection) + fn resolve_multi_table_projection( + &self, + projection: &[SelectItem], + tables: &[&Table], + table_names: &[String], + ) -> SqawkResult { + let num_tables = tables.len(); + let name_refs: Vec<&str> = table_names.iter().map(|s| s.as_str()).collect(); + + // Check for SELECT * + for item in projection { + if matches!(item, SelectItem::Wildcard(_)) { + let (table_cols, schema) = Self::build_wildcard_schema(tables, &name_refs); + return Ok((table_cols, Vec::new(), schema, true)); + } + } + + // Handle explicit column selection - track in SELECT order + let mut output_cols: Vec<(usize, usize)> = Vec::new(); + let mut schema = ResultSchema::new(); + + for item in projection { + let (tbl_name, col_name, output_name) = match item { + SelectItem::UnnamedExpr(expr) => { + let (t, c) = self.extract_join_column_ref(expr)?; + let out = format!("{}.{}", t, c); + (t, c, out) + } + SelectItem::ExprWithAlias { expr, alias } => { + let (t, c) = self.extract_join_column_ref(expr)?; + (t, c, alias.value.clone()) + } + SelectItem::Wildcard(_) => unreachable!("Handled above"), + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Unsupported projection item in multi-table JOIN".into(), + )); + } + }; + + let (tbl_idx, col_idx) = + Self::find_column_in_tables(&tbl_name, &col_name, tables, &name_refs)?; + output_cols.push((tbl_idx, col_idx)); + schema.add_column( + output_name, + tables[tbl_idx].column_metadata()[col_idx].data_type, + ); + } + + Ok((vec![Vec::new(); num_tables], output_cols, schema, false)) + } + + /// Compile a WHERE condition for multi-table joins + fn compile_multi_table_condition( + &mut self, + expr: &Expr, + tables: &[&Table], + table_names: &[String], + ) -> SqawkResult { + match expr { + Expr::BinaryOp { left, op, right } => { + match op { + BinaryOperator::And => { + let left_reg = + self.compile_multi_table_condition(left, tables, table_names)?; + let right_reg = + self.compile_multi_table_condition(right, tables, table_names)?; + Ok(self.emit_and(left_reg, right_reg)) + } + BinaryOperator::Or => { + let left_reg = + self.compile_multi_table_condition(left, tables, table_names)?; + let right_reg = + self.compile_multi_table_condition(right, tables, table_names)?; + Ok(self.emit_or(left_reg, right_reg)) + } + // Comparison operators + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq => { + let left_reg = self.allocate_register(); + let right_reg = self.allocate_register(); + self.compile_multi_table_operand(left, tables, table_names, left_reg)?; + self.compile_multi_table_operand(right, tables, table_names, right_reg)?; + self.emit_comparison(op, left_reg, right_reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported binary operator in multi-table WHERE: {:?}", + op + ))), + } + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression type in multi-table WHERE: {:?}", + expr + ))), + } + } + + /// Compile an operand for a multi-table condition (column ref or literal) + fn compile_multi_table_operand( + &mut self, + expr: &Expr, + tables: &[&Table], + table_names: &[String], + target_reg: i64, + ) -> SqawkResult<()> { + match expr { + Expr::CompoundIdentifier(parts) => { + if parts.len() == 2 { + let tbl_name = &parts[0].value; + let col_name = &parts[1].value; + + // Find which table this column belongs to + for (i, table_name) in table_names.iter().enumerate() { + if tbl_name.eq_ignore_ascii_case(table_name) { + let columns = tables[i].columns(); + if let Some(col_idx) = columns + .iter() + .position(|c| c.eq_ignore_ascii_case(col_name)) + { + self.emit( + OpCode::Column, + i as i64, + col_idx as i64, + target_reg, + None, + &format!("r[{}] = {}.{}", target_reg, table_name, col_name), + ); + return Ok(()); + } else { + return Err(SqawkError::ColumnNotFound(col_name.clone())); + } + } + } + Err(SqawkError::TableNotFound(tbl_name.clone())) + } else { + Err(SqawkError::UnsupportedSqlFeature(format!( + "Compound identifier with {} parts not supported", + parts.len() + ))) + } + } + Expr::Value(value) => { + match value { + Value::Number(num, _) => { + if let Ok(int_val) = num.parse::() { + self.emit( + OpCode::Integer, + int_val, + target_reg, + 0, + None, + &format!("r[{}] = {}", target_reg, int_val), + ); + } else { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Non-integer literal: {}", + num + ))); + } + } + Value::SingleQuotedString(s) => { + self.emit( + OpCode::String, + 0, + target_reg, + 0, + Some(s.clone()), + &format!("r[{}] = '{}'", target_reg, s), + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported literal type: {:?}", + value + ))); + } + } + Ok(()) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported operand type in multi-table condition: {:?}", + expr + ))), + } + } + /// Compile a multi-table JOIN (3+ tables) + pub(crate) fn compile_multi_join( + &mut self, + table_with_joins: &TableWithJoins, + projection: &[SelectItem], + where_clause: &Option, + ) -> SqawkResult<()> { + // Collect all table names and aliases + let (first_table_name, first_table_ref) = match &table_with_joins.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let tref = alias + .as_ref() + .map(|a| a.name.value.clone()) + .unwrap_or_else(|| tname.clone()); + (tname, tref) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported in multi-join".into(), + )) + } + }; + + let mut table_names = vec![first_table_name]; + let mut table_refs = vec![first_table_ref]; + let mut join_conditions: Vec<&Expr> = Vec::new(); + + for join in &table_with_joins.joins { + let (table_name, table_ref) = match &join.relation { + sqlparser::ast::TableFactor::Table { name, alias, .. } => { + let tname = self.get_table_name(name)?; + let tref = alias + .as_ref() + .map(|a| a.name.value.clone()) + .unwrap_or_else(|| tname.clone()); + (tname, tref) + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only simple table scans supported in multi-join".into(), + )) + } + }; + + // Only support INNER JOIN for multi-table joins + match &join.join_operator { + JoinOperator::Inner(JoinConstraint::On(expr)) => { + join_conditions.push(expr); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Only INNER JOIN with ON clause supported in multi-table joins".into(), + )) + } + } + + table_names.push(table_name); + table_refs.push(table_ref); + } + + let num_tables = table_names.len(); + self.add_comment(&format!( + "Multi-table INNER JOIN: {}", + table_names.join(" x ") + )); + + // Verify all tables exist and collect table references + for name in &table_names { + if !self.database.has_table(name) { + return Err(SqawkError::TableNotFound(name.clone())); + } + } + + // Get table references for projection resolution + let tables: Vec<&Table> = table_names + .iter() + .map(|name| self.database.get_table(name)) + .collect::, _>>()?; + + // Resolve projection to determine output columns + // Use table_refs (aliases) for column resolution + let (_table_col_indices, output_col_refs, schema, is_wildcard) = + self.resolve_multi_table_projection(projection, &tables, &table_refs)?; + + self.program.set_result_schema(schema); + + // Collect column counts for each table (always load all columns for JOIN condition evaluation) + let col_counts: Vec = tables.iter().map(|t| t.column_count()).collect(); + + // Open all tables + for (i, name) in table_names.iter().enumerate() { + self.emit( + OpCode::OpenRead, + i as i64, + (i + 1) as i64, + 0, + Some(name.clone()), + &format!("Open table {} (cursor {})", name, i), + ); + } + + // Allocate registers for all table columns (needed for JOIN condition evaluation) + let mut table_start_regs = Vec::new(); + for count in &col_counts { + let start_reg = self.allocate_registers(*count); + table_start_regs.push(start_reg); + } + + // Track loop starts and placeholders for jump patching + let mut loop_starts = Vec::new(); + let mut loop_end_placeholders = Vec::new(); + + // Create nested loops for all tables + for i in 0..num_tables { + // Rewind cursor + let placeholder = self.program.len(); + let opcode = if i == 0 { + OpCode::Rewind + } else { + OpCode::RewindInner + }; + self.emit( + opcode, + i as i64, + 0, // Placeholder + 0, + None, + &format!("Rewind cursor {} ({})", i, table_names[i]), + ); + loop_end_placeholders.push(placeholder); + + // Loop start + loop_starts.push(self.program.len()); + + // Load ALL columns for this table (needed for JOIN condition evaluation) + for col_idx in 0..col_counts[i] { + self.emit( + OpCode::Column, + i as i64, + col_idx as i64, + table_start_regs[i] + col_idx as i64, + None, + &format!( + "r[{}] = {}.col[{}]", + table_start_regs[i] + col_idx as i64, + table_names[i], + col_idx + ), + ); + } + } + + // Compile and evaluate all join conditions + // We'll use placeholders and patch jump addresses after we know the final layout + let mut skip_to_next_placeholders: Vec = Vec::new(); + + for (i, cond) in join_conditions.iter().enumerate() { + // Use table_refs (aliases) for column resolution + let cond_reg = + self.compile_multi_join_condition(cond, &table_refs, &table_start_regs)?; + + // If condition fails, skip to next innermost iteration (placeholder) + let skip_addr = self.program.len(); + skip_to_next_placeholders.push(skip_addr); + self.emit( + OpCode::IfZ, + cond_reg, + 0, // Placeholder - will be patched + 0, + None, + &format!("Skip if JOIN condition {} fails", i + 1), + ); + } + + // Compile WHERE clause if present (additional filtering on top of JOIN conditions) + if let Some(where_expr) = where_clause { + if self.verbose { + eprintln!("Compiling WHERE clause for multi-join: {:?}", where_expr); + } + // Use table_refs (aliases) for column resolution + let where_reg = self.compile_multi_table_condition(where_expr, &tables, &table_refs)?; + + // If WHERE fails, skip to next innermost iteration (placeholder) + let skip_addr = self.program.len(); + skip_to_next_placeholders.push(skip_addr); + self.emit( + OpCode::IfZ, + where_reg, + 0, // Placeholder - will be patched + 0, + None, + "Skip if WHERE condition fails", + ); + } + + // Output row based on projection type + if is_wildcard { + // SELECT * - output all columns + let total_cols: usize = col_counts.iter().sum(); + self.emit( + OpCode::ResultRow, + table_start_regs[0], + total_cols as i64, + 0, + None, + "Output multi-join result row", + ); + } else { + // Explicit projection - load columns in SELECT order into output registers + let output_start_reg = self.allocate_registers(output_col_refs.len()); + + // Load each column in SELECT order + for (out_i, (tbl_idx, col_idx)) in output_col_refs.iter().enumerate() { + // Use the already-loaded register values + let src_reg = table_start_regs[*tbl_idx] + *col_idx as i64; + self.emit( + OpCode::Copy, + src_reg, + output_start_reg + out_i as i64, + 0, + None, + &format!( + "r[{}] = r[{}] ({}.col[{}])", + output_start_reg + out_i as i64, + src_reg, + table_names[*tbl_idx], + col_idx + ), + ); + } + + self.emit( + OpCode::ResultRow, + output_start_reg, + output_col_refs.len() as i64, + 0, + None, + "Output multi-join result row", + ); + } + + // Patch all skip placeholders to jump to the innermost Next (which is next) + let skip_target = self.program.len(); + for placeholder in &skip_to_next_placeholders { + if let Some(inst) = self.program.instructions.get_mut(*placeholder) { + inst.p2 = skip_target as i64; + } + } + + // Close nested loops (innermost first) + let mut loop_ends = Vec::new(); + for i in (0..num_tables).rev() { + self.emit( + OpCode::Next, + i as i64, + loop_starts[i] as i64, + 0, + None, + &format!("Next row from {}", table_names[i]), + ); + loop_ends.push(self.program.len()); + } + loop_ends.reverse(); + + // Close all cursors + for i in 0..num_tables { + self.emit( + OpCode::Close, + i as i64, + 0, + 0, + None, + &format!("Close cursor {}", i), + ); + } + + let final_end = self.program.len(); + + // Patch jump addresses + for (i, placeholder) in loop_end_placeholders.iter().enumerate() { + if let Some(inst) = self.program.instructions.get_mut(*placeholder) { + // Each rewind jumps to just after its corresponding Next (i.e., the next outer loop's end) + // The outermost loop jumps to final_end + if i == 0 { + inst.p2 = final_end as i64; + } else { + inst.p2 = loop_ends[i - 1] as i64; + } + } + } + + Ok(()) + } + + /// Compile a join condition for multi-table joins + fn compile_multi_join_condition( + &mut self, + expr: &Expr, + table_names: &[String], + table_start_regs: &[i64], + ) -> SqawkResult { + match expr { + Expr::BinaryOp { left, op, right } => { + let left_reg = + self.compile_multi_join_operand(left, table_names, table_start_regs)?; + let right_reg = + self.compile_multi_join_operand(right, table_names, table_start_regs)?; + self.emit_comparison(op, left_reg, right_reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported multi-join condition: {:?}", + expr + ))), + } + } + + /// Compile an operand for multi-table join conditions + fn compile_multi_join_operand( + &mut self, + expr: &Expr, + table_names: &[String], + table_start_regs: &[i64], + ) -> SqawkResult { + match expr { + Expr::CompoundIdentifier(parts) => { + if parts.len() == 2 { + let table_alias = parts[0].value.to_lowercase(); + let col_name = parts[1].value.to_lowercase(); + + // Find which table this refers to + for (i, name) in table_names.iter().enumerate() { + if name.to_lowercase() == table_alias { + let table = self.database.get_table(name)?; + if let Some(col_idx) = table.column_index(&col_name) { + // The column value should already be loaded in register + let reg = table_start_regs[i] + col_idx as i64; + return Ok(reg); + } + } + } + } + Err(SqawkError::UnsupportedSqlFeature(format!( + "Could not resolve column reference: {:?}", + expr + ))) + } + Expr::Identifier(ident) => { + // Unqualified column name - search all tables + let col_name = ident.value.to_lowercase(); + for (i, name) in table_names.iter().enumerate() { + let table = self.database.get_table(name)?; + if let Some(col_idx) = table.column_index(&col_name) { + let reg = table_start_regs[i] + col_idx as i64; + return Ok(reg); + } + } + Err(SqawkError::UnsupportedSqlFeature(format!( + "Could not find column: {}", + col_name + ))) + } + Expr::Value(val) => { + let reg = self.allocate_register(); + match val { + sqlparser::ast::Value::Number(n, _) => { + if let Ok(i) = n.parse::() { + self.emit(OpCode::Integer, i, reg, 0, None, "Load constant"); + } else { + return Err(SqawkError::UnsupportedSqlFeature( + "Only integer constants supported".into(), + )); + } + } + sqlparser::ast::Value::SingleQuotedString(s) => { + self.emit( + OpCode::String, + 0, + reg, + 0, + Some(s.clone()), + "Load string constant", + ); + } + _ => { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported value type: {:?}", + val + ))) + } + } + Ok(reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported operand in multi-join: {:?}", + expr + ))), + } + } + + /// Resolve projection for join - returns (left_col_indices, right_col_indices, schema) + fn resolve_join_projection( + &self, + projection: &[SelectItem], + left_table: &Table, + right_table: &Table, + left_table_name: &str, + right_table_name: &str, + ) -> SqawkResult<(Vec, Vec, ResultSchema)> { + let tables: Vec<&Table> = vec![left_table, right_table]; + let table_names: Vec<&str> = vec![left_table_name, right_table_name]; + + // For SELECT *, return all columns from both tables + for item in projection { + if matches!(item, SelectItem::Wildcard(_)) { + let (table_cols, schema) = Self::build_wildcard_schema(&tables, &table_names); + return Ok((table_cols[0].clone(), table_cols[1].clone(), schema)); + } + } + + // Handle explicit column selection + let mut left_cols = Vec::new(); + let mut right_cols = Vec::new(); + let mut schema = ResultSchema::new(); + + for item in projection { + let (tbl_name, col_name, output_name) = match item { + SelectItem::UnnamedExpr(expr) => { + let (t, c) = self.extract_join_column_ref(expr)?; + let out = format!("{}.{}", t, c); + (t, c, out) + } + SelectItem::ExprWithAlias { expr, alias } => { + let (t, c) = self.extract_join_column_ref(expr)?; + (t, c, alias.value.clone()) + } + SelectItem::Wildcard(_) => unreachable!("Handled above"), + _ => { + return Err(SqawkError::UnsupportedSqlFeature( + "Unsupported projection item in JOIN".into(), + )); + } + }; + + let (tbl_idx, col_idx) = + Self::find_column_in_tables(&tbl_name, &col_name, &tables, &table_names)?; + if tbl_idx == 0 { + left_cols.push(col_idx); + } else { + right_cols.push(col_idx); + } + schema.add_column( + output_name, + tables[tbl_idx].column_metadata()[col_idx].data_type, + ); + } + + Ok((left_cols, right_cols, schema)) + } + + /// Extract table and column name from a compound identifier (e.g., users.name) + fn extract_join_column_ref(&self, expr: &Expr) -> SqawkResult<(String, String)> { + match expr { + Expr::CompoundIdentifier(parts) => { + if parts.len() == 2 { + Ok((parts[0].value.clone(), parts[1].value.clone())) + } else { + Err(SqawkError::UnsupportedSqlFeature(format!( + "Expected table.column reference, got {} parts", + parts.len() + ))) + } + } + Expr::Identifier(ident) => { + // Unqualified column - would need to search both tables + // For now, return an error asking for qualified names + Err(SqawkError::UnsupportedSqlFeature(format!( + "Column '{}' must be qualified with table name in JOIN (e.g., table.{})", + ident.value, ident.value + ))) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported expression in JOIN projection: {:?}", + expr + ))), + } + } + + /// Compile a join condition expression + fn compile_join_condition( + &mut self, + expr: &Expr, + left_table: &Table, + right_table: &Table, + left_cursor: usize, + right_cursor: usize, + ) -> SqawkResult { + // Use table names as references (no aliases) + let left_ref = left_table.name().to_string(); + let right_ref = right_table.name().to_string(); + self.compile_join_condition_with_refs( + expr, + left_table, + right_table, + left_cursor, + right_cursor, + &left_ref, + &right_ref, + ) + } + + #[allow(clippy::too_many_arguments)] + fn compile_join_condition_with_refs( + &mut self, + expr: &Expr, + left_table: &Table, + right_table: &Table, + left_cursor: usize, + right_cursor: usize, + left_ref: &str, + right_ref: &str, + ) -> SqawkResult { + // Handle a.col = b.col style conditions + match expr { + Expr::BinaryOp { left, op, right } => { + let left_reg = self.compile_join_operand_with_refs( + left, + left_table, + right_table, + left_cursor, + right_cursor, + left_ref, + right_ref, + )?; + let right_reg = self.compile_join_operand_with_refs( + right, + left_table, + right_table, + left_cursor, + right_cursor, + left_ref, + right_ref, + )?; + self.emit_comparison(op, left_reg, right_reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported join condition expression: {:?}", + expr + ))), + } + } + + /// Compile a join operand with explicit table references (aliases or names) + #[allow(clippy::too_many_arguments)] + fn compile_join_operand_with_refs( + &mut self, + expr: &Expr, + left_table: &Table, + right_table: &Table, + left_cursor: usize, + right_cursor: usize, + left_ref: &str, + right_ref: &str, + ) -> SqawkResult { + match expr { + Expr::CompoundIdentifier(parts) => { + // Handle table.column syntax + if parts.len() == 2 { + let table_alias = parts[0].value.to_lowercase(); + let col_name = parts[1].value.to_lowercase(); + + // Try to find column in left or right table + // Check against table references (which may be aliases) + let left_ref_lower = left_ref.to_lowercase(); + let right_ref_lower = right_ref.to_lowercase(); + + if table_alias == left_ref_lower { + if let Some(col_idx) = left_table.column_index(&col_name) { + let reg = self.allocate_register(); + self.emit( + OpCode::Column, + left_cursor as i64, + col_idx as i64, + reg, + None, + &format!("Load {}.{} for join", table_alias, col_name), + ); + return Ok(reg); + } + } else if table_alias == right_ref_lower { + if let Some(col_idx) = right_table.column_index(&col_name) { + let reg = self.allocate_register(); + self.emit( + OpCode::Column, + right_cursor as i64, + col_idx as i64, + reg, + None, + &format!("Load {}.{} for join", table_alias, col_name), + ); + return Ok(reg); + } + } + + Err(SqawkError::UnsupportedSqlFeature(format!( + "Column {}.{} not found in join tables", + table_alias, col_name + ))) + } else { + Err(SqawkError::UnsupportedSqlFeature( + "Expected table.column in join condition".into(), + )) + } + } + Expr::Identifier(ident) => { + // Unqualified column name - try to find in either table + let col_name = ident.value.to_lowercase(); + + // Try left table first + if let Some(col_idx) = left_table.column_index(&col_name) { + let reg = self.allocate_register(); + self.emit( + OpCode::Column, + left_cursor as i64, + col_idx as i64, + reg, + None, + &format!("Load {} from left for join", col_name), + ); + return Ok(reg); + } + + // Try right table + if let Some(col_idx) = right_table.column_index(&col_name) { + let reg = self.allocate_register(); + self.emit( + OpCode::Column, + right_cursor as i64, + col_idx as i64, + reg, + None, + &format!("Load {} from right for join", col_name), + ); + return Ok(reg); + } + + Err(SqawkError::UnsupportedSqlFeature(format!( + "Column {} not found in either join table", + col_name + ))) + } + Expr::Value(val) => { + // Literal value in join condition + let reg = self.allocate_register(); + match val { + Value::Number(n, _) => { + if let Ok(i) = n.parse::() { + self.emit(OpCode::Integer, i, reg, 0, None, "Load literal for join"); + } + } + Value::SingleQuotedString(s) => { + self.emit( + OpCode::String, + 0, + reg, + 0, + Some(s.clone()), + "Load string literal for join", + ); + } + _ => {} + } + Ok(reg) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported join operand: {:?}", + expr + ))), + } + } + + /// Resolve a column expression to (table_idx, col_idx) for multi-table queries + fn resolve_multi_table_column( + &self, + expr: &Expr, + tables: &[&Table], + table_refs: &[String], + ) -> SqawkResult<(usize, usize)> { + match expr { + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let table_ref = &parts[0].value; + let col_name = &parts[1].value; + + // Find the table by reference (alias or name) + let tbl_idx = table_refs + .iter() + .position(|r| r.eq_ignore_ascii_case(table_ref)) + .ok_or_else(|| SqawkError::TableNotFound(table_ref.clone()))?; + + // Find the column in that table + let col_idx = tables[tbl_idx] + .column_index(col_name) + .ok_or_else(|| SqawkError::ColumnNotFound(col_name.clone()))?; + + Ok((tbl_idx, col_idx)) + } + Expr::Identifier(ident) => { + // Unqualified column - search all tables + let col_name = &ident.value; + for (tbl_idx, table) in tables.iter().enumerate() { + if let Some(col_idx) = table.column_index(col_name) { + return Ok((tbl_idx, col_idx)); + } + } + Err(SqawkError::ColumnNotFound(col_name.clone())) + } + _ => Err(SqawkError::UnsupportedSqlFeature(format!( + "Unsupported column expression in multi-table query: {:?}", + expr + ))), + } + } + + /// Build result schema for a multi-table aggregate query + fn build_multi_table_agg_schema( + &self, + projection: &[SelectItem], + tables: &[&Table], + table_refs: &[String], + ) -> ResultSchema { + let mut schema = ResultSchema::new(); + + for item in projection { + let (name, data_type) = match item { + SelectItem::UnnamedExpr(expr) => { + self.get_multi_table_expr_schema(expr, tables, table_refs) + } + SelectItem::ExprWithAlias { expr, alias } => { + let (_, dt) = self.get_multi_table_expr_schema(expr, tables, table_refs); + (alias.value.clone(), dt) + } + _ => ("?".to_string(), crate::table::DataType::Text), + }; + schema.add_column(name, data_type); + } + + schema + } + + /// Get schema info for a single expression in a multi-table context + fn get_multi_table_expr_schema( + &self, + expr: &Expr, + tables: &[&Table], + table_refs: &[String], + ) -> (String, crate::table::DataType) { + match expr { + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let table_ref = &parts[0].value; + let col_name = &parts[1].value; + + if let Some(tbl_idx) = table_refs + .iter() + .position(|r| r.eq_ignore_ascii_case(table_ref)) + { + if let Some(col_idx) = tables[tbl_idx].column_index(col_name) { + let meta = &tables[tbl_idx].column_metadata()[col_idx]; + return (format!("{}.{}", table_ref, col_name), meta.data_type); + } + } + ( + format!("{}.{}", table_ref, col_name), + crate::table::DataType::Text, + ) + } + Expr::Identifier(ident) => { + let col_name = &ident.value; + for table in tables.iter() { + if let Some(col_idx) = table.column_index(col_name) { + let meta = &table.column_metadata()[col_idx]; + return (col_name.clone(), meta.data_type); + } + } + (col_name.clone(), crate::table::DataType::Text) + } + Expr::Function(func) => { + let func_name = func.name.to_string().to_uppercase(); + // Aggregates typically return numeric types + let data_type = match func_name.as_str() { + "COUNT" => crate::table::DataType::Integer, + "SUM" | "AVG" => crate::table::DataType::Float, + _ => crate::table::DataType::Text, + }; + (func_name, data_type) + } + _ => ("expr".to_string(), crate::table::DataType::Text), + } + } +} diff --git a/src/vm/compiler_tests.rs b/src/vm/compiler_tests.rs new file mode 100644 index 0000000..b7a7e7b --- /dev/null +++ b/src/vm/compiler_tests.rs @@ -0,0 +1,369 @@ +//! Tests for SQL statement compiler +//! +//! Extracted from compiler.rs for maintainability. + +use crate::database::Database; +use crate::error::SqawkError; +use crate::table::Table; +use crate::vm::bytecode::OpCode; +use crate::vm::compiler::SqlCompiler; + +/// Create a test database with a simple table +fn create_test_database() -> Database { + let mut database = Database::new(); + let mut test_table = Table::new("users", vec![], None); + + // Add columns: id (INTEGER), age (INTEGER), name (TEXT) + test_table.add_column("id".to_string(), "INTEGER".to_string()); + test_table.add_column("age".to_string(), "INTEGER".to_string()); + test_table.add_column("name".to_string(), "TEXT".to_string()); + + // Add test data + test_table + .add_row(vec![ + crate::table::Value::Integer(1), + crate::table::Value::Integer(25), + crate::table::Value::String("Alice".to_string().into()), + ]) + .expect("Failed to add test row"); + + test_table + .add_row(vec![ + crate::table::Value::Integer(2), + crate::table::Value::Integer(35), + crate::table::Value::String("Bob".to_string().into()), + ]) + .expect("Failed to add test row"); + + database + .add_table("users".to_string(), test_table) + .expect("Failed to add table"); + database +} + +#[test] +fn test_compile_select_with_where_gt() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, false); // Use non-verbose mode for predictable output + + // Test: SELECT * FROM users WHERE age > 30 + let sql = "SELECT * FROM users WHERE age > 30"; + let program = compiler.compile(sql).expect("Compilation failed"); + + // Verify the generated bytecode contains the expected instructions + let instructions = &program.instructions; + + // Should have: Init, OpenRead, Rewind, Column (for all columns), + // Column (for age), Integer (30), Gt, IfZ, ResultRow, Next, Close, Halt + assert!(instructions.len() >= 8, "Expected at least 8 instructions"); + + // Find key instructions + let gt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Gt); + let ifz_found = instructions.iter().any(|inst| inst.opcode == OpCode::IfZ); + let integer_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::Integer && inst.p1 == 30); + let halt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Halt); + + assert!(gt_found, "Expected Gt comparison instruction"); + assert!(ifz_found, "Expected IfZ conditional jump instruction"); + assert!(integer_found, "Expected Integer instruction with value 30"); + assert!(halt_found, "Expected Halt instruction"); +} + +#[test] +fn test_compile_select_with_where_eq() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE age = 25 + let sql = "SELECT * FROM users WHERE age = 25"; + let program = compiler.compile(sql).expect("Compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Eq comparison and the correct literal value + let eq_found = instructions.iter().any(|inst| inst.opcode == OpCode::Eq); + let value_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::Integer && inst.p1 == 25); + + assert!(eq_found, "Expected Eq comparison instruction"); + assert!(value_found, "Expected Integer instruction with value 25"); +} + +#[test] +fn test_compile_select_with_where_string() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE name = 'Alice' + let sql = "SELECT * FROM users WHERE name = 'Alice'"; + let program = compiler.compile(sql).expect("Compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Eq comparison and String instruction + let eq_found = instructions.iter().any(|inst| inst.opcode == OpCode::Eq); + let string_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::String && inst.p4.as_deref() == Some("Alice")); + + assert!(eq_found, "Expected Eq comparison instruction"); + assert!( + string_found, + "Expected String instruction with value 'Alice'" + ); +} + +#[test] +fn test_compile_select_with_where_lt() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE age < 30 + let sql = "SELECT * FROM users WHERE age < 30"; + let program = compiler.compile(sql).expect("Compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Lt comparison + let lt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Lt); + assert!(lt_found, "Expected Lt comparison instruction"); +} + +#[test] +fn test_compile_select_without_where() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users (no WHERE clause) + let sql = "SELECT * FROM users"; + let program = compiler.compile(sql).expect("Compilation failed"); + + let instructions = &program.instructions; + + // Should NOT have comparison or conditional jump instructions + let comparison_found = instructions.iter().any(|inst| { + matches!( + inst.opcode, + OpCode::Gt | OpCode::Lt | OpCode::Eq | OpCode::Ne | OpCode::Ge | OpCode::Le + ) + }); + let ifz_found = instructions.iter().any(|inst| inst.opcode == OpCode::IfZ); + + assert!( + !comparison_found, + "Should not have comparison instructions without WHERE" + ); + assert!(!ifz_found, "Should not have conditional jump without WHERE"); +} + +#[test] +fn test_column_not_found_error() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, false); + + // Test: SELECT * FROM users WHERE invalid_column > 10 + let sql = "SELECT * FROM users WHERE invalid_column > 10"; + let result = compiler.compile(sql); + + // Should fail with column not found error + assert!(result.is_err(), "Expected compilation to fail"); + match result.unwrap_err() { + SqawkError::ColumnNotFound(col) => { + assert_eq!(col, "invalid_column"); + } + other => panic!("Expected ColumnNotFound error, got {:?}", other), + } +} + +#[test] +fn test_in_subquery_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, false); + + // Test IN (SELECT ...) subquery - now supported in Phase 4F + let sql = "SELECT * FROM users WHERE age IN (SELECT age FROM users)"; + let result = compiler.compile(sql); + + // Should compile successfully now + assert!( + result.is_ok(), + "Expected compilation to succeed: {:?}", + result.unwrap_err() + ); +} + +#[test] +fn test_between_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE age BETWEEN 20 AND 30 + let sql = "SELECT * FROM users WHERE age BETWEEN 20 AND 30"; + let program = compiler.compile(sql).expect("BETWEEN compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Ge and Le comparison instructions (for BETWEEN) + let ge_found = instructions.iter().any(|inst| inst.opcode == OpCode::Ge); + let le_found = instructions.iter().any(|inst| inst.opcode == OpCode::Le); + + assert!(ge_found, "Expected Ge comparison instruction for BETWEEN"); + assert!(le_found, "Expected Le comparison instruction for BETWEEN"); +} + +#[test] +fn test_in_list_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE age IN (25, 30, 35) + let sql = "SELECT * FROM users WHERE age IN (25, 30, 35)"; + let program = compiler.compile(sql).expect("IN list compilation failed"); + + let instructions = &program.instructions; + + // Verify we have multiple Eq comparison instructions (one for each list item) + let eq_count = instructions + .iter() + .filter(|inst| inst.opcode == OpCode::Eq) + .count(); + + assert!( + eq_count >= 3, + "Expected at least 3 Eq comparison instructions for IN list" + ); +} + +#[test] +fn test_like_pattern_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE name LIKE 'A%' + let sql = "SELECT * FROM users WHERE name LIKE 'A%'"; + let program = compiler.compile(sql).expect("LIKE compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Like instruction with the correct pattern + let like_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::Like && inst.p4.as_deref() == Some("A%")); + + assert!(like_found, "Expected Like instruction with pattern 'A%'"); +} + +#[test] +fn test_ilike_pattern_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: SELECT * FROM users WHERE name ILIKE 'alice' + let sql = "SELECT * FROM users WHERE name ILIKE 'alice'"; + let program = compiler.compile(sql).expect("ILIKE compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Like instruction with case-insensitive flag (bit 1 set = 2) + let ilike_found = instructions.iter().any(|inst| { + inst.opcode == OpCode::Like && inst.p2 == 2 // case_insensitive flag + }); + + assert!( + ilike_found, + "Expected Like instruction with case-insensitive flag" + ); +} + +#[test] +fn test_case_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: Simple CASE expression in WHERE clause + let sql = "SELECT * FROM users WHERE CASE WHEN age > 30 THEN 1 ELSE 0 END = 1"; + let program = compiler.compile(sql).expect("CASE compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Gt comparison for the WHEN condition + let gt_found = instructions.iter().any(|inst| inst.opcode == OpCode::Gt); + // Verify we have conditional jump (IfZ) for the CASE branching + let ifz_found = instructions.iter().any(|inst| inst.opcode == OpCode::IfZ); + // Verify we have Goto for jumping to the end + let goto_found = instructions.iter().any(|inst| inst.opcode == OpCode::Goto); + + assert!( + gt_found, + "Expected Gt comparison instruction for CASE WHEN condition" + ); + assert!( + ifz_found, + "Expected IfZ conditional jump instruction for CASE" + ); + assert!( + goto_found, + "Expected Goto instruction for CASE branch jumps" + ); +} + +#[test] +fn test_cast_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: CAST in WHERE clause + let sql = "SELECT * FROM users WHERE CAST(age AS TEXT) = '25'"; + let program = compiler.compile(sql).expect("CAST compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Cast instruction with TEXT type + let cast_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::Cast && inst.p4.as_deref() == Some("TEXT")); + + assert!(cast_found, "Expected Cast instruction with TEXT type"); +} + +#[test] +fn test_coalesce_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: COALESCE in WHERE clause + let sql = "SELECT * FROM users WHERE COALESCE(name, 'default') = 'Alice'"; + let program = compiler.compile(sql).expect("COALESCE compilation failed"); + + let instructions = &program.instructions; + + // Verify we have IsNull instruction for NULL checking + let is_null_found = instructions + .iter() + .any(|inst| inst.opcode == OpCode::IsNull); + + assert!(is_null_found, "Expected IsNull instruction for COALESCE"); +} + +#[test] +fn test_nullif_compilation() { + let database = create_test_database(); + let mut compiler = SqlCompiler::new(&database, true); + + // Test: NULLIF in WHERE clause + let sql = "SELECT * FROM users WHERE NULLIF(name, 'Alice') IS NULL"; + let program = compiler.compile(sql).expect("NULLIF compilation failed"); + + let instructions = &program.instructions; + + // Verify we have Eq instruction for NULLIF comparison + let eq_found = instructions.iter().any(|inst| inst.opcode == OpCode::Eq); + // Verify we have Null instruction for setting NULL result + let null_found = instructions.iter().any(|inst| inst.opcode == OpCode::Null); + + assert!(eq_found, "Expected Eq instruction for NULLIF comparison"); + assert!(null_found, "Expected Null instruction for NULLIF result"); +} diff --git a/src/vm/compiler_window.rs b/src/vm/compiler_window.rs new file mode 100644 index 0000000..dbcf100 --- /dev/null +++ b/src/vm/compiler_window.rs @@ -0,0 +1,637 @@ +//! Window function compilation +//! +//! This module extends SqlCompiler with window function compilation methods. + +use sqlparser::ast::{Expr, Select, SelectItem, Value, WindowType}; + +use super::bytecode::{OpCode, ResultSchema}; +use super::compiler::SqlCompiler; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::{DataType, Table}; + +impl<'a> SqlCompiler<'a> { + pub(crate) fn compile_select_with_window( + &mut self, + select: &Select, + table: &Table, + table_name: &str, + ) -> SqawkResult<()> { + let cursor_idx = 0i64; + let ephemeral_cursor = 1i64; + + self.add_comment("Window function query"); + + // Analyze the projection to find window functions and their specs + let (window_funcs, mut base_columns) = + self.analyze_window_projection(&select.projection, table)?; + + if window_funcs.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "No window functions found in projection".to_string(), + )); + } + + // Get partition and order columns from the first window function (source table indices) + // (All window functions should have the same OVER clause for now) + let (partition_cols, order_cols, order_asc) = + self.extract_window_spec(&window_funcs[0].1, table)?; + + // Add partition/order columns to base_columns if not already present + for &col_idx in &partition_cols { + if !base_columns.contains(&col_idx) { + base_columns.push(col_idx); + } + } + for &col_idx in &order_cols { + if !base_columns.contains(&col_idx) { + base_columns.push(col_idx); + } + } + + // Map source table indices to ephemeral cursor positions + let partition_positions: Vec = partition_cols + .iter() + .filter_map(|&src_idx| base_columns.iter().position(|&bc| bc == src_idx)) + .collect(); + let order_positions: Vec = order_cols + .iter() + .filter_map(|&src_idx| base_columns.iter().position(|&bc| bc == src_idx)) + .collect(); + + let total_col_count = base_columns.len(); + + // Build result schema + let schema = self.build_window_result_schema(&select.projection, table); + self.program.set_result_schema(schema); + + // Build sort spec: partition columns first, then order columns (using ephemeral positions) + let mut sort_spec_parts = Vec::new(); + for &pos in &partition_positions { + sort_spec_parts.push(format!("{}:asc", pos)); + } + for (i, &pos) in order_positions.iter().enumerate() { + let dir = if order_asc.get(i).copied().unwrap_or(true) { + "asc" + } else { + "desc" + }; + sort_spec_parts.push(format!("{}:{}", pos, dir)); + } + let sort_spec = sort_spec_parts.join(","); + + // Open ephemeral cursor for collecting rows + self.emit( + OpCode::OpenEphemeral, + ephemeral_cursor, + total_col_count as i64, + 0, + Some(sort_spec), + "Open ephemeral for window function", + ); + + // Open source table + self.emit( + OpCode::OpenRead, + cursor_idx, + 1, + 0, + Some(table_name.to_string()), + "", + ); + + // Rewind source table + let rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, cursor_idx, 0, 0, None, ""); + + let collect_loop_start = self.program.len(); + + // Allocate registers for base columns + let start_reg = self.allocate_registers(total_col_count); + + // Load all base columns + for (i, &col_idx) in base_columns.iter().enumerate() { + self.emit( + OpCode::Column, + cursor_idx, + col_idx as i64, + start_reg + i as i64, + None, + "", + ); + } + + // Apply WHERE filter if present + if let Some(where_expr) = &select.selection { + let cond_reg = self.compile_where_condition(where_expr, table, cursor_idx as usize)?; + self.emit( + OpCode::IfZ, + cond_reg, + (self.program.len() + 2) as i64, + 0, + None, + "", + ); + } + + // Insert row into ephemeral + self.emit( + OpCode::IdxInsert, + ephemeral_cursor, + start_reg, + total_col_count as i64, + None, + "", + ); + + // Next row in source + self.emit( + OpCode::Next, + cursor_idx, + collect_loop_start as i64, + 0, + None, + "", + ); + + let after_collect = self.program.len(); + + // Patch rewind + if let Some(inst) = self.program.instructions.get_mut(rewind_addr) { + inst.p2 = after_collect as i64; + } + + // Sort the ephemeral (by partition + order keys) + self.emit(OpCode::Sort, ephemeral_cursor, 0, 0, None, ""); + + let sort_jump_target = self.program.len(); + + // Rewind ephemeral for output pass + let output_rewind_addr = self.program.len(); + self.emit(OpCode::Rewind, ephemeral_cursor, 0, 0, None, ""); + + // Allocate registers for output row + let output_col_count = select.projection.len(); + let output_start_reg = self.allocate_registers(output_col_count); + + // Allocate registers for window state + let row_num_reg = self.allocate_register(); // Current row number in partition + let prev_partition_reg = self.allocate_register(); // Previous partition key value + let rank_reg = self.allocate_register(); // Current rank value + let prev_order_reg = self.allocate_register(); // Previous order key value for rank + let dense_rank_reg = self.allocate_register(); // Dense rank value + let rows_at_rank_reg = self.allocate_register(); // Number of rows at current rank + + // Initialize window state BEFORE the loop + self.emit( + OpCode::Integer, + 0, + row_num_reg, + 0, + None, + "Initialize row number", + ); + self.emit( + OpCode::Null, + 0, + prev_partition_reg, + 0, + None, + "Initialize prev partition key", + ); + self.emit(OpCode::Integer, 0, rank_reg, 0, None, "Initialize rank"); + self.emit( + OpCode::Null, + 0, + prev_order_reg, + 0, + None, + "Initialize prev order key", + ); + self.emit( + OpCode::Integer, + 0, + dense_rank_reg, + 0, + None, + "Initialize dense rank", + ); + self.emit( + OpCode::Integer, + 0, + rows_at_rank_reg, + 0, + None, + "Initialize rows at rank", + ); + + // Allocate registers for aggregate window functions (SUM, AVG, COUNT, MIN, MAX) + let agg_sum_reg = self.allocate_register(); // Running sum + let agg_count_reg = self.allocate_register(); // Running count + let agg_min_reg = self.allocate_register(); // Running min + let agg_max_reg = self.allocate_register(); // Running max + + // Initialize aggregate state + self.emit( + OpCode::Integer, + 0, + agg_sum_reg, + 0, + None, + "Initialize aggregate sum", + ); + self.emit( + OpCode::Integer, + 0, + agg_count_reg, + 0, + None, + "Initialize aggregate count", + ); + self.emit( + OpCode::Null, + 0, + agg_min_reg, + 0, + None, + "Initialize aggregate min", + ); + self.emit( + OpCode::Null, + 0, + agg_max_reg, + 0, + None, + "Initialize aggregate max", + ); + + // Find if we have aggregate window functions and extract their column positions + let mut agg_col_pos: Option = None; + for (func_name, _) in &window_funcs { + let ft = self.get_window_func_type(func_name); + if (5..=9).contains(&ft) { + // It's an aggregate function - find the column position + // Look through projection to find the argument + for item in &select.projection { + if let SelectItem::UnnamedExpr(Expr::Function(func)) + | SelectItem::ExprWithAlias { + expr: Expr::Function(func), + .. + } = item + { + if func.name.to_string().to_uppercase() == *func_name && func.over.is_some() + { + if !func.args.is_empty() { + if let Ok(Expr::Identifier(ident)) = + self.extract_function_arg_expr(&func.args[0]) + { + if let Some(src_idx) = table.column_index(&ident.value) { + agg_col_pos = + base_columns.iter().position(|&c| c == src_idx); + } + } + } + break; + } + } + } + break; + } + } + + // WindowAggStep computes window values - we encode state registers in P4 + let window_state_spec = format!( + "row_num:{},prev_part:{},rank:{},prev_ord:{},dense_rank:{},rows_at_rank:{},agg_sum:{},agg_count:{},agg_min:{},agg_max:{},agg_col:{}", + row_num_reg, prev_partition_reg, rank_reg, prev_order_reg, dense_rank_reg, rows_at_rank_reg, + agg_sum_reg, agg_count_reg, agg_min_reg, agg_max_reg, agg_col_pos.unwrap_or(0) + ); + + // Build window spec for WindowAggStep (using ephemeral cursor positions) + let part_cols_str = partition_positions + .iter() + .map(|c| c.to_string()) + .collect::>() + .join(","); + let ord_cols_str = order_positions + .iter() + .map(|c| c.to_string()) + .collect::>() + .join(","); + let window_spec_base = format!( + "WINDOW:part={}:ord={}:state={}", + part_cols_str, ord_cols_str, window_state_spec + ); + + // NOW start the output loop (after state initialization) + let output_loop_start = self.program.len(); + + // Emit ONE WindowAggStep at start of loop to update state for this row + // Get the first window function type for the step + let first_func_type = if !window_funcs.is_empty() { + self.get_window_func_type(&window_funcs[0].0) + } else { + 0 + }; + self.emit( + OpCode::WindowAggStep, + first_func_type, + 0, + ephemeral_cursor, + Some(window_spec_base.clone()), + "Update window state for current row", + ); + + // Compile the output projection + let mut output_idx = 0; + for item in &select.projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + match expr { + Expr::Function(func) if func.over.is_some() => { + // Window function - just emit WindowValue (state already updated above) + let func_name = func.name.to_string().to_uppercase(); + let func_type = self.get_window_func_type(&func_name); + + // For LAG/LEAD, extract column position, offset, and partition info + let p4 = if func_name == "LAG" || func_name == "LEAD" { + // LAG/LEAD(column, offset, default) + // Get column position in ephemeral cursor + let col_pos = if !func.args.is_empty() { + if let Ok(Expr::Identifier(ident)) = + self.extract_function_arg_expr(&func.args[0]) + { + if let Some(src_idx) = table.column_index(&ident.value) { + base_columns + .iter() + .position(|&c| c == src_idx) + .unwrap_or(0) + } else { + 0 + } + } else { + 0 + } + } else { + 0 + }; + + // Get offset (default 1) + let offset = if func.args.len() >= 2 { + if let Ok(Expr::Value(Value::Number(n, _))) = + self.extract_function_arg_expr(&func.args[1]) + { + n.parse::().unwrap_or(1) + } else { + 1 + } + } else { + 1 + }; + + // Partition columns for boundary detection (using ephemeral positions) + let part_str = partition_positions + .iter() + .map(|p| p.to_string()) + .collect::>() + .join(","); + + Some(format!( + "{}:cursor={}:col={}:offset={}:parts={}", + func_name, ephemeral_cursor, col_pos, offset, part_str + )) + } else { + None + }; + + self.emit( + OpCode::WindowValue, + row_num_reg, + output_start_reg + output_idx as i64, + func_type, + p4, + &format!("Get {} value", func_name), + ); + } + Expr::Identifier(ident) => { + // Regular column reference + if let Some(col_idx) = table.column_index(&ident.value) { + if let Some(pos) = base_columns.iter().position(|&c| c == col_idx) { + self.emit( + OpCode::Column, + ephemeral_cursor, + pos as i64, + output_start_reg + output_idx as i64, + None, + "", + ); + } + } + } + _ => { + // Other expressions - compile as usual + self.compile_where_operand( + expr, + table, + ephemeral_cursor as usize, + output_start_reg + output_idx as i64, + )?; + } + } + } + SelectItem::Wildcard(_) => { + // Load all columns + for (i, &_col_idx) in base_columns.iter().enumerate() { + self.emit( + OpCode::Column, + ephemeral_cursor, + i as i64, + output_start_reg + output_idx as i64, + None, + "", + ); + output_idx += 1; + } + continue; + } + _ => {} + } + output_idx += 1; + } + + // Output the row + self.emit( + OpCode::ResultRow, + output_start_reg, + output_col_count as i64, + 0, + None, + "", + ); + + // Next row in ephemeral + self.emit( + OpCode::Next, + ephemeral_cursor, + output_loop_start as i64, + 0, + None, + "", + ); + + let after_output = self.program.len(); + + // Patch output rewind + if let Some(inst) = self.program.instructions.get_mut(output_rewind_addr) { + inst.p2 = after_output as i64; + } + + // Patch sort jump (empty case) + if let Some(inst) = self.program.instructions.get_mut(sort_jump_target - 1) { + inst.p2 = after_output as i64; + } + + // Halt + self.emit(OpCode::Halt, 0, 0, 0, None, ""); + + Ok(()) + } + + /// Analyze projection to extract window functions and base columns needed + #[allow(clippy::type_complexity)] + fn analyze_window_projection( + &self, + projection: &[SelectItem], + table: &Table, + ) -> SqawkResult<(Vec<(String, Option)>, Vec)> { + let mut window_funcs = Vec::new(); + let mut base_columns = Vec::new(); + + for item in projection { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => { + match expr { + Expr::Function(func) if func.over.is_some() => { + let name = func.name.to_string().to_uppercase(); + window_funcs.push((name, func.over.clone())); + + // If it's an aggregate window function, add argument column + if !func.args.is_empty() { + if let Ok(Expr::Identifier(ident)) = + self.extract_function_arg_expr(&func.args[0]) + { + if let Some(col_idx) = table.column_index(&ident.value) { + if !base_columns.contains(&col_idx) { + base_columns.push(col_idx); + } + } + } + } + } + Expr::Identifier(ident) => { + if let Some(col_idx) = table.column_index(&ident.value) { + if !base_columns.contains(&col_idx) { + base_columns.push(col_idx); + } + } + } + _ => {} + } + } + SelectItem::Wildcard(_) => { + // Include all columns + for i in 0..table.column_count() { + if !base_columns.contains(&i) { + base_columns.push(i); + } + } + } + _ => {} + } + } + + Ok((window_funcs, base_columns)) + } + + /// Extract partition and order columns from window specification + fn extract_window_spec( + &self, + window_type: &Option, + table: &Table, + ) -> SqawkResult<(Vec, Vec, Vec)> { + let mut partition_cols = Vec::new(); + let mut order_cols = Vec::new(); + let mut order_asc = Vec::new(); + + if let Some(WindowType::WindowSpec(spec)) = window_type { + // Extract PARTITION BY columns + for expr in &spec.partition_by { + if let Expr::Identifier(ident) = expr { + if let Some(col_idx) = table.column_index(&ident.value) { + partition_cols.push(col_idx); + } + } + } + + // Extract ORDER BY columns + for order_expr in &spec.order_by { + if let Expr::Identifier(ident) = &order_expr.expr { + if let Some(col_idx) = table.column_index(&ident.value) { + order_cols.push(col_idx); + order_asc.push(order_expr.asc.unwrap_or(true)); + } + } + } + } + + Ok((partition_cols, order_cols, order_asc)) + } + + /// Build result schema for window function query + fn build_window_result_schema(&self, projection: &[SelectItem], table: &Table) -> ResultSchema { + let mut schema = ResultSchema::new(); + + for item in projection { + match item { + SelectItem::UnnamedExpr(expr) => { + let (name, data_type) = self.infer_expr_schema(expr, table); + schema.add_column(name, data_type); + } + SelectItem::ExprWithAlias { expr, alias } => { + let (_, data_type) = self.infer_expr_schema(expr, table); + schema.add_column(alias.value.clone(), data_type); + } + SelectItem::Wildcard(_) => { + for col in table.column_metadata() { + schema.add_column(col.name.clone(), col.data_type); + } + } + _ => {} + } + } + + schema + } + + /// Infer schema (name, type) for an expression + pub(crate) fn infer_expr_schema(&self, expr: &Expr, table: &Table) -> (String, DataType) { + match expr { + Expr::Identifier(ident) => { + if let Some(col_idx) = table.column_index(&ident.value) { + let col = &table.column_metadata()[col_idx]; + (ident.value.clone(), col.data_type) + } else { + (ident.value.clone(), DataType::Text) + } + } + Expr::Function(func) => { + let name = func.name.to_string(); + // Window functions return Integer (for ROW_NUMBER, RANK) or the input type (for aggregates) + let data_type = match name.to_uppercase().as_str() { + "ROW_NUMBER" | "RANK" | "DENSE_RANK" | "COUNT" => DataType::Integer, + _ => DataType::Float, + }; + (name, data_type) + } + _ => ("expr".to_string(), DataType::Text), + } + } +} diff --git a/src/vm/engine.rs b/src/vm/engine.rs index 2bf6e73..18d2ac4 100644 --- a/src/vm/engine.rs +++ b/src/vm/engine.rs @@ -4,71 +4,384 @@ //! generated by the compiler. It maintains internal state like registers, cursors, //! and the program counter during execution. -use super::bytecode::{Instruction, OpCode, Program, Register}; +use super::bytecode::{ + Instruction, OpCode, Program, Register, AGG_AVG, AGG_COUNT, AGG_MAX, AGG_MIN, AGG_SUM, +}; +use crate::capacity::{ + DEFAULT_ACCUMULATOR_CAPACITY, DEFAULT_CURSOR_CAPACITY, DEFAULT_MODIFICATIONS_CAPACITY, + DEFAULT_REGISTER_CAPACITY, DEFAULT_RESULT_CAPACITY, DEFAULT_SORTER_CAPACITY, + DEFAULT_SORTER_ROWS_CAPACITY, DEFAULT_SORT_KEYS_CAPACITY, +}; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; -use crate::table::{Table, Value}; +use crate::table::{ColumnDefinition, Table, Value}; use std::collections::HashMap; -/// Table cursor for VM execution -struct Cursor { - /// The table being accessed - table: Table, +/// Data backing a cursor - either borrowed table or owned ephemeral data +/// Following SQLite's model where Column/Next/Rewind work for ALL cursor types +enum CursorData<'a> { + /// Table cursor - borrows reference to table + Table { table: &'a Table }, + /// Ephemeral cursor - owns its data + Ephemeral { + rows: Vec>, + col_count: usize, + sort_keys: Vec<(usize, bool)>, // (col_idx, ascending) + sorted: bool, + sequence: i64, + name: Option, // Optional name for pending tables + }, +} + +/// Unified cursor for VM execution +/// +/// Works with both table references and ephemeral data. +/// This follows SQLite's model where Column/Next/Rewind work for any cursor. +struct Cursor<'a> { + data: CursorData<'a>, /// Current row position (0-based) position: usize, /// Whether the cursor is at a valid row valid: bool, } -impl Cursor { - /// Create a new cursor for a table - fn new(table: Table) -> Self { +impl<'a> Cursor<'a> { + /// Create cursor from table reference + fn from_table(table: &'a Table) -> Self { + Self { + data: CursorData::Table { table }, + position: 0, + valid: false, + } + } + + /// Create ephemeral cursor + fn ephemeral(col_count: usize, sort_keys: Vec<(usize, bool)>) -> Self { + Self { + data: CursorData::Ephemeral { + rows: Vec::with_capacity(DEFAULT_SORTER_ROWS_CAPACITY), + col_count, + sort_keys, + sorted: false, + sequence: 0, + name: None, + }, + position: 0, + valid: false, + } + } + + /// Create named ephemeral cursor (for pending tables) + fn named_ephemeral(col_count: usize, table_name: String) -> Self { Self { - table, + data: CursorData::Ephemeral { + rows: Vec::with_capacity(DEFAULT_SORTER_ROWS_CAPACITY), + col_count, + sort_keys: Vec::with_capacity(DEFAULT_SORT_KEYS_CAPACITY), + sorted: false, + sequence: 0, + name: Some(table_name), + }, position: 0, valid: false, } } - /// Move to the first row + /// Move to the first row (Rewind opcode) fn rewind(&mut self) -> bool { self.position = 0; - self.valid = self.position < self.table.row_count(); + self.valid = match &self.data { + CursorData::Table { table } => self.position < table.row_count(), + CursorData::Ephemeral { rows, .. } => !rows.is_empty(), + }; self.valid } - /// Move to the next row + /// Move to the next row (Next opcode) fn next(&mut self) -> bool { if !self.valid { return false; } self.position += 1; - self.valid = self.position < self.table.row_count(); + self.valid = match &self.data { + CursorData::Table { table } => self.position < table.row_count(), + CursorData::Ephemeral { rows, .. } => self.position < rows.len(), + }; self.valid } - /// Get a column value at the current position + /// Get the current row index (for index-based results) + fn current_row_idx(&self) -> Option { + if self.valid { + Some(self.position) + } else { + None + } + } + + /// Get a column value at the current position (Column opcode) fn column(&self, idx: usize) -> Option { - if !self.valid || idx >= self.table.column_count() { + if !self.valid { + return None; + } + match &self.data { + CursorData::Table { table } => { + if idx >= table.column_count() { + return None; + } + let rows = table.rows(); + if self.position < rows.len() { + rows[self.position].get(idx).cloned() + } else { + None + } + } + CursorData::Ephemeral { + rows, col_count, .. + } => { + if idx >= *col_count { + return None; + } + rows.get(self.position)?.get(idx).cloned() + } + } + } + + /// Get the table name (for table cursors or named ephemeral cursors) + fn table_name(&self) -> &str { + match &self.data { + CursorData::Table { table } => table.name(), + CursorData::Ephemeral { name: Some(n), .. } => n, + CursorData::Ephemeral { name: None, .. } => "", + } + } + + /// Get the current row index (alias for current_row_idx) + fn current_row_index(&self) -> Option { + self.current_row_idx() + } + + /// Insert row into ephemeral cursor (IdxInsert opcode) + fn insert_row(&mut self, row: Vec) -> Result<(), SqawkError> { + match &mut self.data { + CursorData::Ephemeral { rows, sorted, .. } => { + rows.push(row); + *sorted = false; + Ok(()) + } + CursorData::Table { .. } => Err(SqawkError::VmError( + "Cannot insert into table cursor".into(), + )), + } + } + + /// Sort ephemeral cursor (Sort opcode) + /// Returns true if has rows (continue), false if empty (should jump) + fn sort(&mut self) -> Result { + match &mut self.data { + CursorData::Ephemeral { + rows, + sort_keys, + sorted, + .. + } => { + if rows.is_empty() { + return Ok(false); // Empty - caller should jump + } + if !*sorted { + let keys = sort_keys.clone(); + rows.sort_by(|a, b| { + for (col_idx, ascending) in &keys { + let cmp = compare_values( + a.get(*col_idx).unwrap_or(&Value::Null), + b.get(*col_idx).unwrap_or(&Value::Null), + ); + if cmp != std::cmp::Ordering::Equal { + return if *ascending { cmp } else { cmp.reverse() }; + } + } + std::cmp::Ordering::Equal + }); + *sorted = true; + } + self.position = 0; + self.valid = true; + Ok(true) // Has rows - caller continues + } + CursorData::Table { .. } => Err(SqawkError::VmError("Cannot sort table cursor".into())), + } + } + + /// Generate next sequence number (Sequence opcode) + fn next_sequence(&mut self) -> Result { + match &mut self.data { + CursorData::Ephemeral { sequence, .. } => { + let seq = *sequence; + *sequence += 1; + Ok(seq) + } + CursorData::Table { .. } => Err(SqawkError::VmError( + "Cannot generate sequence from table cursor".into(), + )), + } + } + + /// Get row count (for debugging/validation) + #[allow(dead_code)] + fn row_count(&self) -> usize { + match &self.data { + CursorData::Table { table } => table.row_count(), + CursorData::Ephemeral { rows, .. } => rows.len(), + } + } + + /// Check if this is an ephemeral cursor + #[allow(dead_code)] + fn is_ephemeral(&self) -> bool { + matches!(self.data, CursorData::Ephemeral { .. }) + } + + /// Get column count + fn column_count(&self) -> usize { + match &self.data { + CursorData::Table { table } => table.column_count(), + CursorData::Ephemeral { col_count, .. } => *col_count, + } + } + + /// Get rows reference (for table cursors, returns table rows; for ephemeral, returns owned rows) + fn rows(&self) -> &[Vec] { + match &self.data { + CursorData::Table { table } => table.rows(), + CursorData::Ephemeral { rows, .. } => rows, + } + } + + /// Get a column value at a specific row offset from current position + /// Positive offset = forward (for LEAD), negative offset = backward (for LAG) + /// Returns None if the offset position is out of bounds + fn column_at_offset(&self, col_idx: usize, offset: i64) -> Option { + if !self.valid { return None; } - // Access rows from the table using the rows() method - let rows = self.table.rows(); - if self.position < rows.len() { - rows[self.position].get(idx).cloned() + let target_pos = (self.position as i64) + offset; + if target_pos < 0 { + return None; + } + + let target_pos = target_pos as usize; + match &self.data { + CursorData::Table { table } => { + if col_idx >= table.column_count() || target_pos >= table.row_count() { + return None; + } + let rows = table.rows(); + rows.get(target_pos)?.get(col_idx).cloned() + } + CursorData::Ephemeral { + rows, col_count, .. + } => { + if col_idx >= *col_count || target_pos >= rows.len() { + return None; + } + rows.get(target_pos)?.get(col_idx).cloned() + } + } + } +} + +/// Compare two Values for ordering +fn compare_values(a: &Value, b: &Value) -> std::cmp::Ordering { + use std::cmp::Ordering; + match (a, b) { + (Value::Null, Value::Null) => Ordering::Equal, + (Value::Null, _) => Ordering::Less, + (_, Value::Null) => Ordering::Greater, + (Value::Integer(a), Value::Integer(b)) => a.cmp(b), + (Value::Float(a), Value::Float(b)) => a.partial_cmp(b).unwrap_or(Ordering::Equal), + (Value::Integer(a), Value::Float(b)) => { + (*a as f64).partial_cmp(b).unwrap_or(Ordering::Equal) + } + (Value::Float(a), Value::Integer(b)) => { + a.partial_cmp(&(*b as f64)).unwrap_or(Ordering::Equal) + } + (Value::String(a), Value::String(b)) => a.cmp(b), + (Value::Boolean(a), Value::Boolean(b)) => a.cmp(b), + _ => format!("{:?}", a).cmp(&format!("{:?}", b)), + } +} + +/// Sorter for ORDER BY operations +struct Sorter { + rows: Vec>, + sort_keys: Vec<(usize, bool)>, // (column_index, ascending) + position: usize, + sorted: bool, +} + +/// Aggregate accumulator for GROUP BY operations +#[derive(Debug, Clone)] +struct AggAccumulator { + /// Function type: 0=COUNT, 1=SUM, 2=AVG, 3=MIN, 4=MAX + func_type: i64, + /// Count of values processed (for COUNT and AVG) + count: i64, + /// Accumulated value (sum for SUM/AVG, min/max for MIN/MAX) + value: Option, +} + +impl Sorter { + fn new(_col_count: usize, sort_keys: Vec<(usize, bool)>) -> Self { + Self { + rows: Vec::with_capacity(DEFAULT_SORTER_ROWS_CAPACITY), + sort_keys, + position: 0, + sorted: false, + } + } + + fn insert(&mut self, row: Vec) { + self.rows.push(row); + self.sorted = false; + } + + fn sort(&mut self) { + let keys = self.sort_keys.clone(); + self.rows.sort_by(|a, b| { + for (col_idx, ascending) in &keys { + if *col_idx >= a.len() || *col_idx >= b.len() { + continue; + } + let cmp = compare_values(&a[*col_idx], &b[*col_idx]); + if cmp != std::cmp::Ordering::Equal { + return if *ascending { cmp } else { cmp.reverse() }; + } + } + std::cmp::Ordering::Equal + }); + self.sorted = true; + self.position = 0; + } + + fn current_row(&self) -> Option<&Vec> { + if self.sorted && self.position < self.rows.len() { + Some(&self.rows[self.position]) } else { None } } + + fn next(&mut self) -> bool { + if !self.sorted { + return false; + } + self.position += 1; + self.position < self.rows.len() + } } -/// Transaction state for the VM engine -/// -/// This enum tracks the current state of a transaction during VM execution. -/// It enables proper validation of transaction operations (e.g., preventing -/// nested transactions or commits without an active transaction). #[derive(Debug, Clone, Copy, PartialEq)] enum TransactionState { /// No active transaction - the default state when no transaction has been started @@ -81,41 +394,77 @@ enum TransactionState { RolledBack, } +/// Column definition for CREATE TABLE +#[derive(Debug, Clone)] +pub struct ColumnDef { + pub name: String, + pub data_type: String, +} + +/// Pending modification to a table +#[derive(Debug, Clone)] +pub enum TableModification { + Insert { + table_name: String, + row: Vec, + }, + Delete { + table_name: String, + row_index: usize, + }, + CreateTable { + table_name: String, + columns: Vec, + file_path: Option, + delimiter: Option, + }, + DropTable { + table_name: String, + }, + AlterTableAddColumn { + table_name: String, + column_name: String, + column_type: String, + }, + Truncate { + table_name: String, + }, +} + /// SQL VM engine that executes bytecode pub struct VmEngine<'a> { - /// The database containing tables database: &'a Database, - - /// The program being executed program: Program, - - /// Program counter (current instruction address) pc: usize, - - /// VM registers registers: Vec, + cursors: HashMap>, + sorters: HashMap, + accumulators: HashMap, + results: Vec>, - /// Table cursors - cursors: HashMap, + /// Index-based results (Phase 4C parallel path) + /// Each entry contains row indices from each cursor for that result row + /// Option handles NULL rows in outer joins + #[allow(dead_code)] + result_row_indices: Vec>>, - /// Result rows from SELECT statements - results: Vec>, + /// Order of cursor IDs as sources (maps position to cursor ID) + #[allow(dead_code)] + cursor_source_order: Vec, /// Current transaction state (None, Active, Committed, or RolledBack) /// Tracks the lifecycle of a transaction and enforces proper operation sequencing transaction_state: TransactionState, - /// Transaction log tracking changes made during an active transaction - /// - /// This stores the modified data to enable rollback operations: - /// - First element (usize): Cursor ID that made the modification - /// - Second element (Table): Original state of the table before modification - /// - Third element (Vec<(usize, Value)>): List of (column_index, new_value) pairs representing changes - transaction_log: Vec<(usize, Table, Vec<(usize, Value)>)>, + /// Pending modifications to be applied after execution + pending_modifications: Vec, - // Removed unused fields for column_names, modified_tables, and affected_rows /// Whether the engine is in verbose mode verbose: bool, + + /// Once-flag tracking - maps instruction address to has_executed flag + /// Used by the Once opcode to skip repeated initialization + once_flags: HashMap, } impl<'a> VmEngine<'a> { @@ -125,22 +474,39 @@ impl<'a> VmEngine<'a> { database, program: Program::new(), pc: 0, - registers: Vec::new(), - cursors: HashMap::new(), - results: Vec::new(), + registers: Vec::with_capacity(DEFAULT_REGISTER_CAPACITY), + cursors: HashMap::with_capacity(DEFAULT_CURSOR_CAPACITY), + sorters: HashMap::with_capacity(DEFAULT_SORTER_CAPACITY), + accumulators: HashMap::with_capacity(DEFAULT_ACCUMULATOR_CAPACITY), + results: Vec::with_capacity(DEFAULT_RESULT_CAPACITY), + result_row_indices: Vec::with_capacity(DEFAULT_RESULT_CAPACITY), + cursor_source_order: Vec::with_capacity(DEFAULT_CURSOR_CAPACITY), transaction_state: TransactionState::None, - transaction_log: Vec::new(), + pending_modifications: Vec::with_capacity(DEFAULT_MODIFICATIONS_CAPACITY), verbose, + once_flags: HashMap::with_capacity(DEFAULT_CURSOR_CAPACITY), } } + /// Alias for new() - both work with immutable database reference + /// Write operations are accumulated and returned via take_modifications() + pub fn new_mut(database: &'a Database, verbose: bool) -> Self { + Self::new(database, verbose) + } + /// Initialize the VM with a program to execute pub fn init(&mut self, program: Program) { self.program = program; self.pc = 0; - self.registers = Vec::new(); + self.registers = Vec::with_capacity(DEFAULT_REGISTER_CAPACITY); self.cursors.clear(); + self.sorters.clear(); + self.accumulators.clear(); self.results.clear(); + self.result_row_indices.clear(); + self.cursor_source_order.clear(); + self.pending_modifications.clear(); + self.once_flags.clear(); // Allocate enough registers for the program let max_reg = self @@ -152,17 +518,23 @@ impl<'a> VmEngine<'a> { .unwrap_or(10); // Allocate a few extra registers just in case + // Cap max_reg to a reasonable value to avoid overflow (p1/p2/p3 sometimes contain values, not register numbers) + let max_reg = max_reg.min(10000); self.registers = vec![Register::Null; (max_reg + 5) as usize]; // Reset transaction state self.transaction_state = TransactionState::None; - self.transaction_log.clear(); if self.verbose { println!("VM initialized with program:\n{}", self.program); } } + /// Take pending modifications out of the engine + pub fn take_modifications(&mut self) -> Vec { + std::mem::take(&mut self.pending_modifications) + } + /// Execute the current program pub fn execute(&mut self) -> SqawkResult<()> { if self.program.is_empty() { @@ -228,28 +600,63 @@ impl<'a> VmEngine<'a> { OpCode::OpenRead => { // Open a cursor for reading a table let cursor_idx = inst.p1 as usize; - let table_name = inst.p4.clone().unwrap_or_default(); + let table_name = inst.p4.as_deref().unwrap_or(""); // Check if the table exists - if !self.database.has_table(&table_name) { - return Err(SqawkError::TableNotFound(table_name)); + if !self.database.has_table(table_name) { + return Err(SqawkError::TableNotFound(table_name.to_string())); } // Get the table from the database (safe to unwrap since we checked it exists) - let table = self.database.get_table(&table_name).unwrap(); + // Note: We now borrow the table instead of cloning it (Phase 4B optimization) + let table = self.database.get_table(table_name).unwrap(); - // Create a cursor for the table - let cursor = Cursor::new(table.clone()); + // Create a cursor for the table (borrows, no clone) + let cursor = Cursor::from_table(table); self.cursors.insert(cursor_idx, cursor); + // Track cursor source order for index-based results (Phase 4C) + if !self.cursor_source_order.contains(&cursor_idx) { + self.cursor_source_order.push(cursor_idx); + } + Ok(ExecuteResult::Continue) } OpCode::OpenWrite => { - // Not implementing write operations for the MVP - Err(SqawkError::UnsupportedSqlFeature( - "Write operations not supported in VM".to_string(), - )) + // Open a cursor for writing to a table + // Note: Actual modifications are queued and applied after execution + let cursor_idx = inst.p1 as usize; + let table_name = inst.p4.as_deref().unwrap_or(""); + + // Check if the table exists in database or pending modifications + if self.database.has_table(table_name) { + // Get the table from the database + let table = self.database.get_table(table_name).unwrap(); + let cursor = Cursor::from_table(table); + self.cursors.insert(cursor_idx, cursor); + } else { + // Check if there's a pending CreateTable for this table + let pending_create = self.pending_modifications.iter().find(|m| { + matches!(m, TableModification::CreateTable { table_name: tn, .. } if tn == table_name) + }); + + if let Some(TableModification::CreateTable { + columns, + table_name: pending_name, + .. + }) = pending_create.cloned() + { + // Create a named ephemeral cursor with the schema from pending CreateTable + let col_count = columns.len(); + let cursor = Cursor::named_ephemeral(col_count, pending_name); + self.cursors.insert(cursor_idx, cursor); + } else { + return Err(SqawkError::TableNotFound(table_name.to_string())); + } + } + + Ok(ExecuteResult::Continue) } OpCode::Close => { @@ -335,474 +742,2472 @@ impl<'a> VmEngine<'a> { } } - OpCode::Integer => { - // Load integer constant into register - let value = inst.p1; - let register_idx = inst.p2 as usize; + OpCode::InsertRow => { + // Insert row from registers P2..P2+P3 into table at cursor P1 + let cursor_idx = inst.p1 as usize; + let start_reg = inst.p2 as usize; + let col_count = inst.p3 as usize; - if register_idx < self.registers.len() { - self.registers[register_idx] = Register::Integer(value); - Ok(ExecuteResult::Continue) + // Get table name from cursor + let table_name = if let Some(cursor) = self.cursors.get(&cursor_idx) { + cursor.table_name().to_string() } else { - Err(SqawkError::VmError(format!( - "Register index out of bounds: {}", - register_idx - ))) + return Err(SqawkError::VmError(format!( + "Invalid cursor for InsertRow: {}", + cursor_idx + ))); + }; + + // Collect values from registers + let mut row = Vec::with_capacity(col_count); + for i in 0..col_count { + let reg_value = self.get_register(start_reg + i)?; + row.push(Value::from(reg_value)); } - } - OpCode::String => { - // Load string constant into register - let register_idx = inst.p2 as usize; - let string_value = inst.p4.clone().unwrap_or_default(); + // Add to pending modifications + self.pending_modifications + .push(TableModification::Insert { table_name, row }); - if register_idx < self.registers.len() { - self.registers[register_idx] = Register::String(string_value); - Ok(ExecuteResult::Continue) - } else { - Err(SqawkError::VmError(format!( - "Register index out of bounds: {}", - register_idx - ))) + if self.verbose { + println!( + "InsertRow: queued insert of {} columns into cursor {}", + col_count, cursor_idx + ); } + + Ok(ExecuteResult::Continue) } - OpCode::Null => { - // Load NULL into register - let register_idx = inst.p2 as usize; + OpCode::DeleteRow => { + // Delete current row at cursor P1 + let cursor_idx = inst.p1 as usize; - if register_idx < self.registers.len() { - self.registers[register_idx] = Register::Null; - Ok(ExecuteResult::Continue) + // Get table name and current row index from cursor + let (table_name, row_idx) = if let Some(cursor) = self.cursors.get(&cursor_idx) { + (cursor.table_name().to_string(), cursor.current_row_index()) } else { - Err(SqawkError::VmError(format!( - "Register index out of bounds: {}", - register_idx - ))) - } - } - - OpCode::ResultRow => { - // Return a result row from registers - let start_reg = inst.p1 as usize; - let column_count = inst.p2 as usize; + return Err(SqawkError::VmError(format!( + "Invalid cursor for DeleteRow: {}", + cursor_idx + ))); + }; - // Collect values from registers - let mut row = Vec::with_capacity(column_count); - for i in 0..column_count { - let reg_idx = start_reg + i; - if reg_idx < self.registers.len() { - // Convert register to Value - let value = Value::from(self.registers[reg_idx].clone()); - row.push(value); - } else { - return Err(SqawkError::VmError(format!( - "Register index out of bounds: {}", - reg_idx - ))); + if let Some(idx) = row_idx { + // Add to pending modifications + self.pending_modifications.push(TableModification::Delete { + table_name, + row_index: idx, + }); + + if self.verbose { + println!( + "DeleteRow: queued delete of row {} from cursor {}", + idx, cursor_idx + ); } } - // Add the row to results - self.results.push(row); - Ok(ExecuteResult::Continue) } - OpCode::Begin => { - // Begin a transaction - initiates a new atomic unit of work - // All operations performed between BEGIN and COMMIT/ROLLBACK are treated as a single - // logical operation from the perspective of database consistency - if self.transaction_state == TransactionState::Active { + OpCode::CreateTable => { + // Create a new table from P4 specification + // P4 format: "table_name:col1:type1,col2:type2,...|filepath|delimiter" + let spec = inst.p4.clone().unwrap_or_default(); + let parts: Vec<&str> = spec.split('|').collect(); + + if parts.is_empty() { return Err(SqawkError::VmError( - "Transaction already in progress".to_string(), + "CreateTable: missing table specification".to_string(), )); } - // Initialize transaction state to track changes - // - Set state to Active to indicate transaction is in progress - // - Clear any previous transaction log entries - self.transaction_state = TransactionState::Active; - self.transaction_log.clear(); + // Parse table name and columns + let name_and_cols: Vec<&str> = parts[0].split(':').collect(); + if name_and_cols.is_empty() { + return Err(SqawkError::VmError( + "CreateTable: missing table name".to_string(), + )); + } + + let table_name = name_and_cols[0].to_string(); + + // Parse columns (pairs of name:type) + let col_count = (name_and_cols.len() - 1) / 2; + let mut columns = Vec::with_capacity(col_count); + let mut i = 1; + while i + 1 < name_and_cols.len() { + columns.push(ColumnDef { + name: name_and_cols[i].to_string(), + data_type: name_and_cols[i + 1].to_string(), + }); + i += 2; + } + + // Parse optional file path and delimiter + let file_path = parts + .get(1) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let delimiter = parts + .get(2) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + // Add to pending modifications + self.pending_modifications + .push(TableModification::CreateTable { + table_name: table_name.clone(), + columns, + file_path, + delimiter, + }); if self.verbose { - println!("BEGIN TRANSACTION"); + println!("CreateTable: queued creation of table '{}'", table_name); } Ok(ExecuteResult::Continue) } - OpCode::Commit => { - // Commit a transaction - makes all changes permanent - // Finalizes the changes made during the transaction and ensures they - // become permanent, durable parts of the database state - if self.transaction_state != TransactionState::Active { + OpCode::DropTable => { + // Drop a table + // P1 = 1 if IF EXISTS (don't error if table doesn't exist) + // P4 = table name + let if_exists = inst.p1 != 0; + let table_name = inst.p4.as_deref().unwrap_or(""); + + if table_name.is_empty() { return Err(SqawkError::VmError( - "No transaction in progress to commit".to_string(), + "DropTable: missing table name".to_string(), )); } - // Finalize transaction - // - Update state to Committed to indicate successful completion - // - In a WAL implementation, this would flush changes to the main database file - // - In an MVCC implementation, this would make changes visible to new transactions - self.transaction_state = TransactionState::Committed; - - // Clear the transaction log since changes are now permanent - // and don't need to be tracked for potential rollback - self.transaction_log.clear(); + // Check if table exists + let table_exists = self.database.has_table(table_name); - if self.verbose { - println!("COMMIT TRANSACTION"); + if !table_exists && !if_exists { + return Err(SqawkError::TableNotFound(table_name.to_string())); + } + + // Only queue drop if table exists + if table_exists { + self.pending_modifications + .push(TableModification::DropTable { + table_name: table_name.to_string(), + }); + + if self.verbose { + println!("DropTable: queued drop of table '{}'", table_name); + } + } else if self.verbose { + println!( + "DropTable: table '{}' does not exist (IF EXISTS)", + table_name + ); } Ok(ExecuteResult::Continue) } - OpCode::Rollback => { - // Rollback a transaction - abandons all changes made during the transaction - // Undoes all operations since BEGIN, returning the database to its prior state - if self.transaction_state != TransactionState::Active { + OpCode::AlterTableAdd => { + // Add a column to a table + // P4 = "table_name:column_name:column_type" + let spec = inst.p4.as_deref().unwrap_or(""); + let parts: Vec<&str> = spec.split(':').collect(); + + if parts.len() < 3 { return Err(SqawkError::VmError( - "No transaction in progress to rollback".to_string(), + "AlterTableAdd: invalid specification, expected table:column:type" + .to_string(), )); } - // Revert all changes in the transaction log - // In a full implementation, this would: - // 1. Process the transaction log in reverse chronological order (LIFO) - // 2. Restore original data values for each modified row - // 3. Remove any newly inserted rows and restore any deleted rows - // 4. Maintain referential integrity during the rollback process + let table_name = parts[0].to_string(); + let column_name = parts[1].to_string(); + let column_type = parts[2].to_string(); - // Update transaction state to reflect rollback completion - self.transaction_state = TransactionState::RolledBack; - // Clear the transaction log - self.transaction_log.clear(); + // Add to pending modifications + self.pending_modifications + .push(TableModification::AlterTableAddColumn { + table_name: table_name.clone(), + column_name: column_name.clone(), + column_type: column_type.clone(), + }); if self.verbose { - println!("ROLLBACK TRANSACTION"); + println!( + "AlterTableAdd: queued add column '{}' ({}) to table '{}'", + column_name, column_type, table_name + ); } Ok(ExecuteResult::Continue) } - OpCode::SavePoint => { - // Create a savepoint within the current transaction - // A savepoint marks a point within a transaction to which you can later roll back - // without rolling back the entire transaction - if self.transaction_state != TransactionState::Active { + OpCode::Truncate => { + // Remove all rows from a table + // P4 = table name + let table_name = inst.p4.as_deref().unwrap_or(""); + + if table_name.is_empty() { return Err(SqawkError::VmError( - "No active transaction for savepoint".to_string(), + "Truncate: missing table name".to_string(), )); } - // In a full implementation, we would: - // 1. Create a marker in the transaction log to identify this position - // 2. Allow for multiple savepoints with different names - // 3. Support rolling back to any specific savepoint - // 4. Properly handle nested savepoints in a hierarchical manner - let savepoint_name = inst.p4.clone().unwrap_or_else(|| format!("sp_{}", inst.p1)); + // Add to pending modifications + self.pending_modifications + .push(TableModification::Truncate { + table_name: table_name.to_string(), + }); if self.verbose { - println!("SAVEPOINT {}", savepoint_name); + println!("Truncate: queued truncate of table '{}'", table_name); } Ok(ExecuteResult::Continue) } - OpCode::Release => { - // Release a savepoint (commit changes up to the savepoint) - // This permanently applies all changes made since the savepoint was created - // while keeping the transaction active for further operations - if self.transaction_state != TransactionState::Active { - return Err(SqawkError::VmError( - "No active transaction for savepoint release".to_string(), - )); - } + OpCode::StringFunc => { + // Execute a string function + // P1 = src reg, P2 = dest reg, P3 = arg regs (encoded), P4 = "FUNC_NAME:extra_args" + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + let func_spec = inst.p4.clone().unwrap_or_default(); + let parts: Vec<&str> = func_spec.split(':').collect(); + let func_name = parts.first().unwrap_or(&""); + + // Get source value + let src_value = self.get_register(src_reg)?; + let src_str = match src_value { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + Register::Boolean(b) => b.to_string(), + Register::Null => { + // NULL in, NULL out + self.set_register(dest_reg, Register::Null)?; + return Ok(ExecuteResult::Continue); + } + }; + + let result = match *func_name { + "UPPER" => src_str.to_uppercase(), + "LOWER" => src_str.to_lowercase(), + "TRIM" => src_str.trim().to_string(), + "LTRIM" => src_str.trim_start().to_string(), + "RTRIM" => src_str.trim_end().to_string(), + "LENGTH" => { + self.set_register(dest_reg, Register::Integer(src_str.len() as i64))?; + return Ok(ExecuteResult::Continue); + } + "SUBSTR" | "SUBSTRING" => { + // P3 contains start register, parts[1] contains length if present + let start_reg = inst.p3 as usize; + let start = match self.get_register(start_reg)? { + Register::Integer(i) => i as usize, + _ => 1, + }; + // Adjust for 1-based indexing + let start_idx = if start > 0 { start - 1 } else { 0 }; + + // Check if we have a length register specified in parts[1] + if let Some(len_str) = parts.get(1) { + if let Ok(len_reg) = len_str.parse::() { + if let Ok(Register::Integer(len)) = self.get_register(len_reg) { + let len = len as usize; + src_str.chars().skip(start_idx).take(len).collect() + } else { + src_str.chars().skip(start_idx).collect() + } + } else { + src_str.chars().skip(start_idx).collect() + } + } else { + src_str.chars().skip(start_idx).collect() + } + } + "REPLACE" => { + // parts[1] = from_reg, parts[2] = to_reg + if parts.len() >= 3 { + if let (Ok(from_reg), Ok(to_reg)) = + (parts[1].parse::(), parts[2].parse::()) + { + let from_str = match self.get_register(from_reg)? { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + _ => String::new(), + }; + let to_str = match self.get_register(to_reg)? { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + _ => String::new(), + }; + src_str.replace(&from_str, &to_str) + } else { + src_str + } + } else { + src_str + } + } + "CONCAT" => { + // Concatenate strings: parts[1..] contain additional register numbers + let mut result = src_str; + for part in parts.iter().skip(1) { + if let Ok(reg) = part.parse::() { + let val = match self.get_register(reg)? { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + Register::Boolean(b) => b.to_string(), + Register::Null => String::new(), + }; + result.push_str(&val); + } + } + result + } + "LEFT" => { + // LEFT(str, n) - return leftmost n characters + let len_reg = inst.p3 as usize; + let len = match self.get_register(len_reg)? { + Register::Integer(i) => i.max(0) as usize, + _ => 0, + }; + src_str.chars().take(len).collect() + } + "RIGHT" => { + // RIGHT(str, n) - return rightmost n characters + let len_reg = inst.p3 as usize; + let len = match self.get_register(len_reg)? { + Register::Integer(i) => i.max(0) as usize, + _ => 0, + }; + let char_count = src_str.chars().count(); + if len >= char_count { + src_str + } else { + src_str.chars().skip(char_count - len).collect() + } + } + _ => { + return Err(SqawkError::VmError(format!( + "Unknown string function: {}", + func_name + ))); + } + }; + + self.set_register(dest_reg, Register::String(result))?; + + if self.verbose { + println!( + "StringFunc {}: r[{}] -> r[{}]", + func_name, src_reg, dest_reg + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::MathFunc => { + // Execute a math function + // P1 = src reg, P2 = dest reg, P4 = function name (ABS, ROUND, CEIL, FLOOR) + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + let func_name = inst.p4.as_deref().unwrap_or(""); + + // Get source value as a number + let src_value = self.get_register(src_reg)?; + let num = match src_value { + Register::Integer(i) => i as f64, + Register::Float(f) => f, + Register::String(s) => s.parse::().unwrap_or(0.0), + Register::Null => { + // NULL in, NULL out + self.set_register(dest_reg, Register::Null)?; + return Ok(ExecuteResult::Continue); + } + Register::Boolean(b) => { + if b { + 1.0 + } else { + 0.0 + } + } + }; + + let result = match func_name { + "ABS" => num.abs(), + "ROUND" => num.round(), + "CEIL" | "CEILING" => num.ceil(), + "FLOOR" => num.floor(), + _ => { + return Err(SqawkError::VmError(format!( + "Unknown math function: {}", + func_name + ))); + } + }; + + // Store result - preserve integer type if possible + let result_reg = if result.fract() == 0.0 + && result >= i64::MIN as f64 + && result <= i64::MAX as f64 + { + Register::Integer(result as i64) + } else { + Register::Float(result) + }; + self.set_register(dest_reg, result_reg)?; + + if self.verbose { + println!( + "MathFunc {}: r[{}] ({}) -> r[{}] ({})", + func_name, src_reg, num, dest_reg, result + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::DateFunc => { + // Execute a date/time function + // P1 = src reg (optional, for DATE/TIME), P2 = dest reg, P4 = function name + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + let func_name = inst.p4.as_deref().unwrap_or(""); + + let result = match func_name { + "NOW" | "CURRENT_TIMESTAMP" => { + // Return current date and time in ISO 8601 format + use std::time::SystemTime; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default(); + let secs = now.as_secs(); + // Convert to datetime string (simplified - just date for now) + let days_since_epoch = secs / 86400; + let years = 1970 + (days_since_epoch / 365); + let remaining_days = days_since_epoch % 365; + let months = remaining_days / 30 + 1; + let days = remaining_days % 30 + 1; + let hours = (secs % 86400) / 3600; + let minutes = (secs % 3600) / 60; + let seconds = secs % 60; + format!( + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}", + years, months, days, hours, minutes, seconds + ) + } + "DATE" => { + // Extract or validate date from input + let src_value = self.get_register(src_reg)?; + match src_value { + Register::String(s) => { + // Try to extract just the date part (YYYY-MM-DD) + if s.len() >= 10 { + s[..10].to_string() + } else { + s + } + } + Register::Null => { + self.set_register(dest_reg, Register::Null)?; + return Ok(ExecuteResult::Continue); + } + _ => { + // Convert to string and try to extract date + match src_value { + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + _ => String::new(), + } + } + } + } + "TIME" => { + // Extract or validate time from input + let src_value = self.get_register(src_reg)?; + match src_value { + Register::String(s) => { + // Try to extract time part (HH:MM:SS) + if s.contains(' ') { + // Datetime format, extract time after space + s.split(' ').nth(1).unwrap_or(&s).to_string() + } else { + // Already a time string or return as-is + s + } + } + Register::Null => { + self.set_register(dest_reg, Register::Null)?; + return Ok(ExecuteResult::Continue); + } + _ => String::new(), + } + } + "CURRENT_DATE" => { + // Return current date only + use std::time::SystemTime; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default(); + let days_since_epoch = now.as_secs() / 86400; + let years = 1970 + (days_since_epoch / 365); + let remaining_days = days_since_epoch % 365; + let months = remaining_days / 30 + 1; + let days = remaining_days % 30 + 1; + format!("{:04}-{:02}-{:02}", years, months, days) + } + "CURRENT_TIME" => { + // Return current time only + use std::time::SystemTime; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default(); + let secs = now.as_secs(); + let hours = (secs % 86400) / 3600; + let minutes = (secs % 3600) / 60; + let seconds = secs % 60; + format!("{:02}:{:02}:{:02}", hours, minutes, seconds) + } + _ => { + return Err(SqawkError::VmError(format!( + "Unknown date function: {}", + func_name + ))); + } + }; + + self.set_register(dest_reg, Register::String(result))?; + + if self.verbose { + println!("DateFunc {}: -> r[{}]", func_name, dest_reg); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Add => { + // dest = left + right + let left_reg = inst.p1 as usize; + let right_reg = inst.p2 as usize; + let dest_reg = inst.p3 as usize; + + let result = self.arithmetic_op(left_reg, right_reg, "+")?; + self.set_register(dest_reg, result)?; + + if self.verbose { + println!("Add: r[{}] + r[{}] -> r[{}]", left_reg, right_reg, dest_reg); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Subtract => { + // dest = left - right + let left_reg = inst.p1 as usize; + let right_reg = inst.p2 as usize; + let dest_reg = inst.p3 as usize; + + let result = self.arithmetic_op(left_reg, right_reg, "-")?; + self.set_register(dest_reg, result)?; + + if self.verbose { + println!( + "Subtract: r[{}] - r[{}] -> r[{}]", + left_reg, right_reg, dest_reg + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Multiply => { + // dest = left * right + let left_reg = inst.p1 as usize; + let right_reg = inst.p2 as usize; + let dest_reg = inst.p3 as usize; + + let result = self.arithmetic_op(left_reg, right_reg, "*")?; + self.set_register(dest_reg, result)?; + + if self.verbose { + println!( + "Multiply: r[{}] * r[{}] -> r[{}]", + left_reg, right_reg, dest_reg + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Divide => { + // dest = left / right + let left_reg = inst.p1 as usize; + let right_reg = inst.p2 as usize; + let dest_reg = inst.p3 as usize; + + let result = self.arithmetic_op(left_reg, right_reg, "/")?; + self.set_register(dest_reg, result)?; + + if self.verbose { + println!( + "Divide: r[{}] / r[{}] -> r[{}]", + left_reg, right_reg, dest_reg + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Remainder => { + // dest = left % right + let left_reg = inst.p1 as usize; + let right_reg = inst.p2 as usize; + let dest_reg = inst.p3 as usize; + + let result = self.arithmetic_op(left_reg, right_reg, "%")?; + self.set_register(dest_reg, result)?; + + if self.verbose { + println!( + "Remainder: r[{}] %% r[{}] -> r[{}]", + left_reg, right_reg, dest_reg + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::WindowAggStep => { + // Window aggregate step: update window state based on current row + // P1 = func type, P2 = value reg, P3 = cursor, P4 = window spec + let func_type = inst.p1; + let _value_reg = inst.p2 as usize; + let cursor_id = inst.p3 as usize; + let window_spec = inst.p4.clone().unwrap_or_default(); + + // Parse state registers from window spec + let state_part = window_spec.split(":state=").nth(1).unwrap_or(""); + let mut row_num_reg = 0usize; + let mut prev_part_reg = 0usize; + let mut rank_reg = 0usize; + let mut prev_ord_reg = 0usize; + let mut dense_rank_reg = 0usize; + let mut agg_sum_reg = 0usize; + let mut agg_count_reg = 0usize; + let mut agg_min_reg = 0usize; + let mut agg_max_reg = 0usize; + let mut agg_col = 0usize; + + for part in state_part.split(',') { + let kv: Vec<&str> = part.split(':').collect(); + if kv.len() == 2 { + let val = kv[1].parse::().unwrap_or(0); + match kv[0] { + "row_num" => row_num_reg = val, + "prev_part" => prev_part_reg = val, + "rank" => rank_reg = val, + "prev_ord" => prev_ord_reg = val, + "dense_rank" => dense_rank_reg = val, + "agg_sum" => agg_sum_reg = val, + "agg_count" => agg_count_reg = val, + "agg_min" => agg_min_reg = val, + "agg_max" => agg_max_reg = val, + "agg_col" => agg_col = val, + _ => {} + } + } + } + + // Parse partition and order column indices + let part_str = window_spec + .split(":part=") + .nth(1) + .and_then(|s| s.split(':').next()) + .unwrap_or(""); + let ord_str = window_spec + .split(":ord=") + .nth(1) + .and_then(|s| s.split(':').next()) + .unwrap_or(""); + + let part_cols: Vec = + part_str.split(',').filter_map(|s| s.parse().ok()).collect(); + let ord_cols: Vec = + ord_str.split(',').filter_map(|s| s.parse().ok()).collect(); + + // Get current partition key value (first partition column if any) + let current_part_val = if !part_cols.is_empty() { + if let Some(cursor) = self.cursors.get(&cursor_id) { + cursor.column(part_cols[0]).unwrap_or(Value::Null) + } else { + Value::Null + } + } else { + Value::Null // No partition = all rows in same partition + }; + + // Get current order key value + let current_ord_val = if !ord_cols.is_empty() { + if let Some(cursor) = self.cursors.get(&cursor_id) { + cursor.column(ord_cols[0]).unwrap_or(Value::Null) + } else { + Value::Null + } + } else { + Value::Null + }; + + // Check if partition changed + // Use a special marker to detect first row: check if row_num is 0 + let is_first_row = match self.get_register(row_num_reg)? { + Register::Integer(n) => n == 0, + _ => true, + }; + + let prev_part_val = self.get_register(prev_part_reg)?; + let partition_changed = if is_first_row { + true // First row is always a "new partition" + } else { + match (&prev_part_val, ¤t_part_val) { + (Register::Null, Value::Null) => false, // Both null = same partition (or no partition) + (Register::Null, _) => true, // Was null, now has value = new partition + (_, Value::Null) => true, // Had value, now null = new partition + (Register::Integer(p), Value::Integer(c)) => *p != *c, + (Register::String(p), Value::String(c)) => p != c, + (Register::Float(p), Value::Float(c)) => *p != *c, + _ => true, // Different types = partition changed + } + }; + + // Check if order value changed (for RANK) + let prev_ord_val = self.get_register(prev_ord_reg)?; + let order_changed = if is_first_row { + true + } else { + match (&prev_ord_val, ¤t_ord_val) { + (Register::Null, Value::Null) => false, + (Register::Null, _) => true, + (_, Value::Null) => true, + (Register::Integer(p), Value::Integer(c)) => *p != *c, + (Register::String(p), Value::String(c)) => p != c, + (Register::Float(p), Value::Float(c)) => *p != *c, + _ => true, + } + }; + + // Update window state based on function type + match func_type { + 0 => { + // ROW_NUMBER: increment, reset on partition change + let current = if partition_changed { + 1 + } else { + match self.get_register(row_num_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + }; + self.set_register(row_num_reg, Register::Integer(current))?; + } + 1 => { + // RANK: like row_number but same value for ties + let row_num = if partition_changed { + 1 + } else { + match self.get_register(row_num_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + }; + self.set_register(row_num_reg, Register::Integer(row_num))?; + + if partition_changed || order_changed { + self.set_register(rank_reg, Register::Integer(row_num))?; + } + // else rank stays the same (tie) + } + 2 => { + // DENSE_RANK: increment only when value changes + let row_num = if partition_changed { + 1 + } else { + match self.get_register(row_num_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + }; + self.set_register(row_num_reg, Register::Integer(row_num))?; + + let dense = if partition_changed { + 1 + } else if order_changed { + match self.get_register(dense_rank_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + } else { + match self.get_register(dense_rank_reg)? { + Register::Integer(n) => n, + _ => 1, + } + }; + self.set_register(dense_rank_reg, Register::Integer(dense))?; + } + 5..=9 => { + // Aggregate window functions: SUM(5), AVG(6), COUNT(7), MIN(8), MAX(9) + // Track row number for partition detection + let current = if partition_changed { + 1 + } else { + match self.get_register(row_num_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + }; + self.set_register(row_num_reg, Register::Integer(current))?; + + // Get the value from the aggregate column + let agg_value = if let Some(cursor) = self.cursors.get(&cursor_id) { + cursor.column(agg_col).unwrap_or(Value::Null) + } else { + Value::Null + }; + + // Reset aggregates on partition change + if partition_changed { + self.set_register(agg_sum_reg, Register::Integer(0))?; + self.set_register(agg_count_reg, Register::Integer(0))?; + self.set_register(agg_min_reg, Register::Null)?; + self.set_register(agg_max_reg, Register::Null)?; + } + + // Update aggregates based on current value + if !matches!(agg_value, Value::Null) { + // Update count + let new_count = match self.get_register(agg_count_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + }; + self.set_register(agg_count_reg, Register::Integer(new_count))?; + + // Update sum + let val_num = match &agg_value { + Value::Integer(n) => *n as f64, + Value::Float(f) => *f, + Value::String(s) => s.parse::().unwrap_or(0.0), + _ => 0.0, + }; + let current_sum = match self.get_register(agg_sum_reg)? { + Register::Integer(n) => n as f64, + Register::Float(f) => f, + _ => 0.0, + }; + self.set_register(agg_sum_reg, Register::Float(current_sum + val_num))?; + + // Update min + let should_update_min = match self.get_register(agg_min_reg)? { + Register::Null => true, + Register::Integer(n) => val_num < (n as f64), + Register::Float(f) => val_num < f, + _ => false, + }; + if should_update_min { + self.set_register(agg_min_reg, Register::Float(val_num))?; + } + + // Update max + let should_update_max = match self.get_register(agg_max_reg)? { + Register::Null => true, + Register::Integer(n) => val_num > (n as f64), + Register::Float(f) => val_num > f, + _ => false, + }; + if should_update_max { + self.set_register(agg_max_reg, Register::Float(val_num))?; + } + } + } + _ => { + // Other window functions - just track row number + let current = if partition_changed { + 1 + } else { + match self.get_register(row_num_reg)? { + Register::Integer(n) => n + 1, + _ => 1, + } + }; + self.set_register(row_num_reg, Register::Integer(current))?; + } + } + + // Store current partition/order values for next iteration + self.set_register(prev_part_reg, Register::from(current_part_val))?; + self.set_register(prev_ord_reg, Register::from(current_ord_val))?; + + if self.verbose { + println!( + "WindowAggStep: func_type={}, partition_changed={}", + func_type, partition_changed + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::WindowValue => { + // Get current window function value + // P1 = state base reg, P2 = dest reg, P3 = func type + // P4 = for LAG/LEAD: "LAG:cursor=N:col=N:offset=N:parts=N,N,..." + let state_base_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + let func_type = inst.p3; + + let value = match func_type { + 0 => { + // ROW_NUMBER - value is in state_base_reg (row_num_reg) + self.get_register(state_base_reg)? + } + 1 => { + // RANK - value is in rank_reg (state_base_reg + 2) + self.get_register(state_base_reg + 2)? + } + 2 => { + // DENSE_RANK - value is in dense_rank_reg (state_base_reg + 4) + self.get_register(state_base_reg + 4)? + } + 3 | 4 => { + // LAG (3) or LEAD (4) + // Parse P4 to get cursor, column, offset, partition columns + let p4 = inst.p4.clone().unwrap_or_default(); + let mut cursor_id = 1usize; + let mut col_pos = 0usize; + let mut offset = 1i64; + let mut partition_cols: Vec = + Vec::with_capacity(DEFAULT_SORT_KEYS_CAPACITY); + + for part in p4.split(':') { + if let Some(val) = part.strip_prefix("cursor=") { + cursor_id = val.parse().unwrap_or(1); + } else if let Some(val) = part.strip_prefix("col=") { + col_pos = val.parse().unwrap_or(0); + } else if let Some(val) = part.strip_prefix("offset=") { + offset = val.parse().unwrap_or(1); + } else if let Some(parts_str) = part.strip_prefix("parts=") { + if !parts_str.is_empty() { + partition_cols = parts_str + .split(',') + .filter_map(|s| s.parse::().ok()) + .collect(); + } + } + } + + // Get the current row number in partition (1-based) + let row_num = match self.get_register(state_base_reg)? { + Register::Integer(n) => n, + _ => 1, + }; + + // Calculate the row offset + let row_offset = if func_type == 3 { -offset } else { offset }; // LAG = negative, LEAD = positive + + // For LAG: check if we have enough rows behind us in this partition + // For LEAD: check if target row exists and is in same partition + if let Some(cursor) = self.cursors.get(&cursor_id) { + let current_pos = cursor.position; + + // For LAG: row_num is 1-based row in partition + // If row_num <= offset, we can't look back that far, return NULL + if func_type == 3 && row_num <= offset { + Register::Null + } else if func_type == 4 { + // For LEAD: need to check if target row is in same partition + let target_pos = (current_pos as i64 + row_offset) as usize; + let rows = cursor.rows(); + + if target_pos >= rows.len() { + Register::Null + } else { + // Check if target row has same partition keys + let current_row = &rows[current_pos]; + let target_row = &rows[target_pos]; + + let same_partition = if partition_cols.is_empty() { + true // No partition = all in same partition + } else { + partition_cols + .iter() + .all(|&col| current_row.get(col) == target_row.get(col)) + }; + + if same_partition { + target_row + .get(col_pos) + .cloned() + .map(Register::from) + .unwrap_or(Register::Null) + } else { + Register::Null + } + } + } else { + // LAG with valid offset + cursor + .column_at_offset(col_pos, row_offset) + .map(Register::from) + .unwrap_or(Register::Null) + } + } else { + Register::Null + } + } + 5 => { + // SUM - return running sum + // agg_sum is at state_base_reg + 6 + self.get_register(state_base_reg + 6)? + } + 6 => { + // AVG - return running average (sum / count) + // agg_sum is at state_base_reg + 6, agg_count is at state_base_reg + 7 + let sum = match self.get_register(state_base_reg + 6)? { + Register::Float(f) => f, + Register::Integer(n) => n as f64, + _ => 0.0, + }; + let count = match self.get_register(state_base_reg + 7)? { + Register::Integer(n) => n, + _ => 1, + }; + if count > 0 { + Register::Float(sum / count as f64) + } else { + Register::Null + } + } + 7 => { + // COUNT - return running count + // agg_count is at state_base_reg + 7 + self.get_register(state_base_reg + 7)? + } + 8 => { + // MIN - return running min + // agg_min is at state_base_reg + 8 + self.get_register(state_base_reg + 8)? + } + 9 => { + // MAX - return running max + // agg_max is at state_base_reg + 9 + self.get_register(state_base_reg + 9)? + } + _ => { + // Default to row number + self.get_register(state_base_reg)? + } + }; + + self.set_register(dest_reg, value)?; + + if self.verbose { + println!("WindowValue: func_type={}, dest=r[{}]", func_type, dest_reg); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Copy => { + // Copy register P1 to register P2 + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + + let value = self.get_register(src_reg)?; + self.set_register(dest_reg, value)?; + + if self.verbose { + println!("Copy: r[{}] -> r[{}]", src_reg, dest_reg); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Integer => { + // Load integer constant into register + // Special case: p1=-1 means "use current result count as the value" + let value = if inst.p1 == -1 { + self.results.len() as i64 + } else { + inst.p1 + }; + let register_idx = inst.p2 as usize; + + if register_idx < self.registers.len() { + self.registers[register_idx] = Register::Integer(value); + Ok(ExecuteResult::Continue) + } else { + Err(SqawkError::VmError(format!( + "Register index out of bounds: {}", + register_idx + ))) + } + } + + OpCode::String => { + // Load string constant into register + let register_idx = inst.p2 as usize; + let string_value = inst.p4.as_deref().unwrap_or("").to_string(); + + if register_idx < self.registers.len() { + self.registers[register_idx] = Register::String(string_value); + Ok(ExecuteResult::Continue) + } else { + Err(SqawkError::VmError(format!( + "Register index out of bounds: {}", + register_idx + ))) + } + } + + OpCode::Null => { + // Load NULL into register + let register_idx = inst.p2 as usize; + + if register_idx < self.registers.len() { + self.registers[register_idx] = Register::Null; + Ok(ExecuteResult::Continue) + } else { + Err(SqawkError::VmError(format!( + "Register index out of bounds: {}", + register_idx + ))) + } + } + + OpCode::ResultRow => { + // Return a result row from registers + let start_reg = inst.p1 as usize; + let column_count = inst.p2 as usize; + + // Collect values from registers + let mut row = Vec::with_capacity(column_count); + for i in 0..column_count { + let reg_idx = start_reg + i; + if reg_idx < self.registers.len() { + // Convert register to Value + let value = Value::from(self.registers[reg_idx].clone()); + row.push(value); + } else { + return Err(SqawkError::VmError(format!( + "Register index out of bounds: {}", + reg_idx + ))); + } + } + + // Add the row to results + self.results.push(row); + + // Phase 4C: Also record row indices from each cursor (parallel path) + // This collects the current row index from each cursor in source order + let row_indices: Vec> = self + .cursor_source_order + .iter() + .map(|cursor_id| { + self.cursors + .get(cursor_id) + .and_then(|c| c.current_row_idx()) + }) + .collect(); + self.result_row_indices.push(row_indices); + + Ok(ExecuteResult::Continue) + } + + OpCode::Begin => { + // Begin a transaction - initiates a new atomic unit of work + // All operations performed between BEGIN and COMMIT/ROLLBACK are treated as a single + // logical operation from the perspective of database consistency + if self.transaction_state == TransactionState::Active { + return Err(SqawkError::VmError( + "Transaction already in progress".to_string(), + )); + } + + // Initialize transaction state to track changes + self.transaction_state = TransactionState::Active; + + if self.verbose { + println!("BEGIN TRANSACTION"); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Commit => { + // Commit a transaction - makes all changes permanent + // Finalizes the changes made during the transaction and ensures they + // become permanent, durable parts of the database state + if self.transaction_state != TransactionState::Active { + return Err(SqawkError::VmError( + "No transaction in progress to commit".to_string(), + )); + } + + // Finalize transaction + self.transaction_state = TransactionState::Committed; + + if self.verbose { + println!("COMMIT TRANSACTION"); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Rollback => { + // Rollback a transaction - abandons all changes made during the transaction + // Undoes all operations since BEGIN, returning the database to its prior state + if self.transaction_state != TransactionState::Active { + return Err(SqawkError::VmError( + "No transaction in progress to rollback".to_string(), + )); + } + + // Update transaction state to reflect rollback completion + self.transaction_state = TransactionState::RolledBack; + + if self.verbose { + println!("ROLLBACK TRANSACTION"); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::SavePoint => { + // Create a savepoint within the current transaction + // A savepoint marks a point within a transaction to which you can later roll back + // without rolling back the entire transaction + if self.transaction_state != TransactionState::Active { + return Err(SqawkError::VmError( + "No active transaction for savepoint".to_string(), + )); + } + + // In a full implementation, we would: + // 1. Create a marker in the transaction log to identify this position + // 2. Allow for multiple savepoints with different names + // 3. Support rolling back to any specific savepoint + // 4. Properly handle nested savepoints in a hierarchical manner + let savepoint_name = inst + .p4 + .as_deref() + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("sp_{}", inst.p1)); + + if self.verbose { + println!("SAVEPOINT {}", savepoint_name); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Release => { + // Release a savepoint (commit changes up to the savepoint) + // This permanently applies all changes made since the savepoint was created + // while keeping the transaction active for further operations + if self.transaction_state != TransactionState::Active { + return Err(SqawkError::VmError( + "No active transaction for savepoint release".to_string(), + )); + } + + // In a complete implementation, we would: + // 1. Find the specified savepoint in our transaction log + // 2. Make all changes permanent up to that point + // 3. Remove this savepoint and any subsequent nested savepoints + // 4. Maintain the active transaction state for further operations + let savepoint_name = inst + .p4 + .as_deref() + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("sp_{}", inst.p1)); + + if self.verbose { + println!("RELEASE SAVEPOINT {}", savepoint_name); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Lt | OpCode::Le | OpCode::Gt | OpCode::Ge | OpCode::Eq | OpCode::Ne => { + // Unified comparison operations + self.execute_comparison(inst) + } + + OpCode::IfZ => { + // Jump if register P1 contains 0 + // P1: Register to test + // P2: Jump destination (instruction index) + + // Get the register value to test + let reg = self.get_register(inst.p1 as usize)?; + + // Check if the register contains 0 (or logical false) + let is_zero = match reg { + Register::Integer(val) => val == 0, + Register::Float(val) => val == 0.0, + Register::String(ref val) => val.is_empty(), + Register::Null => true, // NULL is considered "zero" for this purpose + Register::Boolean(val) => !val, // false is considered "zero" + }; + + // If the register is zero/false, jump to P2 + if is_zero { + // Jump to the target instruction + return Ok(ExecuteResult::Jump(inst.p2 as usize)); + } + + // Otherwise, continue to the next instruction + Ok(ExecuteResult::Continue) + } + + OpCode::IfPos => { + // Jump if register P1 contains a positive value (> 0) + // P1: Register to test + // P2: Jump destination (instruction index) + + // Get the register value to test + let reg = self.get_register(inst.p1 as usize)?; + + // Check if the register contains a positive value + let is_positive = match reg { + Register::Integer(val) => val > 0, + Register::Float(val) => val > 0.0, + // Boolean true is considered positive, false is not + Register::Boolean(val) => val, + // String and null aren't treated as positive + Register::String(_) | Register::Null => false, + }; + + // If the register is positive, jump to P2 + if is_positive { + return Ok(ExecuteResult::Jump(inst.p2 as usize)); + } + + // Otherwise, continue to the next instruction + Ok(ExecuteResult::Continue) + } + + OpCode::IfNeg => { + // Jump if register P1 contains a negative value (< 0) + // P1: Register to test + // P2: Jump destination (instruction index) + + // Get the register value to test + let reg = self.get_register(inst.p1 as usize)?; + + // Check if the register contains a negative value + let is_negative = match reg { + Register::Integer(val) => val < 0, + Register::Float(val) => val < 0.0, + // Boolean values aren't treated as negative + Register::Boolean(_) => false, + // String and null aren't treated as negative + Register::String(_) | Register::Null => false, + }; + + // If the register is negative, jump to P2 + if is_negative { + return Ok(ExecuteResult::Jump(inst.p2 as usize)); + } + + // Otherwise, continue to the next instruction + Ok(ExecuteResult::Continue) + } + + OpCode::Like => { + // LIKE pattern matching (P1 LIKE P4 pattern, result in P3) + // P1: Register containing value to test + // P2: Flags (bit 0 = negated, bit 1 = case_insensitive) + // P3: Destination register for result (1 for match, 0 for no match) + // P4: Pattern string with SQL LIKE wildcards (% = any sequence, _ = any single char) + + let value_reg = inst.p1 as usize; + let flags = inst.p2; + let result_reg = inst.p3 as usize; + let pattern = inst.p4.clone().unwrap_or_default(); + + let negated = (flags & 1) != 0; + let case_insensitive = (flags & 2) != 0; + + // Get the value to test + let value = self.get_register(value_reg)?; + let value_str = match value { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + Register::Boolean(b) => b.to_string(), + Register::Null => { + // NULL LIKE anything is NULL (treated as false for filtering) + self.set_register(result_reg, Register::Integer(0))?; + return Ok(ExecuteResult::Continue); + } + }; + + // Convert SQL LIKE pattern to regex + let regex_pattern = self.like_pattern_to_regex(&pattern, case_insensitive); + + // Perform the match + let matched = match regex::Regex::new(®ex_pattern) { + Ok(re) => re.is_match(&value_str), + Err(_) => false, + }; - // In a complete implementation, we would: - // 1. Find the specified savepoint in our transaction log - // 2. Make all changes permanent up to that point - // 3. Remove this savepoint and any subsequent nested savepoints - // 4. Maintain the active transaction state for further operations - let savepoint_name = inst.p4.clone().unwrap_or_else(|| format!("sp_{}", inst.p1)); + // Apply negation if needed + let result = if negated { !matched } else { matched }; + + self.set_register(result_reg, Register::Integer(if result { 1 } else { 0 }))?; + + Ok(ExecuteResult::Continue) + } + + OpCode::Glob => { + // GLOB pattern matching (Unix-style wildcards) + // P1: Register containing value to test + // P2: Flags (bit 0 = negated) + // P3: Destination register for result + // P4: Pattern string with GLOB wildcards (* = any sequence, ? = any single char) + + let value_reg = inst.p1 as usize; + let flags = inst.p2; + let result_reg = inst.p3 as usize; + let pattern = inst.p4.clone().unwrap_or_default(); + + let negated = (flags & 1) != 0; + + // Get the value to test + let value = self.get_register(value_reg)?; + let value_str = match value { + Register::String(s) => s, + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + Register::Boolean(b) => b.to_string(), + Register::Null => { + self.set_register(result_reg, Register::Integer(0))?; + return Ok(ExecuteResult::Continue); + } + }; + + // Convert GLOB pattern to regex (case-sensitive) + let regex_pattern = self.glob_pattern_to_regex(&pattern); + + // Perform the match + let matched = match regex::Regex::new(®ex_pattern) { + Ok(re) => re.is_match(&value_str), + Err(_) => false, + }; + + // Apply negation if needed + let result = if negated { !matched } else { matched }; + + self.set_register(result_reg, Register::Integer(if result { 1 } else { 0 }))?; + + Ok(ExecuteResult::Continue) + } + + OpCode::Cast => { + // Cast value in P1 to type specified in P4, result in P2 + // P1: Source register containing value to cast + // P2: Destination register for result + // P4: Target type name ("INTEGER", "TEXT", "REAL", "BOOLEAN") + + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + let target_type = inst.p4.as_deref().unwrap_or("TEXT"); + + let src_value = self.get_register(src_reg)?; + + let result = match target_type.to_uppercase().as_str() { + "INTEGER" | "INT" => match src_value { + Register::Integer(i) => Register::Integer(i), + Register::Float(f) => Register::Integer(f as i64), + Register::String(s) => Register::Integer(s.parse::().unwrap_or(0)), + Register::Boolean(b) => Register::Integer(if b { 1 } else { 0 }), + Register::Null => Register::Null, + }, + "REAL" | "FLOAT" | "DOUBLE" => match src_value { + Register::Integer(i) => Register::Float(i as f64), + Register::Float(f) => Register::Float(f), + Register::String(s) => Register::Float(s.parse::().unwrap_or(0.0)), + Register::Boolean(b) => Register::Float(if b { 1.0 } else { 0.0 }), + Register::Null => Register::Null, + }, + "TEXT" | "VARCHAR" | "CHAR" | "STRING" => match src_value { + Register::Integer(i) => Register::String(i.to_string()), + Register::Float(f) => Register::String(f.to_string()), + Register::String(s) => Register::String(s), + Register::Boolean(b) => Register::String(b.to_string()), + Register::Null => Register::Null, + }, + "BOOLEAN" | "BOOL" => match src_value { + Register::Integer(i) => Register::Boolean(i != 0), + Register::Float(f) => Register::Boolean(f != 0.0), + Register::String(s) => { + let lower = s.to_lowercase(); + Register::Boolean(lower == "true" || lower == "1" || lower == "yes") + } + Register::Boolean(b) => Register::Boolean(b), + Register::Null => Register::Null, + }, + _ => { + // Unknown type - default to TEXT + match src_value { + Register::Integer(i) => Register::String(i.to_string()), + Register::Float(f) => Register::String(f.to_string()), + Register::String(s) => Register::String(s), + Register::Boolean(b) => Register::String(b.to_string()), + Register::Null => Register::Null, + } + } + }; + + self.set_register(dest_reg, result)?; + Ok(ExecuteResult::Continue) + } + + OpCode::IsNull => { + // Check if P1 is NULL, set P2 to 1 if NULL, 0 otherwise + let src_reg = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + + let src_value = self.get_register(src_reg)?; + let is_null = matches!(src_value, Register::Null); + + self.set_register(dest_reg, Register::Integer(if is_null { 1 } else { 0 }))?; + Ok(ExecuteResult::Continue) + } + + // JumpIfTrue and JumpIfFalse have been replaced by SQLite-style opcodes: + // - IfZ (jump if zero/false) + // - IfPos (jump if positive) + // - IfNeg (jump if negative) + OpCode::Noop => { + // No operation + Ok(ExecuteResult::Continue) + } + + OpCode::Distinct => { + // Remove duplicate rows from results + // This is used by UNION (without ALL) to deduplicate combined results + let mut seen = std::collections::HashSet::with_capacity(self.results.len()); + let mut unique_results = Vec::with_capacity(self.results.len()); + + for row in &self.results { + // Create a hashable key from the row values + let key: Vec = row.iter().map(|v| format!("{:?}", v)).collect(); + let key_str = key.join("|"); + + if seen.insert(key_str) { + unique_results.push(row.clone()); + } + } + + self.results = unique_results; + + if self.verbose { + println!( + "DISTINCT: {} unique rows after deduplication", + self.results.len() + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Limit => { + // Limit results to P1 rows, skip first P2 rows (post-processing opcode) + // This is used after DISTINCT to apply LIMIT/OFFSET correctly + let limit = inst.p1 as usize; + let offset = inst.p2 as usize; + + // First apply OFFSET - skip first N rows + if offset > 0 && !self.results.is_empty() { + if offset >= self.results.len() { + self.results.clear(); + } else { + self.results = self.results.drain(offset..).collect(); + } + } + + // Then apply LIMIT - truncate to N rows + if self.results.len() > limit { + self.results.truncate(limit); + } + + if self.verbose { + println!( + "LIMIT: offset {} then truncated to {} rows", + offset, + self.results.len() + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Intersect => { + // Keep only rows that exist in both left and right result sets + // P1 contains the register with the count of left-side results (-1 means no marker) + let left_count = if inst.p1 >= 0 { + match self.get_register(inst.p1 as usize)? { + Register::Integer(n) => n as usize, + _ => { + return Err(SqawkError::VmError( + "Invalid left count register for INTERSECT".to_string(), + )) + } + } + } else { + // If no marker, assume half the results are from each side (fallback) + self.results.len() / 2 + }; + + // Split results into left and right sets + let left_rows: Vec<_> = self.results.iter().take(left_count).cloned().collect(); + let right_rows: Vec<_> = self.results.iter().skip(left_count).cloned().collect(); + + // Create a set of right rows for fast lookup + let right_set: std::collections::HashSet<_> = right_rows + .iter() + .map(|row| { + row.iter() + .map(|v| format!("{:?}", v)) + .collect::>() + .join("|") + }) + .collect(); + + // Keep only left rows that also appear in right + let mut intersection = Vec::with_capacity(left_rows.len()); + let mut seen = std::collections::HashSet::with_capacity(left_rows.len()); + + for row in &left_rows { + let key: Vec = row.iter().map(|v| format!("{:?}", v)).collect(); + let key_str = key.join("|"); + + if right_set.contains(&key_str) && seen.insert(key_str) { + intersection.push(row.clone()); + } + } + + self.results = intersection; + + if self.verbose { + println!("INTERSECT: {} rows in intersection", self.results.len()); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Except => { + // Keep only rows from left that don't exist in right result set + // P1 contains the register with the count of left-side results (-1 means no marker) + let left_count = if inst.p1 >= 0 { + match self.get_register(inst.p1 as usize)? { + Register::Integer(n) => n as usize, + _ => { + return Err(SqawkError::VmError( + "Invalid left count register for EXCEPT".to_string(), + )) + } + } + } else { + // If no marker, assume half the results are from each side (fallback) + self.results.len() / 2 + }; + + // Split results into left and right sets + let left_rows: Vec<_> = self.results.iter().take(left_count).cloned().collect(); + let right_rows: Vec<_> = self.results.iter().skip(left_count).cloned().collect(); + + // Create a set of right rows for fast lookup + let right_set: std::collections::HashSet<_> = right_rows + .iter() + .map(|row| { + row.iter() + .map(|v| format!("{:?}", v)) + .collect::>() + .join("|") + }) + .collect(); + + // Keep only left rows that don't appear in right + let mut difference = Vec::with_capacity(left_rows.len()); + let mut seen = std::collections::HashSet::with_capacity(left_rows.len()); + + for row in &left_rows { + let key: Vec = row.iter().map(|v| format!("{:?}", v)).collect(); + let key_str = key.join("|"); + + if !right_set.contains(&key_str) && seen.insert(key_str) { + difference.push(row.clone()); + } + } + + self.results = difference; + + if self.verbose { + println!("EXCEPT: {} rows in difference", self.results.len()); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::NullRow => { + // Load NULL values into registers P1 through P1+P2-1 + // P1: Start register + // P2: Number of registers to set to NULL + let start_reg = inst.p1 as usize; + let count = inst.p2 as usize; + + for i in 0..count { + self.set_register(start_reg + i, Register::Null)?; + } + + if self.verbose { + println!( + "NullRow: Set registers {} to {} to NULL", + start_reg, + start_reg + count - 1 + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::RewindInner => { + // Rewind cursor P1 for inner loop of nested join + // P1: Cursor index + // P2: Jump address if table is empty + let cursor_idx = inst.p1 as usize; + let jump_addr = inst.p2 as usize; + + if let Some(cursor) = self.cursors.get_mut(&cursor_idx) { + if cursor.rewind() { + // Cursor has rows, continue + Ok(ExecuteResult::Continue) + } else { + // Table is empty, jump to P2 + Ok(ExecuteResult::Jump(jump_addr)) + } + } else { + Err(SqawkError::VmError(format!( + "Cursor {} not found for RewindInner", + cursor_idx + ))) + } + } + + OpCode::MarkMatch => { + // Set register P1 to 1 to indicate a match was found + let reg = inst.p1 as usize; + self.set_register(reg, Register::Integer(1))?; + + if self.verbose { + println!("MarkMatch: Set r[{}] = 1", reg); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::CheckMatch => { + // If register P1 is 0 (no match), jump to P2 + // Then reset register P1 to 0 for next iteration + let reg = inst.p1 as usize; + let jump_addr = inst.p2 as usize; + + let should_jump = match self.get_register(reg)? { + Register::Integer(0) => true, + Register::Integer(_) => false, + Register::Null => true, // Treat NULL as no match + _ => true, // Default to no match for unexpected types + }; + + // Reset the match flag for the next iteration + self.set_register(reg, Register::Integer(0))?; + + if should_jump { + if self.verbose { + println!("CheckMatch: No match found, jumping to {}", jump_addr); + } + Ok(ExecuteResult::Jump(jump_addr)) + } else { + if self.verbose { + println!("CheckMatch: Match found, continuing"); + } + Ok(ExecuteResult::Continue) + } + } + + OpCode::DecrJumpZero => { + // Decrement register P1. If result is zero, jump to P2. + // Used for LIMIT - when counter hits zero, stop outputting rows. + // Note: LIMIT 0 is handled at compile time with a Goto. + let reg = inst.p1 as usize; + let jump_addr = inst.p2 as usize; + + let current = match self.get_register(reg)? { + Register::Integer(n) => n, + _ => 0, + }; + + let new_val = current - 1; + self.set_register(reg, Register::Integer(new_val))?; + + if self.verbose { + println!( + "DecrJumpZero: r[{}] = {} -> {}, jump if zero to {}", + reg, current, new_val, jump_addr + ); + } + + if new_val == 0 { + Ok(ExecuteResult::Jump(jump_addr)) + } else { + Ok(ExecuteResult::Continue) + } + } + + OpCode::SorterOpen => { + // Open a sorter with P1 = sorter ID, P2 = column count, P4 = sort spec + let sorter_id = inst.p1 as usize; + let col_count = inst.p2 as usize; + let sort_spec = inst.p4.clone().unwrap_or_default(); + + // Parse sort specification: "col_idx:asc,col_idx:desc,..." + let sort_keys: Vec<(usize, bool)> = sort_spec + .split(',') + .filter_map(|spec| { + let parts: Vec<&str> = spec.split(':').collect(); + if parts.len() == 2 { + let col_idx = parts[0].parse::().ok()?; + let ascending = parts[1] != "desc"; + Some((col_idx, ascending)) + } else { + None + } + }) + .collect(); + + let sorter = Sorter::new(col_count, sort_keys); + self.sorters.insert(sorter_id, sorter); + + if self.verbose { + println!( + "SorterOpen: Created sorter {} with {} columns, spec: {}", + sorter_id, col_count, sort_spec + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::SorterInsert => { + // Insert row into sorter P1. P2 = start register, P3 = column count + let sorter_id = inst.p1 as usize; + let start_reg = inst.p2 as usize; + let col_count = inst.p3 as usize; + + // Collect values from registers + let mut row = Vec::with_capacity(col_count); + for i in 0..col_count { + let reg_idx = start_reg + i; + let value = Value::from(self.get_register(reg_idx)?); + row.push(value); + } + + if let Some(sorter) = self.sorters.get_mut(&sorter_id) { + sorter.insert(row); + if self.verbose { + println!("SorterInsert: Added row to sorter {}", sorter_id); + } + } else { + return Err(SqawkError::VmError(format!( + "Sorter {} not found", + sorter_id + ))); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::SorterSort => { + // Sort the sorter P1 + let sorter_id = inst.p1 as usize; + + if let Some(sorter) = self.sorters.get_mut(&sorter_id) { + sorter.sort(); + if self.verbose { + println!( + "SorterSort: Sorted {} rows in sorter {}", + sorter.rows.len(), + sorter_id + ); + } + } else { + return Err(SqawkError::VmError(format!( + "Sorter {} not found", + sorter_id + ))); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::SorterData => { + // Copy current sorter P1 row to registers starting at P2. P3 = column count + let sorter_id = inst.p1 as usize; + let start_reg = inst.p2 as usize; + let col_count = inst.p3 as usize; + + let (row_data, position) = { + let sorter = self.sorters.get(&sorter_id).ok_or_else(|| { + SqawkError::VmError(format!("Sorter {} not found", sorter_id)) + })?; + let row = sorter.current_row().ok_or_else(|| { + SqawkError::VmError(format!("No current row in sorter {}", sorter_id)) + })?; + (row.clone(), sorter.position) + }; + + for (i, value) in row_data.iter().take(col_count).enumerate() { + self.set_register(start_reg + i, Register::from(value.clone()))?; + } + + if self.verbose { + println!( + "SorterData: Copied row {} from sorter {} to r[{}..{}]", + position, + sorter_id, + start_reg, + start_reg + col_count - 1 + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::SorterNext => { + // Advance sorter P1. Jump to P2 if more rows, else continue. + let sorter_id = inst.p1 as usize; + let jump_addr = inst.p2 as usize; + + if let Some(sorter) = self.sorters.get_mut(&sorter_id) { + if sorter.next() { + if self.verbose { + println!( + "SorterNext: Advanced to row {} in sorter {}, jumping to {}", + sorter.position, sorter_id, jump_addr + ); + } + Ok(ExecuteResult::Jump(jump_addr)) + } else { + if self.verbose { + println!( + "SorterNext: No more rows in sorter {}, continuing", + sorter_id + ); + } + Ok(ExecuteResult::Continue) + } + } else { + Err(SqawkError::VmError(format!( + "Sorter {} not found", + sorter_id + ))) + } + } + + OpCode::OpenEphemeral => { + // Open ephemeral cursor P1 with P2 columns. P4 = sort key spec. + // Uses unified cursor model - Column/Next/Rewind work for ephemeral cursors. + let cursor_id = inst.p1 as usize; + let col_count = inst.p2 as usize; + let sort_spec = inst.p4.clone().unwrap_or_default(); + + // Parse sort specification: "col_idx:asc,col_idx:desc,..." + let sort_keys: Vec<(usize, bool)> = sort_spec + .split(',') + .filter_map(|spec| { + let parts: Vec<&str> = spec.split(':').collect(); + if parts.len() == 2 { + let col_idx = parts[0].parse::().ok()?; + let ascending = parts[1] != "desc"; + Some((col_idx, ascending)) + } else { + None + } + }) + .collect(); + + // Create ephemeral cursor in the unified cursors map + let cursor = Cursor::ephemeral(col_count, sort_keys); + self.cursors.insert(cursor_id, cursor); + + if self.verbose { + println!( + "OpenEphemeral: Created ephemeral cursor {} with {} columns, spec: {}", + cursor_id, col_count, sort_spec + ); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::IdxInsert => { + // Insert into ephemeral cursor P1. P2 = start register, P3 = column count. + let cursor_id = inst.p1 as usize; + let start_reg = inst.p2 as usize; + let col_count = inst.p3 as usize; + + // Collect values from registers + let mut row = Vec::with_capacity(col_count); + for i in 0..col_count { + let reg_idx = start_reg + i; + let value = Value::from(self.get_register(reg_idx)?); + row.push(value); + } + + if let Some(cursor) = self.cursors.get_mut(&cursor_id) { + cursor.insert_row(row)?; + if self.verbose { + println!("IdxInsert: Added row to cursor {}", cursor_id); + } + } else { + return Err(SqawkError::VmError(format!( + "Cursor {} not found", + cursor_id + ))); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::Sort => { + // Sort ephemeral cursor P1. Jump to P2 if empty, else continue. + let cursor_id = inst.p1 as usize; + let jump_if_empty = inst.p2 as usize; + + if let Some(cursor) = self.cursors.get_mut(&cursor_id) { + let has_rows = cursor.sort()?; + if has_rows { + if self.verbose { + println!( + "Sort: Sorted {} rows in cursor {}", + cursor.row_count(), + cursor_id + ); + } + Ok(ExecuteResult::Continue) + } else { + if self.verbose { + println!( + "Sort: Cursor {} is empty, jumping to {}", + cursor_id, jump_if_empty + ); + } + Ok(ExecuteResult::Jump(jump_if_empty)) + } + } else { + Err(SqawkError::VmError(format!( + "Cursor {} not found", + cursor_id + ))) + } + } + + OpCode::Sequence => { + // P1 = ephemeral cursor, P2 = dest register. Generate next sequence number. + let cursor_id = inst.p1 as usize; + let dest_reg = inst.p2 as usize; + + if let Some(cursor) = self.cursors.get_mut(&cursor_id) { + let seq = cursor.next_sequence()?; + self.set_register(dest_reg, Register::Integer(seq))?; + if self.verbose { + println!( + "Sequence: Generated sequence {} from cursor {} into r[{}]", + seq, cursor_id, dest_reg + ); + } + } else { + return Err(SqawkError::VmError(format!( + "Cursor {} not found", + cursor_id + ))); + } + + Ok(ExecuteResult::Continue) + } + + OpCode::AggStep => { + // Step an aggregate function + // P1 = function type (0=COUNT, 1=SUM, 2=AVG, 3=MIN, 4=MAX) + // P2 = value register + // P3 = accumulator register (used as key for accumulator storage) + let func_type = inst.p1; + let value_reg = inst.p2 as usize; + let acc_reg = inst.p3 as usize; + + // Get the value to accumulate (skip for COUNT(*) where p2=-1) + let value = if inst.p2 >= 0 { + Some(Value::from(self.get_register(value_reg)?)) + } else { + None + }; + + // Check if the accumulator register is Null - if so, reset the accumulator + // This matches SQLite's behavior where Null initialization signals a fresh start + let acc_reg_value = self.get_register(acc_reg)?; + if matches!(acc_reg_value, Register::Null) { + // Reset the accumulator for this register + self.accumulators.insert( + acc_reg, + AggAccumulator { + func_type, + count: 0, + value: None, + }, + ); + // Mark the register as non-Null to indicate accumulator is active + self.registers[acc_reg] = Register::Integer(0); + } + + // Get or create the accumulator + let acc = self.accumulators.entry(acc_reg).or_insert(AggAccumulator { + func_type, + count: 0, + value: None, + }); + + match func_type { + AGG_COUNT => { + // COUNT - count non-NULL values (or all rows if value is None) + if value.is_none() || !matches!(value.as_ref(), Some(Value::Null)) { + acc.count += 1; + } + } + AGG_SUM => { + // SUM + if let Some(val) = value { + if !matches!(val, Value::Null) { + acc.count += 1; + acc.value = Some(match (&acc.value, &val) { + (None, v) => v.clone(), + (Some(Value::Integer(a)), Value::Integer(b)) => { + Value::Integer(a + b) + } + (Some(Value::Float(a)), Value::Float(b)) => Value::Float(a + b), + (Some(Value::Integer(a)), Value::Float(b)) => { + Value::Float(*a as f64 + b) + } + (Some(Value::Float(a)), Value::Integer(b)) => { + Value::Float(a + *b as f64) + } + (Some(existing), _) => existing.clone(), + }); + } + } + } + AGG_AVG => { + // AVG (accumulate sum, divide in AggFinal) + if let Some(val) = value { + if !matches!(val, Value::Null) { + acc.count += 1; + acc.value = Some(match (&acc.value, &val) { + (None, v) => v.clone(), + (Some(Value::Integer(a)), Value::Integer(b)) => { + Value::Integer(a + b) + } + (Some(Value::Float(a)), Value::Float(b)) => Value::Float(a + b), + (Some(Value::Integer(a)), Value::Float(b)) => { + Value::Float(*a as f64 + b) + } + (Some(Value::Float(a)), Value::Integer(b)) => { + Value::Float(a + *b as f64) + } + (Some(existing), _) => existing.clone(), + }); + } + } + } + AGG_MIN => { + // MIN + if let Some(val) = value { + if !matches!(val, Value::Null) { + acc.count += 1; + acc.value = Some(match &acc.value { + None => val, + Some(existing) => { + if compare_values(&val, existing) + == std::cmp::Ordering::Less + { + val + } else { + existing.clone() + } + } + }); + } + } + } + AGG_MAX => { + // MAX + if let Some(val) = value { + if !matches!(val, Value::Null) { + acc.count += 1; + acc.value = Some(match &acc.value { + None => val, + Some(existing) => { + if compare_values(&val, existing) + == std::cmp::Ordering::Greater + { + val + } else { + existing.clone() + } + } + }); + } + } + } + _ => { + return Err(SqawkError::VmError(format!( + "Unknown aggregate function type: {}", + func_type + ))); + } + } if self.verbose { - println!("RELEASE SAVEPOINT {}", savepoint_name); + println!( + "AggStep: func={} acc_reg={} count={}", + func_type, acc_reg, acc.count + ); } Ok(ExecuteResult::Continue) } - OpCode::Lt => { - // Less than comparison (P1 < P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a < b, - (Register::Float(a), Register::Float(b)) => a < b, - (Register::String(a), Register::String(b)) => a < b, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) < *b, - (Register::Float(a), Register::Integer(b)) => *a < (*b as f64), - // Other combinations are not comparable - _ => return Err(SqawkError::VmError( - format!("Cannot compare incompatible types: {:?} and {:?}", reg1, reg2) - )), - }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - - Ok(ExecuteResult::Continue) - } - - OpCode::IfZ => { - // Jump if register P1 contains 0 - // P1: Register to test - // P2: Jump destination (instruction index) - - // Get the register value to test - let reg = self.get_register(inst.p1 as usize)?; - - // Check if the register contains 0 (or logical false) - let is_zero = match reg { - Register::Integer(val) => val == 0, - Register::Float(val) => val == 0.0, - Register::String(ref val) => val.is_empty(), - Register::Null => true, // NULL is considered "zero" for this purpose - Register::Boolean(val) => !val, // false is considered "zero" + OpCode::AggFinal => { + // Finalize an aggregate and store result + // P1 = accumulator register + // P2 = result register + // P3 = function type (0=COUNT, 1=SUM, 2=AVG, 3=MIN, 4=MAX) - used when no accumulator exists + let acc_reg = inst.p1 as usize; + let result_reg = inst.p2 as usize; + let func_type_hint = inst.p3; // Fallback function type + + // First check if the accumulator register is Null - this means no AggStep was called + // (the register was initialized to Null and never updated by AggStep) + let acc_reg_value = self.get_register(acc_reg)?; + let result = if matches!(acc_reg_value, Register::Null) { + // No AggStep was called this iteration - return default based on function type + if func_type_hint == AGG_COUNT { + Register::Integer(0) // COUNT of no rows is 0 + } else { + Register::Null // SUM/AVG/MIN/MAX of no rows is NULL + } + } else if let Some(acc) = self.accumulators.get(&acc_reg) { + match acc.func_type { + AGG_COUNT => { + // COUNT - return the count + Register::Integer(acc.count) + } + AGG_SUM => { + // SUM - return the accumulated sum + match &acc.value { + Some(Value::Integer(i)) => Register::Integer(*i), + Some(Value::Float(f)) => Register::Float(*f), + _ => Register::Null, + } + } + AGG_AVG => { + // AVG - divide sum by count + if acc.count == 0 { + Register::Null + } else { + match &acc.value { + Some(Value::Integer(i)) => { + Register::Float(*i as f64 / acc.count as f64) + } + Some(Value::Float(f)) => Register::Float(f / acc.count as f64), + _ => Register::Null, + } + } + } + AGG_MIN | AGG_MAX => { + // MIN/MAX - return the accumulated value + match &acc.value { + Some(Value::Integer(i)) => Register::Integer(*i), + Some(Value::Float(f)) => Register::Float(*f), + Some(Value::String(s)) => Register::String(s.clone().into_owned()), + Some(Value::Boolean(b)) => Register::Boolean(*b), + _ => Register::Null, + } + } + _ => Register::Null, + } + } else { + // No accumulator found - return appropriate default based on function type + // COUNT with no rows returns 0, others return NULL + if func_type_hint == AGG_COUNT { + Register::Integer(0) // COUNT of no rows is 0 + } else { + Register::Null + } }; - - // If the register is zero/false, jump to P2 - if is_zero { - // Jump to the target instruction - return Ok(ExecuteResult::Jump(inst.p2 as usize)); + + self.set_register(result_reg, result)?; + + if self.verbose { + println!("AggFinal: acc_reg={} result_reg={}", acc_reg, result_reg); } - - // Otherwise, continue to the next instruction + Ok(ExecuteResult::Continue) } - - OpCode::IfPos => { - // Jump if register P1 contains a positive value (> 0) - // P1: Register to test - // P2: Jump destination (instruction index) - - // Get the register value to test - let reg = self.get_register(inst.p1 as usize)?; - - // Check if the register contains a positive value - let is_positive = match reg { - Register::Integer(val) => val > 0, - Register::Float(val) => val > 0.0, - // Boolean true is considered positive, false is not - Register::Boolean(val) => val, - // String and null aren't treated as positive - Register::String(_) | Register::Null => false, - }; - - // If the register is positive, jump to P2 - if is_positive { - return Ok(ExecuteResult::Jump(inst.p2 as usize)); + + OpCode::AggReset => { + // Reset accumulator at register P1 + let acc_reg = inst.p1 as usize; + + // Remove the accumulator so it will be re-initialized on next AggStep + self.accumulators.remove(&acc_reg); + + if self.verbose { + println!("AggReset: cleared accumulator at reg {}", acc_reg); } - - // Otherwise, continue to the next instruction + Ok(ExecuteResult::Continue) } - - OpCode::IfNeg => { - // Jump if register P1 contains a negative value (< 0) - // P1: Register to test - // P2: Jump destination (instruction index) - - // Get the register value to test - let reg = self.get_register(inst.p1 as usize)?; - - // Check if the register contains a negative value - let is_negative = match reg { - Register::Integer(val) => val < 0, - Register::Float(val) => val < 0.0, - // Boolean values aren't treated as negative - Register::Boolean(_) => false, - // String and null aren't treated as negative - Register::String(_) | Register::Null => false, + + OpCode::InitCoroutine => { + // InitCoroutine P1, P2, P3 + // P1 = register to store coroutine return/entry address + // P2 = address to jump to (skip over coroutine body) + // P3 = coroutine entry point (first instruction of coroutine) + let return_reg = inst.p1 as usize; + let skip_addr = inst.p2 as usize; + let entry_point = inst.p3 as usize; + + // Store the entry point in the register - this will be swapped with PC on Yield + self.registers[return_reg] = Register::Integer(entry_point as i64); + + // Jump over the coroutine body to continue main execution + Ok(ExecuteResult::Jump(skip_addr)) + } + + OpCode::Yield => { + // Yield P1 + // P1 = register containing coroutine PC (swap with current PC) + // Cooperative context switch between main code and coroutine + let coro_reg = inst.p1 as usize; + + // Get saved coroutine PC from register + let saved_pc = match &self.registers[coro_reg] { + Register::Integer(pc) => *pc as usize, + _ => { + return Err(SqawkError::VmError( + "Yield: register must contain integer PC".into(), + )) + } }; - - // If the register is negative, jump to P2 - if is_negative { - return Ok(ExecuteResult::Jump(inst.p2 as usize)); + + // Save current PC + 1 (next instruction) into the register + // This is where we'll return after EndCoroutine + self.registers[coro_reg] = Register::Integer((self.pc + 1) as i64); + + // Jump to the saved coroutine location + Ok(ExecuteResult::Jump(saved_pc)) + } + + OpCode::EndCoroutine => { + // EndCoroutine P1 + // P1 = register containing return address (saved by last Yield) + // Jump back to caller, prepare register for next Yield + let coro_reg = inst.p1 as usize; + + // Get return address from register + let return_addr = match &self.registers[coro_reg] { + Register::Integer(pc) => *pc as usize, + _ => { + return Err(SqawkError::VmError( + "EndCoroutine: register must contain integer return address".into(), + )) + } + }; + + // Store next instruction (PC + 1) for the next Yield call + // This will be the coroutine's resume point + self.registers[coro_reg] = Register::Integer((self.pc + 1) as i64); + + // Jump back to caller + Ok(ExecuteResult::Jump(return_addr)) + } + + OpCode::Once => { + // Once P1, P2 + // First time: continue to next instruction + // Subsequently: jump to P2 + // Uses instruction address as unique key for tracking + let skip_addr = inst.p2 as usize; + + if self.once_flags.get(&self.pc).copied().unwrap_or(false) { + // Already executed, skip + Ok(ExecuteResult::Jump(skip_addr)) + } else { + // First time, mark as executed and continue + self.once_flags.insert(self.pc, true); + Ok(ExecuteResult::Continue) } - - // Otherwise, continue to the next instruction - Ok(ExecuteResult::Continue) } - - OpCode::Gt => { - // Greater than comparison (P1 > P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a > b, - (Register::Float(a), Register::Float(b)) => a > b, - (Register::String(a), Register::String(b)) => a > b, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) > *b, - (Register::Float(a), Register::Integer(b)) => *a > (*b as f64), - // Other combinations are not comparable - _ => return Err(SqawkError::VmError( - format!("Cannot compare incompatible types: {:?} and {:?}", reg1, reg2) - )), - }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - - Ok(ExecuteResult::Continue) - } - - OpCode::Ge => { - // Greater than or equal comparison (P1 >= P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a >= b, - (Register::Float(a), Register::Float(b)) => a >= b, - (Register::String(a), Register::String(b)) => a >= b, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) >= *b, - (Register::Float(a), Register::Integer(b)) => *a >= (*b as f64), - // Other combinations are not comparable - _ => return Err(SqawkError::VmError( - format!("Cannot compare incompatible types: {:?} and {:?}", reg1, reg2) - )), - }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - - Ok(ExecuteResult::Continue) - } - - OpCode::Eq => { - // Equal comparison (P1 == P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a == b, - (Register::Float(a), Register::Float(b)) => a == b, - (Register::String(a), Register::String(b)) => a == b, - (Register::Boolean(a), Register::Boolean(b)) => a == b, - (Register::Null, Register::Null) => true, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) == *b, - (Register::Float(a), Register::Integer(b)) => *a == (*b as f64), - // Different types are not equal (except for numeric conversions above) - _ => false, - }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - - Ok(ExecuteResult::Continue) - } - - OpCode::Ne => { - // Not equal comparison (P1 != P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types (opposite of Eq) - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a != b, - (Register::Float(a), Register::Float(b)) => a != b, - (Register::String(a), Register::String(b)) => a != b, - (Register::Boolean(a), Register::Boolean(b)) => a != b, - (Register::Null, Register::Null) => false, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) != *b, - (Register::Float(a), Register::Integer(b)) => *a != (*b as f64), - // Different types are not equal (so they are "not equal" = true) - _ => true, + + OpCode::Exists => { + // Exists P1, P2 + // P1 = result register (set to 1 if cursor P2 has rows, else 0) + // P2 = cursor ID to check + let result_reg = inst.p1 as usize; + let cursor_idx = inst.p2 as usize; + + let has_rows = if let Some(cursor) = self.cursors.get(&cursor_idx) { + cursor.row_count() > 0 + } else { + false }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - + + self.registers[result_reg] = Register::Integer(if has_rows { 1 } else { 0 }); Ok(ExecuteResult::Continue) } - - OpCode::Le => { - // Less than or equal comparison (P1 <= P2, result in P3) - // P1 and P2 are register indices to compare - // P3 is the destination register for the result (1 for true, 0 for false) - - // Get values from registers P1 and P2 - let reg1 = self.get_register(inst.p1 as usize)?; - let reg2 = self.get_register(inst.p2 as usize)?; - - // Perform comparison based on register types - let result = match (®1, ®2) { - (Register::Integer(a), Register::Integer(b)) => a <= b, - (Register::Float(a), Register::Float(b)) => a <= b, - (Register::String(a), Register::String(b)) => a <= b, - // For mixed types, try to convert and compare - (Register::Integer(a), Register::Float(b)) => (*a as f64) <= *b, - (Register::Float(a), Register::Integer(b)) => *a <= (*b as f64), - // Other combinations are not comparable - _ => return Err(SqawkError::VmError( - format!("Cannot compare incompatible types: {:?} and {:?}", reg1, reg2) - )), - }; - - // Store result in destination register (1 for true, 0 for false) - let result_value = if result { Register::Integer(1) } else { Register::Integer(0) }; - self.set_register(inst.p3 as usize, result_value)?; - - Ok(ExecuteResult::Continue) - } - - // JumpIfTrue and JumpIfFalse have been replaced by SQLite-style opcodes: - // - IfZ (jump if zero/false) - // - IfPos (jump if positive) - // - IfNeg (jump if negative) - - OpCode::Noop => { - // No operation + + OpCode::NotExists => { + // NotExists P1, P2 + // P1 = result register (set to 1 if cursor P2 has no rows, else 0) + // P2 = cursor ID to check + let result_reg = inst.p1 as usize; + let cursor_idx = inst.p2 as usize; + + let has_rows = if let Some(cursor) = self.cursors.get(&cursor_idx) { + cursor.row_count() > 0 + } else { + false + }; + + self.registers[result_reg] = Register::Integer(if has_rows { 0 } else { 1 }); Ok(ExecuteResult::Continue) } @@ -821,29 +3226,183 @@ impl<'a> VmEngine<'a> { // Removed unused methods get_results, has_results, get_column_names, // get_affected_rows, and get_modified_tables - /// Create a table from the results + /// Create a table from the results using the result schema pub fn create_result_table(&self) -> SqawkResult> { + let schema = &self.program.result_schema; + + // If we have no results but DO have a schema, create an empty table with headers + // This handles SELECT from empty tables if self.results.is_empty() { + if !schema.is_empty() { + // Build column definitions from schema + let col_defs: Vec = schema + .columns + .iter() + .map(|col| ColumnDefinition { + name: col.name.clone(), + data_type: col.data_type, + }) + .collect(); + let table = Table::new_with_schema("result", col_defs, None, None); + return Ok(Some(table)); + } return Ok(None); } - // Create a new result table - let mut table = Table::new("result", Vec::new(), None); - - // Add columns with dummy names (since we don't have column metadata) + // Build column definitions from schema, filling in generic names if schema is incomplete let col_count = self.results[0].len(); - for i in 0..col_count { - table.add_column(format!("col{}", i), "UNKNOWN".to_string()); - } + let col_defs: Vec = (0..col_count) + .map(|i| { + if i < schema.columns.len() { + ColumnDefinition { + name: schema.columns[i].name.clone(), + data_type: schema.columns[i].data_type, + } + } else { + ColumnDefinition { + name: format!("col{}", i), + data_type: crate::table::DataType::Text, + } + } + }) + .collect(); + + let mut table = Table::new_with_schema("result", col_defs, None, None); // Add rows for row in &self.results { table.add_row(row.clone())?; } + // Phase 4D: Debug validation - verify index tracking is correct + // Only validate in debug builds to avoid performance impact + #[cfg(debug_assertions)] + self.validate_index_tracking(); + Ok(Some(table)) } - + + /// Validate that the index-based result tracking matches the materialized results + /// + /// This method is used in debug builds to ensure the index tracking in Phase 4C + /// is producing correct results before we can fully switch to index-only mode. + #[cfg(debug_assertions)] + fn validate_index_tracking(&self) { + // Skip validation if we have no results or no index tracking + if self.results.is_empty() || self.result_row_indices.is_empty() { + return; + } + + // Verify counts match + if self.results.len() != self.result_row_indices.len() { + eprintln!( + "Index tracking mismatch: {} results vs {} index entries", + self.results.len(), + self.result_row_indices.len() + ); + return; + } + + // For single-table queries (one cursor), validate that indices point to correct rows + if self.cursor_source_order.len() == 1 { + let cursor_id = self.cursor_source_order[0]; + if let Some(cursor) = self.cursors.get(&cursor_id) { + let rows = cursor.rows(); + + for row_indices in self.result_row_indices.iter() { + if let Some(Some(row_idx)) = row_indices.first() { + // Verify the row exists + if *row_idx < rows.len() { + // The index tracking is recording correctly + // (We can't fully validate column values here since ResultRow + // may include computed expressions, not just column references) + } + } + } + } + } + + // For multi-table queries (JOINs), verify the index structure is consistent + if self.cursor_source_order.len() > 1 { + // Verify all row_indices entries have the expected number of cursor entries + for row_indices in &self.result_row_indices { + if row_indices.len() != self.cursor_source_order.len() { + eprintln!( + "JOIN index tracking mismatch: expected {} cursor indices, got {}", + self.cursor_source_order.len(), + row_indices.len() + ); + } + } + } + } + + /// Materialize a JOIN row from tracked indices (Phase 4E) + /// + /// This method reconstructs a result row by reading values directly from + /// source tables using the tracked row indices. This is more memory-efficient + /// than storing cloned row data for each result. + /// + /// # Arguments + /// * `row_idx` - Index into result_row_indices + /// + /// # Returns + /// The materialized row, or None if indices are invalid + #[allow(dead_code)] + fn materialize_join_row_from_indices( + &self, + row_idx: usize, + ) -> Option> { + let indices = self.result_row_indices.get(row_idx)?; + // Estimate total column count from all cursors + let total_cols: usize = self + .cursor_source_order + .iter() + .filter_map(|id| self.cursors.get(id)) + .map(|c| c.column_count()) + .sum(); + let mut row = Vec::with_capacity(total_cols); + + // For each cursor in source order, get the row and append its columns + for (cursor_pos, cursor_id) in self.cursor_source_order.iter().enumerate() { + if let Some(cursor) = self.cursors.get(cursor_id) { + let source_row_idx = indices.get(cursor_pos).copied().flatten(); + + if let Some(row_idx) = source_row_idx { + // Get row from source cursor + let cursor_rows = cursor.rows(); + if let Some(source_row) = cursor_rows.get(row_idx) { + row.extend(source_row.iter().cloned()); + } else { + // Row index out of bounds - fill with NULLs + for _ in 0..cursor.column_count() { + row.push(crate::table::Value::Null); + } + } + } else { + // NULL row (outer join) - fill with NULLs + for _ in 0..cursor.column_count() { + row.push(crate::table::Value::Null); + } + } + } + } + + Some(row) + } + + /// Get the number of cursors (tables) involved in the current query + #[allow(dead_code)] + fn cursor_count(&self) -> usize { + self.cursor_source_order.len() + } + + /// Check if this is a JOIN query (multiple cursors) + #[allow(dead_code)] + fn is_join_query(&self) -> bool { + self.cursor_source_order.len() > 1 + } + /// Get a register value by index /// /// # Arguments @@ -861,7 +3420,7 @@ impl<'a> VmEngine<'a> { ))) } } - + /// Set a register value by index /// /// # Arguments @@ -875,12 +3434,279 @@ impl<'a> VmEngine<'a> { while idx >= self.registers.len() { self.registers.push(Register::Null); } - + // Set the value self.registers[idx] = value; - + Ok(()) } + + /// Execute a comparison operation between two registers + /// + /// # Arguments + /// * `inst` - The instruction containing: + /// - p1: left operand register index + /// - p2: right operand register index + /// - p3: destination register for result (1 for true, 0 for false) + /// - opcode: the comparison type (Lt, Le, Gt, Ge, Eq, Ne) + fn execute_comparison(&mut self, inst: &Instruction) -> SqawkResult { + let reg1 = self.get_register(inst.p1 as usize)?; + let reg2 = self.get_register(inst.p2 as usize)?; + + let result = match inst.opcode { + OpCode::Lt => { + self.compare_registers(®1, ®2, |a, b| a < b, |a, b| a < b, |a, b| a < b)? + } + OpCode::Le => { + self.compare_registers(®1, ®2, |a, b| a <= b, |a, b| a <= b, |a, b| a <= b)? + } + OpCode::Gt => { + self.compare_registers(®1, ®2, |a, b| a > b, |a, b| a > b, |a, b| a > b)? + } + OpCode::Ge => { + self.compare_registers(®1, ®2, |a, b| a >= b, |a, b| a >= b, |a, b| a >= b)? + } + OpCode::Eq => self.compare_registers_eq(®1, ®2)?, + OpCode::Ne => !self.compare_registers_eq(®1, ®2)?, + _ => { + return Err(SqawkError::VmError(format!( + "Invalid comparison opcode: {:?}", + inst.opcode + ))) + } + }; + + let result_value = Register::Integer(if result { 1 } else { 0 }); + self.set_register(inst.p3 as usize, result_value)?; + Ok(ExecuteResult::Continue) + } + + /// Compare two registers for ordering operations (Lt, Le, Gt, Ge) + fn compare_registers( + &self, + reg1: &Register, + reg2: &Register, + cmp_int: Fi, + cmp_float: Ff, + cmp_str: Fs, + ) -> SqawkResult + where + Fi: Fn(i64, i64) -> bool, + Ff: Fn(f64, f64) -> bool, + Fs: Fn(&str, &str) -> bool, + { + match (reg1, reg2) { + (Register::Integer(a), Register::Integer(b)) => Ok(cmp_int(*a, *b)), + (Register::Float(a), Register::Float(b)) => Ok(cmp_float(*a, *b)), + (Register::String(a), Register::String(b)) => Ok(cmp_str(a, b)), + (Register::Integer(a), Register::Float(b)) => Ok(cmp_float(*a as f64, *b)), + (Register::Float(a), Register::Integer(b)) => Ok(cmp_float(*a, *b as f64)), + _ => Err(SqawkError::VmError(format!( + "Cannot compare incompatible types: {:?} and {:?}", + reg1, reg2 + ))), + } + } + + /// Compare two registers for equality (Eq, Ne) + fn compare_registers_eq(&self, reg1: &Register, reg2: &Register) -> SqawkResult { + match (reg1, reg2) { + (Register::Integer(a), Register::Integer(b)) => Ok(a == b), + (Register::Float(a), Register::Float(b)) => Ok(a == b), + (Register::String(a), Register::String(b)) => Ok(a == b), + (Register::Boolean(a), Register::Boolean(b)) => Ok(a == b), + (Register::Null, Register::Null) => Ok(true), + (Register::Integer(a), Register::Float(b)) => Ok((*a as f64) == *b), + (Register::Float(a), Register::Integer(b)) => Ok(*a == (*b as f64)), + // Different types are not equal + _ => Ok(false), + } + } + + /// Perform an arithmetic operation on two register values + /// + /// # Arguments + /// * `left_reg` - Left operand register index + /// * `right_reg` - Right operand register index + /// * `op` - The operator: "+", "-", "*", "/", "%" + /// + /// # Returns + /// The result as a Register value + fn arithmetic_op(&self, left_reg: usize, right_reg: usize, op: &str) -> SqawkResult { + let left = self.get_register(left_reg)?; + let right = self.get_register(right_reg)?; + + // Handle NULL propagation + if matches!(left, Register::Null) || matches!(right, Register::Null) { + return Ok(Register::Null); + } + + // Convert to numbers + let left_num = match left { + Register::Integer(i) => i as f64, + Register::Float(f) => f, + Register::String(s) => s.parse::().unwrap_or(0.0), + Register::Boolean(b) => { + if b { + 1.0 + } else { + 0.0 + } + } + Register::Null => unreachable!(), + }; + + let right_num = match right { + Register::Integer(i) => i as f64, + Register::Float(f) => f, + Register::String(s) => s.parse::().unwrap_or(0.0), + Register::Boolean(b) => { + if b { + 1.0 + } else { + 0.0 + } + } + Register::Null => unreachable!(), + }; + + let result = match op { + "+" => left_num + right_num, + "-" => left_num - right_num, + "*" => left_num * right_num, + "/" => { + if right_num == 0.0 { + return Ok(Register::Null); // Division by zero returns NULL + } + left_num / right_num + } + "%" => { + if right_num == 0.0 { + return Ok(Register::Null); // Modulo by zero returns NULL + } + left_num % right_num + } + _ => { + return Err(SqawkError::VmError(format!( + "Unknown arithmetic operator: {}", + op + ))); + } + }; + + // Preserve integer type if possible + if result.fract() == 0.0 && result >= i64::MIN as f64 && result <= i64::MAX as f64 { + Ok(Register::Integer(result as i64)) + } else { + Ok(Register::Float(result)) + } + } + + /// Convert a SQL LIKE pattern to a regex pattern + /// + /// SQL LIKE uses: + /// - % to match any sequence of zero or more characters + /// - _ to match any single character + /// + /// # Arguments + /// * `pattern` - The SQL LIKE pattern + /// * `case_insensitive` - Whether to perform case-insensitive matching (for ILIKE) + /// + /// # Returns + /// A regex pattern string + fn like_pattern_to_regex(&self, pattern: &str, case_insensitive: bool) -> String { + let mut regex = String::new(); + + // Add case-insensitive flag if needed + if case_insensitive { + regex.push_str("(?i)"); + } + + // Anchor at start + regex.push('^'); + + // Convert each character + let chars: Vec = pattern.chars().collect(); + let mut i = 0; + while i < chars.len() { + match chars[i] { + '%' => regex.push_str(".*"), + '_' => regex.push('.'), + // Escape regex special characters + '.' | '*' | '+' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' + | '\\' => { + regex.push('\\'); + regex.push(chars[i]); + } + c => regex.push(c), + } + i += 1; + } + + // Anchor at end + regex.push('$'); + + regex + } + + /// Convert a GLOB pattern to a regex pattern + /// + /// GLOB uses Unix-style wildcards: + /// - * to match any sequence of zero or more characters + /// - ? to match any single character + /// - [...] for character classes + /// + /// # Arguments + /// * `pattern` - The GLOB pattern + /// + /// # Returns + /// A regex pattern string + fn glob_pattern_to_regex(&self, pattern: &str) -> String { + let mut regex = String::new(); + + // Anchor at start + regex.push('^'); + + // Convert each character + let chars: Vec = pattern.chars().collect(); + let mut i = 0; + while i < chars.len() { + match chars[i] { + '*' => regex.push_str(".*"), + '?' => regex.push('.'), + '[' => { + // Pass through character class as-is (with some escaping) + regex.push('['); + i += 1; + while i < chars.len() && chars[i] != ']' { + if chars[i] == '\\' && i + 1 < chars.len() { + regex.push('\\'); + regex.push(chars[i + 1]); + i += 2; + } else { + regex.push(chars[i]); + i += 1; + } + } + if i < chars.len() { + regex.push(']'); + } + } + // Escape regex special characters (except those handled above) + '.' | '+' | '^' | '$' | '(' | ')' | '{' | '}' | '|' | '\\' => { + regex.push('\\'); + regex.push(chars[i]); + } + c => regex.push(c), + } + i += 1; + } + + // Anchor at end + regex.push('$'); + + regex + } } /// Result of executing an instruction diff --git a/src/vm/mod.rs b/src/vm/mod.rs index a57cbca..57a189f 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -9,26 +9,72 @@ pub mod bytecode; pub mod compiler; +mod compiler_aggregate; +mod compiler_ddl; +mod compiler_dml; +mod compiler_join; +mod compiler_window; pub mod engine; +#[cfg(test)] +mod compiler_tests; #[cfg(test)] mod tests; +use std::collections::HashSet; + +use crate::capacity::DEFAULT_TABLE_CAPACITY; use crate::database::Database; use crate::error::SqawkResult; use crate::table::Table; +/// Result of VM execution including both the result table and modified table names +pub struct VmExecutionResult { + /// The result table (for SELECT queries) + pub table: Option
, + /// Names of tables that were modified (INSERT, UPDATE, DELETE, CREATE TABLE) + pub modified_tables: HashSet, + /// Number of rows affected by the last DML statement (INSERT, UPDATE, DELETE) + pub affected_rows: usize, +} + /// Execute SQL using the VM execution engine /// /// This is the main entry point for VM-based SQL execution in Sqawk. /// It implements a two-phase approach: /// 1. SQL parsing and bytecode generation /// 2. VM execution of bytecode instructions -pub fn execute_vm(sql: &str, database: &Database, verbose: bool) -> SqawkResult> { +/// +/// Returns both the result table and a set of modified table names. +pub fn execute_vm( + sql: &str, + database: &mut Database, + verbose: bool, +) -> SqawkResult { if verbose { println!("VM Engine: Executing SQL via bytecode: {}", sql); } + // In verbose mode, print messages about SQL features being applied + if verbose { + let sql_upper = sql.to_uppercase(); + if sql_upper.contains("DISTINCT") { + eprintln!("Applying DISTINCT"); + } + if sql_upper.contains("ORDER BY") { + eprintln!("Applying ORDER BY"); + } + if sql_upper.contains("LIMIT") || sql_upper.contains("OFFSET") { + eprintln!("Applying LIMIT/OFFSET"); + } + if sql_upper.contains("GROUP BY") { + eprintln!("Applying GROUP BY"); + } + if sql_upper.contains("HAVING") { + eprintln!("Applying HAVING"); + } + } + // PHASE 1: SQL → BYTECODE COMPILATION // Use the SqlCompiler to convert the SQL statement to bytecode @@ -43,7 +89,7 @@ pub fn execute_vm(sql: &str, database: &Database, verbose: bool) -> SqawkResult< // PHASE 2: BYTECODE → EXECUTION & RESULTS // Initialize the VM engine with the bytecode program - let mut vm = engine::VmEngine::new(database, verbose); + let mut vm = engine::VmEngine::new_mut(database, verbose); vm.init(program); if verbose { @@ -57,11 +103,180 @@ pub fn execute_vm(sql: &str, database: &Database, verbose: bool) -> SqawkResult< println!("Phase 2 complete - Execution finished"); } - // Build a table from the execution results + // Get results and modifications before dropping VM (to release database borrow) let result_table = vm.create_result_table(); + let modifications = vm.take_modifications(); + + // Drop VM to release database borrow + drop(vm); + + // PHASE 3: APPLY MODIFICATIONS TO DATABASE + + // Track which tables were modified + let mut modified_tables = HashSet::with_capacity(DEFAULT_TABLE_CAPACITY); + + // Track affected rows count for DML operations + let mut affected_rows: usize = 0; + + // Collect all deletions by table first (to handle index shifting) + let mut deletions_by_table: std::collections::HashMap> = + std::collections::HashMap::new(); - if verbose && result_table.is_ok() { - if let Some(table) = &result_table.as_ref().unwrap() { + // Count inserts by table (for UPDATE detection - UPDATE = DELETE + INSERT pairs) + let mut insert_counts_by_table: std::collections::HashMap = + std::collections::HashMap::new(); + + // First pass: collect deletions and apply non-delete modifications + for modification in modifications { + match modification { + engine::TableModification::Insert { table_name, row } => { + let table = database.get_table_mut(&table_name)?; + table.add_row(row)?; + *insert_counts_by_table + .entry(table_name.clone()) + .or_insert(0) += 1; + modified_tables.insert(table_name); + } + engine::TableModification::Delete { + table_name, + row_index, + } => { + // Collect deletion indices by table + deletions_by_table + .entry(table_name) + .or_default() + .insert(row_index); + } + engine::TableModification::CreateTable { + table_name, + columns, + file_path, + delimiter, + } => { + // Convert engine ColumnDef to table ColumnDefinition + let schema: Vec = columns + .into_iter() + .map(|col| crate::table::ColumnDefinition { + name: col.name, + data_type: match col.data_type.as_str() { + "INTEGER" => crate::table::DataType::Integer, + "REAL" => crate::table::DataType::Float, + "BOOLEAN" => crate::table::DataType::Boolean, + _ => crate::table::DataType::Text, + }, + }) + .collect(); + + // Create the table with schema + let file_path_buf = file_path.map(std::path::PathBuf::from); + let table = crate::table::Table::new_with_schema( + &table_name, + schema, + file_path_buf, + delimiter, + ); + + // Add to database + database.add_table(table_name.clone(), table)?; + modified_tables.insert(table_name); + } + engine::TableModification::DropTable { table_name } => { + // Remove table from database + if !database.remove_table(&table_name) { + return Err(crate::error::SqawkError::TableNotFound(table_name)); + } + // Note: we don't add to modified_tables since the table is gone + } + engine::TableModification::AlterTableAddColumn { + table_name, + column_name, + column_type, + } => { + let table = database.get_table_mut(&table_name)?; + let data_type = match column_type.as_str() { + "INTEGER" => crate::table::DataType::Integer, + "REAL" => crate::table::DataType::Float, + "BOOLEAN" => crate::table::DataType::Boolean, + _ => crate::table::DataType::Text, + }; + table.add_column_with_default(column_name, data_type, crate::table::Value::Null)?; + modified_tables.insert(table_name); + } + engine::TableModification::Truncate { table_name } => { + let table = database.get_table_mut(&table_name)?; + let row_count = table.row_count(); + table.clear_rows()?; + affected_rows += row_count; + modified_tables.insert(table_name); + } + } + } + + // Count pure inserts (not part of UPDATE) as affected rows first + // (before deletions_by_table is consumed) + for (table_name, insert_count) in &insert_counts_by_table { + let delete_count = deletions_by_table + .get(table_name) + .map(|s| s.len()) + .unwrap_or(0); + if delete_count == 0 { + // Pure INSERT (not UPDATE) + affected_rows += insert_count; + } + } + + // Second pass: apply all deletions (filtering out deleted indices in one pass) + for (table_name, indices_to_delete) in deletions_by_table { + let delete_count = indices_to_delete.len(); + let table = database.get_table_mut(&table_name)?; + + // Clone rows with deep conversion to owned values + // This is necessary because mmap storage has Cow::Borrowed strings that + // would become dangling pointers after the storage is replaced + let new_rows: Vec = table + .rows() + .iter() + .enumerate() + .filter(|(idx, _)| !indices_to_delete.contains(idx)) + .map(|(_, row)| { + row.iter() + .map(|value| match value { + crate::table::Value::String(cow) => { + crate::table::Value::String(std::borrow::Cow::Owned(cow.to_string())) + } + v => v.clone(), + }) + .collect() + }) + .collect(); + table.replace_rows(new_rows); + modified_tables.insert(table_name.clone()); + + // If we have equal deletes and inserts for this table, it's an UPDATE + // Otherwise it's a DELETE + let insert_count = insert_counts_by_table + .get(&table_name) + .copied() + .unwrap_or(0); + if insert_count == delete_count && insert_count > 0 { + // UPDATE: count the number of rows updated + affected_rows += delete_count; + if verbose { + eprintln!("Updated {} rows", delete_count); + } + } else { + // DELETE: count the number of rows deleted + affected_rows += delete_count; + if verbose { + eprintln!("Deleted {} rows", delete_count); + } + } + } + + let result = result_table?; + + if verbose { + if let Some(ref table) = result { println!( "Result table created with {} rows and {} columns", table.row_count(), @@ -72,6 +287,10 @@ pub fn execute_vm(sql: &str, database: &Database, verbose: bool) -> SqawkResult< } } - // Return the resulting table (or None for statements with no results) - result_table + // Return both the result table, modified table names, and affected row count + Ok(VmExecutionResult { + table: result, + modified_tables, + affected_rows, + }) } diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 0e3e068..326cbde 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -369,11 +369,17 @@ mod bytecode_tests { // Add some test rows table - .add_row(vec![Value::Integer(1), Value::String("Alice".to_string())]) + .add_row(vec![ + Value::Integer(1), + Value::String("Alice".to_string().into()), + ]) .expect("Failed to add row"); table - .add_row(vec![Value::Integer(2), Value::String("Bob".to_string())]) + .add_row(vec![ + Value::Integer(2), + Value::String("Bob".to_string().into()), + ]) .expect("Failed to add row"); // Add the table to the database @@ -486,7 +492,7 @@ mod bytecode_tests { ); assert_eq!( rows[0][1], - Value::String("Alice".to_string()), + Value::String("Alice".to_string().into()), "First row, second column should be 'Alice'" ); @@ -498,7 +504,7 @@ mod bytecode_tests { ); assert_eq!( rows[1][1], - Value::String("Bob".to_string()), + Value::String("Bob".to_string().into()), "Second row, second column should be 'Bob'" ); } @@ -881,19 +887,19 @@ mod bytecode_tests { mod comparison_tests { use super::*; use crate::vm::tests::bytecode_tests::{create_instruction, execute_bytecode_program}; - + /// Test the Lt (Less Than) comparison opcode #[test] fn test_lt_opcode() { let database = Database::new(); - + // Test cases: [(first_value, second_value, expected_result)] let test_cases = vec![ - (10, 20, 1), // 10 < 20 = true (1) - (20, 10, 0), // 20 < 10 = false (0) - (10, 10, 0), // 10 < 10 = false (0) + (10, 20, 1), // 10 < 20 = true (1) + (20, 10, 0), // 20 < 10 = false (0) + (10, 10, 0), // 10 < 10 = false (0) ]; - + for (val1, val2, expected) in test_cases { let instructions = vec![ // Initialize VM @@ -951,40 +957,40 @@ mod comparison_tests { Some("Stop execution".to_string()), ), ]; - + // Execute the program let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute program with Lt opcode"); - + // Verify the result assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + // Check the result value match &table.rows()[0][0] { Value::Integer(val) => assert_eq!( - *val, expected as i64, - "Expected {} < {} to be {}, got {}", + *val, expected as i64, + "Expected {} < {} to be {}, got {}", val1, val2, expected, val ), other => panic!("Expected Integer type, got {:?}", other), } } } - + /// Test the Le (Less Than or Equal) comparison opcode #[test] fn test_le_opcode() { let database = Database::new(); - + // Test cases: [(first_value, second_value, expected_result)] let test_cases = vec![ - (10, 20, 1), // 10 <= 20 = true (1) - (20, 10, 0), // 20 <= 10 = false (0) - (10, 10, 1), // 10 <= 10 = true (1) + (10, 20, 1), // 10 <= 20 = true (1) + (20, 10, 0), // 20 <= 10 = false (0) + (10, 10, 1), // 10 <= 10 = true (1) ]; - + for (val1, val2, expected) in test_cases { let instructions = vec![ // Initialize VM @@ -1042,40 +1048,40 @@ mod comparison_tests { Some("Stop execution".to_string()), ), ]; - + // Execute the program let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute program with Le opcode"); - + // Verify the result assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + // Check the result value match &table.rows()[0][0] { Value::Integer(val) => assert_eq!( - *val, expected as i64, - "Expected {} <= {} to be {}, got {}", + *val, expected as i64, + "Expected {} <= {} to be {}, got {}", val1, val2, expected, val ), other => panic!("Expected Integer type, got {:?}", other), } } } - + /// Test conditional jump using SQLite-style jump mechanism #[test] fn test_conditional_jump() { let database = Database::new(); - + // Test cases for conditional jumps // (first_value, second_value, expected_result_after_jump) let test_cases = vec![ - (10, 20, 555), // 10 < 20 = true, result = 1, so NOT Jump (555) - (20, 10, 999), // 20 < 10 = false, result = 0, so Jump (999) + (10, 20, 555), // 10 < 20 = true, result = 1, so NOT Jump (555) + (20, 10, 999), // 20 < 10 = false, result = 0, so Jump (999) ]; - + for (val1, val2, expected) in test_cases { let instructions = vec![ // Initialize VM @@ -1133,14 +1139,7 @@ mod comparison_tests { Some("Load true path value (555)".to_string()), ), // Jump to instruction 8 (end) - create_instruction( - OpCode::Goto, - 0, - 8, - 0, - None, - Some("Jump to end".to_string()), - ), + create_instruction(OpCode::Goto, 0, 8, 0, None, Some("Jump to end".to_string())), // False path (Jumped) - load 999 into register 4 create_instruction( OpCode::Integer, @@ -1169,16 +1168,16 @@ mod comparison_tests { Some("Stop execution".to_string()), ), ]; - + // Execute the program let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute program with conditional jump"); - + // Verify the result assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + // Check the conditional jump outcome match &table.rows()[0][0] { Value::Integer(val) => assert_eq!( @@ -1190,19 +1189,19 @@ mod comparison_tests { } } } - + /// Test the IfPos conditional jump opcode #[test] fn test_if_pos_opcode() { let database = Database::new(); - + // Test cases: [(register_value, expected_result_after_jump)] let test_cases = vec![ - (5, 777), // 5 > 0 = true, should jump to 777 - (0, 888), // 0 = 0, should NOT jump, continue to 888 - (-3, 888), // -3 < 0, should NOT jump, continue to 888 + (5, 777), // 5 > 0 = true, should jump to 777 + (0, 888), // 0 = 0, should NOT jump, continue to 888 + (-3, 888), // -3 < 0, should NOT jump, continue to 888 ]; - + for (reg_value, expected) in test_cases { let instructions = vec![ // Initialize VM @@ -1242,14 +1241,7 @@ mod comparison_tests { Some("Load not-jumped value (888)".to_string()), ), // Jump to end - create_instruction( - OpCode::Goto, - 0, - 6, - 0, - None, - Some("Jump to end".to_string()), - ), + create_instruction(OpCode::Goto, 0, 6, 0, None, Some("Jump to end".to_string())), // Jumped path - load 777 into register 2 create_instruction( OpCode::Integer, @@ -1278,16 +1270,16 @@ mod comparison_tests { Some("Stop execution".to_string()), ), ]; - + // Execute the program let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute program with IfPos"); - + // Verify the result assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + // Check the conditional jump outcome match &table.rows()[0][0] { Value::Integer(val) => assert_eq!( @@ -1299,19 +1291,19 @@ mod comparison_tests { } } } - + /// Test the IfNeg conditional jump opcode #[test] fn test_if_neg_opcode() { let database = Database::new(); - + // Test cases: [(register_value, expected_result_after_jump)] let test_cases = vec![ - (-5, 333), // -5 < 0 = true, should jump to 333 - (0, 444), // 0 = 0, should NOT jump, continue to 444 - (3, 444), // 3 > 0, should NOT jump, continue to 444 + (-5, 333), // -5 < 0 = true, should jump to 333 + (0, 444), // 0 = 0, should NOT jump, continue to 444 + (3, 444), // 3 > 0, should NOT jump, continue to 444 ]; - + for (reg_value, expected) in test_cases { let instructions = vec![ // Initialize VM @@ -1351,14 +1343,7 @@ mod comparison_tests { Some("Load not-jumped value (444)".to_string()), ), // Jump to end - create_instruction( - OpCode::Goto, - 0, - 6, - 0, - None, - Some("Jump to end".to_string()), - ), + create_instruction(OpCode::Goto, 0, 6, 0, None, Some("Jump to end".to_string())), // Jumped path - load 333 into register 2 create_instruction( OpCode::Integer, @@ -1387,16 +1372,16 @@ mod comparison_tests { Some("Stop execution".to_string()), ), ]; - + // Execute the program let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute program with IfNeg"); - + // Verify the result assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + // Check the conditional jump outcome match &table.rows()[0][0] { Value::Integer(val) => assert_eq!( @@ -1408,47 +1393,72 @@ mod comparison_tests { } } } - + /// Test Column opcode with simple cursor operations #[test] fn test_column_opcode_basic() { use crate::database::Database; use crate::table::Table; - + // Create a simple test database let mut database = Database::new(); let mut test_table = Table::new("simple_test", vec![], None); - + // Add columns: id, value test_table.add_column("id".to_string(), "INTEGER".to_string()); test_table.add_column("value".to_string(), "INTEGER".to_string()); - + // Add one test row - test_table.add_row(vec![ - crate::table::Value::Integer(1), - crate::table::Value::Integer(42), - ]).expect("Failed to add test row"); - - database.add_table("simple_test".to_string(), test_table).expect("Failed to add table"); - + test_table + .add_row(vec![ + crate::table::Value::Integer(1), + crate::table::Value::Integer(42), + ]) + .expect("Failed to add test row"); + + database + .add_table("simple_test".to_string(), test_table) + .expect("Failed to add table"); + // Simple test: Read the value column (index 1) let instructions = vec![ create_instruction(OpCode::Init, 0, 1, 0, None, Some("Initialize".to_string())), - create_instruction(OpCode::OpenRead, 0, 0, 0, Some("simple_test".to_string()), Some("Open table".to_string())), + create_instruction( + OpCode::OpenRead, + 0, + 0, + 0, + Some("simple_test".to_string()), + Some("Open table".to_string()), + ), create_instruction(OpCode::Rewind, 0, 5, 0, None, Some("Rewind".to_string())), - create_instruction(OpCode::Column, 0, 1, 1, None, Some("Read value column".to_string())), - create_instruction(OpCode::ResultRow, 1, 1, 0, None, Some("Output value".to_string())), + create_instruction( + OpCode::Column, + 0, + 1, + 1, + None, + Some("Read value column".to_string()), + ), + create_instruction( + OpCode::ResultRow, + 1, + 1, + 0, + None, + Some("Output value".to_string()), + ), create_instruction(OpCode::Halt, 0, 0, 0, None, Some("Stop".to_string())), ]; - + let result = execute_bytecode_program(instructions, &database) .expect("Failed to execute column test"); - + // Verify we got the expected value assert!(result.is_some(), "Expected a result table"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 row"); - + match &table.rows()[0][0] { crate::table::Value::Integer(value) => { assert_eq!(*value, 42, "Expected value 42, got {}", value); @@ -1456,56 +1466,94 @@ mod comparison_tests { other => panic!("Expected Integer value, got {:?}", other), } } - + /// Test comparison operations with Column data #[test] fn test_column_with_comparison() { use crate::database::Database; use crate::table::Table; - + // Create test database with two values: one matches, one doesn't let mut database = Database::new(); let mut test_table = Table::new("comparison_test", vec![], None); - + test_table.add_column("id".to_string(), "INTEGER".to_string()); test_table.add_column("score".to_string(), "INTEGER".to_string()); - + // Add two rows: score 25 (should not match > 30), score 35 (should match > 30) - test_table.add_row(vec![ - crate::table::Value::Integer(1), - crate::table::Value::Integer(25), - ]).expect("Failed to add test row"); - - test_table.add_row(vec![ - crate::table::Value::Integer(2), - crate::table::Value::Integer(35), - ]).expect("Failed to add test row"); - - database.add_table("comparison_test".to_string(), test_table).expect("Failed to add table"); - + test_table + .add_row(vec![ + crate::table::Value::Integer(1), + crate::table::Value::Integer(25), + ]) + .expect("Failed to add test row"); + + test_table + .add_row(vec![ + crate::table::Value::Integer(2), + crate::table::Value::Integer(35), + ]) + .expect("Failed to add test row"); + + database + .add_table("comparison_test".to_string(), test_table) + .expect("Failed to add table"); + // Test first row (score = 25, should fail > 30 test) let instructions_row1 = vec![ create_instruction(OpCode::Init, 0, 1, 0, None, Some("Initialize".to_string())), - create_instruction(OpCode::OpenRead, 0, 0, 0, Some("comparison_test".to_string()), Some("Open table".to_string())), + create_instruction( + OpCode::OpenRead, + 0, + 0, + 0, + Some("comparison_test".to_string()), + Some("Open table".to_string()), + ), create_instruction(OpCode::Rewind, 0, 7, 0, None, Some("Rewind".to_string())), - create_instruction(OpCode::Column, 0, 1, 1, None, Some("Read score".to_string())), + create_instruction( + OpCode::Column, + 0, + 1, + 1, + None, + Some("Read score".to_string()), + ), create_instruction(OpCode::Integer, 30, 2, 0, None, Some("Load 30".to_string())), - create_instruction(OpCode::Gt, 1, 2, 3, None, Some("Test score > 30".to_string())), - create_instruction(OpCode::ResultRow, 3, 1, 0, None, Some("Output comparison result".to_string())), + create_instruction( + OpCode::Gt, + 1, + 2, + 3, + None, + Some("Test score > 30".to_string()), + ), + create_instruction( + OpCode::ResultRow, + 3, + 1, + 0, + None, + Some("Output comparison result".to_string()), + ), create_instruction(OpCode::Halt, 0, 0, 0, None, Some("Stop".to_string())), ]; - + let result = execute_bytecode_program(instructions_row1, &database) .expect("Failed to execute comparison test"); - + assert!(result.is_some(), "Expected a result"); let table = result.unwrap(); assert_eq!(table.row_count(), 1, "Expected 1 result row"); - + // Should be 0 (false) since 25 > 30 is false match &table.rows()[0][0] { crate::table::Value::Integer(result_val) => { - assert_eq!(*result_val, 0, "Expected comparison result 0 (false), got {}", result_val); + assert_eq!( + *result_val, 0, + "Expected comparison result 0 (false), got {}", + result_val + ); } other => panic!("Expected Integer comparison result, got {:?}", other), } diff --git a/tests/advanced/mod.rs b/tests/advanced/mod.rs index 4071c3f..e3e272b 100644 --- a/tests/advanced/mod.rs +++ b/tests/advanced/mod.rs @@ -4,7 +4,6 @@ //! covered by the focused test modules. use crate::helpers::create_temp_dir; -use assert_cmd::Command; use predicates::prelude::*; use std::fs; @@ -17,7 +16,7 @@ fn test_sequential_sql_statements() -> Result<(), Box> { fs::write(&file_path, content)?; // Execute multiple statements with sequential dependencies - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("INSERT INTO people (id, name, age) VALUES (4, 'David', 40)") .arg("-s") @@ -45,7 +44,7 @@ fn test_is_null_operator() -> Result<(), Box> { fs::write(&file_path, content)?; // Test IS NULL operator - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM null_test WHERE department IS NULL") .arg(file_path.to_str().unwrap()) @@ -56,7 +55,7 @@ fn test_is_null_operator() -> Result<(), Box> { .stdout(predicate::str::contains("3,Charlie,")); // Test IS NOT NULL operator - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM null_test WHERE name IS NOT NULL") .arg(file_path.to_str().unwrap()) diff --git a/tests/aggregate/mod.rs b/tests/aggregate/mod.rs index da74e7a..16bb117 100644 --- a/tests/aggregate/mod.rs +++ b/tests/aggregate/mod.rs @@ -2,108 +2,88 @@ //! //! Tests for COUNT, SUM, AVG, MIN, MAX functions and their combinations. -use crate::helpers::create_temp_dir; -use assert_cmd::Command; +use crate::helpers::get_employees_file; use predicates::prelude::*; -use std::fs; -use std::io::Write; -use std::path::PathBuf; - -// Helper function to create an aggregates test file -fn create_aggregates_file() -> Result<(tempfile::TempDir, PathBuf), Box> { - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("aggregates.csv"); - - // Create a CSV file for aggregate function testing - let content = "id,name,age,salary,department\n1,Alice,30,70000,Engineering\n2,Bob,25,55000,Marketing\n3,Charlie,35,65000,Engineering\n4,David,40,80000,Sales\n5,Eve,28,60000,Marketing\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} #[test] fn test_basic_aggregate_functions() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aggregates_file()?; + let file_path = get_employees_file(); // Run sqawk with basic aggregate functions - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT COUNT(*), SUM(age), AVG(salary), MIN(age), MAX(salary) FROM aggregates") + .arg("SELECT COUNT(*), SUM(age), AVG(salary), MIN(age), MAX(salary) FROM employees") .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output - actual output uses simpler column names (COUNT,SUM,AVG,MIN,MAX) + // employees.csv has 8 rows: + // COUNT(*) = 8, SUM(age) = 257, AVG(salary) = 67500, MIN(age) = 22, MAX(salary) = 90000 cmd.assert() .success() .stdout(predicate::str::contains("COUNT,SUM,AVG,MIN,MAX")) - .stdout(predicate::str::contains("5,158,66000,25,80000")); + .stdout(predicate::str::contains("8,257,67500,22,90000")); Ok(()) } #[test] fn test_aggregate_functions_with_aliases() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aggregates_file()?; + let file_path = get_employees_file(); // Run sqawk with aggregate functions and aliases - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT COUNT(*) AS count, SUM(salary) AS total_salary, AVG(age) AS avg_age, MIN(salary) AS min_salary, MAX(age) AS max_age FROM aggregates") + .arg("SELECT COUNT(*) AS count, SUM(salary) AS total_salary, AVG(age) AS avg_age, MIN(salary) AS min_salary, MAX(age) AS max_age FROM employees") .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output + // count = 8, total_salary = 540000, avg_age = 32.125, min_salary = 45000, max_age = 45 cmd.assert() .success() .stdout(predicate::str::contains( "count,total_salary,avg_age,min_salary,max_age", )) - .stdout(predicate::str::contains("5,330000,31.6,55000,40")); + .stdout(predicate::str::contains("8,540000,32.125,45000,45")); Ok(()) } #[test] fn test_aggregate_functions_with_filter() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aggregates_file()?; + let file_path = get_employees_file(); // Run sqawk with filtered aggregate functions - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT COUNT(*), SUM(salary), AVG(age), MIN(salary), MAX(age) FROM aggregates WHERE age > 25") + .arg("SELECT COUNT(*), SUM(salary), AVG(age), MIN(salary), MAX(age) FROM employees WHERE age > 25") .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output - should only include rows with age > 25 - // Actual output uses simpler column names (COUNT,SUM,AVG,MIN,MAX) + // Rows with age > 25: Alice(30), Charlie(35), David(40), Eve(28), Frank(32), Grace(45) = 6 rows + // COUNT = 6, SUM(salary) = 440000, AVG(age) = 35, MIN(salary) = 60000, MAX(age) = 45 cmd.assert() .success() .stdout(predicate::str::contains("COUNT,SUM,AVG,MIN,MAX")) - .stdout(predicate::str::contains("4,275000,33.25,60000,40")); + .stdout(predicate::str::contains("6,440000,35,60000,45")); Ok(()) } #[test] -fn test_aggregate_on_basic_table() -> Result<(), Box> { - // We'll use the sample.csv file which is a standard test file - let mut cmd = Command::cargo_bin("sqawk")?; +fn test_aggregate_on_employees() -> Result<(), Box> { + let file_path = get_employees_file(); + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT COUNT(*) AS total_count, AVG(age) AS average_age FROM sample") - .arg("tests/data/sample.csv") + .arg("SELECT COUNT(*) AS total_count, AVG(age) AS average_age FROM employees") + .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output + // COUNT(*) = 8, AVG(age) = 32.125 cmd.assert() .success() .stdout(predicate::str::contains("total_count,average_age")) - .stdout(predicate::str::contains("3,30")); + .stdout(predicate::str::contains("8,32.125")); Ok(()) } diff --git a/tests/alias/mod.rs b/tests/alias/mod.rs index 4d5b1d0..cbb0b5b 100644 --- a/tests/alias/mod.rs +++ b/tests/alias/mod.rs @@ -2,55 +2,35 @@ //! //! Tests for SQL column aliasing functionality with AS keyword. -use crate::helpers::create_temp_dir; -use assert_cmd::Command; +use crate::helpers::get_employees_file; use predicates::prelude::*; -use std::fs; -use std::io::Write; -use std::path::PathBuf; - -// Helper function to create an aliases test file -fn create_aliases_file() -> Result<(tempfile::TempDir, PathBuf), Box> { - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("aliases.csv"); - - // Create a CSV file for alias testing - let content = "id,name,age,department,role\n1,Alice,30,Engineering,Developer\n2,Bob,25,Marketing,Specialist\n3,Charlie,35,Finance,Manager\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} #[test] fn test_basic_column_aliases() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aliases_file()?; + let file_path = get_employees_file(); // Run sqawk with basic column aliases - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT name AS employee_name, age AS employee_age, department AS dept FROM aliases") + .arg("SELECT name AS employee_name, age AS employee_age, department AS dept FROM employees") .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output + // Check output - verify aliased column names and sample data cmd.assert() .success() .stdout(predicate::str::contains("employee_name,employee_age,dept")) .stdout(predicate::str::contains("Alice,30,Engineering")) .stdout(predicate::str::contains("Bob,25,Marketing")) - .stdout(predicate::str::contains("Charlie,35,Finance")); + .stdout(predicate::str::contains("Charlie,35,Engineering")); // Verify SQL execution message appears somewhere in the output (stdout or stderr) let output = cmd.output()?; let stdout = String::from_utf8(output.stdout)?; let stderr = String::from_utf8(output.stderr)?; - let contains_sql = stdout.contains("Executing SQL: SELECT name AS employee_name, age AS employee_age, department AS dept FROM aliases") || - stderr.contains("Executing SQL: SELECT name AS employee_name, age AS employee_age, department AS dept FROM aliases"); + let contains_sql = stdout.contains("Executing SQL: SELECT name AS employee_name, age AS employee_age, department AS dept FROM employees") || + stderr.contains("Executing SQL: SELECT name AS employee_name, age AS employee_age, department AS dept FROM employees"); assert!(contains_sql, "SQL execution message not found in output"); @@ -59,53 +39,57 @@ fn test_basic_column_aliases() -> Result<(), Box> { #[test] fn test_mixed_aliases_and_regular_columns() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aliases_file()?; + let file_path = get_employees_file(); // Run sqawk with a mix of aliased and regular columns - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT id, name AS employee_name, department FROM aliases") + .arg("SELECT id, name AS employee_name, department FROM employees") .arg(file_path.to_str().unwrap()) .arg("-v"); - // Check output + // Check output - verify mixed column names cmd.assert() .success() .stdout(predicate::str::contains("id,employee_name,department")) .stdout(predicate::str::contains("1,Alice,Engineering")) .stdout(predicate::str::contains("2,Bob,Marketing")) - .stdout(predicate::str::contains("3,Charlie,Finance")); + .stdout(predicate::str::contains("3,Charlie,Engineering")); Ok(()) } #[test] fn test_aliases_with_where_clause() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_aliases_file()?; + let file_path = get_employees_file(); // Run sqawk with aliases and a WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT name AS employee_name, age AS employee_age FROM aliases WHERE age > 25") + .arg("SELECT name AS employee_name, age AS employee_age FROM employees WHERE age > 25") .arg(file_path.to_str().unwrap()) .arg("-v"); // Check output - should only include rows with age > 25 + // From employees.csv: Alice(30), Charlie(35), David(40), Eve(28), Frank(32), Grace(45) cmd.assert() .success() .stdout(predicate::str::contains("employee_name,employee_age")) .stdout(predicate::str::contains("Alice,30")) - .stdout(predicate::str::contains("Charlie,35")); + .stdout(predicate::str::contains("Charlie,35")) + .stdout(predicate::str::contains("David,40")); - // Verify we don't see Bob who is age 25 + // Verify we don't see Bob (age 25) or Henry (age 22) let output = cmd.output()?; let stdout = String::from_utf8(output.stdout)?; assert!( !stdout.contains("Bob,25"), "Should not contain Bob (age <= 25)" ); + assert!( + !stdout.contains("Henry,22"), + "Should not contain Henry (age <= 25)" + ); Ok(()) } diff --git a/tests/analyze_profile.py b/tests/analyze_profile.py new file mode 100644 index 0000000..329b3c4 --- /dev/null +++ b/tests/analyze_profile.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Analyze samply profile data and report top hotspots.""" +import json +import gzip +import sys +from collections import defaultdict + +# Get profile path from command line or use default +profile_path = sys.argv[1] if len(sys.argv) > 1 else '/tmp/sqawk_profile.json.gz' +syms_path = profile_path.replace('.json.gz', '.json.syms.json') + +# Load the symbols file +with open(syms_path) as f: + syms_data = json.load(f) + +string_table = syms_data['string_table'] +libs = syms_data['data'] + +# Build address lookup: (lib_idx, address) -> symbol_name +# known_addresses is [[addr, sym_table_idx], ...] +# symbol_table[sym_table_idx]['symbol'] is index into string_table +addr_to_sym = {} +lib_names = [] +for lib_idx, lib in enumerate(libs): + lib_names.append(lib['debug_name']) + sym_table = lib.get('symbol_table', []) + known_addrs = lib.get('known_addresses', []) + + for addr, sym_idx in known_addrs: + if sym_idx < len(sym_table): + sym_entry = sym_table[sym_idx] + name_idx = sym_entry['symbol'] + sym_name = string_table[name_idx] + addr_to_sym[(lib_idx, addr)] = sym_name + +print(f"Built lookup with {len(addr_to_sym)} symbols from {len(libs)} libraries") + +# Load the profile +with gzip.open(profile_path, 'rt') as f: + data = json.load(f) + +thread = data['threads'][0] +strings = thread['stringArray'] +func_names = thread['funcTable']['name'] +func_resources = thread['funcTable']['resource'] +frame_funcs = thread['frameTable']['func'] +stack_prefixes = thread['stackTable']['prefix'] +stack_frames = thread['stackTable']['frame'] +sample_stacks = thread['samples']['stack'] +sample_weights = thread['samples'].get('weight', [1] * len(sample_stacks)) + +# Map resource indices to library indices +res_table = thread['resourceTable'] +res_libs = res_table.get('lib', []) + +def get_func_name(func_idx): + name_idx = func_names[func_idx] + name = strings[name_idx] + + if name.startswith('0x'): + addr = int(name, 16) + res_idx = func_resources[func_idx] + if res_idx is not None and res_idx >= 0 and res_idx < len(res_libs): + lib_idx = res_libs[res_idx] + if lib_idx is not None and (lib_idx, addr) in addr_to_sym: + return addr_to_sym[(lib_idx, addr)] + return name + +# Count samples +self_counts = defaultdict(int) +total_counts = defaultdict(int) + +for i, stack_idx in enumerate(sample_stacks): + if stack_idx is None: + continue + + weight = sample_weights[i] if i < len(sample_weights) else 1 + seen = set() + first = True + + while stack_idx is not None: + frame_idx = stack_frames[stack_idx] + func_idx = frame_funcs[frame_idx] + func_name = get_func_name(func_idx) + + if first: + self_counts[func_name] += weight + first = False + + if func_name not in seen: + total_counts[func_name] += weight + seen.add(func_name) + + stack_idx = stack_prefixes[stack_idx] + +total_samples = sum(self_counts.values()) +print(f"Total samples: {total_samples}") +print() + +print("=" * 100) +print("TOP 10 HOTSPOTS BY SELF TIME (time directly in function)") +print("=" * 100) +sorted_self = sorted(self_counts.items(), key=lambda x: -x[1]) +for i, (name, count) in enumerate(sorted_self[:10], 1): + pct = 100.0 * count / total_samples + print(f"{i:2}. {pct:5.1f}% ({count:6} samples) | {name[:95]}") + +print() +print("=" * 100) +print("TOP 10 HOTSPOTS BY TOTAL TIME (function anywhere on stack)") +print("=" * 100) +sorted_total = sorted(total_counts.items(), key=lambda x: -x[1]) +for i, (name, count) in enumerate(sorted_total[:10], 1): + pct = 100.0 * count / total_samples + print(f"{i:2}. {pct:5.1f}% ({count:6} samples) | {name[:95]}") diff --git a/tests/arithmetic/mod.rs b/tests/arithmetic/mod.rs new file mode 100644 index 0000000..8d0603c --- /dev/null +++ b/tests/arithmetic/mod.rs @@ -0,0 +1,125 @@ +//! Arithmetic operator tests for sqawk VM +//! +//! Tests for +, -, *, /, % operators. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test addition operator +#[test] +fn test_addition() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b\n1,10,5\n2,20,3\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a + b AS sum FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("15")) + .stdout(predicates::str::contains("23")); + + Ok(()) +} + +/// Test subtraction operator +#[test] +fn test_subtraction() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b\n1,10,5\n2,20,3\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a - b AS diff FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("5")) + .stdout(predicates::str::contains("17")); + + Ok(()) +} + +/// Test multiplication operator +#[test] +fn test_multiplication() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b\n1,4,5\n2,3,7\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a * b AS product FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("20")) + .stdout(predicates::str::contains("21")); + + Ok(()) +} + +/// Test division operator +#[test] +fn test_division() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b\n1,20,4\n2,15,3\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a / b AS quotient FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("5")) + .stdout(predicates::str::contains("5")); + + Ok(()) +} + +/// Test modulo operator +#[test] +fn test_modulo() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b\n1,10,3\n2,17,5\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a % b AS remainder FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1")) + .stdout(predicates::str::contains("2")); + + Ok(()) +} + +/// Test unary minus +#[test] +fn test_unary_minus() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,5\n2,-3\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT -value AS negated FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("-5")) + .stdout(predicates::str::contains("3")); + + Ok(()) +} diff --git a/tests/basic/mod.rs b/tests/basic/mod.rs index cd7b3a0..093957d 100644 --- a/tests/basic/mod.rs +++ b/tests/basic/mod.rs @@ -3,12 +3,12 @@ //! This file contains fundamental end-to-end tests for the sqawk application. use crate::helpers::{ - create_custom_csv, create_temp_dir, get_static_sample_file, prepare_test_file, run_test_case, + create_custom_csv, create_temp_dir, get_people_file, run_test_case, run_test_case_with_static_file, SqawkTestCase, }; use predicates::prelude::PredicateBooleanExt; -use std::fs; // Import trait for .not() +use std::fs; #[test] fn test_basic_select() -> Result<(), Box> { @@ -40,14 +40,16 @@ fn test_filtered_select() -> Result<(), Box> { #[test] fn test_insert() -> Result<(), Box> { - // Removed unused test_case definition - - // We need to verify the file was modified, so we'll use a custom test function + // Create temp file for write test let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // Build the command - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("INSERT INTO people (id, name, age) VALUES (4, 'Dave', 40)") .arg("-s") @@ -71,6 +73,46 @@ fn test_insert() -> Result<(), Box> { Ok(()) } +#[test] +fn test_insert_select() -> Result<(), Box> { + // Create temp files - source and target + let temp_dir = create_temp_dir()?; + let source_path = create_custom_csv( + temp_dir.path(), + "source.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; + let target_path = create_custom_csv(temp_dir.path(), "target.csv", "id,name,age\n")?; + + // Test INSERT...SELECT with WHERE + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("INSERT INTO target SELECT * FROM source WHERE age > 30") + .arg("-s") + .arg("SELECT * FROM target") + .arg("--write") + .arg(source_path.to_str().unwrap()) + .arg(target_path.to_str().unwrap()); + + // Check output - should have Alice and Charlie (age > 30) + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,age")) + .stdout(predicates::str::contains("1,Alice,32")) + .stdout(predicates::str::contains("3,Charlie,35")); + + // Verify Bob is not in output (age = 25) + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("2,Bob,25"), + "Bob should not be in output: {}", + stdout + ); + + Ok(()) +} + #[test] fn test_custom_table_name() -> Result<(), Box> { let test_case = SqawkTestCase { @@ -102,18 +144,21 @@ fn test_invalid_sql() -> Result<(), Box> { #[test] fn test_multiple_files() -> Result<(), Box> { - // This test demonstrates using multiple files with custom data let temp_dir = create_temp_dir()?; // Create people.csv file - let people_file = prepare_test_file(temp_dir.path())?; + let people_file = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // Create scores.csv file let scores_content = "id,score\n1,95\n2,75\n3,85\n"; let scores_file = create_custom_csv(temp_dir.path(), "scores.csv", scores_content)?; // Use direct command execution since our SQL is simpler - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM scores WHERE score > 80") .arg(format!("scores={}", scores_file.to_str().unwrap())) @@ -160,10 +205,8 @@ fn test_where_less_than() -> Result<(), Box> { #[test] fn test_static_file_query() -> Result<(), Box> { // This test demonstrates using the existing static test file - // This avoids creating temporary files for simple queries, - // improving test performance. let test_case = SqawkTestCase { - sql: "SELECT * FROM sample WHERE age > 25".to_string(), + sql: "SELECT * FROM people WHERE age > 25".to_string(), expected_stdout: vec![ "id,name,age".to_string(), "1,Alice,32".to_string(), @@ -173,19 +216,22 @@ fn test_static_file_query() -> Result<(), Box> { ..Default::default() }; - // Use the static sample.csv file - run_test_case_with_static_file(test_case, get_static_sample_file()) + // Use the static people.csv file + run_test_case_with_static_file(test_case, get_people_file()) } #[test] fn test_delete_with_where() -> Result<(), Box> { - // This test verifies the DELETE functionality with a WHERE clause - // We need to verify the file was modified, so we'll use a custom test function + // Create temp file for write test let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // First verify we have 3 rows initially - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM people") .arg(file_path.to_str().unwrap()); @@ -197,14 +243,13 @@ fn test_delete_with_where() -> Result<(), Box> { .stdout(predicates::str::contains("3,Charlie,35")); // Now execute DELETE with WHERE - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("DELETE FROM people WHERE age > 30") .arg("-s") .arg("SELECT * FROM people") // Check result after deletion .arg(file_path.to_str().unwrap()) - .arg("-v") // Verbose mode to see more details - ; // Prevent file modifications + .arg("-v"); // Verbose mode to see more details // Verify deletion worked correctly cmd.assert() @@ -226,12 +271,16 @@ fn test_delete_with_where() -> Result<(), Box> { #[test] fn test_delete_all() -> Result<(), Box> { - // This test verifies the DELETE functionality without a WHERE clause (deletes all rows) + // Create temp file for write test let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // First verify we have rows initially - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM people") .arg(file_path.to_str().unwrap()); @@ -242,14 +291,13 @@ fn test_delete_all() -> Result<(), Box> { .stdout(predicates::str::contains("1,Alice,32")); // Now execute DELETE without WHERE - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("DELETE FROM people") .arg("-s") .arg("SELECT * FROM people") // Check result after deletion .arg(file_path.to_str().unwrap()) - .arg("-v") // Verbose mode to see more details - ; // Prevent file modifications + .arg("-v"); // Verbose mode to see more details // Verify deletion worked correctly - should only show header, no rows cmd.assert() diff --git a/tests/between/mod.rs b/tests/between/mod.rs new file mode 100644 index 0000000..ec99716 --- /dev/null +++ b/tests/between/mod.rs @@ -0,0 +1,131 @@ +//! BETWEEN operator tests for sqawk VM +//! +//! This file contains tests for SQL BETWEEN operator. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test basic BETWEEN with integer values +#[test] +fn test_between_integers() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n4,David,40\n5,Eve,20\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test BETWEEN with integer range using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE age BETWEEN 25 AND 35") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,25")) + .stdout(predicates::str::contains("2,Bob,30")) + .stdout(predicates::str::contains("3,Charlie,35")); + + Ok(()) +} + +/// Test BETWEEN excluding boundary values +#[test] +fn test_between_boundaries() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,20\n3,30\n4,40\n5,50\n"; + let file_path = create_custom_csv(temp_dir.path(), "numbers.csv", content)?; + + // Test BETWEEN includes boundaries (20 and 40 should be included) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM numbers WHERE value BETWEEN 20 AND 40") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,20")) + .stdout(predicates::str::contains("3,30")) + .stdout(predicates::str::contains("4,40")); + + Ok(()) +} + +/// Test NOT BETWEEN +#[test] +fn test_not_between() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,score\n1,50\n2,75\n3,100\n4,25\n5,90\n"; + let file_path = create_custom_csv(temp_dir.path(), "scores.csv", content)?; + + // Test NOT BETWEEN - should return values outside the range + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM scores WHERE score NOT BETWEEN 60 AND 80") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,50")) + .stdout(predicates::str::contains("3,100")) + .stdout(predicates::str::contains("4,25")) + .stdout(predicates::str::contains("5,90")); + + Ok(()) +} + +/// Test BETWEEN with no matches +#[test] +fn test_between_no_matches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,5\n2,10\n3,15\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test BETWEEN with range that has no matches - VM mode returns empty output + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE value BETWEEN 100 AND 200") + .arg(file_path.to_str().unwrap()); + + // VM mode with no results returns empty output (no header) + cmd.assert().success(); + + Ok(()) +} + +/// Test BETWEEN with single value in range +#[test] +fn test_between_single_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,price\n1,99\n2,150\n3,201\n"; + let file_path = create_custom_csv(temp_dir.path(), "products.csv", content)?; + + // Test BETWEEN with exactly one value in range + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM products WHERE price BETWEEN 100 AND 200") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,150")); + + Ok(()) +} + +/// Test BETWEEN with equal low and high bounds +#[test] +fn test_between_equal_bounds() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,num\n1,5\n2,10\n3,15\n"; + let file_path = create_custom_csv(temp_dir.path(), "nums.csv", content)?; + + // Test BETWEEN where low == high (should match exact value) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM nums WHERE num BETWEEN 10 AND 10") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,10")); + + Ok(()) +} diff --git a/tests/case/mod.rs b/tests/case/mod.rs new file mode 100644 index 0000000..9c02467 --- /dev/null +++ b/tests/case/mod.rs @@ -0,0 +1,130 @@ +//! CASE WHEN expression tests for sqawk VM +//! +//! This file contains tests for SQL CASE WHEN expressions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test simple CASE with integer conditions +#[test] +fn test_case_simple_integers() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,status\n1,1\n2,2\n3,3\n4,1\n"; + let file_path = create_custom_csv(temp_dir.path(), "records.csv", content)?; + + // Test simple CASE expression (CASE status WHEN 1 THEN 'active' ...) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM records WHERE CASE status WHEN 1 THEN 1 WHEN 2 THEN 0 ELSE 0 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,1")) + .stdout(predicates::str::contains("4,1")); + + Ok(()) +} + +/// Test searched CASE with comparison conditions +#[test] +fn test_case_searched_greater_than() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,50\n3,100\n4,25\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test searched CASE expression (CASE WHEN value > 50 THEN ...) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CASE WHEN value > 50 THEN 1 ELSE 0 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("3,100")); + + Ok(()) +} + +/// Test CASE with multiple WHEN branches +#[test] +fn test_case_multiple_when_branches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,grade\n1,95\n2,85\n3,75\n4,65\n5,55\n"; + let file_path = create_custom_csv(temp_dir.path(), "grades.csv", content)?; + + // Test CASE with multiple WHEN branches - filter for A grades (>= 90) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM grades WHERE CASE WHEN grade > 89 THEN 1 WHEN grade > 79 THEN 0 ELSE 0 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,95")); + + Ok(()) +} + +/// Test CASE with ELSE clause +#[test] +fn test_case_with_else() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,type\n1,premium\n2,standard\n3,basic\n4,premium\n"; + let file_path = create_custom_csv(temp_dir.path(), "accounts.csv", content)?; + + // Test CASE where ELSE applies - non-premium accounts + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM accounts WHERE CASE type WHEN 'premium' THEN 0 ELSE 1 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,standard")) + .stdout(predicates::str::contains("3,basic")); + + Ok(()) +} + +/// Test CASE with no matches going to ELSE +#[test] +fn test_case_no_when_match_uses_else() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,code\n1,X\n2,Y\n3,Z\n"; + let file_path = create_custom_csv(temp_dir.path(), "codes.csv", content)?; + + // No code matches A or B, so all go to ELSE (1) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM codes WHERE CASE code WHEN 'A' THEN 0 WHEN 'B' THEN 0 ELSE 1 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,X")) + .stdout(predicates::str::contains("2,Y")) + .stdout(predicates::str::contains("3,Z")); + + Ok(()) +} + +/// Test CASE with equality condition +#[test] +fn test_case_searched_equality() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,status\n1,active\n2,pending\n3,inactive\n4,active\n"; + let file_path = create_custom_csv(temp_dir.path(), "items.csv", content)?; + + // Test searched CASE with equality + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM items WHERE CASE WHEN status = 'active' THEN 1 ELSE 0 END = 1") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,active")) + .stdout(predicates::str::contains("4,active")); + + Ok(()) +} diff --git a/tests/cast/mod.rs b/tests/cast/mod.rs new file mode 100644 index 0000000..7e386a0 --- /dev/null +++ b/tests/cast/mod.rs @@ -0,0 +1,127 @@ +//! CAST type conversion tests for sqawk VM +//! +//! This file contains tests for SQL CAST type conversion expressions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test CAST integer to text +#[test] +fn test_cast_integer_to_text() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,100\n2,200\n3,300\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test CAST integer to TEXT for comparison + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST(value AS TEXT) = '100'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,100")); + + Ok(()) +} + +/// Test CAST text to integer +#[test] +fn test_cast_text_to_integer() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,50\n2,150\n3,250\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test CAST literal to INTEGER for comparison + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST('100' AS INTEGER) < value") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,150")) + .stdout(predicates::str::contains("3,250")); + + Ok(()) +} + +/// Test CAST with VARCHAR type +#[test] +fn test_cast_to_varchar() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,num\n1,42\n2,99\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test CAST to VARCHAR (should work like TEXT) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST(num AS VARCHAR) = '42'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,42")); + + Ok(()) +} + +/// Test CAST column to integer for comparison +#[test] +fn test_cast_column_to_integer() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,strnum\n1,100\n2,200\n3,50\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Cast string column to integer for numeric comparison + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST(strnum AS INTEGER) > 75") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,100")) + .stdout(predicates::str::contains("2,200")); + + Ok(()) +} + +/// Test CAST preserves integer values +#[test] +fn test_cast_integer_to_integer() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,20\n3,30\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // CAST integer to integer should preserve value + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST(value AS INT) = 20") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,20")); + + Ok(()) +} + +/// Test multiple CAST expressions +#[test] +fn test_cast_both_sides() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,code\n1,A\n2,B\n3,C\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Cast id to text for string comparison + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE CAST(id AS TEXT) = '2'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,B")); + + Ok(()) +} diff --git a/tests/coalesce/mod.rs b/tests/coalesce/mod.rs new file mode 100644 index 0000000..e711733 --- /dev/null +++ b/tests/coalesce/mod.rs @@ -0,0 +1,130 @@ +//! COALESCE and NULLIF function tests for sqawk VM +//! +//! This file contains tests for SQL COALESCE and NULLIF functions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test COALESCE with first non-null value +#[test] +fn test_coalesce_first_non_null() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,nickname\n1,Alice,Ally\n2,Bob,\n3,Charlie,Chuck\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test COALESCE returns first non-NULL value + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE COALESCE(nickname, name) = 'Ally'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,Ally")); + + Ok(()) +} + +/// Test COALESCE falls through to second value when first is NULL +#[test] +fn test_coalesce_fallback_to_second() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,primary_val,secondary_val\n1,,fallback1\n2,main2,fallback2\n"; + let file_path = create_custom_csv(temp_dir.path(), "values.csv", content)?; + + // When primary is NULL/empty, COALESCE should use secondary + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM values WHERE COALESCE(primary_val, secondary_val) = 'fallback1'") + .arg(file_path.to_str().unwrap()); + + // VM outputs NULL for null values + cmd.assert() + .success() + .stdout(predicates::str::contains("1,NULL,fallback1")); + + Ok(()) +} + +/// Test COALESCE with literal default +#[test] +fn test_coalesce_with_literal_default() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,val\n1,\n2,present\n3,\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test COALESCE with literal default value + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE COALESCE(val, 'default') = 'default'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,")) + .stdout(predicates::str::contains("3,")); + + Ok(()) +} + +/// Test NULLIF returns NULL when values equal +#[test] +fn test_nullif_returns_null_when_equal() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // NULLIF(name, 'Alice') should return NULL for Alice + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE NULLIF(name, 'Alice') IS NULL") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")); + + Ok(()) +} + +/// Test NULLIF returns first value when not equal +#[test] +fn test_nullif_returns_value_when_not_equal() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,code\n1,A\n2,B\n3,C\n"; + let file_path = create_custom_csv(temp_dir.path(), "codes.csv", content)?; + + // NULLIF(code, 'A') returns NULL for 'A', returns original value for others + // So NULLIF(code, 'A') IS NOT NULL should match 'B' and 'C' + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM codes WHERE NULLIF(code, 'A') IS NOT NULL") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,B")) + .stdout(predicates::str::contains("3,C")); + + Ok(()) +} + +/// Test COALESCE with multiple arguments +#[test] +fn test_coalesce_multiple_args() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,a,b,c\n1,,,third\n2,,second,third\n3,first,second,third\n"; + let file_path = create_custom_csv(temp_dir.path(), "multi.csv", content)?; + + // COALESCE(a, b, c) should return first non-null + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM multi WHERE COALESCE(a, b, c) = 'third'") + .arg(file_path.to_str().unwrap()); + + // VM outputs NULL for null values + cmd.assert() + .success() + .stdout(predicates::str::contains("1,NULL,NULL,third")); + + Ok(()) +} diff --git a/tests/comparison/mod.rs b/tests/comparison/mod.rs index 6a17387..0f6db7c 100644 --- a/tests/comparison/mod.rs +++ b/tests/comparison/mod.rs @@ -4,26 +4,17 @@ //! to ensure they work correctly with integer types. use predicates::prelude::PredicateBooleanExt; -use std::path::PathBuf; -/// Get path to the static sample test CSV file (for read-only tests) -fn get_test_data_path() -> PathBuf { - PathBuf::from("tests/data/sample.csv") -} - -/// Get path to the static boundaries test CSV file (for read-only tests) -fn get_boundaries_data_path() -> PathBuf { - PathBuf::from("tests/data/boundaries.csv") -} +use crate::helpers::{get_boundaries_file, get_sample_file}; // Test cases for each comparison operator #[test] fn test_equals_operator() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age = 32") .arg(file_path); @@ -38,10 +29,10 @@ fn test_equals_operator() -> Result<(), Box> { #[test] fn test_not_equals_operator() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age != 32") .arg(file_path); @@ -58,10 +49,10 @@ fn test_not_equals_operator() -> Result<(), Box> { #[test] fn test_greater_than_operator() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age > 32") .arg(file_path); @@ -78,10 +69,10 @@ fn test_greater_than_operator() -> Result<(), Box> { #[test] fn test_less_than_operator() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age < 32") .arg(file_path); @@ -98,10 +89,10 @@ fn test_less_than_operator() -> Result<(), Box> { #[test] fn test_greater_than_or_equal_operator() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age >= 32") .arg(file_path); @@ -118,10 +109,10 @@ fn test_greater_than_or_equal_operator() -> Result<(), Box Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age <= 32") .arg(file_path); @@ -138,10 +129,10 @@ fn test_less_than_or_equal_operator() -> Result<(), Box> #[test] fn test_equals_with_no_matches() -> Result<(), Box> { - let file_path = get_test_data_path(); + let file_path = get_sample_file(); // Using the sample.csv file which has columns: id,name,age - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample WHERE age = 40") .arg(file_path); @@ -167,10 +158,10 @@ fn test_equals_with_no_matches() -> Result<(), Box> { #[test] fn test_comparison_boundary_values() -> Result<(), Box> { // Use the static boundaries.csv file with extreme integer values - let file_path = get_boundaries_data_path(); + let file_path = get_boundaries_file(); // Test with max integer value - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM boundaries WHERE value = 9223372036854775807") .arg(file_path.clone()); @@ -181,7 +172,7 @@ fn test_comparison_boundary_values() -> Result<(), Box> { .stdout(predicates::str::contains("4,9223372036854775807")); // Test with min integer value - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM boundaries WHERE value = -9223372036854775808") .arg(file_path.clone()); @@ -192,7 +183,7 @@ fn test_comparison_boundary_values() -> Result<(), Box> { .stdout(predicates::str::contains("5,-9223372036854775808")); // Test greater than zero - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM boundaries WHERE value > 0") .arg(file_path.clone()); @@ -204,7 +195,7 @@ fn test_comparison_boundary_values() -> Result<(), Box> { .stdout(predicates::str::contains("4,9223372036854775807")); // Test less than zero - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM boundaries WHERE value < 0") .arg(file_path.clone()); diff --git a/tests/correlated/mod.rs b/tests/correlated/mod.rs new file mode 100644 index 0000000..60c5cf6 --- /dev/null +++ b/tests/correlated/mod.rs @@ -0,0 +1,236 @@ +//! Correlated subquery tests for sqawk VM +//! +//! This file contains tests for correlated subqueries. +//! Phase 4B: Correlated subquery support with runtime execution + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test correlated EXISTS subquery - users with orders +#[test] +fn test_correlated_exists_users_with_orders() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,1,150\n3,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id) + // Alice (id=1) and Bob (id=2) have orders, Charlie (id=3) does not + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")); + + // Verify Charlie is not in output + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Charlie"), + "Charlie should not be in output: {}", + stdout + ); + + Ok(()) +} + +/// Test correlated NOT EXISTS subquery - users without orders +#[test] +fn test_correlated_not_exists_users_without_orders() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,1,150\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id) + // Only Alice (id=1) has orders, so Bob and Charlie should be returned + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + // Verify Alice is not in output + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Alice"), + "Alice should not be in output: {}", + stdout + ); + + Ok(()) +} + +/// Test correlated EXISTS with additional filter condition +#[test] +fn test_correlated_exists_with_filter() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,50\n2,1,150\n3,2,75\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id AND amount > 100) + // Only Alice (id=1) has an order with amount > 100 (150) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id AND amount > 100)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")); + + // Verify Bob and Charlie are not in output + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Bob"), + "Bob should not be in output: {}", + stdout + ); + assert!( + !stdout.contains("Charlie"), + "Charlie should not be in output: {}", + stdout + ); + + Ok(()) +} + +/// Test correlated scalar subquery with MAX +#[test] +fn test_correlated_scalar_subquery_max() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + // Each employee has a department, find employees with max salary in their department + let employees_content = "id,name,dept,salary\n1,Alice,eng,100\n2,Bob,eng,120\n3,Charlie,sales,90\n4,Diana,sales,110\n"; + let employees_path = create_custom_csv(temp_dir.path(), "employees.csv", employees_content)?; + + // Test: SELECT * FROM employees e WHERE salary = (SELECT MAX(salary) FROM employees WHERE dept = e.dept) + // Bob (120) is max in eng, Diana (110) is max in sales + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM employees e WHERE salary = (SELECT MAX(salary) FROM employees WHERE dept = e.dept)") + .arg(employees_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Bob")) + .stdout(predicates::str::contains("Diana")); + + // Verify Alice and Charlie are not in output + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Alice"), + "Alice should not be in output: {}", + stdout + ); + assert!( + !stdout.contains("Charlie"), + "Charlie should not be in output: {}", + stdout + ); + + Ok(()) +} + +/// Test correlated scalar subquery with COUNT +#[test] +fn test_correlated_scalar_subquery_count() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + // Alice has 3 orders, Bob has 1, Charlie has 0 + let orders_content = "order_id,user_id\n1,1\n2,1\n3,1\n4,2\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE (SELECT COUNT(*) FROM orders WHERE orders.user_id = users.id) > 1 + // Only Alice has more than 1 order + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE (SELECT COUNT(*) FROM orders WHERE orders.user_id = users.id) > 1") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")); + + // Verify Bob and Charlie are not in output + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Bob"), + "Bob should not be in output: {}", + stdout + ); + assert!( + !stdout.contains("Charlie"), + "Charlie should not be in output: {}", + stdout + ); + + Ok(()) +} + +/// Test correlated EXISTS with same table (self-referential) +#[test] +fn test_correlated_exists_same_table() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + // Employees with manager_id referencing other employees + let employees_content = "id,name,manager_id\n1,Alice,0\n2,Bob,1\n3,Charlie,1\n4,Diana,2\n"; + let employees_path = create_custom_csv(temp_dir.path(), "employees.csv", employees_content)?; + + // Test: Find managers (employees who manage someone) + // SELECT * FROM employees e WHERE EXISTS (SELECT 1 FROM employees WHERE manager_id = e.id) + // Alice (id=1) manages Bob and Charlie, Bob (id=2) manages Diana + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM employees e WHERE EXISTS (SELECT 1 FROM employees WHERE manager_id = e.id)") + .arg(employees_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("Bob")); + + // Verify Charlie and Diana are not in output (they don't manage anyone) + let output = cmd.output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Charlie"), + "Charlie should not be in output: {}", + stdout + ); + assert!( + !stdout.contains("Diana"), + "Diana should not be in output: {}", + stdout + ); + + Ok(()) +} diff --git a/tests/data/animals.csv b/tests/data/animals.csv deleted file mode 100644 index 40486cb..0000000 --- a/tests/data/animals.csv +++ /dev/null @@ -1,5 +0,0 @@ -id,name,type,age -1,Rover,dog,5 -2,Whiskers,cat,3 -3,Hopper,rabbit,2 -4,Bubbles,fish,1 \ No newline at end of file diff --git a/tests/data/categories.csv b/tests/data/categories.csv deleted file mode 100644 index cabef0e..0000000 --- a/tests/data/categories.csv +++ /dev/null @@ -1,6 +0,0 @@ -category_id,name,department -1,Electronics,Tech -2,Furniture,Home -3,Kitchen,Home -4,Books,Media -5,Clothing,Fashion \ No newline at end of file diff --git a/tests/data/employees.csv b/tests/data/employees.csv new file mode 100644 index 0000000..cb056aa --- /dev/null +++ b/tests/data/employees.csv @@ -0,0 +1,9 @@ +id,name,age,salary,department,role +1,Alice,30,70000,Engineering,Developer +2,Bob,25,55000,Marketing,Specialist +3,Charlie,35,65000,Engineering,Manager +4,David,40,80000,Sales,Director +5,Eve,28,60000,Marketing,Analyst +6,Frank,32,75000,Engineering,Developer +7,Grace,45,90000,Sales,Director +8,Henry,22,45000,HR,Intern diff --git a/tests/data/people.csv b/tests/data/people.csv new file mode 100644 index 0000000..c7b0ec4 --- /dev/null +++ b/tests/data/people.csv @@ -0,0 +1,4 @@ +id,name,age +1,Alice,32 +2,Bob,25 +3,Charlie,35 diff --git a/tests/data/products_with_prices.csv b/tests/data/products_with_prices.csv deleted file mode 100644 index ffe5289..0000000 --- a/tests/data/products_with_prices.csv +++ /dev/null @@ -1,6 +0,0 @@ -product_id,name,price,category -1,Laptop,1200.00,Electronics -2,Smartphone,800.00,Electronics -3,Desk Chair,150.00,Furniture -4,Coffee Maker,85.50,Kitchen -5,Headphones,120.00,Electronics \ No newline at end of file diff --git a/tests/data/set_a.csv b/tests/data/set_a.csv new file mode 100644 index 0000000..b64aa5b --- /dev/null +++ b/tests/data/set_a.csv @@ -0,0 +1,4 @@ +id,name +1,Alice +2,Bob +3,Charlie diff --git a/tests/data/set_b.csv b/tests/data/set_b.csv new file mode 100644 index 0000000..ad3a231 --- /dev/null +++ b/tests/data/set_b.csv @@ -0,0 +1,4 @@ +id,name +2,Bob +3,Charlie +4,David diff --git a/tests/data/strings.csv b/tests/data/strings.csv new file mode 100644 index 0000000..c7319a4 --- /dev/null +++ b/tests/data/strings.csv @@ -0,0 +1,6 @@ +id,text,mixed_case,padded_text,email +1,apple,ApPlE, trimme ,john@example.com +2,banana,BaNaNa, needs space ,jane@example.com +3,cherry,ChErRy, whitespace ,bob@test.org +4,date,DaTe, extra ,alice@company.co.uk +5,elderberry,ElDeRbErRy, padding ,admin@website.net diff --git a/tests/date_functions/mod.rs b/tests/date_functions/mod.rs new file mode 100644 index 0000000..012a177 --- /dev/null +++ b/tests/date_functions/mod.rs @@ -0,0 +1,85 @@ +//! Date/time function tests for sqawk VM +//! +//! Tests for DATE, TIME, NOW functions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test NOW() function returns a timestamp +#[test] +fn test_now() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id\n1\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT NOW() AS current_time FROM data") + .arg(file_path.to_str().unwrap()); + + // NOW() should return something with date-like format (YYYY-MM-DD) + cmd.assert() + .success() + .stdout(predicates::str::contains("202")); // Year starts with 202x + + Ok(()) +} + +/// Test DATE function extracts date from datetime string +#[test] +fn test_date_extract() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,datetime\n1,2024-01-15 10:30:00\n2,2024-06-20 14:45:00\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT DATE(datetime) AS dt FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2024-01-15")) + .stdout(predicates::str::contains("2024-06-20")); + + Ok(()) +} + +/// Test TIME function extracts time from datetime string +#[test] +fn test_time_extract() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,datetime\n1,2024-01-15 10:30:00\n2,2024-06-20 14:45:00\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT TIME(datetime) AS tm FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("10:30:00")) + .stdout(predicates::str::contains("14:45:00")); + + Ok(()) +} + +/// Test CURRENT_DATE returns today's date +#[test] +fn test_current_date() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id\n1\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT CURRENT_DATE AS today FROM data") + .arg(file_path.to_str().unwrap()); + + // Should return date in YYYY-MM-DD format + cmd.assert() + .success() + .stdout(predicates::str::contains("202")); // Year starts with 202x + + Ok(()) +} diff --git a/tests/ddl/mod.rs b/tests/ddl/mod.rs new file mode 100644 index 0000000..be84cf1 --- /dev/null +++ b/tests/ddl/mod.rs @@ -0,0 +1,417 @@ +//! Tests for DDL (Data Definition Language) operations +//! +//! This file contains tests for DROP TABLE, ALTER TABLE, TRUNCATE, CREATE TABLE AS SELECT, +//! and CREATE TABLE with LOCATION clause. + +use crate::helpers::{create_custom_csv, create_temp_dir}; +use std::fs; + +#[test] +fn test_drop_table_basic() -> Result<(), Box> { + // Test that DROP TABLE queues the table for removal + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n", + )?; + + // DROP TABLE should succeed (table queued for removal) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("DROP TABLE people") + .arg("-v") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("DropTable: queued drop")); + + Ok(()) +} + +#[test] +fn test_drop_table_nonexistent() -> Result<(), Box> { + // Test that DROP TABLE fails for non-existent table + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", "id,name,age\n1,Alice,32\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("DROP TABLE nonexistent") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .failure() + .stderr(predicates::str::contains("not found")); + + Ok(()) +} + +#[test] +fn test_drop_table_if_exists() -> Result<(), Box> { + // Test that DROP TABLE IF EXISTS doesn't error for non-existent tables + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n", + )?; + + // Drop non-existent table with IF EXISTS (should not error) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("DROP TABLE IF EXISTS nonexistent") + .arg("-v") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("does not exist (IF EXISTS)")); + + Ok(()) +} + +#[test] +fn test_drop_table_if_exists_existing() -> Result<(), Box> { + // Test that DROP TABLE IF EXISTS works for existing tables + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n", + )?; + + // Drop existing table with IF EXISTS + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("DROP TABLE IF EXISTS people") + .arg("-v") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("DropTable: queued drop")); + + Ok(()) +} + +#[test] +fn test_truncate_table() -> Result<(), Box> { + // Test that TRUNCATE removes all rows from the table + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; + + // Truncate the table and verify it's empty + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("TRUNCATE TABLE people") + .arg("-s") + .arg("SELECT COUNT(*) as cnt FROM people") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("0")); + + Ok(()) +} + +#[test] +fn test_truncate_verbose() -> Result<(), Box> { + // Test TRUNCATE in verbose mode + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("TRUNCATE TABLE people") + .arg("-v") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Truncate: queued truncate")); + + Ok(()) +} + +#[test] +fn test_alter_table_add_column() -> Result<(), Box> { + // Test that ALTER TABLE ADD COLUMN adds a new column with NULL values + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", "id,name\n1,Alice\n2,Bob\n")?; + + // Add a new column and verify + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("ALTER TABLE people ADD COLUMN age INTEGER") + .arg("-s") + .arg("SELECT * FROM people") + .arg(file_path.to_str().unwrap()); + + // New column should appear with NULL values (shown as empty) + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,age")); + + Ok(()) +} + +#[test] +fn test_alter_table_add_text_column() -> Result<(), Box> { + // Test ALTER TABLE ADD COLUMN with TEXT type + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", "id,name\n1,Alice\n2,Bob\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("ALTER TABLE people ADD COLUMN email TEXT") + .arg("-s") + .arg("SELECT * FROM people") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,email")); + + Ok(()) +} + +#[test] +fn test_create_table_as_select() -> Result<(), Box> { + // Test CREATE TABLE AS SELECT creates a new table from query results + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age,department\n1,Alice,32,Engineering\n2,Bob,25,Sales\n3,Charlie,35,Engineering\n", + )?; + + // Create new table from SELECT with WHERE clause + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("CREATE TABLE engineers AS SELECT id, name FROM people WHERE department = 'Engineering'") + .arg("-s") + .arg("SELECT * FROM engineers") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name")) + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("3,Charlie")); + + Ok(()) +} + +#[test] +fn test_create_table_as_select_all_columns() -> Result<(), Box> { + // Test CREATE TABLE AS SELECT with all columns + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n", + )?; + + // Create new table with SELECT * + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("CREATE TABLE backup AS SELECT * FROM people") + .arg("-s") + .arg("SELECT * FROM backup") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,age")) + .stdout(predicates::str::contains("1,Alice,32")) + .stdout(predicates::str::contains("2,Bob,25")); + + Ok(()) +} + +#[test] +fn test_create_table_as_select_with_filter() -> Result<(), Box> { + // Test CREATE TABLE AS SELECT with age filter + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; + + // Create new table with filtered data + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("CREATE TABLE seniors AS SELECT id, name FROM people WHERE age > 30") + .arg("-s") + .arg("SELECT COUNT(*) as cnt FROM seniors") + .arg(file_path.to_str().unwrap()); + + // Should have Alice (32) and Charlie (35) = 2 rows + cmd.assert() + .success() + .stdout(predicates::str::contains("2")); + + Ok(()) +} + +#[test] +fn test_create_table_verbose() -> Result<(), Box> { + // Test CREATE TABLE AS SELECT in verbose mode + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", "id,name\n1,Alice\n2,Bob\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("CREATE TABLE copy AS SELECT * FROM people") + .arg("-v") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("CreateTable")); + + Ok(()) +} + +// ============================================================================= +// CREATE TABLE with LOCATION tests +// ============================================================================= + +#[test] +fn test_create_table_with_location_creates_empty_table() -> Result<(), Box> { + // Test CREATE TABLE with LOCATION creates an empty table with specified schema + let temp_dir = create_temp_dir()?; + let output_file = temp_dir.path().join("output.csv"); + + let dummy_file = create_custom_csv(temp_dir.path(), "dummy.csv", "x\n1\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg(format!( + "CREATE TABLE mydata (id INT, name TEXT, value INT) LOCATION '{}'", + output_file.to_str().unwrap() + )) + .arg("-s") + .arg("SELECT * FROM mydata") + .arg(dummy_file.to_str().unwrap()); + + // Table exists but is empty (only header row in output) + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,value")); + + Ok(()) +} + +#[test] +fn test_create_table_with_location_insert_and_write() -> Result<(), Box> { + // Test INSERT into table with LOCATION, then write back + let temp_dir = create_temp_dir()?; + let output_file = temp_dir.path().join("output.csv"); + + let dummy_file = create_custom_csv(temp_dir.path(), "dummy.csv", "x\n1\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg(format!( + "CREATE TABLE mydata (id INT, name TEXT) LOCATION '{}'", + output_file.to_str().unwrap() + )) + .arg("-s") + .arg("INSERT INTO mydata VALUES (1, 'Alice'), (2, 'Bob')") + .arg("-s") + .arg("SELECT * FROM mydata") + .arg("--write") + .arg(dummy_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("Bob")); + + // Verify the file was created with the data + let contents = fs::read_to_string(&output_file)?; + assert!(contents.contains("Alice"), "File should contain Alice"); + assert!(contents.contains("Bob"), "File should contain Bob"); + + Ok(()) +} + +#[test] +fn test_create_table_with_location_and_delimiter() -> Result<(), Box> { + // Test CREATE TABLE with LOCATION and custom tab delimiter + let temp_dir = create_temp_dir()?; + let output_file = temp_dir.path().join("output.tsv"); + + let dummy_file = create_custom_csv(temp_dir.path(), "dummy.csv", "x\n1\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg(format!( + "CREATE TABLE mydata (id INT, name TEXT) LOCATION '{}' STORED AS TEXTFILE WITH (DELIMITER='\t')", + output_file.to_str().unwrap() + )) + .arg("-s") + .arg("INSERT INTO mydata VALUES (1, 'Alice')") + .arg("--write") + .arg(dummy_file.to_str().unwrap()); + + cmd.assert().success(); + + // Verify the file was created with tab delimiter + let contents = fs::read_to_string(&output_file)?; + assert!(contents.contains('\t'), "File should use tab delimiter"); + assert!(contents.contains("Alice"), "File should contain Alice"); + + Ok(()) +} + +#[test] +fn test_create_table_with_location_multiple_operations() -> Result<(), Box> { + // Test CREATE TABLE with LOCATION followed by INSERT, UPDATE, SELECT + let temp_dir = create_temp_dir()?; + let output_file = temp_dir.path().join("output.csv"); + + let dummy_file = create_custom_csv(temp_dir.path(), "dummy.csv", "x\n1\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg(format!( + "CREATE TABLE mydata (id INT, name TEXT, value INT) LOCATION '{}'", + output_file.to_str().unwrap() + )) + .arg("-s") + .arg("INSERT INTO mydata VALUES (1, 'Alice', 100), (2, 'Bob', 200)") + .arg("-s") + .arg("UPDATE mydata SET value = 999 WHERE name = 'Alice'") + .arg("-s") + .arg("SELECT name, value FROM mydata WHERE value > 500") + .arg("--write") + .arg(dummy_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("999")); + + // Verify the file contains the updated data + let contents = fs::read_to_string(&output_file)?; + assert!( + contents.contains("999"), + "File should contain updated value" + ); + + Ok(()) +} diff --git a/tests/delimiter/mod.rs b/tests/delimiter/mod.rs index d415693..63e7af9 100644 --- a/tests/delimiter/mod.rs +++ b/tests/delimiter/mod.rs @@ -2,52 +2,16 @@ //! //! Tests for different file delimiter options (-F flag) with tab and colon separators. -use crate::helpers::create_temp_dir; -use assert_cmd::Command; -use std::fs; -use std::io::Write; -use std::path::PathBuf; - -// Helper function to create a tab-delimited test file -fn create_tab_delimited_file() -> Result<(tempfile::TempDir, PathBuf), Box> { - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("employees.tsv"); - - // Create a tab-delimited file for testing - let content = "id\tname\tsalary\tdepartment\n1\tAlice\t75000\tEngineering\n2\tBob\t65000\tMarketing\n3\tCharlie\t85000\tEngineering\n4\tDavid\t60000\tSales\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} - -// Helper function to create a colon-delimited test file -fn create_colon_delimited_file() -> Result<(tempfile::TempDir, PathBuf), Box> -{ - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("contacts.txt"); - - // Create a colon-delimited file for testing - let content = "id:name:email:phone\n1:Alice:alice@example.com:555-1234\n2:Bob:bob@example.com:555-5678\n3:Charlie:charlie@example.com:555-9012\n5:David:david@example.com:555-3456\n10:Eve:eve@example.com:555-7890\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} +use crate::helpers::{get_contacts_file, get_employees_tsv_file}; #[test] fn test_tab_delimiter() -> Result<(), Box> { - // Create a tab-delimited file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_tab_delimited_file()?; + let file_path = get_employees_tsv_file(); // Run sqawk with tab delimiter - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT * FROM employees WHERE salary > 70000") + .arg("SELECT id, name, salary, department FROM employees WHERE salary > 70000") .arg("-F") .arg("\t") .arg(file_path.to_str().unwrap()) @@ -61,27 +25,35 @@ fn test_tab_delimiter() -> Result<(), Box> { let stdout = String::from_utf8(output.stdout)?; // Check that header and desired rows are present + // employees.tsv has: id, name, department, salary, age + // Rows with salary > 70000: John Smith (75000), Bob Johnson (80000), Alice Williams (85000), + // David Miller (90000), Frank Wilson (82000), Henry Martin (78000) + // Output should be tab-delimited (matching input) + assert!( + stdout.contains("id\tname\tsalary\tdepartment"), + "Header row should be present with tab delimiter" + ); assert!( - stdout.contains("id,name,salary,department"), - "Header row should be present" + stdout.contains("John Smith") && stdout.contains("75000"), + "Row for John Smith should be present (salary > 70000)" ); assert!( - stdout.contains("1,Alice,75000,Engineering"), - "Row for Alice should be present (salary > 70000)" + stdout.contains("Bob Johnson") && stdout.contains("80000"), + "Row for Bob Johnson should be present (salary > 70000)" ); assert!( - stdout.contains("3,Charlie,85000,Engineering"), - "Row for Charlie should be present (salary > 70000)" + stdout.contains("Alice Williams") && stdout.contains("85000"), + "Row for Alice Williams should be present (salary > 70000)" ); // Check that we don't see rows with salary <= 70000 assert!( - !stdout.contains("Bob") || !stdout.contains("65000"), - "Should not contain Bob with salary 65000 (salary <= 70000)" + !stdout.contains("Jane Doe") || !stdout.contains("65000"), + "Should not contain Jane Doe with salary 65000 (salary <= 70000)" ); assert!( - !stdout.contains("David") || !stdout.contains("60000"), - "Should not contain David with salary 60000 (salary <= 70000)" + !stdout.contains("Charlie Brown") || !stdout.contains("60000"), + "Should not contain Charlie Brown with salary 60000 (salary <= 70000)" ); Ok(()) @@ -89,11 +61,10 @@ fn test_tab_delimiter() -> Result<(), Box> { #[test] fn test_colon_delimiter() -> Result<(), Box> { - // Create a colon-delimited file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_colon_delimited_file()?; + let file_path = get_contacts_file(); // Run sqawk with colon delimiter - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, name, email FROM contacts WHERE id > 5") .arg("-F") @@ -109,31 +80,34 @@ fn test_colon_delimiter() -> Result<(), Box> { let stdout = String::from_utf8(output.stdout)?; // Check that header and desired rows are present + // contacts.txt has: id:name:email:phone:city + // Rows with id > 5: David Miller (6), Emily Davis (7), Frank Wilson (8), Grace Taylor (9), Henry Martin (10) + // Output should be colon-delimited (matching input) assert!( - stdout.contains("id,name,email"), - "Header row should be present" + stdout.contains("id:name:email"), + "Header row should be present with colon delimiter" ); assert!( - stdout.contains("10,Eve,eve@example.com"), - "Row for Eve should be present (id > 5)" + stdout.contains("David Miller") && stdout.contains("david.miller@example.com"), + "Row for David Miller should be present (id > 5)" ); - - // Check that we don't see rows with id <= 5 assert!( - !stdout.contains("Alice") || !stdout.contains("alice@example.com"), - "Should not contain Alice (id <= 5)" + stdout.contains("Henry Martin") && stdout.contains("henry.martin@example.com"), + "Row for Henry Martin should be present (id > 5)" ); + + // Check that we don't see rows with id <= 5 assert!( - !stdout.contains("Bob") || !stdout.contains("bob@example.com"), - "Should not contain Bob (id <= 5)" + !stdout.contains("John Smith") || !stdout.contains("john.smith@example.com"), + "Should not contain John Smith (id <= 5)" ); assert!( - !stdout.contains("Charlie") || !stdout.contains("charlie@example.com"), - "Should not contain Charlie (id <= 5)" + !stdout.contains("Jane Doe") || !stdout.contains("jane.doe@example.com"), + "Should not contain Jane Doe (id <= 5)" ); assert!( - !stdout.contains("David") || !stdout.contains("david@example.com"), - "Should not contain David (id <= 5)" + !stdout.contains("Bob Johnson") || !stdout.contains("bob.johnson@example.com"), + "Should not contain Bob Johnson (id <= 5)" ); Ok(()) diff --git a/tests/distinct/mod.rs b/tests/distinct/mod.rs index 8d35768..4c4c4c9 100644 --- a/tests/distinct/mod.rs +++ b/tests/distinct/mod.rs @@ -3,27 +3,12 @@ //! Tests for DISTINCT SELECT queries in various contexts including //! with WHERE clauses, ORDER BY clauses, and JOINs. -use assert_cmd::Command; -use std::path::PathBuf; -// No longer using predicates in our tests - -// Helper function to get the path to static test files -fn get_duplicates_file() -> PathBuf { - PathBuf::from("tests/data/duplicates.csv") -} - -fn get_users_file() -> PathBuf { - PathBuf::from("tests/data/users.csv") -} - -fn get_orders_file() -> PathBuf { - PathBuf::from("tests/data/orders.csv") -} +use crate::helpers::{get_duplicates_file, get_orders_file, get_users_file}; #[test] fn test_distinct_basic() -> Result<(), Box> { // Test the basic DISTINCT functionality with a simple query - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT name, department FROM duplicates") .arg(get_duplicates_file().to_str().unwrap()) @@ -65,7 +50,7 @@ fn test_distinct_basic() -> Result<(), Box> { #[test] fn test_distinct_with_where() -> Result<(), Box> { // Test DISTINCT with a WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT name, role FROM duplicates WHERE department = 'Engineering'") .arg(get_duplicates_file().to_str().unwrap()) @@ -124,7 +109,7 @@ fn test_distinct_with_where() -> Result<(), Box> { #[test] fn test_distinct_with_order_by() -> Result<(), Box> { // Test DISTINCT with ORDER BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT name, department FROM duplicates ORDER BY department, name") .arg(get_duplicates_file().to_str().unwrap()) @@ -187,7 +172,7 @@ fn test_distinct_with_order_by() -> Result<(), Box> { #[test] fn test_distinct_with_join() -> Result<(), Box> { // Test DISTINCT with JOIN operations - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT users.name, orders.product_id FROM users INNER JOIN orders ON users.id = orders.user_id") .arg(get_users_file().to_str().unwrap()) @@ -243,7 +228,7 @@ fn test_distinct_with_join() -> Result<(), Box> { #[test] fn test_distinct_with_select_star() -> Result<(), Box> { // Test DISTINCT with SELECT * (should deduplicate identical rows but not filter ID differences) - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT * FROM duplicates WHERE department = 'Engineering'") .arg(get_duplicates_file().to_str().unwrap()) @@ -306,7 +291,7 @@ fn test_distinct_with_select_star() -> Result<(), Box> { #[test] fn test_distinct_without_id() -> Result<(), Box> { // Test DISTINCT without ID column to ensure proper deduplication - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT name, department, role FROM duplicates") .arg(get_duplicates_file().to_str().unwrap()) diff --git a/tests/except/mod.rs b/tests/except/mod.rs new file mode 100644 index 0000000..fadb8ea --- /dev/null +++ b/tests/except/mod.rs @@ -0,0 +1,179 @@ +//! EXCEPT tests for sqawk VM +//! +//! This file contains tests for SQL EXCEPT set operation. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test EXCEPT returns rows in left but not in right +#[test] +fn test_except_basic() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,name\n2,Bob\n3,Charlie\n4,David\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // EXCEPT should only return Alice (in a but not in b) + assert!( + output_str.contains("1,Alice"), + "Alice should be in EXCEPT result" + ); + assert!( + !output_str.contains("2,Bob"), + "Bob should not be in EXCEPT result" + ); + assert!( + !output_str.contains("3,Charlie"), + "Charlie should not be in EXCEPT result" + ); + assert!( + !output_str.contains("4,David"), + "David should not be in EXCEPT result" + ); + + Ok(()) +} + +/// Test EXCEPT with disjoint tables (no common rows) +#[test] +fn test_except_disjoint_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n3,Charlie\n4,David\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // EXCEPT of disjoint tables should return all rows from a + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")); + + Ok(()) +} + +/// Test EXCEPT with identical tables +#[test] +fn test_except_identical_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // EXCEPT of identical tables should return empty result + assert!( + !output_str.contains("Alice"), + "Alice should not be in result" + ); + assert!(!output_str.contains("Bob"), "Bob should not be in result"); + + Ok(()) +} + +/// Test EXCEPT with first table being subset +#[test] +fn test_except_subset_as_first() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n2,Bob\n"; + let content_b = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // Since Bob is in both, result should be empty + assert!(!output_str.contains("Bob"), "Bob should not be in result"); + + Ok(()) +} + +/// Test EXCEPT with second table being subset +#[test] +fn test_except_subset_as_second() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,name\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // Alice and Charlie should be in result, Bob should not + assert!(output_str.contains("1,Alice"), "Alice should be in result"); + assert!( + output_str.contains("3,Charlie"), + "Charlie should be in result" + ); + assert!(!output_str.contains("2,Bob"), "Bob should not be in result"); + + Ok(()) +} + +/// Test EXCEPT removes duplicates from left side +#[test] +fn test_except_removes_duplicates() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + // Table a has Alice twice + let content_a = "id,name\n1,Alice\n1,Alice\n2,Bob\n"; + // Table b has Bob + let content_b = "id,name\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a EXCEPT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // Alice should appear only once (EXCEPT deduplicates) + let alice_count = output_str.matches("1,Alice").count(); + assert_eq!( + alice_count, 1, + "Alice should appear only once in EXCEPT result" + ); + assert!(!output_str.contains("2,Bob"), "Bob should not be in result"); + + Ok(()) +} diff --git a/tests/exists/mod.rs b/tests/exists/mod.rs new file mode 100644 index 0000000..b1f8b5f --- /dev/null +++ b/tests/exists/mod.rs @@ -0,0 +1,194 @@ +//! EXISTS/NOT EXISTS subquery tests for sqawk VM +//! +//! This file contains tests for EXISTS subqueries. +//! Phase 4F: Subquery support + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test basic EXISTS subquery - table has rows +#[test] +fn test_exists_subquery_has_rows() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders) + // Since orders table has rows, all users should be returned + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + Ok(()) +} + +/// Test EXISTS subquery with WHERE filter - matching rows +#[test] +fn test_exists_subquery_with_where_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE amount > 150) + // order 2 has amount=200 > 150, so EXISTS is true + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE amount > 150)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + Ok(()) +} + +/// Test EXISTS subquery with WHERE filter - no matching rows +#[test] +fn test_exists_subquery_with_where_no_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE amount > 500) + // No orders with amount > 500, so EXISTS is false - no users returned + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE amount > 500)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + // No results expected + cmd.assert().success(); + + Ok(()) +} + +/// Test NOT EXISTS subquery - table has rows +#[test] +fn test_not_exists_subquery_has_rows() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders) + // Since orders table has rows, NOT EXISTS is false - no users returned + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + // No results expected + cmd.assert().success(); + + Ok(()) +} + +/// Test NOT EXISTS subquery - no matching rows +#[test] +fn test_not_exists_subquery_no_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders WHERE amount > 1000) + // No orders with amount > 1000, so NOT EXISTS is true - all users returned + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE NOT EXISTS (SELECT 1 FROM orders WHERE amount > 1000)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + Ok(()) +} + +/// Test EXISTS with single-row table +#[test] +fn test_exists_single_row() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let data_content = "id,value\n1,100\n2,200\n3,300\n"; + let data_path = create_custom_csv(temp_dir.path(), "data.csv", data_content)?; + + let single_content = "flag\n1\n"; + let single_path = create_custom_csv(temp_dir.path(), "single.csv", single_content)?; + + // Test: SELECT * FROM data WHERE EXISTS (SELECT 1 FROM single) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE EXISTS (SELECT 1 FROM single)") + .arg(data_path.to_str().unwrap()) + .arg(single_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,100")) + .stdout(predicates::str::contains("2,200")) + .stdout(predicates::str::contains("3,300")); + + Ok(()) +} + +/// Test EXISTS combined with other WHERE conditions +#[test] +fn test_exists_with_other_conditions() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let users_content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + let orders_content = "order_id,user_id,amount\n1,1,100\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Test: SELECT * FROM users WHERE age > 25 AND EXISTS (SELECT 1 FROM orders) + // EXISTS is true, but we also filter by age > 25 + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE age > 25 AND EXISTS (SELECT 1 FROM orders)") + .arg(users_path.to_str().unwrap()) + .arg(orders_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,Bob,30")) + .stdout(predicates::str::contains("3,Charlie,35")); + + Ok(()) +} diff --git a/tests/full_join/mod.rs b/tests/full_join/mod.rs new file mode 100644 index 0000000..8166c4d --- /dev/null +++ b/tests/full_join/mod.rs @@ -0,0 +1,151 @@ +//! FULL JOIN tests for sqawk VM +//! +//! This file contains tests for SQL FULL OUTER JOIN operations. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test FULL JOIN returns all rows from both tables with matches +#[test] +fn test_full_join_with_matches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")); + + Ok(()) +} + +/// Test FULL JOIN returns NULLs for non-matching rows on both sides +#[test] +fn test_full_join_with_nulls_both_sides() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n4,Marketing\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // Charlie (id=3) has no match in right table -> right columns NULL + // Marketing (id=4) has no match in left table -> left columns NULL + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")) + .stdout(predicates::str::contains("3,Charlie,NULL,NULL")) + .stdout(predicates::str::contains("NULL,NULL,4,Marketing")); + + Ok(()) +} + +/// Test FULL JOIN with completely disjoint tables +#[test] +fn test_full_join_disjoint() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n3,Engineering\n4,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // No matches, all rows should have NULLs on the opposite side + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,NULL,NULL")) + .stdout(predicates::str::contains("2,Bob,NULL,NULL")) + .stdout(predicates::str::contains("NULL,NULL,3,Engineering")) + .stdout(predicates::str::contains("NULL,NULL,4,Sales")); + + Ok(()) +} + +/// Test FULL OUTER JOIN syntax (equivalent to FULL JOIN) +#[test] +fn test_full_outer_join_syntax() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n"; + let content_b = "id,dept\n1,Engineering\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL OUTER JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")); + + Ok(()) +} + +/// Test FULL JOIN with empty left table +#[test] +fn test_full_join_empty_left() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // All right rows should be returned with NULL for left columns + cmd.assert() + .success() + .stdout(predicates::str::contains("NULL,NULL,1,Engineering")) + .stdout(predicates::str::contains("NULL,NULL,2,Sales")); + + Ok(()) +} + +/// Test FULL JOIN with empty right table +#[test] +fn test_full_join_empty_right() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a FULL JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // All left rows should be returned with NULL for right columns + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,NULL,NULL")) + .stdout(predicates::str::contains("2,Bob,NULL,NULL")); + + Ok(()) +} diff --git a/tests/group_by/mod.rs b/tests/group_by/mod.rs index f3052bb..4e8b0e3 100644 --- a/tests/group_by/mod.rs +++ b/tests/group_by/mod.rs @@ -2,35 +2,15 @@ //! //! Tests for SQL GROUP BY clause with various aggregate functions. -use crate::helpers::create_temp_dir; -use assert_cmd::Command; +use crate::helpers::get_departments_file; use predicates::prelude::*; -use std::fs; -use std::io::Write; -use std::path::PathBuf; - -// Helper function to create a departments test file -fn create_departments_file() -> Result<(tempfile::TempDir, PathBuf), Box> { - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("departments.csv"); - - // Create a CSV file for GROUP BY testing - let content = "id,name,department,salary\n1,Alice,Engineering,75000\n2,Bob,Marketing,65000\n3,Charlie,Engineering,85000\n4,David,Sales,60000\n5,Eve,Marketing,70000\n6,Frank,Engineering,90000\n7,Grace,Sales,65000\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} #[test] fn test_basic_group_by() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_departments_file()?; + let file_path = get_departments_file(); // Run sqawk with basic GROUP BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS count, SUM(salary) AS total_salary, AVG(salary) AS avg_salary FROM departments GROUP BY department") .arg(file_path.to_str().unwrap()) @@ -68,11 +48,10 @@ fn test_basic_group_by() -> Result<(), Box> { #[test] fn test_group_by_with_order_by() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_departments_file()?; + let file_path = get_departments_file(); // Run sqawk with GROUP BY and ORDER BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary FROM departments GROUP BY department ORDER BY avg_salary DESC") .arg(file_path.to_str().unwrap()) @@ -123,11 +102,10 @@ fn test_group_by_with_order_by() -> Result<(), Box> { #[test] fn test_group_by_with_complex_expressions() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_departments_file()?; + let file_path = get_departments_file(); // Run sqawk with complex GROUP BY expressions - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS employee_count, SUM(salary) AS total_salary, AVG(salary) AS avg_salary, MIN(salary) AS min_salary, MAX(salary) AS max_salary FROM departments GROUP BY department ORDER BY avg_salary DESC") .arg(file_path.to_str().unwrap()) @@ -193,11 +171,10 @@ fn test_group_by_with_complex_expressions() -> Result<(), Box Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_departments_file()?; + let file_path = get_departments_file(); // Run sqawk with GROUP BY and HAVING - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary FROM departments GROUP BY department HAVING COUNT(*) > 2") .arg(file_path.to_str().unwrap()) @@ -237,11 +214,10 @@ fn test_group_by_with_having() -> Result<(), Box> { #[test] fn test_group_by_with_having_avg() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_departments_file()?; + let file_path = get_departments_file(); // Run sqawk with GROUP BY and HAVING with AVG function - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS employee_count, AVG(salary) AS avg_salary FROM departments GROUP BY department HAVING AVG(salary) > 70000") .arg(file_path.to_str().unwrap()) diff --git a/tests/headerless/mod.rs b/tests/headerless/mod.rs new file mode 100644 index 0000000..d1bee25 --- /dev/null +++ b/tests/headerless/mod.rs @@ -0,0 +1,194 @@ +//! Tests for headerless file auto-detection +//! +//! This file contains tests for automatic detection of files without headers +//! and the automatic generation of column names (a, b, c, ...). + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +#[test] +fn test_headerless_auto_detect_numeric_first_row() -> Result<(), Box> { + // Test auto-detection when first row contains numeric values + let temp_dir = create_temp_dir()?; + + // First row has numbers - should be detected as data, not headers + let data_file = create_custom_csv( + temp_dir.path(), + "numbers.csv", + "1,100,200\n2,150,250\n3,200,300\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a, b, c FROM numbers") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("a,b,c")) + .stdout(predicates::str::contains("1,100,200")); + + Ok(()) +} + +#[test] +fn test_headerless_passwd_style() -> Result<(), Box> { + // Test auto-detection for /etc/passwd style files + let temp_dir = create_temp_dir()?; + + // passwd-style: starts with "root" which triggers headerless detection + let data_file = create_custom_csv( + temp_dir.path(), + "passwd", + "root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-F:") + .arg("-s") + .arg("SELECT a, c, f FROM passwd") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("root")) + .stdout(predicates::str::contains("/root")); + + Ok(()) +} + +#[test] +fn test_headerless_path_in_first_row() -> Result<(), Box> { + // Test auto-detection when first row contains a path + let temp_dir = create_temp_dir()?; + + // First field starts with "/" - should be detected as data + let data_file = create_custom_csv( + temp_dir.path(), + "paths.csv", + "/usr/bin,executable,1000\n/etc/config,config,500\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a, b FROM paths") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("/usr/bin")) + .stdout(predicates::str::contains("executable")); + + Ok(()) +} + +#[test] +fn test_headerless_alphabetic_columns() -> Result<(), Box> { + // Test that column names follow a, b, c, ... pattern + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv(temp_dir.path(), "data.csv", "1,2,3,4,5\n6,7,8,9,10\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a, b, c, d, e FROM data") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("a,b,c,d,e")) + .stdout(predicates::str::contains("1,2,3,4,5")); + + Ok(()) +} + +#[test] +fn test_headerless_filter_query() -> Result<(), Box> { + // Test filtering on auto-generated column names + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv( + temp_dir.path(), + "data.csv", + "1,Alice,100\n2,Bob,200\n3,Charlie,50\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT b FROM data WHERE c > 75") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("Bob")); + + Ok(()) +} + +#[test] +fn test_headerless_aggregate() -> Result<(), Box> { + // Test aggregation with auto-generated column names + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv( + temp_dir.path(), + "data.csv", + "1,North,100\n2,South,200\n3,North,150\n4,South,250\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT b, SUM(c) FROM data GROUP BY b") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("North")) + .stdout(predicates::str::contains("South")); + + Ok(()) +} + +#[test] +fn test_headerless_with_comments() -> Result<(), Box> { + // Test headerless file with comment lines + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv( + temp_dir.path(), + "data.csv", + "# This is a comment\n1,Alice,100\n# Another comment\n2,Bob,200\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("Bob")); + + Ok(()) +} + +#[test] +fn test_headerless_asterisk_field() -> Result<(), Box> { + // Test auto-detection when field contains asterisk (like passwd x field) + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv(temp_dir.path(), "data.csv", "user1,*,1001\nuser2,*,1002\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT a, c FROM data") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("user1")) + .stdout(predicates::str::contains("1001")); + + Ok(()) +} diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index f57bca8..26f96ec 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -6,12 +6,20 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; -use std::process::Command; -use assert_cmd::prelude::*; use predicates::prelude::*; use tempfile::TempDir; +/// Create a Command for running the sqawk binary +/// +/// Uses the cargo_bin_cmd! macro which is compatible with custom cargo build directories. +#[macro_export] +macro_rules! sqawk_cmd { + () => { + assert_cmd::cargo::cargo_bin_cmd!("sqawk") + }; +} + /// Represents a test case for sqawk pub struct SqawkTestCase { /// The SQL query to execute @@ -44,14 +52,9 @@ impl Default for SqawkTestCase { } } -/// Run a test using a temporary csv file created specifically for this test +/// Run a test using the standard people.csv static fixture pub fn run_test_case(test_case: SqawkTestCase) -> Result<(), Box> { - // Create a temporary directory for test files - let temp_dir = create_temp_dir()?; - let test_file = prepare_test_file(temp_dir.path())?; - - // Call the common test function - run_test_case_with_file(test_case, test_file) + run_test_case_with_static_file(test_case, get_people_file()) } /// Run a test using a static test file that already exists in the project @@ -88,8 +91,8 @@ fn run_test_case_with_file( test_case: SqawkTestCase, test_file: PathBuf, ) -> Result<(), Box> { - // Build the command - let mut cmd = Command::cargo_bin("sqawk")?; + // Build the command using cargo_bin_cmd! macro (compatible with custom build dirs) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); // Add SQL statement cmd.arg("-s").arg(&test_case.sql); @@ -152,27 +155,63 @@ pub fn create_temp_dir() -> Result> { } } -/// Helper function to create a standard test CSV file with people data -/// This uses a template identical to the static sample.csv test file -pub fn prepare_test_file(dir: &Path) -> Result> { - let file_path = dir.join("people.csv"); - let content = "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n"; - fs::write(&file_path, content)?; - Ok(file_path) +/// Path to the employees.csv fixture (comprehensive test data with salary, department, role) +pub fn get_employees_file() -> PathBuf { + PathBuf::from("tests/data/employees.csv") } -/// Helper function to create a test CSV file with people data including category_id -/// Note: Currently not used, but keeping for potential future tests -#[allow(dead_code)] -pub fn prepare_test_file_with_category(dir: &Path) -> Result> { - let file_path = dir.join("people.csv"); - let content = "id,name,age,category_id\n1,Alice,32,1\n2,Bob,25,2\n3,Charlie,35,1\n"; - fs::write(&file_path, content)?; - Ok(file_path) +/// Path to the people.csv fixture (simple test data: id, name, age) +pub fn get_people_file() -> PathBuf { + PathBuf::from("tests/data/people.csv") +} + +/// Path to the departments.csv fixture +pub fn get_departments_file() -> PathBuf { + PathBuf::from("tests/data/departments.csv") +} + +/// Path to the duplicates.csv fixture +pub fn get_duplicates_file() -> PathBuf { + PathBuf::from("tests/data/duplicates.csv") +} + +/// Path to the strings.csv fixture (for string function tests) +pub fn get_strings_file() -> PathBuf { + PathBuf::from("tests/data/strings.csv") +} + +/// Path to boundaries.csv fixture (for comparison tests) +pub fn get_boundaries_file() -> PathBuf { + PathBuf::from("tests/data/boundaries.csv") +} + +/// Path to users.csv fixture (for join tests) +pub fn get_users_file() -> PathBuf { + PathBuf::from("tests/data/users.csv") +} + +/// Path to orders.csv fixture (for join tests) +pub fn get_orders_file() -> PathBuf { + PathBuf::from("tests/data/orders.csv") +} + +/// Path to products.csv fixture (for join tests) +pub fn get_products_file() -> PathBuf { + PathBuf::from("tests/data/products.csv") +} + +/// Path to employees.tsv fixture (for delimiter tests) +pub fn get_employees_tsv_file() -> PathBuf { + PathBuf::from("tests/data/employees.tsv") +} + +/// Path to contacts.txt fixture (for colon-delimiter tests) +pub fn get_contacts_file() -> PathBuf { + PathBuf::from("tests/data/contacts.txt") } -/// Helper function to get the path to the static sample.csv test file -pub fn get_static_sample_file() -> PathBuf { +/// Path to sample.csv fixture (simple test data: id, name, age) +pub fn get_sample_file() -> PathBuf { PathBuf::from("tests/data/sample.csv") } diff --git a/tests/in_list/mod.rs b/tests/in_list/mod.rs new file mode 100644 index 0000000..459c04d --- /dev/null +++ b/tests/in_list/mod.rs @@ -0,0 +1,131 @@ +//! IN list operator tests for sqawk VM +//! +//! This file contains tests for SQL IN (list) operator. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test basic IN with integer values +#[test] +fn test_in_list_integers() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n4,David,40\n5,Eve,20\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test IN with integer list using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE age IN (25, 35, 40)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,25")) + .stdout(predicates::str::contains("3,Charlie,35")) + .stdout(predicates::str::contains("4,David,40")); + + Ok(()) +} + +/// Test IN with string values +#[test] +fn test_in_list_strings() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,city\n1,Alice,NYC\n2,Bob,LA\n3,Charlie,Chicago\n4,David,NYC\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test IN with string list using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE city IN ('NYC', 'LA')") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,NYC")) + .stdout(predicates::str::contains("2,Bob,LA")) + .stdout(predicates::str::contains("4,David,NYC")); + + Ok(()) +} + +/// Test NOT IN +#[test] +fn test_not_in_list() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,status\n1,active\n2,pending\n3,inactive\n4,active\n5,archived\n"; + let file_path = create_custom_csv(temp_dir.path(), "records.csv", content)?; + + // Test NOT IN using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM records WHERE status NOT IN ('active', 'archived')") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,pending")) + .stdout(predicates::str::contains("3,inactive")); + + Ok(()) +} + +/// Test IN with single value (equivalent to =) +#[test] +fn test_in_list_single_value() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,20\n3,30\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test IN with single value using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE value IN (20)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,20")); + + Ok(()) +} + +/// Test IN with no matches +#[test] +fn test_in_list_no_matches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,num\n1,5\n2,10\n3,15\n"; + let file_path = create_custom_csv(temp_dir.path(), "nums.csv", content)?; + + // Test IN with values that don't match + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM nums WHERE num IN (100, 200, 300)") + .arg(file_path.to_str().unwrap()); + + // VM mode with no results returns empty output + cmd.assert().success(); + + Ok(()) +} + +/// Test IN with all values matching +#[test] +fn test_in_list_all_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,code\n1,A\n2,B\n3,C\n"; + let file_path = create_custom_csv(temp_dir.path(), "codes.csv", content)?; + + // Test IN that matches all rows + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM codes WHERE code IN ('A', 'B', 'C')") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,A")) + .stdout(predicates::str::contains("2,B")) + .stdout(predicates::str::contains("3,C")); + + Ok(()) +} diff --git a/tests/in_subquery/mod.rs b/tests/in_subquery/mod.rs new file mode 100644 index 0000000..d02d1f6 --- /dev/null +++ b/tests/in_subquery/mod.rs @@ -0,0 +1,171 @@ +//! IN (SELECT ...) subquery tests for sqawk VM +//! +//! This file contains tests for IN subqueries. +//! Phase 4F: Subquery support + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test basic IN subquery +#[test] +fn test_in_subquery_basic() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + // Create orders table + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n3,1,150\n4,3,300\n5,4,250\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Create users table + let users_content = "id,name,active\n1,Alice,1\n2,Bob,0\n3,Charlie,1\n4,David,1\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + // Test: SELECT * FROM orders WHERE user_id IN (SELECT id FROM users) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM orders WHERE user_id IN (SELECT id FROM users)") + .arg(orders_path.to_str().unwrap()) + .arg(users_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,1,100")) + .stdout(predicates::str::contains("2,2,200")) + .stdout(predicates::str::contains("3,1,150")) + .stdout(predicates::str::contains("4,3,300")) + .stdout(predicates::str::contains("5,4,250")); + + Ok(()) +} + +/// Test IN subquery with WHERE clause in subquery +#[test] +fn test_in_subquery_with_where() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + // Create orders table + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n3,1,150\n4,3,300\n5,4,250\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Create users table - only active users have active=1 + let users_content = "id,name,active\n1,Alice,1\n2,Bob,0\n3,Charlie,1\n4,David,0\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + // Test: SELECT * FROM orders WHERE user_id IN (SELECT id FROM users WHERE active = 1) + // Only users 1 (Alice) and 3 (Charlie) are active + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM orders WHERE user_id IN (SELECT id FROM users WHERE active = 1)") + .arg(orders_path.to_str().unwrap()) + .arg(users_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,1,100")) + .stdout(predicates::str::contains("3,1,150")) + .stdout(predicates::str::contains("4,3,300")); + + Ok(()) +} + +/// Test NOT IN subquery +#[test] +fn test_not_in_subquery() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + // Create orders table + let orders_content = "order_id,user_id,amount\n1,1,100\n2,2,200\n3,5,150\n4,3,300\n5,6,250\n"; + let orders_path = create_custom_csv(temp_dir.path(), "orders.csv", orders_content)?; + + // Create users table + let users_content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n4,David\n"; + let users_path = create_custom_csv(temp_dir.path(), "users.csv", users_content)?; + + // Test: SELECT * FROM orders WHERE user_id NOT IN (SELECT id FROM users) + // Users 5 and 6 are not in the users table + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM orders WHERE user_id NOT IN (SELECT id FROM users)") + .arg(orders_path.to_str().unwrap()) + .arg(users_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("3,5,150")) + .stdout(predicates::str::contains("5,6,250")); + + Ok(()) +} + +/// Test IN subquery with empty subquery result +#[test] +fn test_in_subquery_empty_result() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let data_content = "id,value\n1,10\n2,20\n3,30\n"; + let data_path = create_custom_csv(temp_dir.path(), "data.csv", data_content)?; + + let empty_content = "id,value\n1,100\n2,200\n"; + let empty_path = create_custom_csv(temp_dir.path(), "empty.csv", empty_content)?; + + // Test: SELECT * FROM data WHERE value IN (SELECT value FROM empty WHERE value > 1000) + // Subquery returns no rows + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE value IN (SELECT value FROM empty WHERE value > 1000)") + .arg(data_path.to_str().unwrap()) + .arg(empty_path.to_str().unwrap()); + + // No matches expected when subquery is empty + cmd.assert().success(); + + Ok(()) +} + +/// Test IN subquery on same table (self-referential) +#[test] +fn test_in_subquery_same_table() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let content = "id,name,manager_id\n1,CEO,0\n2,Alice,1\n3,Bob,1\n4,Charlie,2\n5,David,3\n"; + let file_path = create_custom_csv(temp_dir.path(), "employees.csv", content)?; + + // Test: Find all employees who are managers + // SELECT * FROM employees WHERE id IN (SELECT manager_id FROM employees) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM employees WHERE id IN (SELECT manager_id FROM employees)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,CEO,0")) + .stdout(predicates::str::contains("2,Alice,1")) + .stdout(predicates::str::contains("3,Bob,1")); + + Ok(()) +} + +/// Test IN subquery with string values +#[test] +fn test_in_subquery_strings() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + + let products_content = "id,name,category\n1,Laptop,Electronics\n2,Mouse,Electronics\n3,Desk,Furniture\n4,Chair,Furniture\n5,Book,Other\n"; + let products_path = create_custom_csv(temp_dir.path(), "products.csv", products_content)?; + + let categories_content = "name,active\nElectronics,1\nFurniture,0\n"; + let categories_path = create_custom_csv(temp_dir.path(), "categories.csv", categories_content)?; + + // Test: SELECT * FROM products WHERE category IN (SELECT name FROM categories WHERE active = 1) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM products WHERE category IN (SELECT name FROM categories WHERE active = 1)") + .arg(products_path.to_str().unwrap()) + .arg(categories_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Laptop,Electronics")) + .stdout(predicates::str::contains("2,Mouse,Electronics")); + + Ok(()) +} diff --git a/tests/intersect/mod.rs b/tests/intersect/mod.rs new file mode 100644 index 0000000..22c3b53 --- /dev/null +++ b/tests/intersect/mod.rs @@ -0,0 +1,165 @@ +//! INTERSECT tests for sqawk VM +//! +//! This file contains tests for SQL INTERSECT set operation. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test INTERSECT returns only common rows +#[test] +fn test_intersect_common_rows() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,name\n2,Bob\n3,Charlie\n4,David\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a INTERSECT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // INTERSECT should only return Bob and Charlie (common to both) + cmd.assert() + .success() + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + // Verify Alice and David are NOT in the result + let output = cmd.output()?.stdout; + let output_str = String::from_utf8_lossy(&output); + assert!( + !output_str.contains("1,Alice"), + "Alice should not be in INTERSECT result" + ); + assert!( + !output_str.contains("4,David"), + "David should not be in INTERSECT result" + ); + + Ok(()) +} + +/// Test INTERSECT with disjoint tables (no common rows) +#[test] +fn test_intersect_disjoint_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n3,Charlie\n4,David\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a INTERSECT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // INTERSECT of disjoint tables should return only header + assert!( + !output_str.contains("Alice"), + "Alice should not be in INTERSECT of disjoint tables" + ); + assert!( + !output_str.contains("Bob"), + "Bob should not be in INTERSECT of disjoint tables" + ); + assert!( + !output_str.contains("Charlie"), + "Charlie should not be in INTERSECT of disjoint tables" + ); + assert!( + !output_str.contains("David"), + "David should not be in INTERSECT of disjoint tables" + ); + + Ok(()) +} + +/// Test INTERSECT with identical tables +#[test] +fn test_intersect_identical_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a INTERSECT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // INTERSECT of identical tables should return all rows (deduplicated) + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")); + + Ok(()) +} + +/// Test INTERSECT with one table being subset of another +#[test] +fn test_intersect_subset_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,name\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a INTERSECT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // Only Bob should be in the result + assert!(output_str.contains("2,Bob"), "Bob should be in result"); + assert!( + !output_str.contains("1,Alice"), + "Alice should not be in result" + ); + assert!( + !output_str.contains("3,Charlie"), + "Charlie should not be in result" + ); + + Ok(()) +} + +/// Test INTERSECT removes duplicates +#[test] +fn test_intersect_removes_duplicates() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + // Table a has Bob twice + let content_a = "id,name\n2,Bob\n2,Bob\n"; + // Table b has Bob once + let content_b = "id,name\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a INTERSECT SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // INTERSECT should deduplicate, so Bob should appear only once + let bob_count = output_str.matches("2,Bob").count(); + assert_eq!( + bob_count, 1, + "Bob should appear only once in INTERSECT result" + ); + + Ok(()) +} diff --git a/tests/join/mod.rs b/tests/join/mod.rs index d7b2707..95099ed 100644 --- a/tests/join/mod.rs +++ b/tests/join/mod.rs @@ -2,27 +2,14 @@ //! //! Tests for cross joins, inner joins, and multi-table joins. -use assert_cmd::Command; use predicates::prelude::*; -use std::path::PathBuf; -// Helper function to get the path to static test files for joins -fn get_users_file() -> PathBuf { - PathBuf::from("tests/data/users.csv") -} - -fn get_orders_file() -> PathBuf { - PathBuf::from("tests/data/orders.csv") -} - -fn get_products_file() -> PathBuf { - PathBuf::from("tests/data/products.csv") -} +use crate::helpers::{get_orders_file, get_products_file, get_users_file}; #[test] fn test_cross_join() -> Result<(), Box> { // Test a basic cross join between two tables - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM users, orders") .arg(get_users_file().to_str().unwrap()) @@ -41,7 +28,7 @@ fn test_cross_join() -> Result<(), Box> { .stdout(predicate::str::contains("Jane,jane@example.com")); // Test with LIMIT separately - this will only show the first user due to how cross join ordering works - let mut cmd2 = Command::cargo_bin("sqawk")?; + let mut cmd2 = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd2.arg("-s") .arg("SELECT * FROM users, orders LIMIT 5") .arg(get_users_file().to_str().unwrap()) @@ -55,7 +42,7 @@ fn test_cross_join() -> Result<(), Box> { .stderr(predicate::str::contains("Applying LIMIT/OFFSET")); // Test a filtered query to specifically get Jane's data - let mut cmd3 = Command::cargo_bin("sqawk")?; + let mut cmd3 = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd3.arg("-s") .arg("SELECT users.name, users.email FROM users, orders WHERE users.id = 2 LIMIT 1") .arg(get_users_file().to_str().unwrap()) @@ -72,7 +59,7 @@ fn test_cross_join() -> Result<(), Box> { #[test] fn test_inner_join() -> Result<(), Box> { // Test an inner join between users and orders using WHERE condition - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM users, orders WHERE users.id = orders.user_id") .arg(get_users_file().to_str().unwrap()) @@ -104,7 +91,7 @@ fn test_inner_join() -> Result<(), Box> { #[test] fn test_inner_join_with_projection() -> Result<(), Box> { // Test an inner join with column projection - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, orders.product_id, orders.date FROM users, orders WHERE users.id = orders.user_id") .arg(get_users_file().to_str().unwrap()) @@ -126,7 +113,7 @@ fn test_inner_join_with_projection() -> Result<(), Box> { #[test] fn test_three_table_join() -> Result<(), Box> { // Test a three-way join between users, orders, and products - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, products.name, products.price, orders.date FROM users, orders, products WHERE orders.user_id = users.id AND orders.product_id = products.product_id") .arg(get_users_file().to_str().unwrap()) @@ -151,7 +138,7 @@ fn test_three_table_join() -> Result<(), Box> { #[test] fn test_join_with_additional_filtering() -> Result<(), Box> { // Test a join with additional non-join filtering in the WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, products.name, products.price FROM users, orders, products WHERE users.id = orders.user_id AND products.product_id = orders.product_id AND products.price > 500") .arg(get_users_file().to_str().unwrap()) @@ -192,7 +179,7 @@ fn test_join_order_preservation() -> Result<(), Box> { // Test that join result order is preserved based on input order // This test verifies that the order of rows in the result set - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, orders.id FROM users, orders WHERE users.id = orders.user_id") .arg(get_users_file().to_str().unwrap()) diff --git a/tests/join_on/mod.rs b/tests/join_on/mod.rs index 22384a2..82d3d03 100644 --- a/tests/join_on/mod.rs +++ b/tests/join_on/mod.rs @@ -2,27 +2,14 @@ //! //! Tests for SQL JOIN with the ON clause, as opposed to using WHERE for join conditions. -use assert_cmd::Command; use predicates::prelude::*; -use std::path::PathBuf; -// Helper functions to get the path to standard test files -fn get_users_file() -> PathBuf { - PathBuf::from("tests/data/users.csv") -} - -fn get_orders_file() -> PathBuf { - PathBuf::from("tests/data/orders.csv") -} - -fn get_products_file() -> PathBuf { - PathBuf::from("tests/data/products.csv") -} +use crate::helpers::{get_orders_file, get_products_file, get_users_file}; #[test] fn test_inner_join_on_basic() -> Result<(), Box> { // Test a basic INNER JOIN with ON clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, orders.product_id, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id") .arg(get_users_file().to_str().unwrap()) @@ -63,7 +50,7 @@ fn test_inner_join_on_basic() -> Result<(), Box> { #[test] fn test_inner_join_on_with_where() -> Result<(), Box> { // Test INNER JOIN with ON and additional WHERE filtering - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name, orders.product_id, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id WHERE orders.product_id > 102") .arg(get_users_file().to_str().unwrap()) @@ -99,7 +86,7 @@ fn test_inner_join_on_with_where() -> Result<(), Box> { #[test] fn test_three_way_inner_join_on() -> Result<(), Box> { // Test a three-way INNER JOIN with ON clauses - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name AS user, products.name AS product, products.price, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id INNER JOIN products ON orders.product_id = products.product_id") .arg(get_users_file().to_str().unwrap()) @@ -126,7 +113,7 @@ fn test_three_way_inner_join_on() -> Result<(), Box> { #[test] fn test_complex_operations_with_join_on() -> Result<(), Box> { // Test a complex query with JOIN ON, WHERE, and ORDER BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT users.name AS customer, orders.date AS purchase_date, products.name AS item, products.price AS price FROM users INNER JOIN orders ON users.id = orders.user_id INNER JOIN products ON orders.product_id = products.product_id WHERE products.price > 100 ORDER BY price DESC") .arg(get_users_file().to_str().unwrap()) diff --git a/tests/left_join/mod.rs b/tests/left_join/mod.rs new file mode 100644 index 0000000..d813c01 --- /dev/null +++ b/tests/left_join/mod.rs @@ -0,0 +1,123 @@ +//! LEFT JOIN tests for sqawk VM +//! +//! This file contains tests for SQL LEFT OUTER JOIN operations. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test LEFT JOIN returns all rows from left table with matches +#[test] +fn test_left_join_with_matches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a LEFT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")); + + Ok(()) +} + +/// Test LEFT JOIN returns NULL for non-matching rows +#[test] +fn test_left_join_with_nulls() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a LEFT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // Charlie (id=3) has no match, so right columns should be NULL + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")) + .stdout(predicates::str::contains("3,Charlie,NULL,NULL")); + + Ok(()) +} + +/// Test LEFT JOIN with all non-matching rows +#[test] +fn test_left_join_all_nulls() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n3,Engineering\n4,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a LEFT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // No matches, all rows should have NULL for right columns + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,NULL,NULL")) + .stdout(predicates::str::contains("2,Bob,NULL,NULL")); + + Ok(()) +} + +/// Test LEFT OUTER JOIN syntax (equivalent to LEFT JOIN) +#[test] +fn test_left_outer_join_syntax() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n"; + let content_b = "id,dept\n1,Engineering\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a LEFT OUTER JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")); + + Ok(()) +} + +/// Test LEFT JOIN with empty right table +#[test] +fn test_left_join_empty_right() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a LEFT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // All left rows should be returned with NULL for right columns + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,NULL,NULL")) + .stdout(predicates::str::contains("2,Bob,NULL,NULL")); + + Ok(()) +} diff --git a/tests/like/mod.rs b/tests/like/mod.rs new file mode 100644 index 0000000..4f83c4f --- /dev/null +++ b/tests/like/mod.rs @@ -0,0 +1,175 @@ +//! LIKE and ILIKE pattern matching tests for sqawk VM +//! +//! This file contains tests for SQL LIKE and ILIKE pattern matching operations. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test basic LIKE pattern with % wildcard (matches any sequence) +#[test] +fn test_like_percent_prefix() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,city\n1,Alice,Atlanta\n2,Bob,Boston\n3,Charlie,Chicago\n4,David,Denver\n5,Alex,Austin\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test LIKE with prefix pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE name LIKE 'A%'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,Atlanta")) + .stdout(predicates::str::contains("5,Alex,Austin")); + + Ok(()) +} + +/// Test LIKE pattern with % suffix +#[test] +fn test_like_percent_suffix() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,email\n1,Alice,alice@example.com\n2,Bob,bob@test.org\n3,Charlie,charlie@example.com\n"; + let file_path = create_custom_csv(temp_dir.path(), "users.csv", content)?; + + // Test LIKE with suffix pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM users WHERE email LIKE '%@example.com'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,alice@example.com")) + .stdout(predicates::str::contains("3,Charlie,charlie@example.com")); + + Ok(()) +} + +/// Test LIKE pattern with % on both sides (contains) +#[test] +fn test_like_percent_contains() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,description\n1,The quick brown fox\n2,A lazy dog\n3,Quick and easy\n4,Slow and steady\n"; + let file_path = create_custom_csv(temp_dir.path(), "items.csv", content)?; + + // Test LIKE with contains pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM items WHERE description LIKE '%and%'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("3,Quick and easy")) + .stdout(predicates::str::contains("4,Slow and steady")); + + Ok(()) +} + +/// Test LIKE pattern with _ wildcard (matches single character) +#[test] +fn test_like_underscore() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,code\n1,A1B\n2,A2B\n3,A12B\n4,A3B\n5,AAB\n"; + let file_path = create_custom_csv(temp_dir.path(), "codes.csv", content)?; + + // Test LIKE with underscore pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM codes WHERE code LIKE 'A_B'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,A1B")) + .stdout(predicates::str::contains("2,A2B")) + .stdout(predicates::str::contains("4,A3B")); + + Ok(()) +} + +/// Test NOT LIKE pattern +#[test] +fn test_not_like() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n3,Alex\n4,Charlie\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test NOT LIKE using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE name NOT LIKE 'A%'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("4,Charlie")); + + Ok(()) +} + +/// Test ILIKE case-insensitive pattern matching +#[test] +fn test_ilike_case_insensitive() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,ALICE\n2,alice\n3,Alice\n4,Bob\n5,ALEX\n"; + let file_path = create_custom_csv(temp_dir.path(), "names.csv", content)?; + + // Test ILIKE case-insensitive pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM names WHERE name ILIKE 'al%'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,ALICE")) + .stdout(predicates::str::contains("2,alice")) + .stdout(predicates::str::contains("3,Alice")) + .stdout(predicates::str::contains("5,ALEX")); + + Ok(()) +} + +/// Test exact match pattern (no wildcards) +#[test] +fn test_like_exact_match() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,status\n1,active\n2,inactive\n3,active\n4,pending\n"; + let file_path = create_custom_csv(temp_dir.path(), "records.csv", content)?; + + // Test LIKE with exact pattern using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM records WHERE status LIKE 'active'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,active")) + .stdout(predicates::str::contains("3,active")); + + Ok(()) +} + +/// Test LIKE with special regex characters that need escaping +#[test] +fn test_like_special_chars() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,test.csv\n2,test_csv\n3,test+csv\n4,testXcsv\n"; + let file_path = create_custom_csv(temp_dir.path(), "files.csv", content)?; + + // Test LIKE with literal dot using VM + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM files WHERE value LIKE 'test.csv'") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,test.csv")); + + Ok(()) +} diff --git a/tests/limit_offset/mod.rs b/tests/limit_offset/mod.rs index 97ae235..ccc1c77 100644 --- a/tests/limit_offset/mod.rs +++ b/tests/limit_offset/mod.rs @@ -3,26 +3,12 @@ //! Tests for SQL queries with LIMIT and OFFSET clauses in various contexts //! including with ORDER BY, aggregate functions, and GROUP BY. -use assert_cmd::Command; -use std::path::PathBuf; - -// Helper function to get the path to static test files -fn get_sample_file() -> PathBuf { - PathBuf::from("tests/data/sample.csv") -} - -fn get_departments_file() -> PathBuf { - PathBuf::from("tests/data/departments.csv") -} - -fn get_duplicates_file() -> PathBuf { - PathBuf::from("tests/data/duplicates.csv") -} +use crate::helpers::{get_departments_file, get_duplicates_file, get_sample_file}; #[test] fn test_limit_basic() -> Result<(), Box> { // Test basic LIMIT functionality - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample LIMIT 2") .arg(get_sample_file().to_str().unwrap()) @@ -64,7 +50,7 @@ fn test_limit_basic() -> Result<(), Box> { #[test] fn test_limit_with_offset() -> Result<(), Box> { // Test LIMIT with OFFSET - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample LIMIT 2 OFFSET 1") .arg(get_sample_file().to_str().unwrap()) @@ -109,7 +95,7 @@ fn test_limit_with_offset() -> Result<(), Box> { #[test] fn test_limit_with_order_by() -> Result<(), Box> { // Test LIMIT with ORDER BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample ORDER BY age DESC LIMIT 2") .arg(get_sample_file().to_str().unwrap()) @@ -161,39 +147,19 @@ fn test_limit_with_order_by() -> Result<(), Box> { #[test] fn test_limit_with_aggregates() -> Result<(), Box> { // Test LIMIT with aggregate functions and GROUP BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT department, COUNT(*) AS count, AVG(salary) AS avg_salary FROM departments GROUP BY department ORDER BY avg_salary DESC LIMIT 2") - .arg(get_departments_file().to_str().unwrap()) - .arg("-v"); + .arg(get_departments_file().to_str().unwrap()); // Execute the command and capture its output let output = cmd.output()?; let success = output.status.success(); - let stderr = String::from_utf8(output.stderr)?; let stdout = String::from_utf8(output.stdout)?; // Verify the command succeeded assert!(success, "Command failed"); - // Verify stderr contains expected messages - assert!( - stderr.contains("Applying aggregate functions"), - "Missing 'Applying aggregate functions' in stderr" - ); - assert!( - stderr.contains("Applying GROUP BY"), - "Missing 'Applying GROUP BY' in stderr" - ); - assert!( - stderr.contains("Applying ORDER BY"), - "Missing 'Applying ORDER BY' in stderr" - ); - assert!( - stderr.contains("Applying LIMIT/OFFSET"), - "Missing 'Applying LIMIT/OFFSET' in stderr" - ); - // Check output contains header and expected departments assert!( stdout.contains("department,count,avg_salary"), @@ -224,7 +190,7 @@ fn test_limit_with_aggregates() -> Result<(), Box> { #[test] fn test_distinct_with_limit() -> Result<(), Box> { // Test DISTINCT with LIMIT and OFFSET - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT DISTINCT department, role FROM duplicates ORDER BY department ASC LIMIT 3 OFFSET 1") .arg(get_duplicates_file().to_str().unwrap()) @@ -290,7 +256,7 @@ fn test_distinct_with_limit() -> Result<(), Box> { #[test] fn test_zero_limit() -> Result<(), Box> { // Test LIMIT 0 (should return only the header) - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample LIMIT 0") .arg(get_sample_file().to_str().unwrap()) @@ -338,7 +304,7 @@ fn test_zero_limit() -> Result<(), Box> { #[test] fn test_offset_beyond_table_size() -> Result<(), Box> { // Test OFFSET beyond table size (should return only the header) - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM sample LIMIT 10 OFFSET 10") .arg(get_sample_file().to_str().unwrap()) diff --git a/tests/math_functions/mod.rs b/tests/math_functions/mod.rs new file mode 100644 index 0000000..7f9295a --- /dev/null +++ b/tests/math_functions/mod.rs @@ -0,0 +1,89 @@ +//! Math function tests for sqawk VM +//! +//! Tests for ABS, ROUND, CEIL, FLOOR functions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test ABS function with negative value +#[test] +fn test_abs_negative() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,-5\n2,10\n3,-15\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT id, ABS(value) AS abs_val FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,5")) + .stdout(predicates::str::contains("2,10")) + .stdout(predicates::str::contains("3,15")); + + Ok(()) +} + +/// Test ROUND function +#[test] +fn test_round() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,3.7\n2,3.2\n3,3.5\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT id, ROUND(value) AS rounded FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,4")) + .stdout(predicates::str::contains("2,3")) + .stdout(predicates::str::contains("3,4")); + + Ok(()) +} + +/// Test CEIL function +#[test] +fn test_ceil() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,3.1\n2,3.9\n3,4.0\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT id, CEIL(value) AS ceiled FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,4")) + .stdout(predicates::str::contains("2,4")) + .stdout(predicates::str::contains("3,4")); + + Ok(()) +} + +/// Test FLOOR function +#[test] +fn test_floor() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,3.1\n2,3.9\n3,4.0\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT id, FLOOR(value) AS floored FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,3")) + .stdout(predicates::str::contains("2,3")) + .stdout(predicates::str::contains("3,4")); + + Ok(()) +} diff --git a/tests/mod.rs b/tests/mod.rs index f1b324f..30f518c 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -14,11 +14,53 @@ mod update; // UPDATE statement tests mod advanced; // Tests for advanced SQL features and edge cases mod aggregate; // Tests for aggregate functions (COUNT, SUM, AVG, MIN, MAX) mod alias; // Tests for column aliases (AS keyword) +mod between; // Tests for BETWEEN operator (VM implementation) +mod case; // Tests for CASE WHEN expressions (VM implementation) +mod cast; // Tests for CAST type conversion (VM implementation) +mod coalesce; // Tests for COALESCE and NULLIF functions (VM implementation) mod csv_handler; // Tests for CSV handler features (comments, error recovery) mod delimiter; // Tests for delimiter options (-F flag) mod group_by; // Tests for GROUP BY functionality +mod in_list; // Tests for IN list operator (VM implementation) mod join_on; // Tests for JOIN ON syntax (as opposed to WHERE for joins) +mod like; // Tests for LIKE/ILIKE pattern matching (VM implementation) mod limit_offset; // Tests for LIMIT and OFFSET clauses + +// Phase 2: Set Operations (VM implementation) +mod except; // Tests for EXCEPT set operation +mod intersect; // Tests for INTERSECT set operation +mod union; // Tests for UNION/UNION ALL set operations + +// Phase 3: Outer Joins (VM implementation) +mod full_join; // Tests for FULL OUTER JOIN +mod left_join; // Tests for LEFT OUTER JOIN +mod right_join; // Tests for RIGHT OUTER JOIN + +// Phase 4F: Subquery Support (VM implementation) +mod exists; // Tests for EXISTS/NOT EXISTS subqueries +mod in_subquery; // Tests for IN (SELECT ...) subqueries +mod subquery; // Tests for scalar subqueries with aggregates + +// Phase 4B: Correlated Subquery Support (VM implementation) +mod correlated; // Tests for correlated subqueries with runtime execution + +// Phase 7: Advanced Expressions (VM implementation) +mod arithmetic; // Tests for arithmetic operators (+, -, *, /, %) +mod date_functions; // Tests for date/time functions (DATE, TIME, NOW) +mod math_functions; // Tests for math functions (ABS, ROUND, CEIL, FLOOR) +mod string_functions_ext; // Tests for extended string functions (CONCAT, LEFT, RIGHT) + +// Phase 8: Window Functions (VM implementation) +mod window; // Tests for window functions (ROW_NUMBER, RANK, DENSE_RANK) + +// Phase 10: DDL Enhancements (VM implementation) +mod ddl; // Tests for DDL operations (DROP TABLE, ALTER TABLE, TRUNCATE, CREATE TABLE AS SELECT, CREATE TABLE LOCATION) + +// Table definition modes +mod headerless; // Tests for headerless file auto-detection (a, b, c column naming) +mod tabledef; // Tests for --tabledef CLI option + +mod output; // Tests for stdout and file output (atomic writes) mod repl; // Tests for interactive REPL functionality with pre-generated input mod string_functions; // Tests for string functions (UPPER, LOWER, TRIM, SUBSTR, REPLACE) diff --git a/tests/order_by/mod.rs b/tests/order_by/mod.rs index 37fae0c..12db873 100644 --- a/tests/order_by/mod.rs +++ b/tests/order_by/mod.rs @@ -3,7 +3,6 @@ //! Tests for sorting functionality with different column combinations and directions. use crate::helpers::*; -use assert_cmd::Command; use predicates::prelude::*; /// Test ordering by a single column in ascending order (implicit) @@ -53,7 +52,7 @@ fn test_order_by_multiple_columns() -> Result<(), Box> { let file_path = create_custom_csv(temp_dir.path(), "employees.csv", content)?; // Build the command - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM employees ORDER BY dept ASC, salary DESC") .arg(format!("employees={}", file_path.to_str().unwrap())); @@ -94,7 +93,7 @@ fn test_order_by_with_where() -> Result<(), Box> { let file_path = create_custom_csv(temp_dir.path(), "employees.csv", content)?; // Build the command - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT name, age FROM employees WHERE dept = 'IT' ORDER BY age DESC") .arg(format!("employees={}", file_path.to_str().unwrap())); @@ -132,7 +131,7 @@ fn test_order_by_with_projection() -> Result<(), Box> { let file_path = create_custom_csv(temp_dir.path(), "employees.csv", content)?; // Build the command - include age in the SELECT since we need it for ORDER BY - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT name, salary, age FROM employees ORDER BY age DESC") .arg(format!("employees={}", file_path.to_str().unwrap())); diff --git a/tests/output/mod.rs b/tests/output/mod.rs new file mode 100644 index 0000000..d9f220d --- /dev/null +++ b/tests/output/mod.rs @@ -0,0 +1,303 @@ +//! Tests for output functionality +//! +//! This module tests both stdout output and file writeback behavior, +//! including the atomic write pattern (temp file + rename). + +use crate::helpers::{create_custom_csv, create_temp_dir}; +use std::fs; + +// ============================================================================ +// STDOUT OUTPUT TESTS +// ============================================================================ + +#[test] +fn test_stdout_select_csv() -> Result<(), Box> { + // Test that SELECT output goes to stdout in CSV format + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.csv", + "id,name,value\n1,Alice,100\n2,Bob,200\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,value")) + .stdout(predicates::str::contains("1,Alice,100")) + .stdout(predicates::str::contains("2,Bob,200")); + + Ok(()) +} + +#[test] +fn test_stdout_select_with_delimiter() -> Result<(), Box> { + // Test that SELECT output uses the correct delimiter for TSV files + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.tsv", + "id\tname\tvalue\n1\tAlice\t100\n2\tBob\t200\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-F") + .arg("\t") + .arg("-s") + .arg("SELECT * FROM data") + .arg(file_path.to_str().unwrap()); + + // Output should be tab-delimited + cmd.assert() + .success() + .stdout(predicates::str::contains("id\tname\tvalue")) + .stdout(predicates::str::contains("1\tAlice\t100")) + .stdout(predicates::str::contains("2\tBob\t200")); + + Ok(()) +} + +#[test] +fn test_stdout_empty_result() -> Result<(), Box> { + // Test that queries with no matching rows still output headers + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.csv", + "id,name,value\n1,Alice,100\n2,Bob,200\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE id = 999") + .arg(file_path.to_str().unwrap()); + + // Should succeed with just headers (no data rows match) + cmd.assert() + .success() + .stdout(predicates::str::contains("id,name,value")); + + Ok(()) +} + +// ============================================================================ +// FILE OUTPUT TESTS (ATOMIC WRITEBACK) +// ============================================================================ + +#[test] +fn test_file_write_csv() -> Result<(), Box> { + // Test that --write flag correctly saves CSV modifications to file + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.csv", + "id,name,value\n1,Alice,100\n2,Bob,200\n", + )?; + + // Update a value and write back + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("UPDATE data SET value = 150 WHERE name = 'Alice'") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify file was updated + let content = fs::read_to_string(&file_path)?; + assert!( + content.contains("1,Alice,150"), + "File should contain updated value" + ); + assert!( + !content.contains("1,Alice,100"), + "File should not contain old value" + ); + assert!( + content.contains("2,Bob,200"), + "Other rows should be unchanged" + ); + + Ok(()) +} + +#[test] +fn test_file_write_tsv() -> Result<(), Box> { + // Test that --write flag correctly saves TSV modifications to file + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.tsv", + "id\tname\tvalue\n1\tAlice\t100\n2\tBob\t200\n", + )?; + + // Update a value and write back + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-F") + .arg("\t") + .arg("-s") + .arg("UPDATE data SET value = 150 WHERE name = 'Alice'") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify file was updated with tab delimiter preserved + let content = fs::read_to_string(&file_path)?; + assert!( + content.contains("1\tAlice\t150"), + "File should contain updated value with tabs" + ); + assert!( + !content.contains("1\tAlice\t100"), + "File should not contain old value" + ); + assert!( + content.contains("2\tBob\t200"), + "Other rows should be unchanged" + ); + + Ok(()) +} + +#[test] +fn test_file_write_insert() -> Result<(), Box> { + // Test INSERT with --write correctly persists new rows + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", "id,name,value\n1,Alice,100\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("INSERT INTO data (id, name, value) VALUES (2, 'Bob', 200)") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify file contains the new row + let content = fs::read_to_string(&file_path)?; + assert!(content.contains("1,Alice,100"), "Original row should exist"); + assert!(content.contains("2,Bob,200"), "New row should be added"); + + Ok(()) +} + +#[test] +fn test_file_write_delete() -> Result<(), Box> { + // Test DELETE with --write correctly removes rows from file + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.csv", + "id,name,value\n1,Alice,100\n2,Bob,200\n3,Charlie,300\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("DELETE FROM data WHERE name = 'Bob'") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify Bob was deleted + let content = fs::read_to_string(&file_path)?; + assert!(content.contains("1,Alice,100"), "Alice should remain"); + assert!(!content.contains("2,Bob,200"), "Bob should be deleted"); + assert!(content.contains("3,Charlie,300"), "Charlie should remain"); + + Ok(()) +} + +#[test] +fn test_file_no_write_by_default() -> Result<(), Box> { + // Test that modifications are NOT written without --write flag + let temp_dir = create_temp_dir()?; + let original_content = "id,name,value\n1,Alice,100\n2,Bob,200\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", original_content)?; + + // Update without --write flag + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("UPDATE data SET value = 999 WHERE name = 'Alice'") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify file was NOT modified + let content = fs::read_to_string(&file_path)?; + assert_eq!( + content, original_content, + "File should be unchanged without --write" + ); + + Ok(()) +} + +#[test] +fn test_file_write_preserves_original_on_read_only_query() -> Result<(), Box> +{ + // Test that SELECT with --write doesn't modify file (no changes to persist) + let temp_dir = create_temp_dir()?; + let original_content = "id,name,value\n1,Alice,100\n2,Bob,200\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", original_content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify file is unchanged (SELECT doesn't modify data) + let content = fs::read_to_string(&file_path)?; + assert_eq!( + content, original_content, + "File should be unchanged after SELECT" + ); + + Ok(()) +} + +#[test] +fn test_file_write_multiple_operations() -> Result<(), Box> { + // Test multiple SQL operations with single --write at end + let temp_dir = create_temp_dir()?; + let file_path = create_custom_csv( + temp_dir.path(), + "data.csv", + "id,name,value\n1,Alice,100\n2,Bob,200\n3,Charlie,300\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("UPDATE data SET value = 150 WHERE name = 'Alice'") + .arg("-s") + .arg("DELETE FROM data WHERE name = 'Bob'") + .arg("-s") + .arg("INSERT INTO data (id, name, value) VALUES (4, 'Diana', 400)") + .arg("--write") + .arg(file_path.to_str().unwrap()); + + cmd.assert().success(); + + // Verify all operations were applied + let content = fs::read_to_string(&file_path)?; + assert!( + content.contains("1,Alice,150"), + "Alice value should be updated" + ); + assert!(!content.contains("2,Bob,200"), "Bob should be deleted"); + assert!( + content.contains("3,Charlie,300"), + "Charlie should be unchanged" + ); + assert!(content.contains("4,Diana,400"), "Diana should be added"); + + Ok(()) +} diff --git a/tests/repl/mod.rs b/tests/repl/mod.rs index b0b3cfc..48695b5 100644 --- a/tests/repl/mod.rs +++ b/tests/repl/mod.rs @@ -1,3 +1,4 @@ +use std::fs; use std::io::Write; use std::process::{Command, Stdio}; use std::thread; @@ -83,17 +84,26 @@ fn test_repl_table_operations() { #[test] fn test_repl_write_toggle() { + // Create a temporary directory for this test + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + let temp_file_path = temp_dir.path().join("sample.csv"); + + // Copy sample.csv to the temporary file + let original_content = + fs::read_to_string("tests/data/sample.csv").expect("Failed to read sample.csv"); + fs::write(&temp_file_path, &original_content).expect("Failed to write temp file"); + // Commands to test the .write toggle let test_commands = ".write\nUPDATE sample SET age = 32 WHERE id = 1;\n.write\nSELECT * FROM sample;\n.exit\n"; - // Start the sqawk process with sample data loaded + // Start the sqawk process with temp data loaded let mut process = Command::new("cargo") .args([ "run", "--bin", "sqawk", - "tests/data/sample.csv", + temp_file_path.to_str().unwrap(), "--interactive", ]) .stdin(Stdio::piped()) diff --git a/tests/repl/test_repl_from_file.rs b/tests/repl/test_repl_from_file.rs index f36649e..92a2e14 100644 --- a/tests/repl/test_repl_from_file.rs +++ b/tests/repl/test_repl_from_file.rs @@ -1,11 +1,9 @@ -use assert_cmd::Command; - #[test] fn test_repl_from_file() { // Removed unused temporary directory // Run sqawk with input from repl_commands.txt in the tmp directory - let mut cmd = Command::cargo_bin("sqawk").unwrap(); + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); let assert = cmd .arg("--interactive") .arg("tests/data/sample.csv") diff --git a/tests/right_join/mod.rs b/tests/right_join/mod.rs new file mode 100644 index 0000000..18f77a7 --- /dev/null +++ b/tests/right_join/mod.rs @@ -0,0 +1,123 @@ +//! RIGHT JOIN tests for sqawk VM +//! +//! This file contains tests for SQL RIGHT OUTER JOIN operations. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test RIGHT JOIN returns all rows from right table with matches +#[test] +fn test_right_join_with_matches() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a RIGHT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")); + + Ok(()) +} + +/// Test RIGHT JOIN returns NULL for non-matching rows +#[test] +fn test_right_join_with_nulls() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n3,Marketing\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a RIGHT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // Marketing (id=3) has no match in left table, so left columns should be NULL + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")) + .stdout(predicates::str::contains("2,Bob,2,Sales")) + .stdout(predicates::str::contains("NULL,NULL,3,Marketing")); + + Ok(()) +} + +/// Test RIGHT JOIN with all non-matching rows +#[test] +fn test_right_join_all_nulls() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,dept\n3,Engineering\n4,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a RIGHT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // No matches, all rows should have NULL for left columns + cmd.assert() + .success() + .stdout(predicates::str::contains("NULL,NULL,3,Engineering")) + .stdout(predicates::str::contains("NULL,NULL,4,Sales")); + + Ok(()) +} + +/// Test RIGHT OUTER JOIN syntax (equivalent to RIGHT JOIN) +#[test] +fn test_right_outer_join_syntax() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n"; + let content_b = "id,dept\n1,Engineering\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a RIGHT OUTER JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice,1,Engineering")); + + Ok(()) +} + +/// Test RIGHT JOIN with empty left table +#[test] +fn test_right_join_empty_left() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n"; + let content_b = "id,dept\n1,Engineering\n2,Sales\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a RIGHT JOIN b ON a.id = b.id") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // All right rows should be returned with NULL for left columns + cmd.assert() + .success() + .stdout(predicates::str::contains("NULL,NULL,1,Engineering")) + .stdout(predicates::str::contains("NULL,NULL,2,Sales")); + + Ok(()) +} diff --git a/tests/string_functions/mod.rs b/tests/string_functions/mod.rs index 6dc429f..6aca4d7 100644 --- a/tests/string_functions/mod.rs +++ b/tests/string_functions/mod.rs @@ -7,40 +7,15 @@ //! The implementation of these string functions is tested in the unit tests in src/string_functions.rs, //! while these integration tests focus on using the functions in WHERE clauses with the full sqawk command. -use crate::helpers::create_temp_dir; -use assert_cmd::Command; +use crate::helpers::{get_employees_file, get_strings_file}; use predicates::prelude::*; -use std::fs; -use std::io::Write; -use std::path::PathBuf; - -// Helper function to create a strings test file -fn create_strings_file() -> Result<(tempfile::TempDir, PathBuf), Box> { - let temp_dir = create_temp_dir()?; - let file_path = temp_dir.path().join("strings.csv"); - - // Create a CSV file for string function testing - let content = "id,text,mixed_case,padded_text,email\n\ - 1,apple,ApPlE, trimme ,john@example.com\n\ - 2,banana,BaNaNa, needs space ,jane@example.com\n\ - 3,cherry,ChErRy, whitespace ,bob@test.org\n\ - 4,date,DaTe, extra ,alice@company.co.uk\n\ - 5,elderberry,ElDeRbErRy, padding ,admin@website.net\n"; - - let mut file = fs::File::create(&file_path)?; - file.write_all(content.as_bytes())?; - - // Return both the TempDir (to keep it alive) and the file path - Ok((temp_dir, file_path)) -} #[test] fn test_upper_function_in_where() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with UPPER function in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, text FROM strings WHERE UPPER(text) = 'APPLE'") .arg(file_path.to_str().unwrap()) @@ -59,11 +34,10 @@ fn test_upper_function_in_where() -> Result<(), Box> { #[test] fn test_lower_function_in_where() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with LOWER function in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, mixed_case FROM strings WHERE LOWER(mixed_case) = 'apple'") .arg(file_path.to_str().unwrap()) @@ -90,11 +64,10 @@ fn test_trim_function_in_where() -> Result<(), Box> { #[test] fn test_substr_function_in_where() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with SUBSTR function (2 arguments) in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, text FROM strings WHERE SUBSTR(text, 1, 1) = 'a'") .arg(file_path.to_str().unwrap()) @@ -109,7 +82,7 @@ fn test_substr_function_in_where() -> Result<(), Box> { .stdout(predicate::str::contains("3,cherry").not()); // Run sqawk with SUBSTR function (3 arguments) in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, text FROM strings WHERE SUBSTR(text, 2, 3) = 'ate'") .arg(file_path.to_str().unwrap()) @@ -128,11 +101,10 @@ fn test_substr_function_in_where() -> Result<(), Box> { #[test] fn test_replace_function_in_where() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with REPLACE function in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, email FROM strings WHERE REPLACE(email, 'example.com', 'test.com') = 'john@test.com'") .arg(file_path.to_str().unwrap()) @@ -151,11 +123,10 @@ fn test_replace_function_in_where() -> Result<(), Box> { #[test] fn test_combining_string_functions_in_where() -> Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with combinations of string functions in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, text FROM strings WHERE UPPER(SUBSTR(text, 1, 1)) = 'A'") .arg(file_path.to_str().unwrap()) @@ -174,11 +145,10 @@ fn test_combining_string_functions_in_where() -> Result<(), Box Result<(), Box> { - // Create test data file - keep temp_dir alive for the test duration - let (_temp_dir, file_path) = create_strings_file()?; + let file_path = get_strings_file(); // Run sqawk with string functions in WHERE clause - let mut cmd = Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT id, text FROM strings WHERE UPPER(text) = 'APPLE' OR UPPER(text) = 'BANANA'") .arg(file_path.to_str().unwrap()) @@ -198,12 +168,13 @@ fn test_string_functions_with_where_clause() -> Result<(), Box Result<(), Box> { - // We'll use the sample.csv file which is a standard test file - let mut cmd = Command::cargo_bin("sqawk")?; +fn test_string_functions_with_employees_data() -> Result<(), Box> { + let file_path = get_employees_file(); + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") - .arg("SELECT id, name FROM sample WHERE UPPER(name) = 'ALICE'") - .arg("tests/data/sample.csv") + .arg("SELECT id, name FROM employees WHERE UPPER(name) = 'ALICE'") + .arg(file_path.to_str().unwrap()) .arg("-v"); // Check output diff --git a/tests/string_functions_ext/mod.rs b/tests/string_functions_ext/mod.rs new file mode 100644 index 0000000..af04476 --- /dev/null +++ b/tests/string_functions_ext/mod.rs @@ -0,0 +1,67 @@ +//! Extended string function tests for sqawk VM +//! +//! Tests for CONCAT, LEFT, RIGHT functions. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test CONCAT function with multiple arguments +#[test] +fn test_concat() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,first,last\n1,John,Doe\n2,Jane,Smith\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT CONCAT(first, ' ', last) AS full_name FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("John Doe")) + .stdout(predicates::str::contains("Jane Smith")); + + Ok(()) +} + +/// Test LEFT function +#[test] +fn test_left() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alexander\n2,Bob\n3,Charlotte\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT LEFT(name, 3) AS short FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Ale")) + .stdout(predicates::str::contains("Bob")) + .stdout(predicates::str::contains("Cha")); + + Ok(()) +} + +/// Test RIGHT function +#[test] +fn test_right() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alexander\n2,Bob\n3,Charlotte\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT RIGHT(name, 3) AS ending FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("der")) + .stdout(predicates::str::contains("Bob")) + .stdout(predicates::str::contains("tte")); + + Ok(()) +} diff --git a/tests/subquery/mod.rs b/tests/subquery/mod.rs new file mode 100644 index 0000000..665aca9 --- /dev/null +++ b/tests/subquery/mod.rs @@ -0,0 +1,156 @@ +//! Scalar subquery tests for sqawk VM +//! +//! This file contains tests for scalar subqueries with aggregate functions. +//! Phase 4F: Subquery support + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test scalar subquery with MAX aggregate +#[test] +fn test_scalar_subquery_max() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n4,David,40\n5,Eve,20\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test: SELECT * FROM people WHERE age = (SELECT MAX(age) FROM people) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE age = (SELECT MAX(age) FROM people)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("4,David,40")); + + Ok(()) +} + +/// Test scalar subquery with MIN aggregate +#[test] +fn test_scalar_subquery_min() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n4,David,40\n5,Eve,20\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test: SELECT * FROM people WHERE age = (SELECT MIN(age) FROM people) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE age = (SELECT MIN(age) FROM people)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("5,Eve,20")); + + Ok(()) +} + +/// Test scalar subquery with COUNT aggregate +#[test] +fn test_scalar_subquery_count() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,100\n2,200\n3,300\n4,400\n5,500\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test: SELECT * FROM data WHERE id = (SELECT COUNT(*) FROM data) + // COUNT(*) = 5, so should match id=5 + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE id = (SELECT COUNT(*) FROM data)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("5,500")); + + Ok(()) +} + +/// Test scalar subquery with SUM aggregate +#[test] +fn test_scalar_subquery_sum() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,20\n3,30\n4,40\n5,100\n"; + let _file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Test: SELECT * FROM data WHERE value = (SELECT SUM(value) FROM data WHERE id <= 3) + // SUM of 10+20+30 = 60, but 60 is not in the table, so no results + // Let's test with a matching value + let content2 = "id,value\n1,10\n2,20\n3,30\n4,60\n"; + let file_path2 = create_custom_csv(temp_dir.path(), "data2.csv", content2)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data2 WHERE value = (SELECT SUM(value) FROM data2 WHERE id <= 3)") + .arg(file_path2.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("4,60")); + + Ok(()) +} + +/// Test scalar subquery with AVG aggregate +#[test] +fn test_scalar_subquery_avg() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,score\n1,80\n2,90\n3,100\n4,90\n"; + let file_path = create_custom_csv(temp_dir.path(), "scores.csv", content)?; + + // AVG(score) = (80+90+100+90)/4 = 90 + // Should match id=2 and id=4 where score=90 + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM scores WHERE score = (SELECT AVG(score) FROM scores)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("2,90")) + .stdout(predicates::str::contains("4,90")); + + Ok(()) +} + +/// Test scalar subquery returning no rows (NULL case) +#[test] +fn test_scalar_subquery_empty_result() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,value\n1,10\n2,20\n3,30\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + // Subquery with impossible WHERE returns no rows + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM data WHERE value = (SELECT MAX(value) FROM data WHERE value > 1000)") + .arg(file_path.to_str().unwrap()); + + // MAX on empty set is NULL, so no matches expected + cmd.assert().success(); + + Ok(()) +} + +/// Test scalar subquery with greater than comparison +#[test] +fn test_scalar_subquery_greater_than() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n4,David,40\n5,Eve,20\n"; + let file_path = create_custom_csv(temp_dir.path(), "people.csv", content)?; + + // Test: SELECT * FROM people WHERE age > (SELECT AVG(age) FROM people) + // AVG(age) = (25+30+35+40+20)/5 = 30 + // Should return people with age > 30: Charlie(35), David(40) + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM people WHERE age > (SELECT AVG(age) FROM people)") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("3,Charlie,35")) + .stdout(predicates::str::contains("4,David,40")); + + Ok(()) +} diff --git a/tests/tabledef/mod.rs b/tests/tabledef/mod.rs new file mode 100644 index 0000000..f738c97 --- /dev/null +++ b/tests/tabledef/mod.rs @@ -0,0 +1,165 @@ +//! Tests for --tabledef CLI option +//! +//! This file contains tests for defining column names for headerless files +//! using the --tabledef command-line option. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +#[test] +fn test_tabledef_basic() -> Result<(), Box> { + // Test basic --tabledef usage with a headerless file + let temp_dir = create_temp_dir()?; + + // Create a headerless file (no header row) + let data_file = create_custom_csv( + temp_dir.path(), + "data.csv", + "1,Alice,100\n2,Bob,200\n3,Charlie,300\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("--tabledef=data:id,name,value") + .arg("-s") + .arg("SELECT name, value FROM data WHERE value > 150") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("name,value")) + .stdout(predicates::str::contains("Bob")) + .stdout(predicates::str::contains("Charlie")); + + Ok(()) +} + +#[test] +fn test_tabledef_with_custom_delimiter() -> Result<(), Box> { + // Test --tabledef with colon-delimited file (like /etc/passwd) + let temp_dir = create_temp_dir()?; + + // Create a passwd-like file + let data_file = create_custom_csv( + temp_dir.path(), + "passwd", + "root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin\nuser:x:1000:1000:User:/home/user:/bin/bash\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-F:") + .arg("--tabledef=passwd:username,password,uid,gid,gecos,home,shell") + .arg("-s") + .arg("SELECT username, home FROM passwd WHERE uid >= 1000") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("user")) + .stdout(predicates::str::contains("/home/user")); + + Ok(()) +} + +#[test] +fn test_tabledef_multiple_tables() -> Result<(), Box> { + // Test multiple --tabledef options for different tables + let temp_dir = create_temp_dir()?; + + let users_file = create_custom_csv(temp_dir.path(), "users.csv", "1,Alice\n2,Bob\n")?; + + let orders_file = create_custom_csv( + temp_dir.path(), + "orders.csv", + "101,1,500\n102,2,300\n103,1,200\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("--tabledef=users:id,name") + .arg("--tabledef=orders:order_id,user_id,amount") + .arg("-s") + .arg("SELECT users.name, orders.amount FROM users, orders WHERE users.id = orders.user_id") + .arg(users_file.to_str().unwrap()) + .arg(orders_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice")) + .stdout(predicates::str::contains("500")); + + Ok(()) +} + +#[test] +fn test_tabledef_with_table_alias() -> Result<(), Box> { + // Test --tabledef with explicit table name assignment + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv( + temp_dir.path(), + "raw_data.csv", + "1,Apple,1.50\n2,Banana,0.75\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("--tabledef=products:id,name,price") + .arg("-s") + .arg("SELECT name, price FROM products") + .arg(format!("products={}", data_file.to_str().unwrap())); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Apple")) + .stdout(predicates::str::contains("1.5")); + + Ok(()) +} + +#[test] +fn test_tabledef_aggregate_query() -> Result<(), Box> { + // Test --tabledef with aggregate functions + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv( + temp_dir.path(), + "sales.csv", + "2024-01,North,1000\n2024-01,South,1500\n2024-02,North,1200\n2024-02,South,1800\n", + )?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("--tabledef=sales:month,region,amount") + .arg("-s") + .arg("SELECT region, SUM(amount) as total FROM sales GROUP BY region") + .arg(data_file.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("North")) + .stdout(predicates::str::contains("South")); + + Ok(()) +} + +#[test] +fn test_tabledef_with_write() -> Result<(), Box> { + // Test --tabledef with --write flag + let temp_dir = create_temp_dir()?; + + let data_file = create_custom_csv(temp_dir.path(), "data.csv", "1,Alice,100\n2,Bob,200\n")?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("--tabledef=data:id,name,value") + .arg("-s") + .arg("UPDATE data SET value = 999 WHERE name = 'Alice'") + .arg("--write") + .arg(data_file.to_str().unwrap()); + + cmd.assert().success(); + + // Verify the file was updated + let contents = std::fs::read_to_string(&data_file)?; + assert!( + contents.contains("999"), + "File should contain updated value" + ); + + Ok(()) +} diff --git a/tests/union/mod.rs b/tests/union/mod.rs new file mode 100644 index 0000000..4fc9184 --- /dev/null +++ b/tests/union/mod.rs @@ -0,0 +1,161 @@ +//! UNION and UNION ALL tests for sqawk VM +//! +//! This file contains tests for SQL UNION and UNION ALL set operations. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test UNION ALL combines all rows from both tables +#[test] +fn test_union_all_combines_all_rows() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n2,Bob\n3,Charlie\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION ALL SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + // UNION ALL should include Bob twice (once from each table) + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")); + + // Count occurrences of Bob to verify duplicates are kept + let output = cmd.output()?.stdout; + let output_str = String::from_utf8_lossy(&output); + let bob_count = output_str.matches("2,Bob").count(); + assert_eq!(bob_count, 2, "UNION ALL should keep duplicate rows"); + + Ok(()) +} + +/// Test UNION removes duplicate rows +#[test] +fn test_union_removes_duplicates() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n2,Bob\n3,Charlie\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // UNION should include Bob only once (deduplication) + let bob_count = output_str.matches("2,Bob").count(); + assert_eq!(bob_count, 1, "UNION should deduplicate rows"); + + Ok(()) +} + +/// Test UNION with disjoint tables (no common rows) +#[test] +fn test_union_disjoint_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n3,Charlie\n4,David\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")) + .stdout(predicates::str::contains("3,Charlie")) + .stdout(predicates::str::contains("4,David")); + + Ok(()) +} + +/// Test UNION with identical tables +#[test] +fn test_union_identical_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // With identical tables, UNION should only return 2 unique rows + let alice_count = output_str.matches("1,Alice").count(); + let bob_count = output_str.matches("2,Bob").count(); + assert_eq!(alice_count, 1, "Alice should appear once after UNION"); + assert_eq!(bob_count, 1, "Bob should appear once after UNION"); + + Ok(()) +} + +/// Test UNION ALL with identical tables +#[test] +fn test_union_all_identical_tables() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION ALL SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // UNION ALL should keep all 4 rows (2 from each table) + let alice_count = output_str.matches("1,Alice").count(); + let bob_count = output_str.matches("2,Bob").count(); + assert_eq!(alice_count, 2, "Alice should appear twice with UNION ALL"); + assert_eq!(bob_count, 2, "Bob should appear twice with UNION ALL"); + + Ok(()) +} + +/// Test UNION with empty second table +#[test] +fn test_union_with_empty_table() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content_a = "id,name\n1,Alice\n2,Bob\n"; + let content_b = "id,name\n"; + let file_a = create_custom_csv(temp_dir.path(), "a.csv", content_a)?; + let file_b = create_custom_csv(temp_dir.path(), "b.csv", content_b)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT * FROM a UNION SELECT * FROM b") + .arg(file_a.to_str().unwrap()) + .arg(file_b.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("1,Alice")) + .stdout(predicates::str::contains("2,Bob")); + + Ok(()) +} diff --git a/tests/update/mod.rs b/tests/update/mod.rs index 652c4c2..80a9403 100644 --- a/tests/update/mod.rs +++ b/tests/update/mod.rs @@ -2,7 +2,7 @@ //! //! This file contains tests for the SQL UPDATE statement. -use crate::helpers::{create_temp_dir, prepare_test_file}; +use crate::helpers::{create_custom_csv, create_temp_dir}; use std::fs; @@ -10,10 +10,14 @@ use std::fs; fn test_update_with_where() -> Result<(), Box> { // This test verifies the UPDATE functionality with a WHERE clause let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // First verify we have initial data - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM people") .arg(file_path.to_str().unwrap()); @@ -25,7 +29,7 @@ fn test_update_with_where() -> Result<(), Box> { .stdout(predicates::str::contains("3,Charlie,35")); // Now execute UPDATE with WHERE - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("UPDATE people SET age = 31 WHERE name = 'Alice'") .arg("-s") @@ -55,10 +59,14 @@ fn test_update_with_where() -> Result<(), Box> { fn test_update_all_rows() -> Result<(), Box> { // This test verifies the UPDATE functionality without a WHERE clause (updates all rows) let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // First verify we have initial data - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("SELECT * FROM people") .arg(file_path.to_str().unwrap()); @@ -69,7 +77,7 @@ fn test_update_all_rows() -> Result<(), Box> { .stdout(predicates::str::contains("1,Alice,32")); // Now execute UPDATE without WHERE - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("UPDATE people SET age = 40") // Update all ages to 40 .arg("-s") @@ -99,10 +107,14 @@ fn test_update_all_rows() -> Result<(), Box> { fn test_update_and_write() -> Result<(), Box> { // This test verifies that UPDATE with --write flag persists changes to the file let temp_dir = create_temp_dir()?; - let file_path = prepare_test_file(temp_dir.path())?; + let file_path = create_custom_csv( + temp_dir.path(), + "people.csv", + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n", + )?; // Execute UPDATE with --write flag - let mut cmd = assert_cmd::Command::cargo_bin("sqawk")?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-s") .arg("UPDATE people SET name = 'Alicia' WHERE name = 'Alice'") .arg("-s") diff --git a/tests/window/mod.rs b/tests/window/mod.rs new file mode 100644 index 0000000..fd22f71 --- /dev/null +++ b/tests/window/mod.rs @@ -0,0 +1,365 @@ +//! Window function tests for sqawk VM +//! +//! Tests for ROW_NUMBER, RANK, DENSE_RANK, LAG, LEAD, SUM/AVG/COUNT OVER with OVER clause. + +use crate::helpers::{create_custom_csv, create_temp_dir}; + +/// Test ROW_NUMBER() without partition +#[test] +fn test_row_number_simple() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name\n1,Alice\n2,Bob\n3,Charlie\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, ROW_NUMBER() OVER () AS rn FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,1")) + .stdout(predicates::str::contains("Bob,2")) + .stdout(predicates::str::contains("Charlie,3")); + + Ok(()) +} + +/// Test ROW_NUMBER() with PARTITION BY +#[test] +fn test_row_number_partition() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,dept\n1,Alice,Sales\n2,Bob,Sales\n3,Charlie,IT\n4,Diana,IT\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, dept, ROW_NUMBER() OVER (PARTITION BY dept) AS rn FROM data") + .arg(file_path.to_str().unwrap()); + + // IT should have 1,2 and Sales should have 1,2 + cmd.assert() + .success() + .stdout(predicates::str::contains(",IT,1")) + .stdout(predicates::str::contains(",IT,2")) + .stdout(predicates::str::contains(",Sales,1")) + .stdout(predicates::str::contains(",Sales,2")); + + Ok(()) +} + +/// Test ROW_NUMBER() with ORDER BY +#[test] +fn test_row_number_order() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,score\n1,Alice,80\n2,Bob,90\n3,Charlie,70\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, score, ROW_NUMBER() OVER (ORDER BY score DESC) AS rn FROM data") + .arg(file_path.to_str().unwrap()); + + // Bob (90) should be first, Alice (80) second, Charlie (70) third + cmd.assert() + .success() + .stdout(predicates::str::contains("Bob,90,1")) + .stdout(predicates::str::contains("Alice,80,2")) + .stdout(predicates::str::contains("Charlie,70,3")); + + Ok(()) +} + +/// Test RANK() with ties +#[test] +fn test_rank_ties() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,score\n1,Alice,90\n2,Bob,85\n3,Charlie,90\n4,Diana,80\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, score, RANK() OVER (ORDER BY score DESC) AS rnk FROM data") + .arg(file_path.to_str().unwrap()); + + // Alice and Charlie both 90 get rank 1, Bob gets 3 (not 2), Diana gets 4 + cmd.assert() + .success() + .stdout(predicates::str::contains(",90,1")) + .stdout(predicates::str::contains("Bob,85,3")) + .stdout(predicates::str::contains("Diana,80,4")); + + Ok(()) +} + +/// Test DENSE_RANK() +#[test] +fn test_dense_rank() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,score\n1,Alice,90\n2,Bob,85\n3,Charlie,90\n4,Diana,80\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, score, DENSE_RANK() OVER (ORDER BY score DESC) AS drnk FROM data") + .arg(file_path.to_str().unwrap()); + + // Alice and Charlie get 1, Bob gets 2 (not 3), Diana gets 3 + cmd.assert() + .success() + .stdout(predicates::str::contains(",90,1")) + .stdout(predicates::str::contains("Bob,85,2")) + .stdout(predicates::str::contains("Diana,80,3")); + + Ok(()) +} + +/// Test LAG() without partition +#[test] +fn test_lag_simple() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,60000\n3,Charlie,70000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, LAG(salary) OVER (ORDER BY salary) AS prev_salary FROM data") + .arg(file_path.to_str().unwrap()); + + // First row has NULL, others have previous salary + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,NULL")) + .stdout(predicates::str::contains("Bob,60000,50000")) + .stdout(predicates::str::contains("Charlie,70000,60000")); + + Ok(()) +} + +/// Test LEAD() without partition +#[test] +fn test_lead_simple() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,60000\n3,Charlie,70000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, LEAD(salary) OVER (ORDER BY salary) AS next_salary FROM data") + .arg(file_path.to_str().unwrap()); + + // Last row has NULL, others have next salary + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,60000")) + .stdout(predicates::str::contains("Bob,60000,70000")) + .stdout(predicates::str::contains("Charlie,70000,NULL")); + + Ok(()) +} + +/// Test LAG() with PARTITION BY +#[test] +fn test_lag_partition() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,dept,salary\n1,Alice,Sales,50000\n2,Bob,Sales,60000\n3,Charlie,IT,70000\n4,Diana,IT,80000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, dept, salary, LAG(salary) OVER (PARTITION BY dept ORDER BY salary) AS prev_salary FROM data") + .arg(file_path.to_str().unwrap()); + + // First in each partition gets NULL + cmd.assert() + .success() + .stdout(predicates::str::contains("Charlie,IT,70000,NULL")) + .stdout(predicates::str::contains("Diana,IT,80000,70000")) + .stdout(predicates::str::contains("Alice,Sales,50000,NULL")) + .stdout(predicates::str::contains("Bob,Sales,60000,50000")); + + Ok(()) +} + +/// Test LEAD() with PARTITION BY +#[test] +fn test_lead_partition() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,dept,salary\n1,Alice,Sales,50000\n2,Bob,Sales,60000\n3,Charlie,IT,70000\n4,Diana,IT,80000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, dept, salary, LEAD(salary) OVER (PARTITION BY dept ORDER BY salary) AS next_salary FROM data") + .arg(file_path.to_str().unwrap()); + + // Last in each partition gets NULL + cmd.assert() + .success() + .stdout(predicates::str::contains("Charlie,IT,70000,80000")) + .stdout(predicates::str::contains("Diana,IT,80000,NULL")) + .stdout(predicates::str::contains("Alice,Sales,50000,60000")) + .stdout(predicates::str::contains("Bob,Sales,60000,NULL")); + + Ok(()) +} + +/// Test LAG() with offset > 1 +#[test] +fn test_lag_offset() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,55000\n3,Charlie,60000\n4,Diana,65000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg( + "SELECT name, salary, LAG(salary, 2) OVER (ORDER BY salary) AS prev_2_salary FROM data", + ) + .arg(file_path.to_str().unwrap()); + + // First two rows have NULL, others have salary from 2 rows back + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,NULL")) + .stdout(predicates::str::contains("Bob,55000,NULL")) + .stdout(predicates::str::contains("Charlie,60000,50000")) + .stdout(predicates::str::contains("Diana,65000,55000")); + + Ok(()) +} + +/// Test LEAD() with offset > 1 +#[test] +fn test_lead_offset() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,55000\n3,Charlie,60000\n4,Diana,65000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, LEAD(salary, 2) OVER (ORDER BY salary) AS next_2_salary FROM data") + .arg(file_path.to_str().unwrap()); + + // Last two rows have NULL, others have salary from 2 rows ahead + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,60000")) + .stdout(predicates::str::contains("Bob,55000,65000")) + .stdout(predicates::str::contains("Charlie,60000,NULL")) + .stdout(predicates::str::contains("Diana,65000,NULL")); + + Ok(()) +} + +/// Test SUM() OVER with ORDER BY (running sum) +#[test] +fn test_sum_over_order() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,55000\n3,Charlie,60000\n4,Diana,65000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, SUM(salary) OVER (ORDER BY salary) AS running_sum FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,50000")) + .stdout(predicates::str::contains("Bob,55000,105000")) + .stdout(predicates::str::contains("Charlie,60000,165000")) + .stdout(predicates::str::contains("Diana,65000,230000")); + + Ok(()) +} + +/// Test AVG() OVER with ORDER BY (running average) +#[test] +fn test_avg_over_order() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,score\n1,Alice,100\n2,Bob,80\n3,Charlie,60\n4,Diana,40\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, score, AVG(score) OVER (ORDER BY score DESC) AS running_avg FROM data") + .arg(file_path.to_str().unwrap()); + + // Descending order: Alice(100), Bob(80), Charlie(60), Diana(40) + // Running avg: 100, 90, 80, 70 + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,100,100")) + .stdout(predicates::str::contains("Bob,80,90")) + .stdout(predicates::str::contains("Charlie,60,80")) + .stdout(predicates::str::contains("Diana,40,70")); + + Ok(()) +} + +/// Test COUNT() OVER with ORDER BY (running count) +#[test] +fn test_count_over_order() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,55000\n3,Charlie,60000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, COUNT(salary) OVER (ORDER BY salary) AS running_count FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,1")) + .stdout(predicates::str::contains("Bob,55000,2")) + .stdout(predicates::str::contains("Charlie,60000,3")); + + Ok(()) +} + +/// Test MIN() and MAX() OVER with ORDER BY +#[test] +fn test_min_max_over_order() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,salary\n1,Alice,50000\n2,Bob,60000\n3,Charlie,55000\n4,Diana,70000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, salary, MIN(salary) OVER (ORDER BY salary) AS running_min, MAX(salary) OVER (ORDER BY salary) AS running_max FROM data") + .arg(file_path.to_str().unwrap()); + + // Sorted: Alice(50000), Charlie(55000), Bob(60000), Diana(70000) + cmd.assert() + .success() + .stdout(predicates::str::contains("Alice,50000,50000,50000")) + .stdout(predicates::str::contains("Charlie,55000,50000,55000")) + .stdout(predicates::str::contains("Bob,60000,50000,60000")) + .stdout(predicates::str::contains("Diana,70000,50000,70000")); + + Ok(()) +} + +/// Test SUM() OVER with PARTITION BY +#[test] +fn test_sum_over_partition() -> Result<(), Box> { + let temp_dir = create_temp_dir()?; + let content = "id,name,dept,salary\n1,Alice,Sales,50000\n2,Bob,Sales,60000\n3,Charlie,IT,70000\n4,Diana,IT,80000\n"; + let file_path = create_custom_csv(temp_dir.path(), "data.csv", content)?; + + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s") + .arg("SELECT name, dept, salary, SUM(salary) OVER (PARTITION BY dept ORDER BY salary) AS dept_sum FROM data") + .arg(file_path.to_str().unwrap()); + + cmd.assert() + .success() + .stdout(predicates::str::contains("Charlie,IT,70000,70000")) + .stdout(predicates::str::contains("Diana,IT,80000,150000")) + .stdout(predicates::str::contains("Alice,Sales,50000,50000")) + .stdout(predicates::str::contains("Bob,Sales,60000,110000")); + + Ok(()) +}