diff --git a/compliance/ADAPTER_INTERFACE.md b/compliance/ADAPTER_INTERFACE.md new file mode 100644 index 00000000..d2f0a539 --- /dev/null +++ b/compliance/ADAPTER_INTERFACE.md @@ -0,0 +1,85 @@ + + +# Adapter Interface Contract + +**Version**: 1.0 + +--- + +## Purpose + +This document defines the boundary between the **test harness** and an **OSI implementation adapter**. It is the authoritative reference for what adapters may and must not do. + +## CLI Contract + +```bash + sql --model --query-file --dialect +``` + +| Stream | Content | +|--------|---------| +| **stdout** | Generated SQL string only — no headers, no decoration | +| **stderr** | Error messages on failure | +| **exit code** | 0 on success, non-zero on error | + +## Adapter Boundary Rules + +An adapter is a **thin translator** between the harness CLI contract and the implementation's programmatic API. It performs **format conversion only**. + +### An adapter MUST: + +1. Parse CLI arguments (`--model`, `--query-file`, `--dialect`) +2. Translate the query JSON into the implementation's query object (e.g., `LODQuery`) +3. Invoke the implementation's planner and transpiler +4. Print the resulting SQL to stdout +5. Print errors to stderr and exit non-zero on failure + +### An adapter MUST NOT: + +1. **Validate query semantics** — duplicate dimensions, ambiguous fields, ORDER BY validity, EXCLUDE dimension membership. These are planner responsibilities. If the implementation doesn't validate them, **file a bug on the implementation**. + +2. **Rewrite SQL expressions** — dialect function translation (IFF→CASE, ZEROIFNULL→COALESCE), expression normalization. These are transpiler responsibilities. If the transpiler mishandles a function, **fix the transpiler**. + +3. **Substitute parameters** — resolving `:param`/`$param` placeholders, type-aware quoting. These are parsing-layer responsibilities. If the parser doesn't handle parameters, **extend the parser**. + +4. **Handle special query shapes** — dimensions-only queries, empty-measure queries. These are planner responsibilities. If the planner can't handle a valid query shape, **fix the planner**. + +5. **Work around implementation bugs** — injecting synthetic measures, monkey-patching frozen models, generating SQL directly. If the implementation produces wrong SQL, **fix the implementation**. + +6. **Contain business logic** — aggregation classification, grain resolution, safety checks, filter routing. All BI semantics belong in the implementation. + +### An adapter MAY: + +1. Set up import paths to locate the implementation (`sys.path` manipulation) +2. Register dialect plugins (e.g., importing `osi.frontend.dialect` to register with sqlglot) +3. Translate between the JSON query format and the implementation's API types (this is format conversion, not business logic) + +## Size Guidance + +A compliant adapter should be about a page of code (or less). If an adapter exceeds this, it is almost certainly doing work that belongs in the implementation. The reference Python adapter is ~160 lines. + +## For AI Agents + +When working on adapters for the OSI compliance suite, you MUST follow these rules: + +1. **Never add business logic to an adapter.** If a test fails because the implementation doesn't handle something, fix the implementation — not the adapter. +2. **Never add more than 10 lines to an adapter without checking this document.** If you're adding validation, rewriting, parameter handling, or SQL generation, you're in the wrong file. +3. **The adapter line count is a hard signal.** If it exceeds 200 lines, something is wrong. +4. **When in doubt, the implementation is the right place.** The adapter exists only because different implementations have different API shapes. Everything else is implementation work. diff --git a/compliance/README.md b/compliance/README.md index 7a59b046..23a3c6eb 100644 --- a/compliance/README.md +++ b/compliance/README.md @@ -1 +1,33 @@ -This is for compliance tests + + +# Ossie Compliance + +This directory holds the compliance test suites that validate an Ossie +implementation against a proposal's semantics, plus the shared runner +that executes them. + +- [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md) — the CLI contract an + implementation must satisfy to plug into a compliance suite. +- [`harness/`](harness/) — the engine-agnostic runner / reporter / DB + manager shared by every per-version suite below. +- [`foundation/`](foundation/) — the compliance suite for the + Foundation proposal (`osi_version: "0.1"`). See its + [README](foundation/README.md) for current status — this is + currently a bootstrap slice, not the full suite. diff --git a/compliance/foundation/.gitignore b/compliance/foundation/.gitignore new file mode 100644 index 00000000..f1c84633 --- /dev/null +++ b/compliance/foundation/.gitignore @@ -0,0 +1,9 @@ +# Everything in results/ is per-run runner output, written by +# `python -m harness.runner` into a subdirectory of results/ (the +# default is `results/latest/`; the Makefile under +# impl/python/ uses `results/osi_python/` and `results/osi_python_all/`). +# +# Only the curated baseline at results/REPORT.md is tracked. Repo-wide +# Python / cache / venv rules live in the root .gitignore. +results/* +!results/REPORT.md diff --git a/compliance/foundation/DATA_TESTS.md b/compliance/foundation/DATA_TESTS.md new file mode 100644 index 00000000..d4ce4b9e --- /dev/null +++ b/compliance/foundation/DATA_TESTS.md @@ -0,0 +1,2258 @@ + + +# DATA_TESTS.md — Concrete Test Vectors for the Foundation Compliance Suite + +This document is the **concrete realization** of the Foundation Conformance +Decisions catalogued in `Proposed_OSI_Semantics.md` Appendix B. Each +**T-NNN** entry below is a runnable test vector: a model, a dataset, a query, +and the row set or error code an implementation MUST produce. Appendix B's +**D-NNN** entries say *what* an implementation must do; this file says *what +inputs prove it*. + +Implementations are encouraged to translate each entry into a fixture under +the published Foundation compliance suite — for `osi_python` this is +[`compliance/foundation-v0.1/tests/`](/tests/) +(one folder per `T-NNN` containing `metadata.yaml + model.yaml + +query.json + gold_rows.json`). Implementations on other engines run the +same suite through the standard adapter contract documented in +[`compliance/foundation-v0.1/ADAPTER_INTERFACE.md`](../../compliance/ADAPTER_INTERFACE.md). +The expected outputs are normative and implementation-independent. + +> **Bootstrap-slice note.** The compliance suite currently ported into +> this repo only ships fixtures for the T-005 family +> (`compliance/foundation-v0.1/tests/cross_grain/moderate/`). Every +> other `T-NNN` entry below is normative documentation for tests that +> will land — with their own fixtures — in follow-up PRs. See +> `compliance/foundation-v0.1/README.md` for current status. + +--- + +## 1. Conventions + +### 1.1 Test entry shape + +Every test has the same five fields: + +| Field | Meaning | +|:---|:---| +| **Anchors** | The spec section(s) and `D-NNN` decision(s) the test pins. | +| **Fixture** | The named model fixture (§3) the test runs against. | +| **Data** | The concrete rows. For brevity, fixture-shared data lives in §3 and is referenced by name. Test-local data is shown inline. | +| **Query** | The semantic query — written in the Foundation surface used by tests (a thin YAML around §5's `Aggregation Query` / `Scalar Query` shapes). | +| **Expected** | Either a row set (presented as a markdown table, order-insensitive unless noted) or `error: ` with the diagnostic content the engine MUST surface. | + +A test passes iff the engine's observable output (rows or typed error) +matches the **Expected** field. Tests do **not** assert on SQL string, +plan shape, CTE layout, or any other internal artefact (§11.1 of the +semantics spec). + +### 1.2 Row-set comparison + +- **Order-insensitive** by default — sort both sides by all output columns before comparing. +- **NULL-aware** — `NULL == NULL` for the purposes of the test (the SQL three-valued logic does not apply to assertions). +- **Type-aware** — numeric values are compared by value, not by representation; `100`, `100.0`, and `1e2` are equal. String values must match exactly. + +### 1.3 Error comparison + +An error result is a triple `(code, anchor, must_contain)`: + +```yaml +error: + code: E_AGGREGATE_IN_WHERE + anchor: §5.3 # the spec section the engine MUST cite in its diagnostic + must_contain: # substrings the diagnostic MUST include (case-insensitive) + - "aggregate" + - "Where" +``` + +An engine MAY include additional diagnostic content. The code and the +`must_contain` substrings are the only assertions; wording is otherwise free. + +### 1.4 Determinism guarantee + +Every test that produces a row set MUST be deterministic: running it +twice on the same fixture MUST produce byte-identical SQL and the same +multiset of rows. This is also `D-014`'s test shape, applied uniformly. + +--- + +### 1.5 NULL handling, empty groups, and `0` (D-033) + +The Foundation follows **standard SQL** for aggregates over empty +input row sets (§6.11 / D-033): + +| Aggregate family | Empty-input result | +|:---|:---| +| `COUNT(*)`, `COUNT(x)`, `COUNT(DISTINCT x)` | `0` | +| `SUM`, `AVG`, `MIN`, `MAX`, `MEDIAN`, percentiles, `STDDEV`, `VARIANCE` | `NULL` | + +Tests in this catalog use these results literally — but **only for +dim values that actually appear in the result**. Per §1.6 (Plan A for +single-measure shapes), a dim value with no matching fact row is +dropped from the result entirely; there is no empty-aggregate cell +because there is no row. + +The empty-aggregate rule applies to: + +- **Multi-measure stitch cells** where one branch has no contribution + for a dim value present via the other branch (T-011, T-045) — + `SUM` → `NULL`, `COUNT` → `0`. +- **Dim values present via the `NULL`-key orphan bucket** when the + fact row has no matching dim row (T-047, T-053 etc.) — same rule + applies; the orphan-bucket row is part of the result, and its + cells follow standard SQL. + +Models that prefer `0` for a `SUM` cell MUST declare it explicitly: +`COALESCE(SUM(amount), 0) AS revenue`. The Foundation does NOT +auto-rewrite `SUM` to `COALESCE(SUM, 0)`. + +--- + +### 1.6 Resolved — single-measure shapes follow Plan A (preserve facts) + +Earlier drafts of this catalog had an unresolved tension between two +flagship tests: + +- **T-001** (`Dims: [customers.region]; Measures: [SUM(orders.amount)]`) + expected a `NORTH` row with `revenue = NULL` — implying the planner + preserves the dimension domain even when no fact rows match. +- **T-006** (`Dims: [customers.region]; Measures: [COUNT(*)]; Where + orders.status = 'completed'`) expected NORTH to *not* appear — + implying the planner anchors on the fact and drops dim values with + no matching fact. + +Both could not be correct under one planner. The Architect resolved +this in favour of **Plan A — preserve facts, drop unmatched dim values +— for single-measure aggregation queries**, with the **multi-measure +shape using FULL OUTER stitch (§6.6 row 3) to preserve both sides** as +a deliberate contrast. The spec was updated to match (§6.6, §6.2 step 7, +D-001 example), and existing tests were brought into alignment. + +#### Decision summary + +| Query shape | Default join | Which dim values appear in the result | +|:---|:---|:---| +| Single-measure aggregation: `Dims: [dim.X]; Measures: [SUM(fact.Y)]` | `fact LEFT JOIN dim` (§6.6 row 1) | Distinct `dim.X` reached by a surviving `fact` row, plus a `NULL`-key bucket for orphan facts. **A dim value with no matching fact does NOT appear.** | +| Multi-measure aggregation: `Dims: [dim.X]; Measures: [SUM(factA.Y), SUM(factB.Y)]` | Each measure independently resolved per row 1, then `FULL OUTER` stitch on shared dims (§6.6 row 3) | Union of dim values reachable from *either* branch. A dim value appears if either fact pulls it in. | + +This rule is now consistent across every test in the catalog: + +- **T-001, T-005a-e, T-026, T-028, T-047, T-053**: single-measure or + single-fact-source queries — NORTH does not appear. +- **T-006**: single-measure query with a fact-level WHERE — NORTH + does not appear. +- **T-011, T-045**: multi-measure stitch — NORTH appears via the + branch that has data for customer 4. + +#### How a user gets the "preserve all dim values" behaviour + +If a user wants every dim value to appear (BI-tool convention), they +make the query multi-measure by adding a measure sourced from the dim +dataset itself: + +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(customers.id) AS customer_count +``` + +This converts the shape into a multi-measure stitch — the `customers` +branch contributes every distinct region (because `COUNT(customers.id)` +is defined for every region in `customers`), and the FULL OUTER +preserves NORTH with `revenue = NULL`, `customer_count = 1`. + +This is the BI-tool "all regions appear" behaviour, surfaced +*deliberately* through the query shape rather than as a hidden +planner default. The user opts in by asking for it. + +#### Why Plan A (not preserve-dims by default) + +- **Matches raw SQL.** `SELECT dim.X, SUM(fact.Y) FROM fact LEFT JOIN + dim ... GROUP BY dim.X` produces exactly the single-measure Plan A + result. The Foundation does not introduce surprises relative to + the SQL the user could write by hand. +- **Matches Snowflake Semantic Views.** Snowflake's default plan + shape is also Plan A — porting Snowflake models to OSI is a no-op + for this dimension behaviour. +- **Composes cleanly with multi-measure stitch.** §6.6 row 3 already + mandates FULL OUTER for multi-measure shapes; row 1 says LEFT for + single-measure shapes. The contrast is the mental model: "the + multi-measure stitch is the *only* place both sides are preserved + — and that's because there is no other correct shape for merging + two independently-aggregated facts." +- **Surfaces orphan facts.** Plan A keeps the `NULL`-key bucket for + orphan fact rows (e.g. order 105 with `customer_id = 99`). This is + a data-quality signal — preserve-dims would lose it. + +#### Trade-off accepted + +A first-time BI user typing a single-measure query and *expecting* to +see every region might be surprised when a region with no fact data +is absent. The mitigation is the documented multi-measure pattern +above. The Foundation prefers "predictable, matches raw SQL" over +"BI-tool-magical". + +> **Note for test authors.** When writing a new test that groups by a +> dimension over a single-fact source, write the expected rows +> following Plan A: include only dim values reached by a fact row, +> plus the `NULL`-key bucket for orphans. If you want every dim value +> in the expected rows, the test must be multi-measure (with one of +> the measures sourced from the dim dataset). + +--- + +## 2. How to add a new test + +1. Pick the lowest-numbered free `T-NNN`. +2. Link it to a `D-NNN` in `Proposed_OSI_Semantics.md` Appendix B — if no `D-NNN` covers the behaviour, add one there first. +3. Write the test against an existing fixture (§3) if possible; create a new fixture only when the existing ones can't express the shape. +4. Cite the test from the relevant spec § so the doc and the test grow together. + +--- + +## 3. Common fixtures + +### 3.1 Fixture **F-PRELUDE** — single-fact star with multi-fact extension + +Mirrors the *Prelude model* in `Proposed_OSI_Semantics.md` Appendix A. + +```yaml +datasets: + - name: customers + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: region, type: string } + - { name: segment, type: string } + + - name: orders + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: customer_id, type: integer } + - { name: amount, type: decimal(10,2) } + - { name: status, type: string } + + - name: returns + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: customer_id, type: integer } + - { name: amount, type: decimal(10,2) } + + - name: premium_customers + primary_key: [id] + fields: + - { name: id, type: integer } + +relationships: + - { name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id] } # N:1 + - { name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id] } # N:1 +``` + +**Data:** + +`customers`: + +| id | region | segment | +|:--:|:-------|:--------| +| 1 | EAST | retail | +| 2 | EAST | retail | +| 3 | WEST | wholesale | +| 4 | NORTH | retail | + +`orders` (note: orphan order 105 with `customer_id = 99` not in `customers`): + +| id | customer_id | amount | status | +|:---:|:-----------:|-------:|:-----------| +| 101 | 1 | 100.00 | completed | +| 102 | 1 | 50.00 | completed | +| 103 | 2 | 200.00 | pending | +| 104 | 3 | 75.00 | completed | +| 105 | 99 | 30.00 | completed | ← orphan: customer 99 not in customers + +`returns`: + +| id | customer_id | amount | +|:---:|:-----------:|-------:| +| 201 | 1 | 10.00 | +| 202 | 3 | 5.00 | +| 203 | 4 | 15.00 | ← customer 4 has a return but no order + +`premium_customers`: + +| id | +|:--:| +| 1 | +| 3 | + +### 3.2 Fixture **F-BRIDGE** — M:N through a bridge + +Mirrors *Mini-model M2* in Appendix A. This is the fixture for the +flagship `T-015` bridge-deduplication test. + +```yaml +datasets: + - name: actors + primary_key: [actor_id] + fields: + - { name: actor_id, type: integer } + - { name: name, type: string } + - { name: height, type: integer } + + - name: movies + primary_key: [movie_id] + fields: + - { name: movie_id, type: integer } + - { name: title, type: string } + - { name: gross, type: decimal(10,2) } + + - name: appearances + primary_key: [actor_id, movie_id] + role: bridge + fields: + - { name: actor_id, type: integer } + - { name: movie_id, type: integer } + +relationships: + - { name: app_to_actor, from: appearances, to: actors, from_columns: [actor_id], to_columns: [actor_id] } # N:1 + - { name: app_to_movie, from: appearances, to: movies, from_columns: [movie_id], to_columns: [movie_id] } # N:1 +``` + +**Data:** + +`actors`: + +| actor_id | name | height | +|:--------:|:------|-------:| +| 1 | Alice | 170 | +| 2 | Bob | 170 | +| 3 | Carol | 180 | + +`movies`: + +| movie_id | title | gross | +|:--------:|:-------|-------:| +| 10 | Action | 100.00 | +| 11 | Drama | 200.00 | +| 12 | Comedy | 50.00 | + +`appearances`: + +| actor_id | movie_id | +|:--------:|:--------:| +| 1 | 10 | +| 1 | 11 | +| 2 | 10 | ← M10 (Action) has two actors at height 170 +| 3 | 12 | + +### 3.3 Fixture **F-BRIDGE-NONE** — variant of F-BRIDGE without the bridge + +Same `actors` and `movies` data as F-BRIDGE, but no `appearances` dataset +and an undeclared M:N edge between `actors` and `movies`. Used to test +that the engine fails closed instead of guessing a join path. + +```yaml +datasets: + - name: actors + primary_key: [actor_id] + fields: [ { name: actor_id }, { name: name }, { name: height } ] + - name: movies + primary_key: [movie_id] + fields: [ { name: movie_id }, { name: title }, { name: gross } ] + +# Note: no `relationships:` block. The model declares no edge; the +# engine MUST NOT silently fabricate one. +``` + +### 3.4 Fixture **F-AMBIG** — two relationships between the same datasets + +Mirrors *Mini-model M3* in Appendix A. Used for `E_AMBIGUOUS_PATH`. + +```yaml +datasets: + - name: orders + primary_key: [id] + fields: + - { name: id } + - { name: placed_by_id } + - { name: fulfilled_by_id } + - { name: amount } + + - name: users + primary_key: [id] + fields: [ { name: id }, { name: region } ] + +relationships: + - { name: order_placed_by, from: orders, to: users, from_columns: [placed_by_id], to_columns: [id] } + - { name: order_fulfilled_by, from: orders, to: users, from_columns: [fulfilled_by_id], to_columns: [id] } +``` + +**Data:** + +`orders`: + +| id | placed_by_id | fulfilled_by_id | amount | +|:---:|:------------:|:---------------:|-------:| +| 301 | 1 | 2 | 100.00 | +| 302 | 2 | 2 | 50.00 | + +`users`: + +| id | region | +|:--:|:-------| +| 1 | EAST | +| 2 | WEST | + +### 3.5 Fixture **F-NOPATH** — two disconnected datasets + +Mirrors *Mini-model M1* in Appendix A. No relationships are declared. + +```yaml +datasets: + - name: orders + primary_key: [id] + fields: [ { name: id }, { name: customer_id }, { name: amount } ] + + - name: inventory_movements + primary_key: [movement_id] + fields: [ { name: movement_id }, { name: warehouse_id }, { name: quantity } ] +``` + +--- + +## 4. Tests + +### 4.A Query shape + +#### T-001 — Aggregation-query cardinality is `DISTINCT(Dimensions)` + +**Anchors:** §5.1.1, §5.2 · D-001 +**Fixture:** F-PRELUDE + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue +``` + +**Expected (single-measure ⇒ Plan A per §6.6 row 1 and §6.2 step 7):** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NULL | 30.00 | ← orphan order 105 (customer_id = 99 not in customers) + +Row count = `COUNT(DISTINCT region)` over the regions reached by an +`orders` row, plus one for the orphan-customer bucket. NORTH (customer +4's region) does **not** appear: customer 4 has no orders, and a +single-measure aggregation query follows the fact's natural LEFT join +shape (`orders LEFT JOIN customers`) — a dim value reachable only +from the dim domain is *not* in the result. + +Engines that "carry" `customer_id` into the output grain because the +measure is sourced from `orders` fail this test on row count (they +would group by `customer_id`, not `region`, and produce four data +rows + orphan). Engines that produce a `NORTH | NULL` row are +applying the multi-measure stitch shape to a single-measure query — +also wrong (see §1.6). + +If the user wants NORTH to appear with `revenue = NULL`, they must +make the query multi-measure (e.g. `Measures: [SUM(orders.amount) AS +revenue, COUNT(customers.id) AS customer_count]`) — the FULL OUTER +stitch in §6.6 row 3 then preserves every region present in +`customers`. See T-011 for the canonical multi-measure shape. + +#### T-002 — Mixing `Dimensions` and `Fields` is rejected + +**Anchors:** §5.1 · D-010 +**Fixture:** F-PRELUDE + +**Query:** +```yaml +Dimensions: [customers.region] +Fields: [customers.id] +Measures: [SUM(orders.amount)] +``` + +**Expected:** +```yaml +error: + code: E_MIXED_QUERY_SHAPE + anchor: §5.1 + must_contain: ["Dimensions", "Fields", "scalar"] +``` + +#### T-003 — Bare metric reference inside `Fields` is rejected + +**Anchors:** §5.1.2 · D-011 +**Fixture:** F-PRELUDE. Assume `orders.total_revenue = SUM(amount)` is declared. + +**Query:** +```yaml +Fields: [orders.id, orders.total_revenue] +``` + +**Expected:** +```yaml +error: + code: E_AGGREGATE_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["scalar", "total_revenue", "aggregate"] +``` + +A scalar query asks for row-level values; a metric is by definition an +aggregate at query grain, which has no defined value at the row grain +of `orders`. The engine MUST NOT silently aggregate, broadcast, or +"window" the metric. + +--- + +### 4.B Field & metric grain + +#### T-004 — Implicit home-grain aggregation in a field expression + +**Anchors:** §4.3 · D-003, D-015 +**Fixture:** F-PRELUDE. Add this field to the model: + +```yaml +fields: + - name: customers.lifetime_value + expression: "SUM(orders.amount)" +``` + +**Query (scalar):** +```yaml +Fields: [customers.region, customers.id, customers.lifetime_value] +``` + +**Expected:** + +| region | id | lifetime_value | +|:-------|:--:|---------------:| +| EAST | 1 | 150.00 | ← SUM(100, 50) +| EAST | 2 | 200.00 | +| WEST | 3 | 75.00 | +| NORTH | 4 | NULL | ← no orders (or 0, depending on engine's LEFT-aggregate convention — both are acceptable; test allows either) + +The inner `SUM` is evaluated at the **home grain of `customers`** (one +sum per customer row), regardless of any grouping the consuming query +applies. This is what makes `lifetime_value` reusable across queries. + +**Compilation-strategy equivalence (D-015):** an engine MAY compile the +inner `SUM` as (a) a correlated subquery, (b) a `LATERAL` / `CROSS APPLY`, +or (c) a pre-aggregated CTE joined back to `customers`. The test passes +under any of the three. + +#### T-005 — Cross-grain aggregation: single-step and nested forms + +This is the **flagship test family for D-020** — the most-debated +semantic in cross-vendor BI portability. The Foundation accepts **both** +single-step and explicit-nested forms over a `1 : N` edge; they give +different numerical answers for non-distributive aggregates (the +single-step form gives the "all rows at once" answer, the nested form +gives the "per-home-row first" answer). For distributive aggregates +they give identical answers, so either form is acceptable for style. + +**Anchors:** §4.5 form (1), §6.1 Semantic 2 · D-020 + +**Fixture:** F-PRELUDE. + +--- + +##### T-005a — Single-step cross-grain `SUM` over `1 : N` is accepted (distributive) + +**Metric declaration:** +```yaml +metrics: + - name: customers.total_order_amount + expression: "SUM(orders.amount)" # single-step; resolves at query grain +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.total_order_amount] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | total_order_amount | +|:-------|-------------------:| +| EAST | 350.00 | ← orders 101, 102, 103 +| WEST | 75.00 | ← order 104 +| NULL | 30.00 | ← orphan order 105 + +No error. For distributive aggregates the single-step form is +equivalent to the explicit-nested form `SUM(SUM(orders.amount))` — both +return identical numbers (cross-validated by T-005d below). + +--- + +##### T-005b — Single-step cross-grain `AVG` over `1 : N` is accepted (heavy-weighted) + +**Metric declaration:** +```yaml +metrics: + - name: customers.avg_order_amount + expression: "AVG(orders.amount)" # single-step; standard SQL semantics +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.avg_order_amount] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** the heavy-customer-weighted SQL average. + +| region | avg_order_amount | +|:-------|-----------------:| +| EAST | 116.67 | ← AVG(100, 50, 200) = 350/3 — three orders in EAST +| WEST | 75.00 | ← AVG(75) — one order in WEST +| NULL | 30.00 | ← orphan order 105 + +No error. Note that EAST's answer (`116.67`) is **not** the unweighted +average across customers — Customer 1 averages `75` over two orders, +Customer 2 averages `200` over one order, and the unweighted answer +would be `(75 + 200) / 2 = 137.5`. That second number is what the +**nested** form produces; see T-005c. + +--- + +##### T-005c — Explicit nested cross-grain `AVG` over `1 : N` is accepted (unweighted) + +The Snowflake-style explicit form is **still accepted**; it just gives a +different (and meaningfully different) answer for non-distributive +aggregates. The Foundation does not force this style — but neither does +it reject it. Users porting from Snowflake Semantic Views can leave their +nested expressions unchanged. + +**Metric declaration:** +```yaml +metrics: + - name: customers.avg_of_per_customer_avg + expression: "AVG(AVG(orders.amount))" # inner: per-customer avg; outer: avg across customers +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.avg_of_per_customer_avg] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** the unweighted average across customers in each region. + +| region | avg_of_per_customer_avg | +|:-------|------------------------:| +| EAST | 137.50 | ← AVG(Cust1=75, Cust2=200) = 275/2 — two customers in EAST +| WEST | 75.00 | ← AVG(Cust3=75) — one customer in WEST +| NULL | 30.00 | ← orphan order's customer is unknown; treated as one customer-of-one-order + +The user gets to pick: write `AVG(orders.amount)` for the +heavy-weighted answer (T-005b) or `AVG(AVG(orders.amount))` for the +unweighted answer (T-005c). Both are conformant. The two numbers differ +for non-distributive aggregates — this difference is the central +BI-portability decision in D-020. + +--- + +##### T-005d — Single-step and nested `SUM` are numerically equivalent (distributive) + +Run both forms against identical data and assert the row sets are +identical. + +**Metric A (single-step):** +```yaml +- name: customers.sum_single + expression: "SUM(orders.amount)" +``` + +**Metric B (nested):** +```yaml +- name: customers.sum_nested + expression: "SUM(SUM(orders.amount))" +``` + +**Query A:** +```yaml +Dimensions: [customers.region] +Measures: [customers.sum_single] +``` + +**Query B:** +```yaml +Dimensions: [customers.region] +Measures: [customers.sum_nested] +``` + +**Expected:** identical row sets (after column-name normalisation). +Both forms are single-measure aggregation queries ⇒ Plan A ⇒ NORTH +is absent and the orphan `NULL`-region row is present. This test is +what makes "port your Snowflake nested form unchanged" portable in +the distributive case — and is the canonical demonstration that +nested-vs-single is purely a stylistic choice when the aggregate is +distributive. + +--- + +##### T-005e — `COUNT(DISTINCT)` single-step over `1 : N` is accepted + +**Anchors:** §4.5, §6.1 · D-020 case (d), D-022 caveat + +**Metric declaration:** +```yaml +metrics: + - name: customers.distinct_order_statuses + expression: "COUNT(DISTINCT orders.status)" +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.distinct_order_statuses] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** + +| region | distinct_order_statuses | +|:-------|------------------------:| +| EAST | 2 | ← `completed` (101, 102) + `pending` (103) +| WEST | 1 | ← `completed` (104) +| NULL | 1 | ← orphan 105 has `completed` + +No `E_UNSAFE_REAGGREGATION`. This pins D-020 (d) and the D-022 caveat +explicitly: a holistic aggregate over a plain `1 : N` edge is one SQL +aggregate over the joined rows — well-defined per D-020. The +`E_UNSAFE_REAGGREGATION` error only fires when the plan would require +*decomposing* a holistic aggregate across grains, which happens on M:N +(D-022) or fan-out shapes the engine cannot pre-aggregate safely. + +#### T-006 — `COUNT(*)` over a dataset is well-defined at the home grain + +**Anchors:** §7.2 · D-016 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - COUNT(*) AS order_count +Where: + - orders.status = 'completed' +``` + +**Expected:** + +| region | order_count | +|:-------|------------:| +| EAST | 2 | ← orders 101, 102 (102? wait 103 is pending). 101+102 completed. 103 pending excluded. +| WEST | 1 | ← order 104 +| NULL | 1 | ← orphan order 105 (status=completed) + +`COUNT(*)` is the count of rows of the **measure-bearing dataset** +that survive the predicates — here, `orders`. Engines that emit +`COUNT()` as a transparent rewrite (e.g. +Snowflake's `COUNT(*)`-rejection workaround) MUST produce the same +numbers. + +--- + +### 4.C Predicate routing + +#### T-007 — Aggregate in `Where` is rejected + +**Anchors:** §5.3 · D-005, D-012a +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount)] +Where: + - SUM(orders.amount) > 100 # aggregate inside Where +``` + +**Expected:** +```yaml +error: + code: E_AGGREGATE_IN_WHERE + anchor: §5.3 + must_contain: ["aggregate", "Where", "Having"] # diagnostic should hint at Having +``` + +The diagnostic MUST identify the offending aggregate by name and SHOULD +suggest moving the predicate to `Having`. + +#### T-008 — Pure row-level predicate in `Having` is rejected + +**Anchors:** §5.3 · D-012b +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount)] +Having: + - customers.region = 'EAST' # row-level dimension predicate +``` + +**Expected:** +```yaml +error: + code: E_NON_AGGREGATE_IN_HAVING + anchor: §5.3 + must_contain: ["Having", "Where"] +``` + +Note this test is about *routing*, not behaviour. An engine that +silently treats `Having` as `Where` produces the right numbers but +breaks the contract, and so fails the test on the error code, not +the rows. + +#### T-009 — Mixed-level boolean predicate is rejected + +**Anchors:** §5.3 · D-012c +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Having: + - customers.region = 'EAST' AND SUM(orders.amount) > 100 +``` + +**Expected:** +```yaml +error: + code: E_MIXED_PREDICATE_LEVEL + anchor: §5.3 + must_contain: ["row-level", "aggregate", "split"] +``` + +The diagnostic SHOULD suggest splitting the predicate (row-level half +→ `Where`, aggregate half → `Having`). + +--- + +### 4.D Default join behaviour + +#### T-010 — `N:1` enrichment defaults to `LEFT`; orphans appear under `NULL` + +**Anchors:** §6.4 · D-004a +**Fixture:** F-PRELUDE. + +Covered indirectly by T-001's expected rows — the `region = NULL` row +exists *only because* the default join from `orders → customers` is +`LEFT` and order 105's `customer_id = 99` does not match. An engine +that defaults to `INNER` returns one fewer row and fails the test on +row count. This is the single most common silent-correctness failure +in BI tools (Snowflake's "first matching row" rule is the worst case; +see Appendix B note B-1). + +#### T-011 — Multi-fact composition defaults to `FULL OUTER` on shared dims + +**Anchors:** §6.4 · D-004b +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - SUM(returns.amount) AS return_total +``` + +**Expected (multi-measure ⇒ FULL OUTER stitch per §6.6 row 3 / §6.2 +step 7; *both* sides preserved):** + +| region | revenue | return_total | +|:-------|--------:|-------------:| +| EAST | 350.00 | 10.00 | ← both facts have data +| WEST | 75.00 | 5.00 | ← both facts have data +| NORTH | NULL | 15.00 | ← only `returns` has data (customer 4); revenue is `SUM` over zero rows = `NULL` (§6.11 / D-033) +| NULL | 30.00 | NULL | ← only `orders` has data (orphan 105); return_total is `SUM` over zero rows = `NULL` + +This is the **multi-measure** counterpart to T-001's single-measure +Plan A: because there are two independently-aggregated facts +(`orders` and `returns`), §6.6 row 3 mandates a `FULL OUTER` between +the two pre-aggregated branches. The union of dim values appearing in +*either* branch is the result. NORTH appears here (via the `returns` +branch) because customer 4 has a return, even though customer 4 has +no orders. Without `returns` as a second measure, this would be a +single-measure query and Plan A would drop NORTH (see T-001). An +engine that defaults to `INNER` between the two pre-aggregated +branches loses NORTH and the orphan-bucket, failing on row count. + +**Companion `COUNT` test:** replacing either `SUM` with `COUNT(*)` or +`COUNT(orders.id)` would produce `0` in the missing-side cell, not +`NULL`, per standard SQL. See T-047 for the explicit +`SUM`-returns-`NULL` / `COUNT`-returns-`0` contrast. + +#### T-012 — Scalar grand total uses `CROSS JOIN` + +**Anchors:** §6.4 · D-004c +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [] # no dimensions ⇒ scalar grain +Measures: + - SUM(orders.amount) AS total_revenue + - SUM(returns.amount) AS total_returns +``` + +**Expected:** exactly one row. + +| total_revenue | total_returns | +|--------------:|--------------:| +| 455.00 | 30.00 | + +`455.00 = 100+50+200+75+30` (all orders including orphan); +`30.00 = 10+5+15` (all returns). The two scalars sit on a single row +joined via `CROSS JOIN` (`1 ⋈ 1`), not on separate rows. + +--- + +### 4.E M:N resolution — the flagship section + +#### T-013 — Two unrelated facts with no shared dimension raise `E3013` + +**Anchors:** §6.6 · D-007 +**Fixture:** F-NOPATH. + +**Query:** +```yaml +Measures: + - SUM(orders.amount) + - SUM(inventory_movements.quantity) +``` + +**Expected:** +```yaml +error: + code: E3013_NO_STITCHING_DIMENSION + anchor: §6.6 + must_contain: ["orders", "inventory_movements", "shared dimension"] +``` + +An engine that emits a Cartesian product here is producing a +plausible-but-wrong total and is the single failure mode Promise 4 is +designed to prevent. + +#### T-014 — `N:N` with no bridge and no shared dim raises `E3012` + +**Anchors:** §6.6 · D-007 +**Fixture:** F-BRIDGE-NONE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: [SUM(movies.gross)] +``` + +**Expected:** +```yaml +error: + code: E3012_MN_NO_SAFE_REWRITE + anchor: §6.6 + must_contain: ["actors", "movies", "bridge"] # diagnostic should suggest the remedy +``` + +#### T-015 — Bridge resolution de-duplicates per `(fact, group)` (FLAGSHIP) + +**Anchors:** §6.6.1, §6.3 Semantic 2 · D-026 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: + - SUM(movies.gross) AS total_gross +``` + +**Expected:** + +| height | total_gross | +|-------:|------------:| +| 170 | 300.00 | ← M10 (100) counted ONCE, plus M11 (200). Not 400. +| 180 | 50.00 | + +**Why this test exists.** The naive flat-join SQL +`actors ⋈ appearances ⋈ movies GROUP BY actors.height` produces +`(170 → 400, 180 → 50)` because M10's gross is replicated by Alice's +*and* Bob's appearances. The Foundation's bridge plan, per §6.6.1, +de-duplicates at the `(movie_id, height)` level before the final +`SUM`, and so M10 contributes 100 to height 170 exactly once. This +is **Semantic 2** in operation: no row of `movies` (the measure's +home dataset) contributes more than once to any output group. + +**Vendor cross-check.** + +| Tool | Same data, same query, expected output | +|:---|:---| +| Looker (symmetric aggregates on, declared PK on each view) | `170 → 300, 180 → 50` | +| Tableau (Relationships, 2020.2+) | `170 → 300, 180 → 50` | +| Power BI (bridge with single-direction filter + `SUMX(DISTINCT(...))`) | `170 → 300, 180 → 50` | +| Naive SQL join (no de-duplication) | `170 → 400, 180 → 50` — **wrong** | + +An OSI implementation that matches the naive-SQL number on this fixture +is non-conformant on D-026, regardless of how it generates the SQL. + +#### T-016 — Bare non-distributive aggregate over a bridge is rejected + +**Anchors:** §4.5, §6.1, §6.6.1 · D-027 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: [AVG(movies.gross)] +``` + +**Expected:** +```yaml +error: + code: E_UNSAFE_REAGGREGATION + anchor: §6.1 + must_contain: ["AVG", "non-distributive", "nested", "AVG(AVG"] +``` + +The diagnostic MUST suggest the nested form (e.g., `AVG(AVG(movies.gross))` +to mean "average across actors of their per-actor average gross"). + +#### T-017 — Nested aggregation over a bridge succeeds with per-endpoint plan + +**Anchors:** §4.5 form (1), §6.6.1 · D-020, D-027 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: + - AVG(AVG(movies.gross)) AS avg_of_per_actor_avg +``` + +**Expected:** + +| height | avg_of_per_actor_avg | +|-------:|---------------------:| +| 170 | 125.00 | ← AVG(per-actor avg) = AVG(Alice=150, Bob=100) = 125 +| 180 | 50.00 | ← AVG(Carol=50) = 50 + +The inner `AVG` runs at actor grain (per actor: Alice averages M10/M11 += 150; Bob averages M10 = 100; Carol averages M12 = 50); the outer `AVG` +runs at query grain (per height). This is the explicit case where the +per-endpoint-first plan is correct and produces different numbers from +T-015's de-duplication plan — and that's the point of requiring nested +aggregation: user intent is now unambiguous. + +#### ~~T-018~~ — Deferred (was: `EXISTS_IN` compiles to `EXISTS`) + +The original T-018 exercised the semi-join filter form +(`EXISTS_IN` / `NOT EXISTS_IN`). That construct has been **removed +from the Foundation** and moved to a separate follow-up proposal that +will specify its surface syntax, NULL-safety guarantees, and +compilation contract in full. This test will be reinstated (and +expanded — positive case, `NOT`-form, NULL on both sides) once the +EXISTS_IN proposal lands. + +Until then: the Foundation has no semi-join construct, and there is +nothing for this slot to test. The slot is retained to keep T-NNN +numbering stable; future re-use of `T-018` for an unrelated test +would be misleading. + +--- + +### 4.F Path resolution & namespacing + +#### T-019 — Two relationships between the same datasets raise `E_AMBIGUOUS_PATH` + +**Anchors:** §6.7 · D-018a +**Fixture:** F-AMBIG. + +**Query:** +```yaml +Dimensions: [users.region] +Measures: [COUNT(orders.id)] +``` + +**Expected:** +```yaml +error: + code: E_AMBIGUOUS_PATH + anchor: §6.7 + must_contain: ["order_placed_by", "order_fulfilled_by"] # both candidate paths named +``` + +The diagnostic MUST list **every** candidate relationship by name so +the user can pick one (the planned remedy is to annotate the query +with the relationship to use; see §10). + +#### T-020 — Bare-name references resolve to the global namespace, not a dataset + +**Anchors:** §4.6 · D-006 +**Fixture:** F-PRELUDE. Declare `orders.total_revenue = SUM(amount)` as a +*dataset-scoped* metric. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [total_revenue] # bare name, not qualified +``` + +**Expected:** +```yaml +error: + code: E_NAME_NOT_FOUND + anchor: §4.6 + must_contain: ["total_revenue", "global", "orders.total_revenue"] +``` + +The bare name MUST NOT silently match `orders.total_revenue`. Using +the dataset-scoped metric requires the qualified reference: + +**Query (positive case):** +```yaml +Measures: [orders.total_revenue] +``` + +Resolves and returns the per-region revenue from T-001. + +--- + +### 4.G Grain error contracts + +#### T-021 — `COUNT(DISTINCT)` under fan-out raises `E_UNSAFE_REAGGREGATION` + +**Anchors:** §6.1 Starting Grain, §7.2 · D-022 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [orders.status] +Measures: [COUNT(DISTINCT customers.id) AS unique_customers] +``` + +**Expected:** +```yaml +error: + code: E_UNSAFE_REAGGREGATION + anchor: §6.1 + must_contain: ["COUNT(DISTINCT)", "holistic", "pre-aggregate"] +``` + +The aggregate is holistic and reads a column from the one-side +(`customers.id`) over a join that fans out to orders. The diagnostic +MUST identify the aggregate, classify it as holistic, and suggest a +pre-aggregation at the customer grain. + +For comparison, swapping in `SUM(orders.amount)` (distributive) on the +same shape succeeds with no error — this is the discriminator between +D-022 and the safe-by-pre-aggregation case. + +#### T-022 — Fan-out in a scalar query raises `E_FAN_OUT_IN_SCALAR_QUERY` + +**Anchors:** §6.1 Final Grain, §5.1.2 · D-023 +**Fixture:** F-PRELUDE. + +**Query (scalar):** +```yaml +Fields: [customers.region, orders.id] +``` + +**Expected:** +```yaml +error: + code: E_FAN_OUT_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["scalar", "fan-out", "customers", "orders"] +``` + +The scalar query has no aggregation step in which to absorb the +fan-out; either the user must aggregate one side or switch to an +aggregation query. The diagnostic MUST suggest both remedies. + +#### T-023 — Row-level reference to a finer grain raises `E_UNAGGREGATED_FINER_GRAIN_REFERENCE` + +**Anchors:** §6.1 Starting Grain · D-024 +**Fixture:** F-PRELUDE. Declare this *invalid* field on `customers`: + +```yaml +fields: + - name: customers.first_order_amount + expression: "orders.amount" # bare row-level reference to higher-grain table +``` + +**Expected (at model load or first query referencing the field):** +```yaml +error: + code: E_UNAGGREGATED_FINER_GRAIN_REFERENCE + anchor: §6.1 + must_contain: ["orders.amount", "aggregate", "customers"] +``` + +The remedy is to either wrap the reference in an aggregate +(`SUM(orders.amount)` — implicit home-grain aggregation per D-003), +or pull the value from a coarser-grain dataset where it is already +at the home grain. The diagnostic MUST surface both options. + +--- + +### 4.H Common-clause semantics + +This section pins behaviour that applies to **both** query shapes: +predicate-list interpretation, `Order By` NULL handling, `Limit` +without `Order By`, and home-grain scalar usability in `Where`. + +#### T-024 — Boolean home-grain scalar usable in `Where` + +**Anchors:** §4.3 routing rules, §6.1 Semantic 2 · D-005 (e) +**Fixture:** F-PRELUDE. + +**Model addition:** +```yaml +fields: + - dataset: customers + name: has_completed_orders + expression: "COUNT(orders.id WHERE orders.status = 'completed') > 0" +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [COUNT(customers.id) AS customer_count] +Where: ["customers.has_completed_orders"] +``` + +**Expected:** + +| region | customer_count | +|:-------|---------------:| +| EAST | 1 | ← Customer 1 (orders 101, 102 completed). Customer 2 has only pending order 103, excluded. +| WEST | 1 | ← Customer 3 (order 104 completed) + +No error. The field `has_completed_orders` contains `COUNT(orders.*) > 0` — +an aggregate over a finer-grain dataset — but it resolves at customer +grain via implicit home-grain aggregation (§4.3), so its **resolved +shape** is a boolean scalar at customer grain. By D-005 (e), it is +therefore valid in `Where` for any query at customer grain or coarser, +where it filters customers pre-aggregation. An engine that classifies +fields by surface syntax (i.e., "contains `COUNT`, therefore route to +`Having`") would raise `E_WRONG_PREDICATE_LOCATION` and fail this test. + +--- + +#### T-026 — Outer `Order By` defaults to `NULLS LAST` + +**Anchors:** §5.1 Common clause semantics, §6.10.2 · D-014, D-029 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Order By: ["customers.region ASC"] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NULL | 30.00 | ← orphan-bucket sorts LAST (NULLS LAST default) + +The non-NULL `region` values sort by ASC; the orphan `NULL`-region +bucket sorts last per the `NULLS LAST` default. + +Compilation MUST emit explicit `NULLS LAST` into the compiled SQL even +though the query body did not say so: + +```yaml +sql_contract: + forbidden_substrings: + - "ORDER BY .* ASC$" # bare ASC without explicit NULLS LAST + required_substrings: + - "ORDER BY .* ASC NULLS LAST" +``` + +This MUST be byte-stable across compilations (D-014) and MUST agree +across dialects (Snowflake / Databricks / Postgres) even though each +dialect's *native* default differs. An engine that omits the explicit +`NULLS LAST` from compiled SQL fails on the SQL contract, even if the +runtime row order happens to match. + +--- + +#### T-028 — `Where: [P1, P2]` is `P1 AND P2` + +**Anchors:** §5.1 Common clause semantics · D-014 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Where: + - "orders.status = 'completed'" + - "orders.amount > 60" +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | revenue | +|:-------|--------:| +| EAST | 100.00 | ← only order 101 (status=completed, amount=100). Order 102 (amount=50) excluded by amount filter. Order 103 (pending) excluded by status filter. +| WEST | 75.00 | ← order 104 (status=completed, amount=75) +| NULL | 200.00 | ← orphan 105 (status=completed, amount=200) + +The result MUST equal what a single equivalent `Where:` entry +`"orders.status = 'completed' AND orders.amount > 60"` produces — same +rows, same numbers, byte-identical compiled SQL (after alias +normalisation). An engine that interprets a predicate list as `OR` +would include order 102 (status=completed, amount=50) and inflate +EAST to `150` — a runtime test failure. + +--- + +#### T-052 — `Limit` without `Order By` compiles and runs + +**Anchors:** §5.1 Common clause semantics · D-014 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Limit: 2 +``` + +**Expected:** + +(a) **No error.** The query compiles successfully — `Limit` without + `Order By` is a legitimate (if non-deterministic) BI query. + +(b) **No warning required.** The Foundation does NOT mandate that + engines emit a diagnostic. An engine that emits a "non-deterministic + Limit" advisory is conformant; an engine that emits no diagnostic at + all is also conformant. + +(c) **Compiled SQL is byte-stable (D-014).** Two compilations of the + same `(model, query, dialect)` MUST produce byte-identical SQL — + the *output rows* may differ between runs of that SQL (since the + engine is free to pick any two rows), but the SQL string itself is + deterministic. + +(d) **Result has exactly two rows**, both with valid `region` values + from the result set `{EAST, WEST, NORTH, NULL}`. Specific row + selection is engine-defined. + +--- + +### 4.I Window functions + +The Foundation supports a defined subset of SQL window functions +(§6.10). This section exercises both the supported shapes (with strong +result expectations) and the rejected shapes (with explicit error +codes). + +#### T-027 — `ORDER BY` inside `OVER (...)` defaults to `NULLS LAST` + +**Anchors:** §6.10.2 · D-029 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC) AS rn" +Where: + - "orders.status = 'completed'" +``` + +**Expected:** the compiled SQL MUST contain `ORDER BY ... DESC NULLS +LAST` inside the `OVER (...)` clause, even though the query body did +not specify it. Engines compiling to Snowflake (native `DESC NULLS +FIRST`) MUST emit the explicit clause to override the dialect default. + +```yaml +sql_contract: + required_substrings: + - "OVER (.*ORDER BY .* DESC NULLS LAST.*)" +``` + +The result row set: every `(customer_id, amount)` group's NULL `amount` +rows (if any existed) would sort after non-NULL rows. In F-PRELUDE all +orders have non-NULL `amount`, so this test exercises the SQL contract +above rather than data NULLs. + +--- + +#### T-029 — Window function in `Where` is rejected + +**Anchors:** §6.10.3 · D-030 (E_WINDOW_IN_WHERE) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [orders.customer_id] +Measures: [SUM(orders.amount) AS revenue] +Where: + - "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount) = 1" +``` + +**Expected:** the engine MUST reject: + +```yaml +error: + code: E_WINDOW_IN_WHERE + anchor: §6.10.3 + must_contain: ["window", "Where", "Qualify", "subquery"] +``` + +The diagnostic MUST mention both supported alternatives: (a) move the +predicate into a `Having` or `Qualify` clause, or (b) wrap the windowed +expression in a derived field and filter that field at a coarser grain. + +--- + +#### T-030 — Nested window functions rejected + +**Anchors:** §6.10.4 · D-031 +**Fixture:** F-PRELUDE. + +**Query (illegal — window over window):** +```yaml +Fields: + - orders.id + - "SUM(SUM(orders.amount) OVER (PARTITION BY orders.customer_id)) OVER ()" +``` + +**Expected:** rejected at compile time: + +```yaml +error: + code: E_NESTED_WINDOW + anchor: §6.10.4 + must_contain: ["nested window", "single window", "derived field"] +``` + +This matches every major SQL dialect (Snowflake, Databricks, Postgres, +BigQuery all reject nested windows). The diagnostic MUST suggest +factoring the inner window into a derived field. + +--- + +#### T-031 — Running-total window accepted (the canonical positive) + +**Anchors:** §6.10.1, §6.10.2 · D-030 (positive) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - orders.amount + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total" +Where: + - "orders.customer_id IS NOT NULL" +Order By: + - "orders.customer_id ASC" + - "orders.id ASC" +``` + +**Expected:** + +| id | customer_id | amount | running_total | +|----:|------------:|-------:|--------------:| +| 101 | 1 | 100.00 | 100.00 | +| 102 | 1 | 50.00 | 150.00 | +| 103 | 2 | 200.00 | 200.00 | +| 104 | 3 | 75.00 | 75.00 | + +The compiled SQL MUST contain explicit `ROWS BETWEEN UNBOUNDED +PRECEDING AND CURRENT ROW` (the literal frame; not a synonym), and the +`OVER (...)` `ORDER BY` MUST include the `NULLS LAST` default +resolution. + +--- + +#### T-032 — `ROW_NUMBER` with outer `Where` filter (qualify-style pattern) + +**Anchors:** §6.10.1, §6.10.3 · D-030 (positive) +**Fixture:** F-PRELUDE. + +**Strategy:** the legal way to "filter by `ROW_NUMBER = 1`" is to +expose the window as a derived field and filter at a coarser grain, +**OR** to use the engine's `Qualify`-style mechanism (deferred — +§10). This test exercises the derived-field strategy. + +**Model addition:** +```yaml +fields: + - dataset: orders + name: order_rank_in_customer + expression: "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC, orders.id ASC)" +``` + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - orders.amount + - orders.order_rank_in_customer +Where: + - "orders.order_rank_in_customer = 1" +``` + +**Expected:** the engine MUST compile the inner window in a CTE / inline +subquery and the outer `Where` MUST filter that subquery's result — +NEVER push the window inside the outer `WHERE`. The result is one row +per customer (their highest-amount order): + +| id | customer_id | amount | order_rank_in_customer | +|----:|------------:|-------:|-----------------------:| +| 101 | 1 | 100.00 | 1 | +| 103 | 2 | 200.00 | 1 | +| 104 | 3 | 75.00 | 1 | +| 105 | 99 | 200.00 | 1 | + +`Order By` was omitted; engines are not required to return rows in any +specific order, but the row *content* MUST match. + +--- + +#### T-033 — Composing onto a windowed metric is rejected + +**Anchors:** §6.10.5 · E_WINDOWED_METRIC_COMPOSITION +**Fixture:** F-PRELUDE. + +**Model addition:** +```yaml +metrics: + - name: orders.running_total_by_customer + expression: "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id)" + - name: orders.running_total_ratio + expression: "orders.running_total_by_customer / SUM(orders.amount)" # composition onto windowed metric +``` + +**Expected:** model load (or first reference to the second metric) MUST +fail: + +```yaml +error: + code: E_WINDOWED_METRIC_COMPOSITION + anchor: §6.10.5 + must_contain: ["window", "composition", "derived field", "§10"] +``` + +The diagnostic MUST acknowledge that the feature is deferred to §10 +(parameterised window composition) — not a permanent prohibition. + +--- + +#### T-034 — `GROUPS` frame mode rejected + +**Anchors:** §6.10.6 · D-032 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.amount GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS recent_sum" +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_FRAME_MODE + anchor: §6.10.6 + must_contain: ["GROUPS", "ROWS", "RANGE", "§10"] +``` + +Diagnostic MUST list the two supported alternatives (`ROWS` and `RANGE`) +and point at §10 as the future home of `GROUPS`. + +--- + +#### T-035 — Parameterised frame bound rejected + +**Anchors:** §6.10.6 · D-032 +**Fixture:** F-PRELUDE; query parameter `:n` declared. + +**Query:** +```yaml +Fields: + - orders.id + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id ROWS BETWEEN :n PRECEDING AND CURRENT ROW) AS recent_sum" +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_FRAME_BOUND + anchor: §6.10.6 + must_contain: ["parameter", "literal integer", "UNBOUNDED", "§10"] +``` + +Diagnostic MUST mention the three accepted bound shapes (integer +literal, `UNBOUNDED`, `CURRENT ROW`) and point at §10. + +--- + +#### T-036 — Window over a `1 : N` fan-out is accepted (no implicit aggregation) + +**Anchors:** §6.10.1, §6.7 · D-022, D-030 (positive) +**Fixture:** F-PRELUDE. + +The interesting question is what happens when a window-aggregate is +defined at the customer-row grain but its argument lives at the orders +grain. Windows do NOT trigger implicit home-grain aggregation — they +require the user to be explicit about which grain the window runs over. +This test exercises the *correct* shape: the window is defined inside a +scalar query at orders grain (its native home), so no fan-out: + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - "RANK() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC) AS rank_by_customer" +``` + +**Expected:** the query compiles at orders grain (the natural home of +the window). The result is one row per order with the customer-internal +rank: + +| id | customer_id | rank_by_customer | +|----:|------------:|-----------------:| +| 101 | 1 | 1 | ← amount 100 > amount 50 +| 102 | 1 | 2 | +| 103 | 2 | 1 | +| 104 | 3 | 1 | +| 105 | 99 | 1 | + +**Companion negative — T-036b:** trying to use the same window +expression as a *field on customers* (i.e., declaring +`customers.rank_in_some_customer = RANK() OVER (...)`) MUST raise +`E_AMBIGUOUS_MEASURE_GRAIN` (D-025) — the field's home grain is +underspecified. + +--- + +### 4.J BI-shape coverage + +These are the four "shape archetypes" every real BI semantic layer +must handle. Without explicit tests, regressions can ship silently. + +#### T-043 — Multi-hop `N : 1` enrichment chain + +**Anchors:** §6.4 (cardinality inference), §6.6 (join defaults) +**Fixture:** F-CHAIN — a three-hop `N : 1` chain +(`order_lines → orders → customers → segments`): + +```yaml +datasets: + - { name: segments, primary_key: [id], fields: [name] } + - { name: customers, primary_key: [id], fields: [segment_id, region] } + - { name: orders, primary_key: [id], fields: [customer_id, amount, status] } + - { name: order_lines, primary_key: [id], fields: [order_id, sku, qty, price] } + +relationships: + - { from: customers, from_keys: [segment_id], to: segments, to_keys: [id], cardinality: 1:1 → many:1 } + - { from: orders, from_keys: [customer_id], to: customers, to_keys: [id], cardinality: N:1 } + - { from: order_lines, from_keys: [order_id], to: orders, to_keys: [id], cardinality: N:1 } +``` + +**Query:** +```yaml +Dimensions: [segments.name] +Measures: [SUM(order_lines.qty * order_lines.price) AS revenue] +``` + +**Expected:** the planner finds a path +`order_lines → orders → customers → segments` (three `N : 1` hops). +Cardinality inference proves no fan-out (every hop is many-to-one), so +the SQL is a chain of `LEFT JOIN`s with no de-duplication step. + +Compiled SQL contract: + +```yaml +sql_contract: + required_substrings: + - "FROM .*order_lines" + - "LEFT JOIN .*orders" + - "LEFT JOIN .*customers" + - "LEFT JOIN .*segments" + forbidden_substrings: + - "DISTINCT" # no de-duplication needed; chain is N:1 all the way +``` + +Wrong join direction (e.g., `customers LEFT JOIN order_lines`) would +fan out; the test fails if the row count differs from +`COUNT(DISTINCT segments.id) + 1` (for orphan segments). + +--- + +#### T-044 — Composite primary key / foreign key + +**Anchors:** D-009 (deferred-key rejection — composite keys ARE supported +in the foundation), §6.4 +**Fixture:** F-COMPOSITE — `inventory` keyed by `(store_id, sku)` with +a fact `sales` referencing the composite: + +```yaml +datasets: + - name: inventory + primary_key: [store_id, sku] + fields: [stock_level, reorder_point] + - name: sales + primary_key: [id] + fields: [store_id, sku, qty, sale_ts] + +relationships: + - from: sales + from_keys: [store_id, sku] + to: inventory + to_keys: [store_id, sku] + cardinality: N:1 +``` + +**Query:** +```yaml +Dimensions: [inventory.reorder_point] +Measures: [SUM(sales.qty) AS units_sold] +``` + +**Expected:** the engine generates a join `ON sales.store_id = +inventory.store_id AND sales.sku = inventory.sku`. The result is one +row per distinct `reorder_point` value. + +Compiled SQL contract: + +```yaml +sql_contract: + required_substrings: + - "ON .*store_id .* AND .*sku" # both key components in ON clause + forbidden_substrings: + - "ON .*store_id\\s+\\)" # only one key component + - "ON .*sku\\s+\\)" +``` + +An engine that uses only one key component fails this test (it would +produce row duplication if any `(store_id, sku)` collision exists, or +silent NULLs). + +--- + +#### T-045 — Two-measure stitch with bridge resolution + +**Anchors:** §6.8 (M:N), §6.8.2 (stitch) · D-026 +**Fixture:** F-BRIDGE — `customers ⇌ orders` with `customers ⇌ +returns` and a separate `customers ⇌ campaigns` many-to-many via +`customer_campaigns` bridge. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(DISTINCT campaigns.id) AS active_campaign_count +``` + +**Expected:** the planner must: + +1. Aggregate `orders` to customer grain (per-customer revenue). +2. Resolve the M:N path `customers → customer_campaigns → campaigns` + with bridge de-duplication per D-026 (count each campaign at most + once per customer). +3. Aggregate each branch to region grain. +4. `FULL OUTER` stitch on region (multi-measure ⇒ §6.6 row 3 / §6.2 + step 7). + +Result: every region present in *either* branch's join path appears +(the union of "regions reached by an order" and "regions reached by +a campaign-association"). A region with neither order nor campaign +activity does NOT appear — same Plan A logic as T-001, just applied +per-branch and then unioned by the FULL OUTER. Missing-side cells +follow standard SQL per §6.11: `revenue` (a `SUM`) returns `NULL` +when no orders contribute to the region; `active_campaign_count` (a +`COUNT DISTINCT`) returns `0` when no campaigns contribute. The +differing empty-set values across the two measure types is the +*point* of the test: an engine that returns the same value for both +(e.g., both `0` or both `NULL`) is doing a Foundation-level rewrite +the spec forbids. + +The CRITICAL invariant: `active_campaign_count` MUST NOT be inflated by +the M:N fan-out. An engine that joins everything first then groups +will inflate the count and fail; the engine MUST aggregate each branch +independently. + +```yaml +sql_contract: + required_substrings: + - "WITH .*orders_branch" # or two named CTEs + - "FULL OUTER JOIN" +``` + +--- + +#### T-046 — Reflexive (self-referential) relationship + +**Anchors:** §6.4 (cardinality), §6.6 (join defaults), §6.9 (path +resolution with roles) +**Fixture:** F-REFLEXIVE — `employees` references itself via +`manager_id`: + +```yaml +datasets: + - name: employees + primary_key: [id] + fields: [name, manager_id, region] + +relationships: + - name: reports_to + from: employees + from_keys: [manager_id] + to: employees + to_keys: [id] + cardinality: N:1 + role: manager # qualifier — the "to" side acts as `manager` +``` + +**Query:** +```yaml +Dimensions: + - "employees{role=manager}.region AS manager_region" +Measures: + - COUNT(employees.id) AS direct_report_count +``` + +**Expected:** the planner generates `employees AS employee LEFT JOIN +employees AS manager ON employee.manager_id = manager.id` and groups by +`manager.region`. Each employee is counted exactly once. + +```yaml +sql_contract: + required_substrings: + - "FROM .*employees" + - "LEFT JOIN .*employees" # self-join with distinct aliases +``` + +An engine that fails to disambiguate the two `employees` references +either errors with `E_AMBIGUOUS_PATH` (if the role qualifier is not +honoured) or silently produces wrong counts. + +--- + +### 4.K Boundary, empty, and NULL cases + +#### T-025 — Scalar query with two unrelated row-level facts is rejected + +**Anchors:** §5.1.2, Appendix B · D-023 (extended) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.amount + - returns.amount + - customers.region +``` + +**Expected:** + +```yaml +error: + code: E_FAN_OUT_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["orders", "returns", "aggregation", "single home"] +``` + +`orders` and `returns` are independent facts both on the many-side of +`customers`. A row-level scalar query referencing fields from both +would have to either fan out (replicate orders rows per return row, or +vice versa) or pick a single home — neither is unambiguous. The error +MUST suggest two resolutions: convert to an aggregation query, or pick +a single fact as the home. (A semi-join filter form is deferred to a +future proposal — see the §6.8 deferred-feature note.) + +--- + +#### T-040 — `E_NO_PATH` when no relationship connects two datasets + +**Anchors:** §6.9 · D-018b +**Fixture:** F-NOPATH — two datasets with no declared relationship +between them: + +```yaml +datasets: + - name: customers + primary_key: [id] + fields: [name, region] + - name: weather_observations # no relationship to customers + primary_key: [id] + fields: [station_id, observation_date, temperature] +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [AVG(weather_observations.temperature) AS avg_temp] +``` + +**Expected:** + +```yaml +error: + code: E_NO_PATH + anchor: §6.9 + must_contain: ["customers", "weather_observations", "no path", "relationship"] +``` + +Diagnostic MUST name both endpoint datasets and explicitly state that +no declared `relationships` entry connects them. + +--- + +#### T-041 — Reserved-name rejection + +**Anchors:** §3 reserved names · D-019 +**Fixture:** F-PRELUDE with a deliberately illegal addition. + +**Model addition (illegal):** +```yaml +fields: + - dataset: customers + name: select # reserved SQL keyword + expression: "'placeholder'" +``` + +**Expected:** model load fails: + +```yaml +error: + code: E_RESERVED_NAME + anchor: §3 + must_contain: ["select", "reserved", "rename"] +``` + +Diagnostic MUST quote the offending name verbatim. + +--- + +#### T-042 — Deferred-key rejection + +**Anchors:** §10 deferred keys · D-009 +**Fixture:** F-PRELUDE with a deliberately illegal addition. + +**Model addition (illegal — `grain: FIXED` is deferred to §10):** +```yaml +metrics: + - name: orders.revenue_at_orders_grain + expression: "SUM(orders.amount)" + grain: FIXED # deferred key +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_KEY_REJECTED + anchor: §10 + must_contain: ["grain", "deferred", "§10"] +``` + +The diagnostic MUST identify which deferred key was used (`grain`) and +cite §10. This is the canonical instance; the full coverage of D-009 +requires one instance per deferred key — see §6 below. + +--- + +#### T-047 — Empty-set aggregate behaviour: `SUM` → `NULL`, `COUNT` → `0` (D-033) + +**Anchors:** §6.11 · D-033 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(orders.id) AS order_count + - COUNT(*) AS row_count +``` + +**Expected (three measures over one fact — still a single fact source, +so Plan A applies; NORTH does not appear):** + +| region | revenue | order_count | row_count | +|:-------|--------:|------------:|----------:| +| EAST | 350.00 | 3 | 3 | +| WEST | 75.00 | 1 | 1 | +| NULL | 30.00 | 1 | 1 | + +> Why no NORTH? All three measures are sourced from `orders` — the +> measure dataset-set is the same for all of them, so step 6 of the +> §6.2 evaluation algorithm produces one combined row set per the +> single-measure Plan A rule. NORTH (customer 4 with no orders) has +> no `orders` rows pulling it into the result. If the user wanted +> NORTH to appear, they would add a measure sourced from a different +> dataset that *does* have data for customer 4 — e.g. +> `COUNT(customers.id) AS customer_count` — which would convert the +> shape into a true multi-measure-stitch query and the FULL OUTER +> rule of §6.6 row 3 would preserve NORTH via the customers branch. + +This pins the standard-SQL split codified by D-033: the `COUNT` family +(`COUNT(*)`, `COUNT(x)`, `COUNT(DISTINCT x)`) returns `0` on an empty +set because it counts rows; every other aggregate (`SUM`, `AVG`, +`MIN`, `MAX`, `MEDIAN`, percentiles, `STDDEV`, `VARIANCE`) returns +`NULL` because there is no defined result. + +**Companion T-053** below covers the `MIN`/`MAX`/`AVG` branch +(non-distributive, also `NULL` over empty). + +--- + +#### T-048 — NULL foreign key in `1 : N` enrichment + +**Anchors:** §6.6 (LEFT default), §6.11 +**Fixture:** F-PRELUDE-NULLFK — F-PRELUDE with one extra order row: + +```sql +-- order 106: amount 80, status completed, customer_id NULL +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +``` + +**Expected:** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NORTH | 0 | +| NULL | 110.00 | ← orphan order 105 (30) + NULL-FK order 106 (80) — both bucketed under `region = NULL` + +The result demonstrates that a NULL foreign key behaves the same as an +unmatched key (orphan row 105 with `customer_id = 99` not in +`customers`): both produce a `NULL`-region bucket. An engine that +silently drops NULL-FK rows during the join would understate the +NULL-bucket by 80 and fail. + +--- + +#### T-049 — NULL in a dimension column + +**Anchors:** §5.1.1 (Result grain), §6.6 +**Fixture:** F-PRELUDE-NULLDIM — F-PRELUDE with customer 4's `region` +set to `NULL` (overriding NORTH). + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue, COUNT(customers.id) AS customer_count] +``` + +**Expected:** + +| region | revenue | customer_count | +|:-------|--------:|---------------:| +| EAST | 350.00 | 2 | +| WEST | 75.00 | 1 | +| NULL | 30.00 | 1 | ← customer 4 (NULL region) + orphan order 105's NULL bucket merge into one row + +The two distinct sources of NULL (the customer with NULL region + the +orphan order with unmatched customer) collapse into a *single* output +row. SQL `GROUP BY` treats all NULLs as one group; the Foundation +preserves this behaviour. An engine that emits two separate NULL rows +fails this test. + +--- + +#### T-050 — Empty aggregation query is rejected + +**Anchors:** §5.1.1 (At least one of `Dimensions` or `Measures` MUST be +non-empty), §6.2 normative evaluation step 1 · `E_EMPTY_AGGREGATION_QUERY` +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [] +Measures: [] +``` + +**Expected:** + +```yaml +error: + code: E_EMPTY_AGGREGATION_QUERY + anchor: §5.1.1 + must_contain: ["Dimensions", "Measures", "at least one"] +``` + +A query with neither dimensions nor measures has no shape at all — +this is distinct from "no dimensions" (which is legal and produces a +grand total) or "no measures" (which is the `Dimensions`-only listing +shape, also legal). The diagnostic MUST make this distinction explicit. + +--- + +#### T-051 — Empty scalar query is rejected + +**Anchors:** §5.1.2, §6.2 normative evaluation step 1 · `E_EMPTY_SCALAR_QUERY` +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: [] +``` + +**Expected:** + +```yaml +error: + code: E_EMPTY_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["Fields", "at least one", "scalar"] +``` + +A scalar query with no fields has no projection; it cannot be compiled +to any SQL. The diagnostic MUST mention that `Fields` requires at least +one entry. + +--- + +#### T-053 — `AVG` / `MIN` / `MAX` over zero matching rows return `NULL` + +**Anchors:** §6.11 · D-033 (non-`COUNT` branch) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - AVG(orders.amount) AS avg_amount + - MIN(orders.amount) AS min_amount + - MAX(orders.amount) AS max_amount +``` + +**Expected (three measures over one fact — Plan A; NORTH absent):** + +| region | avg_amount | min_amount | max_amount | +|:-------|-----------:|-----------:|-----------:| +| EAST | 116.67 | 50.00 | 200.00 | +| WEST | 75.00 | 75.00 | 75.00 | +| NULL | 30.00 | 30.00 | 30.00 | + +Combined with T-047 (`SUM` → `NULL` on empty, `COUNT` → `0`), this +fully pins D-033: only the `COUNT` family returns `0` over an empty +input. The NORTH row is absent for the same reason as in T-001 / T-047 +— a single-fact aggregation query follows Plan A. + +--- + +## 5. Coverage map + +This map is the authoritative cross-reference. Every `D-NNN` decision +in the spec's Appendix B that has observable runtime behaviour MUST +appear here with at least one covering `T-NNN`. + +| D-NNN | Decision | Covered by | +|:------|:---|:---| +| D-001 | Aggregation-query cardinality | T-001, T-050 (empty rejected) | +| D-003 | Implicit home-grain aggregation in fields | T-004, T-024 | +| D-004 | Default join types | T-001 (LEFT), T-010 (LEFT, indirect), T-011 (FULL OUTER), T-012 (CROSS), T-043 (multi-hop N:1 chain) | +| D-005 | Predicate routing by resolved expression shape | T-007, T-024 (e: boolean home-grain scalar in `Where`) | +| D-006 | Bare name → global namespace | T-020 | +| D-007 | M:N MUST be a safe rewrite | T-013, T-014 | +| D-009 | Deferred-key rejection | T-042 (canonical instance — `grain: FIXED`) | +| D-010 | Query shape determined by clauses | T-002 | +| D-011 | Bare metric in `Fields` rejected | T-003 | +| D-012 | Predicate-shape errors | T-007 (in WHERE), T-008 (in HAVING), T-009 (mixed) | +| D-014 | Determinism | §1.4 (applied to every test); explicit in T-026, T-052 (compiled SQL byte-stable) | +| D-015 | Three home-grain compilation strategies equivalent | T-004 | +| D-016 | `COUNT(*)` works | T-006, T-047 | +| ~~D-017~~ | **Deferred** — moved to a separate EXISTS_IN proposal | ~~T-018~~ (deferred slot) | +| D-018 | Path-resolution errors | T-019 (ambiguous), T-040 (E_NO_PATH) | +| D-019 | Reserved names rejected | T-041 | +| D-020 | Single-step AND nested cross-grain aggregation accepted | **T-005a (single-step SUM), T-005b (single-step AVG, heavy-weighted), T-005c (nested AVG, unweighted), T-005d (single-step SUM ≡ nested SUM), T-005e (single-step COUNT DISTINCT), T-017** | +| D-022 | Non-decomposable aggregate over fan-out | T-021, T-005e (caveat: D-022 fires on M:N decomposition, not plain 1:N) | +| D-023 | Fan-out in scalar query | T-022, T-025 (multiple unrelated row-level facts) | +| D-024 | Unaggregated finer-grain reference | T-023 | +| **D-026** | **Bridge resolution de-duplicates per `(fact, group)`** | **T-015 (flagship), T-045 (two-measure bridge)** | +| **D-027** | **Bare non-distributive aggregate over M:N rejected** | **T-016, T-017 (positive)** | +| D-029 | `Order By` defaults to `NULLS LAST` (outer + `OVER`) | T-026 (outer), T-027 (`OVER`) | +| D-030 | Windows: legal placements, `E_WINDOW_IN_WHERE` | T-029 (rejected), T-031 (running total positive), T-032 (qualify pattern), T-036 (window over `1:N`) | +| D-031 | Nested windows rejected | T-030 | +| D-032 | `ROWS` / `RANGE` accepted; `GROUPS` and parameterised bounds deferred | T-031 (positive `ROWS`), T-034 (`GROUPS` rejected), T-035 (param frame bound rejected) | +| **D-033** | **Empty-set aggregate behaviour follows standard SQL: `COUNT` family → `0`, everything else → `NULL`** | **T-011 (stitch missing-side `SUM` = `NULL`, the canonical test now that single-measure shapes drop missing-dim rows entirely per §1.6); T-045 (mixed `SUM`/`COUNT` branches); T-047, T-053 (one-fact multi-measure — `COUNT` = `0` for present dim values; missing dim values themselves are dropped per Plan A)** | +| E_WINDOWED_METRIC_COMPOSITION | Windowed metrics cannot be composed | T-033 | + +### 5.1 BI-shape coverage + +Independent of D-NNN coverage, these are the shapes every production +BI tool must handle. The catalog covers them as follows: + +| Shape | Covered by | +|:---|:---| +| Single-table fact | T-006, T-021, T-031, T-032, T-036 | +| `N : 1` enrichment (one hop) — single-measure (Plan A) | T-001, T-002, T-010 | +| `N : 1` enrichment (multi-hop chain) | T-043 | +| Composite primary/foreign key | T-044 | +| `1 : N` cross-grain aggregation (single-step) | T-005a, T-005b, T-005e, T-017 | +| `1 : N` cross-grain aggregation (nested) | T-005c, T-005d | +| `M : N` via bridge | T-013, T-014, T-015, T-016, T-017, T-045 | +| Multi-measure stitch (`FULL OUTER`, §6.6 row 3) | T-011, T-045 | +| Semi-join filter | ~~T-018~~ (deferred to future EXISTS_IN proposal) | +| Reflexive (self-referential) | T-046 | +| Windowed metrics | T-027, T-031, T-032, T-036 | +| Empty / boundary queries | T-050, T-051, T-052 | +| NULL semantics | T-011 (stitch missing-side), T-047, T-048, T-049, T-053 | + +## 6. Decisions awaiting coverage + +The following `D-NNN` entries do not yet have a `T-NNN` vector. Adding +one is the natural next sprint of this catalog. + +| D-NNN | Why deferred | +|:------|:---| +| D-002 | Multi-measure shared-grain composition — T-011 and T-045 cover the FULL-OUTER stitch path; D-002's "all measures resolve at the query's grain" claim is implicit in both. A direct test pinning the empty-dimensions / two-fact-no-dims case (Snowflake errata #4) is the natural next addition. | +| D-008 | No `UNSAFE`; no per-metric `joins.type` override. Test is a parser-level rejection — better placed in a parser-conformance suite than this row-set catalog. | +| D-009 | T-042 is the canonical instance; full coverage requires one test per deferred key (six total). Mechanically generated; out of scope for this seed but trivial follow-up. | +| D-013 | Same-named expressions reachable via qualification — needs a fixture with a deliberate name collision; planned for the next pass. | +| D-021 | Structured `expression` slot — parser/codegen-conformance. | +| D-025 | `E_AMBIGUOUS_MEASURE_GRAIN` catch-all — the D-025 enumeration lists the known cases; T-036b covers one (windowed field with no declared home grain). Remaining enumerated cases are mechanical follow-ups. | + +When this list shrinks to empty, every `D-NNN` in Appendix B has an +executable witness in this file and the Foundation Conformance Suite is +self-contained. diff --git a/compliance/foundation/README.md b/compliance/foundation/README.md new file mode 100644 index 00000000..c96966ea --- /dev/null +++ b/compliance/foundation/README.md @@ -0,0 +1,166 @@ + + +# OSI Compliance Test Suite — Foundation + +**Targets the +[Ossie Semantic Foundation](https://docs.google.com/document/d/1bEuIb2eUpwesi6cIZ5FOkXMQDh7KAPIjZsWchmLgOAk/edit?usp=sharing) proposal +[`SQL_EXPRESSION_SUBSET`](../core-spec/expression_language.md)** + +The Foundation is the deliberately narrow first-cut of Ossie semantics: +two query shapes, implicit home-grain aggregation, window functions, +and a small deferred-features list — pinned by numbered Conformance +Decisions (D-NNN) throughout the Foundation spec, and an error code +index in its appendix. + +The runner harness lives at [`../harness/`](../harness/) and is +shared with every future per-version suite under `compliance/`. + +## Status of this slice + +This is a **bootstrap slice**, not the full suite. It ports over just +enough of the compliance suite to make the mechanism (test discovery, +metadata schema, decision-coverage tracking, DuckDB-backed row +comparison, reporting) runnable and testable end to end: + +- **Tests present:** `tests/cross_grain/moderate/` only — 4 cases + (`t-005a`, `t-005b`, `t-005d`, `t-005e`) covering D-020 (single-step + cross-grain aggregation over a `1 : N` edge). Every other area listed + under `Layout` below (query shape, bridge/M:N, windows, namespace, + etc.) is the *target* design carried over from the source proposal + and will land — with its own tests — in follow-up PRs. +- **No adapter yet.** `adapters/osi_python_adapter.py` delegates to + `impl/python/conformance/adapter.py`, which does not exist in this + repo yet (`impl/python/` is still a placeholder). Until an + implementation lands, `python -m harness.runner --list` is the way + to exercise this suite — it discovers and validates the test corpus + without needing an engine. The 4 included tests carry + `status: planned` in their `metadata.yaml` (matching their status in + the source proposal — the semantics they pin are accepted but not + yet required-to-pass), so a full `harness.runner` run against a + partial adapter will correctly skip them unless invoked with + `--include-planned`. +- **`decisions.yaml`** keeps every `D-NNN` row from the source + proposal (so `harness`'s own registry-consistency tests still pass), + but only `D-020` has a populated `tests:` list — every other + decision's `tests:` is intentionally empty pending its witness + tests landing in a later PR. +- **Dropped `T-024`.** The source proposal's compliance suite also + shipped `tests/cross_grain/moderate/t-024-boolean-home-grain-scalar-in-where/`. + It was excluded from this slice: its `gold.sql` filters on + `customers.segment = 'PREMIUM'`, a value that does not exist in the + `f_prelude` fixture (segments are `retail` / `wholesale`), so the + query returns zero rows — and the test as implemented doesn't + exercise the aggregate-derived boolean-field case that + `DATA_TESTS.md` §4.G documents for T-024 (`has_completed_orders`, + an aggregate `COUNT(...) > 0` field routed through `Where`). See + the PR description for the full writeup; worth fixing at the + source before it's re-added here. + +## Layout + +``` +compliance/foundation-v0.1/ + README.md # this file + SPEC.md # the specs this suite targets (Foundation v0.1) + pyproject.toml # editable install; depends on ../harness (osi_compliance_harness) + conformance.yaml # test conformance levels; "foundation_v0_1" = required + proposals.yaml # mirrors §10 of the Foundation spec; every deferred is a proposal + decisions.yaml # D-NNN registry with anchor + status; tests: populated only for D-020 in this slice + adapters/ + osi_python_adapter.py # delegates to impl/python/conformance/adapter.py (not yet present) + datasets/ + f_prelude/ # mirrors DATA_TESTS.md §3.1 — the only fixture this slice needs + tests/ + cross_grain/ # T-005a/b/d/e (D-020) — the only populated area in this slice + results/ # created by the runner (gitignored); no committed baseline yet +``` + +The full target layout — `query_shape/`, `scalar_query/`, `bridge/`, +`windows/`, `namespace/`, etc. — is documented in `DATA_TESTS.md` and +`decisions.yaml`; those directories don't exist on disk yet. + +## Quick start + +```bash +# From the repo root +pip install -e compliance/harness +pip install -e compliance/foundation + +cd compliance/foundation + +# List discovered tests without running them (no adapter needed). +# The 4 cross_grain/moderate cases show status "planned" — they're +# skipped by a real run until an adapter lands, unless you pass +# --include-planned. +python -m harness.runner --list --tests tests/ --include-planned + +# Once an OSI implementation adapter exists (see ../ADAPTER_INTERFACE.md): +python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ \ + --include-planned +``` + +The harness ships under [`../harness/`](../harness/) and is installed +as the `osi_compliance_harness` package — every per-version suite under +`compliance/` shares this one runner / reporter / DB manager. The +harness's own mechanics (discovery, adapter invocation, row +comparison, reporting) are covered by its unit tests — see +[`../harness/README.md`](../harness/README.md). + +## Per-test layout + +Each test is a folder containing: + +| File | Purpose | +|:---|:---| +| `metadata.yaml` | `id: T-NNN`, `decision: D-NNN`, `spec_refs`, `required_features`, `expected_error_code`, `xfail_reason` (if applicable). | +| `model.yaml` | The semantic model (typically a thin wrapper around a fixture from `datasets/f_*`). | +| `query.json` | The semantic query, in the new two-shape format (`dimensions` + `measures` for aggregation queries; `fields` for scalar queries). | +| `gold.sql` | A hand-written reference SQL query the harness executes against the fixture data to produce the expected row multiset. Treated by the harness as a row oracle, not as a SQL-string comparison — D-014 is per-engine, not cross-engine. | + +Tests assert on observable behaviour only: + +- `expected_error_code: E_` ⇒ adapter must surface that code in + stderr (substring match — see `compliance/harness/src/harness/runner.py`). +- The harness runs both `gold.sql` and the adapter's emitted SQL against + the shared fixture and compares the resulting row multisets + (order-insensitive unless the query has an `Order By`). + +This means a `gold.sql` is a *witness* of the answer's shape and the +fixture data; the harness never compares SQL strings byte-for-byte +(per D-014, that's a per-engine concern). + +## Decision coverage + +Every `D-NNN` row in the Foundation spec should eventually have at +least one runnable case here. The mapping lives in `decisions.yaml`; +in this slice only `D-020` has one. + +## See also + +- `SPEC.md` — what the suite targets and why. +- `decisions.yaml` — the D-NNN status board. +- `proposals.yaml` — §10 deferred-features registry. +- `../../impl/python/` — where the upstream engine and its + `conformance/adapter.py` will land. +- `../ADAPTER_INTERFACE.md` — the CLI contract every + adapter satisfies. diff --git a/compliance/foundation/SPEC.md b/compliance/foundation/SPEC.md new file mode 100644 index 00000000..04bbf75a --- /dev/null +++ b/compliance/foundation/SPEC.md @@ -0,0 +1,90 @@ + + +# OSI Compliance Test Suite — Spec Coverage + +**Targets:** Foundation `osi_version: "0.1"`. Authoritative spec +documents: + +- [`../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md`](../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md) + — the semantic, query, join, M:N, and window contracts; Appendix B + (`D-001` … `D-033`) and Appendix C (error code index) are the + specific anchors this suite exercises. +- [`../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md`](../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md) + — the `OSI_SQL_2026` default expression dialect. +- [`DATA_TESTS.md`](DATA_TESTS.md) + — the normative test catalog (`T-001` … `T-NNN`) this suite encodes + as runnable cases. + +This suite is the runnable witness layer for the spec. Every +conformance decision in Appendix B has at least one `T-NNN` test here; +every error code in Appendix C has at least one negative test here. + +## What this suite does NOT cover + +- OSI features outside the Foundation surface (LOD grain modes, filter + context, named filters, semi-join filter form, per-metric + `joins.{type, using_relationships}`, non-equijoins, etc.). These + features are deferred per §10 of the Foundation proposal and are + exercised here only as negative cases that expect + `E_DEFERRED_KEY_REJECTED` (D-009). Future per-version compliance + suites under `compliance/` will cover them positively when their + proposals land. +- Engine-internal plan shape (CTE count, join order, alias names, + generated SQL string). The Foundation defines per-engine determinism + only (D-014); cross-engine SQL determinism is not required, so tests + assert on rows or `error.code` only. +- Performance characteristics. The suite is a correctness signal; any + performance regression checks live in the implementation's own + benchmark harness. + +## Compliance levels + +Defined in [`conformance.yaml`](conformance.yaml): + +| Level | Description | +|:---|:---| +| **`foundation_v0_1`** | Required for every Foundation-claiming engine. Every D-NNN in Appendix B must produce the expected outcome. | +| **`foundation_v0_1_strict`** | Adds determinism witnesses (D-014, D-029) — same `(model, query, dialect)` produces byte-identical SQL across runs. Optional. | + +Cross-engine portability is observable behaviour (rows / error codes), +not SQL text. `foundation_v0_1_strict` is per-engine determinism. + +## Decision coverage + +The mapping `D-NNN ↔ T-NNN` lives in [`decisions.yaml`](decisions.yaml). +The runner emits `results/decisions_coverage.md` after every run so +gaps are visible in PR review. + +## Adapter contract + +Adapters live under [`adapters/`](adapters/). The CLI contract is the +one published at +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md): + +``` + sql --model --query-file --dialect +``` + +stdout = generated SQL; stderr = `: ` on +failure; exit code = 0 on success, non-zero on error. The bundled +`osi_python_adapter.py` delegates to +[`../../impl/python/conformance/adapter.py`](../../impl/python/conformance/adapter.py) +so the existing CLI contract stays the source of truth — this suite +adds no new translation logic. diff --git a/compliance/foundation/adapters/osi_python_adapter.py b/compliance/foundation/adapters/osi_python_adapter.py new file mode 100644 index 00000000..76b26a22 --- /dev/null +++ b/compliance/foundation/adapters/osi_python_adapter.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""OSI Foundation compliance adapter for implementations. + +A thin delegator to ``impl/python/conformance/adapter.py``. The upstream +adapter implements the CLI contract published in +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md) (``sql --model M +--query-file Q --dialect D``); this file re-exposes it from inside the +compliance suite so the harness can resolve it from +``adapters/osi_python_adapter.py`` without reaching out of the suite. + +We deliberately do NOT re-implement any conversion logic here. If a suite +test case needs a YAML / JSON shape the upstream adapter does not +understand, fix it in the upstream adapter (a single source of truth) — +never fork the conversion logic. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_ADAPTER_DIR = Path(__file__).resolve().parent +# adapters/ is at compliance/foundation/adapters/. +# Walk up three levels (adapters → foundation → compliance → repo root), +# then descend into impl/python where the upstream adapter lives. +_REPO_ROOT = _ADAPTER_DIR.parent.parent.parent +_IMPL_PYTHON_ROOT = _REPO_ROOT / "impl" / "python" +_IMPL_PYTHON_CONFORMANCE = _IMPL_PYTHON_ROOT / "conformance" + +if not _IMPL_PYTHON_CONFORMANCE.exists(): + sys.stderr.write( + f"osi_python_adapter: cannot find upstream adapter at " + f"{_IMPL_PYTHON_CONFORMANCE}; check that impl/python/ is present " + f"at the OSI repo root.\n", + ) + sys.exit(2) + +# Make the upstream conformance package importable. The upstream adapter +# prepends impl/python/src to sys.path itself, so we only need to expose its +# parent directory here. +sys.path.insert(0, str(_IMPL_PYTHON_ROOT)) + +from conformance.adapter import main # noqa: E402 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compliance/foundation/conformance.yaml b/compliance/foundation/conformance.yaml new file mode 100644 index 00000000..5103d22b --- /dev/null +++ b/compliance/foundation/conformance.yaml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OSI Compliance Test Suite — Conformance Levels (updated Foundation) +# +# This file pins which subsets of the suite an engine MUST pass to claim +# a particular Foundation conformance level. The Foundation spec lives +# at ../../proposals/foundation/Proposed_OSI_Semantics.md (osi_version: 0.1). +# +# Each level lists the test areas that contribute to it; an +# implementation passes the level iff every must_pass case in those +# areas is green. +# +# Cross-engine SQL determinism is NOT required by Foundation v0.1 +# (per D-014); cross-engine portability is observable behaviour +# (rows or error code). The strict level adds per-engine SQL +# determinism witnesses on top of the basic foundation level. + +version: "0.1" + +levels: + foundation_v0_1: + description: > + Foundation v0.1 conformance. Required for any engine that claims + to implement the updated OSI Foundation. Every D-NNN row in + Appendix B of Proposed_OSI_Semantics.md must produce the + expected outcome (row set or error code). + pass_threshold: 100% + areas: + query_shape: all + scalar_query: all + field_metric_grain: all + cross_grain: all + nested_aggregates: all + bridge: all + joins_default: all + predicate_routing: all + namespace: all + windows: all + null_ordering: all + empty_inputs: all + deferred: all + error_taxonomy: all + + foundation_v0_1_strict: + description: > + Foundation v0.1 with per-engine SQL determinism witnesses + (D-014). Optional. Adds: same (model, query, dialect) compiled + twice on the same engine produces byte-identical SQL; outer + Order By and OVER ORDER BY emit explicit NULLS LAST per D-029. + pass_threshold: 100% + includes: [foundation_v0_1] + areas: + determinism: + all: all # D-014, D-029 SQL-text witnesses + +# Extension levels (e.g. foundation_v0_1_with_grain_modes) will be +# added when their feature proposals (§10 of the Foundation spec) +# are ratified. Each ratified proposal becomes a new level here that +# requires foundation_v0_1 plus the proposal's tests. diff --git a/compliance/foundation/datasets/f_prelude/description.md b/compliance/foundation/datasets/f_prelude/description.md new file mode 100644 index 00000000..2e6baa61 --- /dev/null +++ b/compliance/foundation/datasets/f_prelude/description.md @@ -0,0 +1,29 @@ +# F-PRELUDE + +The flagship Foundation fixture — a single-fact star (`customers ← +orders`, `customers ← returns`) with one orphan order (customer_id = +99) and one customer with no orders (id = 4). Every Foundation +semantic that exercises join defaults, NULL group keys, multi-fact +stitch, and basic aggregation runs against this fixture. + +Authoritative source: `../../DATA_TESTS.md §3.1`. + +The deliberate edge cases: + +- **Orphan order (105)**: tests Semantic 1 (`LEFT` enrichment surfaces + orphan orders under `region = NULL`). +- **Customer 4 with returns but no orders**: tests Semantic 3 (the + `NORTH` region appears in the multi-fact stitch even though it has + no orders). +- **Both `EAST` customers (1 and 2) have orders, customer 3 in `WEST` + has orders too**: gives the bridge / chasm tests a non-degenerate + cardinality. + +Aggregate witnesses: + +- `SUM(orders.amount)` total: 455.00 +- `SUM(orders.amount)` by region (LEFT enrichment): EAST=350.00, + WEST=75.00, NULL=30.00. +- `SUM(returns.amount)` by region: EAST=10.00, WEST=5.00, NORTH=15.00. +- `COUNT(orders.id)` total: 5; with `Where status='completed'`: 4. +- Distinct customers: 4. diff --git a/compliance/foundation/datasets/f_prelude/schema.sql b/compliance/foundation/datasets/f_prelude/schema.sql new file mode 100644 index 00000000..c4a8c3c8 --- /dev/null +++ b/compliance/foundation/datasets/f_prelude/schema.sql @@ -0,0 +1,51 @@ +-- F-PRELUDE — single-fact star with multi-fact extension. +-- Source: ../../DATA_TESTS.md §3.1 +-- +-- Used by: T-001, T-002, T-003, T-004, T-005x, T-006, T-011, T-016, +-- T-021, T-029, T-033, and most query-shape / predicate-routing +-- / namespace cases. + +CREATE TABLE customers ( + id INTEGER PRIMARY KEY, + region VARCHAR, + segment VARCHAR +); + +INSERT INTO customers VALUES + (1, 'EAST', 'retail'), + (2, 'EAST', 'retail'), + (3, 'WEST', 'wholesale'), + (4, 'NORTH', 'retail'); + +CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + amount DECIMAL(10, 2), + status VARCHAR +); + +-- Note: order 105 is an orphan (customer_id = 99 is not in customers). +INSERT INTO orders VALUES + (101, 1, 100.00, 'completed'), + (102, 1, 50.00, 'completed'), + (103, 2, 200.00, 'pending'), + (104, 3, 75.00, 'completed'), + (105, 99, 30.00, 'completed'); + +CREATE TABLE returns ( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + amount DECIMAL(10, 2) +); + +-- Note: customer 4 has a return but no orders — Semantic 3 case. +INSERT INTO returns VALUES + (201, 1, 10.00), + (202, 3, 5.00), + (203, 4, 15.00); + +CREATE TABLE premium_customers ( + id INTEGER PRIMARY KEY +); + +INSERT INTO premium_customers VALUES (1), (3); diff --git a/compliance/foundation/decisions.yaml b/compliance/foundation/decisions.yaml new file mode 100644 index 00000000..c3c84fb7 --- /dev/null +++ b/compliance/foundation/decisions.yaml @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Foundation Conformance Decisions — runnable test mapping. +# +# Mirrors the D-NNN decisions called out throughout +# ../../proposals/foundation/Proposed_OSI_Semantics.md. Every D-NNN row +# from the source proposal is kept here (harness's registry-consistency +# tests require full D-NNN coverage), but this bootstrap slice only ships +# witness tests for D-020 — every other decision's tests: is intentionally +# empty pending its tests landing in a follow-up PR. The runner emits +# results/decisions_coverage.md after every run so gaps are visible. +# +# Status: +# * must_pass — the engine MUST pass every T-NNN test pinned to this +# decision; failure is a Foundation conformance failure. +# * xfail — the test is shipped but expected to fail until the +# named sprint flips it; xfail_pinned_to identifies the sprint. + +version: "0.1" + +decisions: + + - id: D-001 + title: Aggregation result cardinality + multi-measure stitch on incompatible roots + spec_ref: "§5.1.1, §6.2, §6.6" + tests: [] + status: must_pass + + - id: D-002 + title: All measures resolve at the query's grain (no per-aggregate native granularity leak) + spec_ref: §6.2 + tests: [] + status: must_pass + + - id: D-003 + title: Implicit home-grain aggregation in field expressions + spec_ref: §4.3.1 + tests: [] + status: must_pass + + - id: D-004 + title: "Default join shapes (LEFT for N:1; FULL OUTER stitch; CROSS JOIN of 1-row scalars)" + spec_ref: §6.6 + tests: [] + status: must_pass + + - id: D-005 + title: "Routing of predicates by resolved expression shape (no role: keyword)" + spec_ref: "§4.3, §6.3" + tests: [] + status: must_pass + + - id: D-006 + title: Bare-name resolves to global namespace; dataset-scoped via dataset.field + spec_ref: §4.6 + tests: [] + status: must_pass + + - id: D-007 + title: "M:N traversals must produce a result equivalent to bridge or stitch; otherwise E3012" + spec_ref: §6.8 + tests: [] + status: must_pass + + - id: D-008 + title: No per-metric joins.type override at Foundation level (deferred) + spec_ref: "§6.6, §6.9" + tests: [] + status: must_pass + + - id: D-009 + title: Deferred relationship-level keys rejected with E_DEFERRED_KEY_REJECTED in default mode + spec_ref: §11 + tests: [] + status: must_pass + + - id: D-010 + title: Aggregation vs Scalar query shape — mixing rejected with E_MIXED_QUERY_SHAPE + spec_ref: §5.1 + tests: [] + status: must_pass + + - id: D-011 + title: Bare metric reference inside Fields rejected with E_AGGREGATE_IN_SCALAR_QUERY + spec_ref: §5.1.2 + tests: [] + status: must_pass + + - id: D-012 + title: Predicate-shape errors (Where vs Having) + spec_ref: §6.3 + tests: [] + status: must_pass + + - id: D-014 + title: Per-engine determinism — same (model, query, dialect) → byte-identical SQL + spec_ref: §11 + tests: [] + status: must_pass + + - id: D-016 + title: COUNT(*) is required and counts rows of the home dataset / query grain + spec_ref: §7 + tests: [] + status: must_pass + + - id: D-018 + title: Path resolution — unique path used; multiple ⇒ E_AMBIGUOUS_PATH; none ⇒ E_NO_PATH + spec_ref: §6.9 + tests: [] + status: must_pass + + - id: D-019 + title: Reserved names (GRAIN, FILTER, QUERY_FILTER) cannot be used as user identifiers + spec_ref: §4.6.2 + tests: [] + status: must_pass + + - id: D-020 + title: "Single-step + nested cross-grain aggregates over 1:N (every aggregate category)" + spec_ref: "§4.5 form (1), §6.1 Semantic 2" + tests: + - tests/cross_grain/moderate/t-005a-single-step-sum/ + - tests/cross_grain/moderate/t-005b-single-step-avg/ + - tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/ + - tests/cross_grain/moderate/t-005e-count-distinct-single-step/ + status: must_pass + + - id: D-021 + title: "expression slot accepts string OR { dialects: [...] } per OSI_SQL_2026" + spec_ref: "§4.3 / §4.5" + tests: [] + status: must_pass + + - id: D-022 + title: Plan that forces decomposition the aggregate cannot survive ⇒ E_UNSAFE_REAGGREGATION (chasm pre-agg / stitch only — bridge plan is single-pass and not in scope) + spec_ref: "§6.2 Starting Grain, §6.7, §6.8.2" + tests: [] + status: must_pass + + - id: D-023 + title: Scalar query — fan-out / incompatible-home rejected with E_FAN_OUT_IN_SCALAR_QUERY + spec_ref: "§6.2 Final Grain, §5.1.2" + tests: [] + status: must_pass + + - id: D-024 + title: Row-level reference at finer grain ⇒ E_UNAGGREGATED_FINER_GRAIN_REFERENCE + spec_ref: §6.2 Starting Grain + tests: [] + status: must_pass + + - id: D-025 + title: Catch-all measure-grain ambiguity ⇒ E_AMBIGUOUS_MEASURE_GRAIN + spec_ref: §6.2 Starting Grain + tests: [] + status: must_pass + + - id: D-026 + title: Bridge resolution materializes distinct (fact, group-key) — Semantic 2 + spec_ref: "§6.1 Semantic 2, §6.8.1" + tests: [] + status: must_pass + + - id: D-027 + title: | + Bare cross-grain aggregate over an N:N edge resolves via the §6.8.1 + bridge-dedup construction in a single pass for every aggregate + category (distributive, algebraic, holistic). The "per-home-row-first" + interpretation requires the nested form AGG(AGG(...)), which is + deferred to §10 and raises E_NESTED_AGGREGATION_DEFERRED. + spec_ref: "§4.5 form (1), §6.8.1" + tests: [] + status: must_pass + + - id: D-028 + title: Standard SQL window functions in Measures / Fields / Order By / Having; Where ⇒ E_WINDOW_IN_WHERE + spec_ref: §6.10.1 + tests: [] + status: must_pass + + - id: D-029 + title: NULLS LAST default for outer Order By + OVER ORDER BY; emitted explicitly into compiled SQL + spec_ref: "§5.1, §6.10.2" + tests: [] + status: must_pass + + - id: D-030 + title: Window over fan-out ⇒ pre-fan-out materialization (or E_WINDOW_OVER_FANOUT_REWRITE) + spec_ref: §6.10.3 + tests: [] + status: must_pass + + - id: D-031 + title: Composing a metric on top of a windowed metric ⇒ E_WINDOWED_METRIC_COMPOSITION + spec_ref: §6.10.5 + tests: [] + status: must_pass + + - id: D-032 + title: "ROWS / RANGE with integer-literal bounds only; GROUPS or :param ⇒ E_DEFERRED_FRAME_MODE" + spec_ref: §6.10.6 + tests: [] + status: must_pass + + - id: D-033 + title: Empty / NULL aggregate behaviour follows standard SQL (COUNT⇒0, others⇒NULL) + spec_ref: "§6.11.1, §6.11.2" + tests: [] + status: must_pass diff --git a/compliance/foundation/proposals.yaml b/compliance/foundation/proposals.yaml new file mode 100644 index 00000000..59fc4a4a --- /dev/null +++ b/compliance/foundation/proposals.yaml @@ -0,0 +1,231 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OSI Foundation v0.1 — Proposal Registry +# ----------------------------------------------------------------------- +# Mirrors §10 of ../../proposals/foundation/Proposed_OSI_Semantics.md. Every +# entry is either: +# +# * status: foundation — IN scope for v0.1; engines MUST implement +# * status: deferred — OUT of scope; rejected at parse time with +# E_DEFERRED_KEY_REJECTED (D-009). +# +# Test metadata.yaml files reference these IDs via `required_features:`. +# A test that references an unknown ID is a CI failure. +# +# Adapters advertise the proposals they implement via +# `enabled_proposals.yaml` (or equivalent) per the harness contract. +# A Foundation-only adapter advertises every `status: foundation` ID +# and rejects every `status: deferred` ID with E_DEFERRED_KEY_REJECTED. + +version: "0.1" + +proposals: + # -------- Foundation (v0.1) — required for every conforming engine -- + - id: two_query_shapes + status: foundation + title: Aggregation vs Scalar query shapes (Fields clause) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + decisions: [D-001, D-010, D-011, D-023] + - id: routing_by_expression_shape + status: foundation + title: Predicate routing by resolved expression shape (no role:) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.3 + decisions: [D-005, D-012] + - id: implicit_home_grain_aggregation_rejection + status: foundation + title: Implicit home-grain aggregation in field expressions is rejected (E_AGGREGATE_IN_FIELD) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.3 + rejection_code: E_AGGREGATE_IN_FIELD + decisions: [D-003] + # D-003 and D-015 are both struck/deferred in Appendix B; the + # Foundation's active surface is the rejection rule (any aggregate + # in a field body MUST raise E_AGGREGATE_IN_FIELD per §4.3 / + # Appendix C). Per-strategy equivalence (D-015) returns when + # §10's grain-aware-functions proposal lands. + - id: cross_grain_aggregates + status: foundation + title: Single-step cross-grain aggregates over 1:N (nested form is deferred per D-020(c)/D-027(d)) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.5 + decisions: [D-020, D-024] + - id: default_join_shapes + status: foundation + title: Default LEFT for N:1, FULL OUTER stitch for incompatible-root multi-fact + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.6 + decisions: [D-001, D-004, D-008] + - id: bridge_dedup + status: foundation + title: Bridge resolution materializes distinct (fact, group-key) in a single-pass aggregate (every category) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8.1 + decisions: [D-026, D-027] + - id: decomposition_safety + status: foundation + title: E_UNSAFE_REAGGREGATION fires only when the chosen plan forces decomposition the aggregate cannot survive (chasm pre-agg / stitch — bridge plan is single-pass and not in scope) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.2 + decisions: [D-022] + - id: identifier_resolution + status: foundation + title: Global / dataset-scoped namespace, ambiguous-path / no-path errors + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.6 + decisions: [D-006, D-018, D-019] + - id: window_functions_standard + status: foundation + title: Standard SQL window functions (ranking / navigation / aggregate-windows; ROWS / RANGE; integer-literal bounds) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10 + decisions: [D-028, D-030, D-031, D-032] + - id: nulls_last_default + status: foundation + title: Default NULLS LAST emission for outer Order By + OVER ORDER BY + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + decisions: [D-014, D-029] + - id: empty_aggregate_standard_sql + status: foundation + title: Standard SQL empty / NULL aggregate behaviour + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.11 + decisions: [D-033] + - id: osi_sql_2026_dialect + status: foundation + title: OSI_SQL_2026 default dialect with per-dialect expression form + spec_ref: ../../proposals/foundation/SQL_EXPRESSION_SUBSET.md + decisions: [D-021, D-016] + - id: windowed_metric_composition + status: foundation + title: Composing a metric on top of a windowed metric is rejected with E_WINDOWED_METRIC_COMPOSITION + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10.5 + rejection_code: E_WINDOWED_METRIC_COMPOSITION + decisions: [D-031] + + # -------- Deferred (§10 of the Foundation spec) --------------------- + # Each deferred entry MUST be rejected at parse time with + # E_DEFERRED_KEY_REJECTED (D-009). The compliance suite ships exactly + - id: explicit_grain_overrides + status: deferred + title: Explicit grain overrides (FIXED / INCLUDE / EXCLUDE / TABLE) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: filter_context_propagation + status: deferred + title: Filter context propagation (reset, filter.expression on metrics) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: metric_composition_grain_filter + status: deferred + title: Metric composition with grain or filter inheritance + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: natural_grain_top_level + status: deferred + title: Model-level natural_grain declaration + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: path_disambiguation_using_relationships + status: deferred + title: Per-metric joins.using_relationships path disambiguation + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: per_metric_joins_type + status: deferred + title: Per-metric joins.type override (INNER / LEFT / FULL) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: non_equijoin_relationships + status: deferred + title: condition / cardinality on relationships (non-equijoin) + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: asof_range_relationships + status: deferred + title: ASOF and Range relationships + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: referential_integrity_keys + status: deferred + title: referential_integrity / from_all_rows_match / to_all_rows_match + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: semi_additive_measures + status: deferred + title: Semi-additive measures over snapshot facts + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: grouping_sets + status: deferred + title: GROUPING SETS / ROLLUP / CUBE + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: pivot_operator + status: deferred + title: PIVOT / UNPIVOT + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: parameterized_window_frame_bounds + status: deferred + title: Parameterized window frame bounds (e.g. ROWS BETWEEN :n PRECEDING ...) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_FRAME_MODE + - id: groups_frame_mode + status: deferred + title: GROUPS frame mode (non-portable) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_FRAME_MODE + - id: ordered_set_aggregates + status: deferred + title: WITHIN GROUP ordered-set aggregates (LISTAGG, PERCENTILE_CONT) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + # ----- moved up: this is a Foundation rejection rule, not deferral + # See the foundation block above for the actual entry. + - id: named_filters + status: deferred + title: Reusable named boolean filters referenced by name + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: dataset_scope_filters + status: deferred + title: Dataset-level filters with scope-based propagation + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: semi_join_filter_form + status: deferred + title: EXISTS_IN / NOT EXISTS_IN semi-join filter (separate proposal pending) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: role_keyword + status: deferred + title: "role: keyword on fields/datasets (replaced by routing-by-shape D-005)" + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: legacy_attr_unsafe_agg_grainagg + status: deferred + title: Legacy ATTR / UNSAFE / AGG / GRAIN_AGG keyword set + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: multi_hop_bridge + status: deferred + title: Multi-hop bridge resolution (more than one bridge between the same endpoints) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: symmetric_aggregates + status: deferred + title: Looker-style symmetric aggregates (codegen-side, not a correctness mechanism) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: snowflake_partition_by_excluding + status: deferred + title: Snowflake ``PARTITION BY ... EXCLUDING`` window extension (vendor-specific, outside OSI_SQL_2026) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#12a + rejection_code: E_DEFERRED_KEY_REJECTED diff --git a/compliance/foundation/pyproject.toml b/compliance/foundation/pyproject.toml new file mode 100644 index 00000000..15aa728f --- /dev/null +++ b/compliance/foundation/pyproject.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "osi_compliance_foundation_v0_1" +version = "0.1.0" +description = "OSI Foundation v0.1 compliance test suite (osi_version 0.1)." +readme = "README.md" +requires-python = ">=3.11" +authors = [{ name = "OSI Foundation contributors" }] +license = { text = "Apache-2.0" } +dependencies = [ + "duckdb>=0.10", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", +] + +# The compliance runner harness is vendored at compliance/harness/ inside +# this OSI repo. To install everything needed to run the suite: +# +# pip install -e ../harness +# pip install -e . +# +# Both packages are editable so iterating on either is one-step. + +[tool.setuptools.packages.find] +where = ["."] +include = ["adapters*"] +exclude = ["tests*", "datasets*", "results*"] diff --git a/compliance/foundation/tests/README.md b/compliance/foundation/tests/README.md new file mode 100644 index 00000000..feb149b1 --- /dev/null +++ b/compliance/foundation/tests/README.md @@ -0,0 +1,87 @@ + + +# Foundation v0.1 compliance tests + +> **Bootstrap-slice note.** Only `cross_grain/moderate/` is physically +> present on disk right now (4 tests: `t-005a`, `t-005b`, `t-005d`, +> `t-005e`, covering D-020). Every other row in the Layout table below +> describes the target design and will land, with its own tests, in +> follow-up PRs — see `../README.md` for current status. + +Each test is a folder containing: + +| File | Purpose | +|:---|:---| +| `metadata.yaml` | `id: T-NNN`, `decision: D-NNN`, `dataset: f_*`, `spec_refs`, `required_features`, `expected_error_code` (negative cases), `xfail_reason` (if pinned to a sprint). | +| `model.yaml` | The semantic model the test runs against. | +| `query.json` | The semantic query, in the new two-shape format (`dimensions` + `measures` for aggregation queries; `fields` for scalar queries). | +| `gold.sql` | Hand-written reference SQL the harness executes against the fixture data to produce the expected row multiset. Positive cases only; negative cases keep a stub `gold.sql` so the harness can locate the directory. | + +Tests assert on observable behaviour only. The harness executes both +`gold.sql` and the adapter's emitted SQL against the shared fixture +and compares the resulting row multisets — order-insensitive unless +the query has an `Order By`. SQL strings are never compared +byte-for-byte (per D-014 that's a per-engine concern). + +## Layout + +| Area | What it covers | Anchor decisions | +|:---|:---|:---| +| `query_shape/` | Aggregation vs Scalar shape, mixing rejection, COUNT(*), expression-form dialects | D-001, D-010, D-002, D-016, D-021 | +| `scalar_query/` | Scalar query specifics: bare metric in Fields, fan-out rejection | D-011, D-023 | +| `field_metric_grain/` | Field expressions with implicit home-grain aggregation | D-003, D-015 | +| `cross_grain/` | Single-step cross-grain aggregates (1:N) | D-020, D-024 | +| `nested_aggregates/` | Nested aggregates over M:N (inner-grain inference) | D-022, D-027 | +| `bridge/` | M:N bridge resolution + de-duplication (Semantic 2) | D-026 (T-015 flagship) | +| `joins_default/` | LEFT for N:1, FULL OUTER stitch, CROSS JOIN of scalars | D-001, D-004, D-008 | +| `predicate_routing/` | Where vs Having shape errors | D-005, D-012 | +| `namespace/` | Global / dataset-scoped resolution, ambiguous / no path, reserved names | D-006, D-018, D-019 | +| `windows/` | Window placement, pre-fan-out, deferred frame modes, composition | D-028, D-030, D-031, D-032 | +| `null_ordering/` | NULLS LAST default emission, per-engine determinism witnesses | D-014, D-029 | +| `empty_inputs/` | Standard SQL empty / NULL aggregate behaviour | D-033 | +| `deferred/` | One negative test per `E_DEFERRED_KEY_REJECTED` family | D-009 | +| `validation_errors/` | Structural validation errors that are NOT deferred features (e.g. `E_FIELD_DEPENDENCY_CYCLE` — fields must form a DAG). | Appendix C | +| `error_taxonomy/` | Remaining Appendix C codes (E_PRIMARY_KEY_REQUIRED, E_AMBIGUOUS_MEASURE_GRAIN, etc.) | Appendix C | + +## Sprint timeline + +- **S-B**: this scaffold + `tests/README.md` (no test cases yet). +- **S-C**: lands `T-001 … T-033` plus the `E_DEFERRED_KEY_REJECTED` + negative tests, derived from `DATA_TESTS.md §4`. +- **S-E**: lands additional `T-NNN` cases from the differential / + edge-case audit before any S-1..S-17 sprint runs. + +## Negative tests + +A test is negative when its `metadata.yaml` carries +`expected_error_code: E_` (no `gold_rows.json`). The runner +asserts that the adapter exited non-zero AND that stderr contains the +named error code. Negative tests do NOT carry `required_features:` +unless the rejection itself depends on a Foundation-level feature; the +goal is for every adapter (Foundation or extended) to produce the same +error. + +## xfail policy + +A case marked `xfail_reason: ` is shipped but expected to fail +until the named sprint flips it to `must_pass`. Every `xfail_reason` +MUST cite the sprint ID (e.g., `xfail_reason: "Sprint S-7 — default +join shape rewrite (D-004)"`) so the rollout can find every red row +when the implementation lands. diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql new file mode 100644 index 00000000..72c1f429 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, SUM(o.amount) AS total_order_amount +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml new file mode 100644 index 00000000..c2e07555 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005a-single-step-sum +description: "Single-step cross-grain SUM over 1:N accepted; identical to nested SUM (distributive)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, distributive] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005a +status: planned +xfail_reason: "Sprint S-5 — single-step cross-grain aggregation (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml new file mode 100644 index 00000000..f40095f4 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: total_order_amount, expression: "SUM(orders.amount)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json new file mode 100644 index 00000000..0cdd1322 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "total_order_amount", + "metric": "total_order_amount" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql new file mode 100644 index 00000000..b02fe99a --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, AVG(o.amount) AS avg_order_amount +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml new file mode 100644 index 00000000..700f35a9 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005b-single-step-avg +description: "Single-step cross-grain AVG over 1:N accepted; heavy-customer-weighted (standard SQL AVG semantics)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, non-distributive] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005b +status: planned +xfail_reason: "Sprint S-5 — single-step cross-grain AVG (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml new file mode 100644 index 00000000..ddbd0a0d --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: avg_order_amount, expression: "AVG(orders.amount)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json new file mode 100644 index 00000000..4a39b104 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "avg_order_amount", + "metric": "avg_order_amount" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql new file mode 100644 index 00000000..45078103 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, SUM(o.amount) AS sum_single +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml new file mode 100644 index 00000000..a46fdd62 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005d-single-step-vs-nested-sum +description: "Single-step SUM and nested SUM(SUM(...)) over 1:N produce identical row sets (distributive)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, distributive, equivalence] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005d +status: planned +xfail_reason: "Sprint S-5 — single-step ≡ nested SUM equivalence (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml new file mode 100644 index 00000000..7dbe616f --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml @@ -0,0 +1,39 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: sum_single, expression: "SUM(orders.amount)"} + # sum_nested intentionally removed — nested aggregation in a metric + # body is deferred (D-027(d)); see Proposed_OSI_Semantics.md §10. diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json new file mode 100644 index 00000000..42ed57b7 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "sum_single", + "metric": "sum_single" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql new file mode 100644 index 00000000..cf18293f --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, COUNT(DISTINCT o.status) AS distinct_order_statuses +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml new file mode 100644 index 00000000..5979567a --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml @@ -0,0 +1,14 @@ +name: t-005e-count-distinct-single-step +description: "Single-step COUNT(DISTINCT) over 1:N accepted (holistic but well-defined on plain 1:N)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" + - "Proposed_OSI_Semantics.md#D-022" +tags: [cross-grain, 1:N, count-distinct, holistic] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005e +status: planned +xfail_reason: "Sprint S-5 — COUNT(DISTINCT) single-step over 1:N (D-020 (d), D-022 caveat)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml new file mode 100644 index 00000000..916557fc --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: distinct_order_statuses, expression: "COUNT(DISTINCT orders.status)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json new file mode 100644 index 00000000..09b7c631 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "distinct_order_statuses", + "metric": "distinct_order_statuses" + } + ] +} diff --git a/compliance/harness/README.md b/compliance/harness/README.md new file mode 100644 index 00000000..cee3a148 --- /dev/null +++ b/compliance/harness/README.md @@ -0,0 +1,54 @@ + + +# OSI Compliance Harness + +Shared runner / reporter / DB manager for the OSI compliance suite. + +This package is the engine behind every per-version compliance suite +under `compliance/`. It is **engine-agnostic** — it does not know about +any specific OSI implementation. Engines plug in via an *adapter* that +implements the CLI contract documented in +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md). + +## Install + +```bash +pip install -e . +``` + +This installs the `harness` package. The compliance suites then depend on +this package to run their tests. + +## Run + +The harness resolves ``--output`` relative to the current working +directory, so run it from the suite root (per-run artifacts then land +under ``/results/latest/`` by default): + +```bash +cd ../foundation +python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ +``` + +See [`../foundation-v0.1/README.md`](../foundation/README.md) for +the suite-level entry point and reporting layout. diff --git a/compliance/harness/pyproject.toml b/compliance/harness/pyproject.toml new file mode 100644 index 00000000..4e7995bd --- /dev/null +++ b/compliance/harness/pyproject.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "osi_compliance_harness" +version = "0.1.0" +description = "Shared runner / reporter / DB manager for the OSI compliance suite." +readme = "README.md" +requires-python = ">=3.11" +authors = [{ name = "OSI Foundation contributors" }] +license = { text = "Apache-2.0" } +dependencies = [ + "duckdb>=0.10", + "pyyaml>=6.0", + "sqlglot>=25", +] + +[project.optional-dependencies] +dev = ["pytest>=7.4"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["harness*"] diff --git a/compliance/harness/src/harness/__init__.py b/compliance/harness/src/harness/__init__.py new file mode 100644 index 00000000..9aa0a35e --- /dev/null +++ b/compliance/harness/src/harness/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""OSI Compliance Test Suite — Test Harness.""" diff --git a/compliance/harness/src/harness/__main__.py b/compliance/harness/src/harness/__main__.py new file mode 100644 index 00000000..69b68036 --- /dev/null +++ b/compliance/harness/src/harness/__main__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Allow running the harness as: python -m harness""" + +from .runner import main + +raise SystemExit(main()) diff --git a/compliance/harness/src/harness/db_manager.py b/compliance/harness/src/harness/db_manager.py new file mode 100644 index 00000000..48b7b4fb --- /dev/null +++ b/compliance/harness/src/harness/db_manager.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""DuckDB database management for the test harness.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import duckdb +import sqlglot + + +class DBManager: + """Manages DuckDB connections and dataset loading.""" + + def __init__(self) -> None: + self._conn: duckdb.DuckDBPyConnection | None = None + self._loaded_datasets: set[str] = set() + + def connect(self) -> duckdb.DuckDBPyConnection: + """Create a fresh in-memory DuckDB connection.""" + self.close() + self._conn = duckdb.connect(":memory:") + self._loaded_datasets.clear() + return self._conn + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + self._loaded_datasets.clear() + + def load_dataset(self, dataset_name: str, datasets_dir: Path) -> None: + """Load a dataset's schema.sql into the current connection.""" + if self._conn is None: + raise RuntimeError("No active database connection") + + if dataset_name in self._loaded_datasets: + return + + schema_path = datasets_dir / dataset_name / "schema.sql" + if not schema_path.exists(): + raise FileNotFoundError(f"Dataset schema not found: {schema_path}") + + # Split with sqlglot rather than a naive ``.split(";")`` so that + # semicolons inside string literals and ``--``/``/* */`` comments + # are handled correctly. ``comments=False`` drops comment-only + # statements (they render as empty text) so we skip them below. + sql_text = schema_path.read_text() + for statement in sqlglot.parse(sql_text, dialect="duckdb"): + if statement is None: + continue + stmt = statement.sql(dialect="duckdb", comments=False).strip() + if stmt: + self._conn.execute(stmt) + + self._loaded_datasets.add(dataset_name) + + def execute_sql(self, sql: str) -> list[dict[str, Any]]: + """Execute SQL and return results as a list of dicts.""" + if self._conn is None: + raise RuntimeError("No active database connection") + + result = self._conn.execute(sql) + columns = [desc[0] for desc in result.description] + rows = result.fetchall() + + return [dict(zip(columns, row)) for row in rows] + + def reset(self) -> None: + """Drop all tables and reset the connection.""" + self.connect() diff --git a/compliance/harness/src/harness/models.py b/compliance/harness/src/harness/models.py new file mode 100644 index 00000000..6e732a4a --- /dev/null +++ b/compliance/harness/src/harness/models.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Data models for the OSI compliance test harness.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from functools import cached_property +from pathlib import Path +from typing import Any + + +class TestStatus(str, Enum): + __test__ = False # not a pytest test class despite the "Test" prefix + + PASS = "pass" + FAIL = "fail" + ERROR = "error" + SKIP = "skip" + + +@dataclass(frozen=True) +class TestCase: + """A single compliance test case loaded from disk.""" + + __test__ = False # not a pytest test class despite the "Test" prefix + + test_id: str + name: str + description: str + area: str + difficulty: str + dataset: str + spec_refs: list[str] + tags: list[str] + model_path: Path + query_path: Path + gold_sql_path: Path + test_dir: Path + expected_error: bool = False + expected_error_code: str = "" + # Default matches the base level in foundation/conformance.yaml. Tests + # may override with any level declared there (e.g. foundation_v0_1_strict). + conformance_level: str = "foundation_v0_1" + status: str = "active" # "active" or "planned" — planned tests skipped unless --include-planned + required_features: list[str] = field(default_factory=list) # Feature IDs; skip if adapter doesn't support + + @cached_property + def has_order_by(self) -> bool: + """Check if the query specifies an order_by clause.""" + qdict = json.loads(self.query_path.read_text()) + return bool(qdict.get("order_by")) + + +@dataclass +class TestResult: + """Result of running a single test case.""" + + __test__ = False # not a pytest test class despite the "Test" prefix + + test_id: str + area: str + difficulty: str + status: TestStatus + spec_refs: list[str] = field(default_factory=list) + error_type: str = "" + error_detail: str = "" + generated_sql: str = "" + generated_rows: list[dict[str, Any]] = field(default_factory=list) + gold_rows: list[dict[str, Any]] = field(default_factory=list) + duration_ms: float = 0.0 + required_features: list[str] = field(default_factory=list) + + +@dataclass +class SuiteResult: + """Aggregate results for a test suite run.""" + + adapter: str + results: list[TestResult] = field(default_factory=list) + # None means "no filter applied" (all proposals implicitly enabled). + adapter_features: frozenset[str] | None = None + + @property + def total(self) -> int: + return len(self.results) + + @property + def passed(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.PASS) + + @property + def failed(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.FAIL) + + @property + def errors(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.ERROR) + + @property + def skipped(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.SKIP) diff --git a/compliance/harness/src/harness/proposals_check.py b/compliance/harness/src/harness/proposals_check.py new file mode 100644 index 00000000..32b525c2 --- /dev/null +++ b/compliance/harness/src/harness/proposals_check.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate ``required_features`` in test metadata against proposals.yaml. + +Run via ``python -m harness.proposals_check`` (see ``__main__`` guard). +CI calls this; a non-zero exit signals an unknown or misspelled proposal +ID, which would otherwise silently skip tests on every adapter. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Iterable + +import yaml + +PROPOSALS_FILE = "proposals.yaml" +TESTS_DIR = "tests" +METADATA_FILE = "metadata.yaml" +# ``foundation`` denotes a proposal that is in scope for Foundation v0.1 +# (the proposal at proposals/foundation/); ``deferred`` denotes a +# §10 deferred feature; ``proposed`` / ``thin_slice`` were the +# pre-Foundation rollout statuses. New rows should use ``foundation`` +# or ``deferred`` — the older two are retained so legacy proposal rows +# parse without churn. +VALID_STATUS = {"foundation", "thin_slice", "proposed", "deferred"} + + +class ProposalsError(Exception): + """Raised when the registry itself is malformed.""" + + +def load_proposal_ids(root: Path) -> set[str]: + """Return the set of declared proposal IDs. + + Raises :class:`ProposalsError` if the registry is missing keys, + has duplicates, or lists an unknown ``status``. + """ + path = root / PROPOSALS_FILE + data = yaml.safe_load(path.read_text()) + if not isinstance(data, dict) or "proposals" not in data: + raise ProposalsError(f"{path}: missing top-level 'proposals:' key") + + ids: set[str] = set() + for entry in data["proposals"]: + if not isinstance(entry, dict): + raise ProposalsError(f"{path}: every proposal must be a mapping") + try: + pid = entry["id"] + status = entry["status"] + except KeyError as exc: + raise ProposalsError( + f"{path}: proposal entry missing required key: {exc}" + ) from exc + if status not in VALID_STATUS: + raise ProposalsError( + f"{path}: proposal {pid!r} has invalid status {status!r}; " + f"expected one of {sorted(VALID_STATUS)}" + ) + if pid in ids: + raise ProposalsError(f"{path}: duplicate proposal id {pid!r}") + ids.add(pid) + return ids + + +def iter_metadata_files(root: Path) -> Iterable[Path]: + """Yield every ``metadata.yaml`` under the tests tree.""" + yield from sorted((root / TESTS_DIR).rglob(METADATA_FILE)) + + +def collect_unknown_references( + root: Path, valid_ids: set[str] +) -> list[tuple[Path, list[str]]]: + """Return [(metadata_path, unknown_ids)] for every offending file.""" + offenders: list[tuple[Path, list[str]]] = [] + for meta_path in iter_metadata_files(root): + meta = yaml.safe_load(meta_path.read_text()) or {} + features = meta.get("required_features", []) or [] + if not isinstance(features, list): + offenders.append((meta_path, [f": {features!r}"])) + continue + unknown = [f for f in features if f not in valid_ids] + if unknown: + offenders.append((meta_path, unknown)) + return offenders + + +def _format_report( + offenders: list[tuple[Path, list[str]]], + valid_ids: set[str], + root: Path, +) -> str: + lines = [ + "ERROR: unknown proposal IDs referenced in test metadata.", + f"Registry: {(root / PROPOSALS_FILE).as_posix()}", + "", + "Offending files:", + ] + for path, unknown in offenders: + rel = path.relative_to(root) + lines.append(f" {rel}") + for u in unknown: + lines.append(f" - {u}") + lines.append("") + lines.append( + "Either add the proposal to proposals.yaml (with a status), " + "or fix the spelling in the metadata file." + ) + lines.append(f"Known IDs ({len(valid_ids)}): {', '.join(sorted(valid_ids))}") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + """Entry point. Returns 0 on success, 1 on any violation.""" + argv = argv if argv is not None else sys.argv[1:] + root = Path(argv[0]).resolve() if argv else Path(__file__).resolve().parent.parent + + try: + valid_ids = load_proposal_ids(root) + except ProposalsError as exc: + print(f"FATAL: {exc}", file=sys.stderr) + return 2 + + offenders = collect_unknown_references(root, valid_ids) + if offenders: + print(_format_report(offenders, valid_ids, root), file=sys.stderr) + return 1 + + count = sum(1 for _ in iter_metadata_files(root)) + print( + f"OK: {count} metadata files validated against {len(valid_ids)} " + f"registered proposal IDs." + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/compliance/harness/src/harness/reporter.py b/compliance/harness/src/harness/reporter.py new file mode 100644 index 00000000..0966eefb --- /dev/null +++ b/compliance/harness/src/harness/reporter.py @@ -0,0 +1,234 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Report generation for the OSI compliance test suite.""" + +from __future__ import annotations + +import csv +import io +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +from .models import SuiteResult, TestResult, TestStatus + +PROPOSALS_FILE_NAME = "proposals.yaml" + + +def write_reports(suite: SuiteResult, output_dir: Path) -> tuple[Path, Path]: + """Write failure CSV and summary MD to output_dir. Returns (csv_path, md_path).""" + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = output_dir / "failures.csv" + md_path = output_dir / "summary.md" + + _write_failures_csv(suite, csv_path) + _write_summary_md(suite, md_path) + + return csv_path, md_path + + +def _write_failures_csv(suite: SuiteResult, path: Path) -> None: + failures = [r for r in suite.results if r.status in (TestStatus.FAIL, TestStatus.ERROR)] + + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow( + [ + "test_id", + "area", + "difficulty", + "spec_refs", + "error_type", + "details", + ] + ) + for r in failures: + writer.writerow( + [ + r.test_id, + r.area, + r.difficulty, + "; ".join(r.spec_refs), + r.error_type, + r.error_detail, + ] + ) + + +def _write_summary_md(suite: SuiteResult, path: Path) -> None: + buf = io.StringIO() + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + buf.write("# OSI Compliance Test Suite — Summary\n\n") + buf.write(f"**Adapter**: {suite.adapter} \n") + buf.write(f"**Date**: {now} \n\n") + + pct = (suite.passed / suite.total * 100) if suite.total > 0 else 0 + buf.write("## Overall\n\n") + buf.write("| Metric | Count |\n") + buf.write("|--------|-------|\n") + buf.write(f"| Total | {suite.total} |\n") + buf.write(f"| Passed | {suite.passed} |\n") + buf.write(f"| Failed | {suite.failed} |\n") + buf.write(f"| Errors | {suite.errors} |\n") + buf.write(f"| Skipped | {suite.skipped} |\n") + buf.write(f"| **Compliance** | **{pct:.1f}%** |\n\n") + + _write_breakdown(buf, "By Area", suite.results, key=lambda r: r.area) + _write_breakdown(buf, "By Difficulty", suite.results, key=lambda r: r.difficulty) + _write_proposals_status(buf, suite) + + failures = [r for r in suite.results if r.status in (TestStatus.FAIL, TestStatus.ERROR)] + if failures: + buf.write("## Failures\n\n") + buf.write("| Test | Area | Difficulty | Error |\n") + buf.write("|------|------|------------|-------|\n") + for r in failures: + detail = r.error_detail[:80].replace("|", "\\|") if r.error_detail else "" + buf.write(f"| {r.test_id} | {r.area} | {r.difficulty} | {detail} |\n") + buf.write("\n") + + path.write_text(buf.getvalue()) + + +def _write_breakdown( + buf: io.StringIO, + title: str, + results: list[TestResult], + key, +) -> None: + groups: dict[str, list[TestResult]] = defaultdict(list) + for r in results: + groups[key(r)].append(r) + + buf.write(f"## {title}\n\n") + buf.write("| Group | Total | Pass | Fail | Error | Skip | % |\n") + buf.write("|-------|-------|------|------|-------|------|---|\n") + + for group_name in sorted(groups.keys()): + group = groups[group_name] + total = len(group) + passed = sum(1 for r in group if r.status == TestStatus.PASS) + failed = sum(1 for r in group if r.status == TestStatus.FAIL) + errored = sum(1 for r in group if r.status == TestStatus.ERROR) + skipped = sum(1 for r in group if r.status == TestStatus.SKIP) + pct = (passed / total * 100) if total > 0 else 0 + buf.write(f"| {group_name} | {total} | {passed} | {failed} " f"| {errored} | {skipped} | {pct:.0f}% |\n") + buf.write("\n") + + +def _load_proposal_registry(*starts: Path) -> dict[str, str]: + """Return ``{proposal_id: status}``, searching upwards for ``proposals.yaml``. + + Walks up from every given start path. Empty dict when the registry + isn't findable; the caller falls back to the inferred set of IDs + seen across results. + """ + tried: set[Path] = set() + for start in starts: + cursor = start.resolve() + for _ in range(6): + if cursor in tried: + break + tried.add(cursor) + candidate = cursor / PROPOSALS_FILE_NAME + if candidate.exists(): + data = yaml.safe_load(candidate.read_text()) or {} + return {p["id"]: p.get("status", "") for p in data.get("proposals", [])} + if cursor.parent == cursor: + break + cursor = cursor.parent + return {} + + +def _write_proposals_status(buf: io.StringIO, suite: SuiteResult) -> None: + """Emit the Proposals-status section. + + For every proposal ID that is either advertised by the adapter or + referenced by at least one result, report: + + * whether the adapter advertised it (``enabled``), + * total tests that require it, + * tests that ran vs. were skipped, + * pass rate among the ones that ran. + """ + registry = _load_proposal_registry(Path.cwd(), Path(__file__).parent) + + referenced: set[str] = set() + for r in suite.results: + referenced.update(r.required_features) + advertised: set[str] = set(suite.adapter_features or ()) + all_ids = sorted(referenced | advertised | set(registry.keys())) + if not all_ids: + return + + counts_total: dict[str, int] = defaultdict(int) + counts_ran: dict[str, int] = defaultdict(int) + counts_passed: dict[str, int] = defaultdict(int) + counts_skipped: dict[str, int] = defaultdict(int) + + for r in suite.results: + for feat in r.required_features: + counts_total[feat] += 1 + if r.status == TestStatus.SKIP and r.error_type == "unsupported_proposal": + counts_skipped[feat] += 1 + else: + counts_ran[feat] += 1 + if r.status == TestStatus.PASS: + counts_passed[feat] += 1 + + buf.write("## Proposals Status\n\n") + if suite.adapter_features is None: + buf.write("_No `--proposals` filter applied; every proposal implicitly enabled._\n\n") + else: + buf.write(f"Adapter advertised {len(advertised)} proposal(s): " f"`{', '.join(sorted(advertised)) or '(none)'}`\n\n") + + buf.write("| Proposal | Status | Enabled | Tests | Ran | Passed | Skipped | Pass% |\n") + buf.write("|----------|--------|---------|-------|-----|--------|---------|-------|\n") + for pid in all_ids: + status = registry.get(pid, "unknown") + enabled = "yes" if (suite.adapter_features is None or pid in advertised) else "no" + total = counts_total.get(pid, 0) + ran = counts_ran.get(pid, 0) + passed = counts_passed.get(pid, 0) + skipped = counts_skipped.get(pid, 0) + pct = f"{(passed / ran * 100):.0f}%" if ran else "—" + buf.write(f"| `{pid}` | {status} | {enabled} | {total} | {ran} | {passed} | {skipped} | {pct} |\n") + buf.write("\n") + + +def format_summary_console(suite: SuiteResult) -> str: + """Format a brief console summary.""" + pct = (suite.passed / suite.total * 100) if suite.total > 0 else 0 + lines = [ + f"\n{'=' * 50}", + f"OSI Compliance Test Results — {suite.adapter}", + f"{'=' * 50}", + f" Total: {suite.total}", + f" Passed: {suite.passed}", + f" Failed: {suite.failed}", + f" Errors: {suite.errors}", + f" Skipped: {suite.skipped}", + f" Compliance: {pct:.1f}%", + ] + if suite.adapter_features is not None: + lines.append(f" Proposals: {', '.join(sorted(suite.adapter_features)) or '(none)'}") + lines.append(f"{'=' * 50}") + return "\n".join(lines) diff --git a/compliance/harness/src/harness/result_compare.py b/compliance/harness/src/harness/result_compare.py new file mode 100644 index 00000000..36421f89 --- /dev/null +++ b/compliance/harness/src/harness/result_compare.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Result comparison logic with order-insensitive and numeric-tolerant matching.""" + +from __future__ import annotations + +import math +from datetime import date, datetime, time +from decimal import Decimal +from typing import Any + +DEFAULT_EPSILON = 0.0001 + + +def compare_results( + generated: list[dict[str, Any]], + gold: list[dict[str, Any]], + *, + ordered: bool = False, + epsilon: float = DEFAULT_EPSILON, +) -> tuple[bool, str]: + """Compare two result sets. + + Returns (is_match, detail_message). + """ + if len(generated) != len(gold): + return False, (f"Row count mismatch: generated {len(generated)} rows, " f"gold {len(gold)} rows") + + if not generated: + return True, "Both result sets are empty" + + gen_cols = set(_normalize_key(k) for k in generated[0].keys()) + gold_cols = set(_normalize_key(k) for k in gold[0].keys()) + + if gen_cols != gold_cols: + missing = gold_cols - gen_cols + extra = gen_cols - gold_cols + parts = [] + if missing: + parts.append(f"missing columns: {sorted(missing)}") + if extra: + parts.append(f"extra columns: {sorted(extra)}") + return False, f"Column mismatch: {'; '.join(parts)}" + + gen_normalized = [_normalize_row(r) for r in generated] + gold_normalized = [_normalize_row(r) for r in gold] + + if not ordered: + gen_normalized = _sort_rows(gen_normalized) + gold_normalized = _sort_rows(gold_normalized) + + for i, (gen_row, gold_row) in enumerate(zip(gen_normalized, gold_normalized)): + match, detail = _compare_rows(gen_row, gold_row, epsilon) + if not match: + return False, f"Row {i} mismatch: {detail}" + + return True, f"All {len(generated)} rows match" + + +def _normalize_key(key: str) -> str: + return key.lower().strip() + + +def _normalize_row(row: dict[str, Any]) -> dict[str, Any]: + return {_normalize_key(k): v for k, v in row.items()} + + +def _sort_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Sort rows by all columns for order-insensitive comparison.""" + if not rows: + return rows + columns = sorted(rows[0].keys()) + + def sort_key(row: dict[str, Any]) -> tuple: + return tuple(_sortable_value(row.get(c)) for c in columns) + + return sorted(rows, key=sort_key) + + +def _sortable_value(val: Any) -> tuple: + """Convert a value to a sortable tuple (type_tag, comparable_value).""" + if val is None: + return (0, "") + if isinstance(val, bool): + return (1, int(val)) + if isinstance(val, (int, float, Decimal)): + f = float(val) + if math.isnan(f): + return (2, float("inf")) + return (2, f) + return (3, str(val)) + + +def _compare_rows( + gen: dict[str, Any], + gold: dict[str, Any], + epsilon: float, +) -> tuple[bool, str]: + for key in gold: + gen_val = gen.get(key) + gold_val = gold.get(key) + + if not _values_equal(gen_val, gold_val, epsilon): + return False, (f"column '{key}': generated={gen_val!r}, gold={gold_val!r}") + return True, "" + + +def _values_equal(a: Any, b: Any, epsilon: float) -> bool: + if a is None and b is None: + return True + if a is None or b is None: + return False + + if isinstance(a, (int, float, Decimal)) and isinstance(b, (int, float, Decimal)): + fa, fb = float(a), float(b) + if fa == fb: + return True + if math.isnan(fa) and math.isnan(fb): + return True + if math.isnan(fa) or math.isnan(fb): + return False + if abs(fa - fb) < epsilon: + return True + denom = max(abs(fa), abs(fb)) + if denom == 0: + return True + return abs(fa - fb) / denom < epsilon + + if isinstance(a, datetime) and isinstance(b, datetime): + return a == b + if isinstance(a, datetime) and isinstance(b, date): + return a.date() == b + if isinstance(a, date) and isinstance(b, datetime): + return a == b.date() + if isinstance(a, date) and isinstance(b, date): + return a == b + + if isinstance(a, time) and isinstance(b, time): + return a == b + + return str(a) == str(b) diff --git a/compliance/harness/src/harness/runner.py b/compliance/harness/src/harness/runner.py new file mode 100644 index 00000000..d65ca43f --- /dev/null +++ b/compliance/harness/src/harness/runner.py @@ -0,0 +1,599 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Main test runner for the OSI compliance test suite.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +from .db_manager import DBManager +from .models import SuiteResult, TestCase, TestResult, TestStatus +from .reporter import format_summary_console, write_reports +from .result_compare import compare_results + +CONFORMANCE_FILE_NAME = "conformance.yaml" +DEFAULT_ADAPTER_TIMEOUT = 60 + + +class AdapterTimeout(Exception): + """Raised when the adapter subprocess exceeds its time budget.""" + + def __init__(self, timeout: int) -> None: + super().__init__(f"adapter exceeded {timeout}s timeout") + self.timeout = timeout + + +def load_conformance_levels(start: Path, *, max_depth: int = 6) -> set[str]: + """Return the conformance-level names declared in the nearest + ``conformance.yaml``, searching upward from ``start``. + + Returns an empty set when no registry is found; callers then accept + any level string rather than blocking on an unknown one. + """ + cursor = start.resolve() + for _ in range(max_depth): + candidate = cursor / CONFORMANCE_FILE_NAME + if candidate.exists(): + data = yaml.safe_load(candidate.read_text()) or {} + return set((data.get("levels") or {}).keys()) + if cursor.parent == cursor: + break + cursor = cursor.parent + return set() + + +def discover_tests( + tests_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + include_planned: bool = False, +) -> list[TestCase]: + """Walk the tests directory and discover all test cases.""" + test_cases: list[TestCase] = [] + + for metadata_path in sorted(tests_dir.rglob("metadata.yaml")): + test_dir = metadata_path.parent + meta = yaml.safe_load(metadata_path.read_text()) + + test_difficulty = meta.get("difficulty", "") + test_area = meta.get("area", "") + + if difficulty and test_difficulty != difficulty: + continue + if area and test_area != area: + continue + + model_path = test_dir / "model.yaml" + query_path = test_dir / "query.json" + gold_sql_path = test_dir / "gold.sql" + + if not all(p.exists() for p in [model_path, query_path, gold_sql_path]): + print( + f"WARN: Skipping {test_dir.name} — missing required files", + file=sys.stderr, + ) + continue + + parts = test_dir.relative_to(tests_dir).parts + test_id = "/".join(parts) + + test_status = meta.get("status", "active") + if test_status == "planned" and not include_planned: + continue + + test_cases.append( + TestCase( + test_id=test_id, + name=meta.get("name", test_dir.name), + description=meta.get("description", ""), + area=test_area, + difficulty=test_difficulty, + dataset=meta.get("dataset", ""), + spec_refs=meta.get("spec_refs", []), + tags=meta.get("tags", []), + model_path=model_path, + query_path=query_path, + gold_sql_path=gold_sql_path, + test_dir=test_dir, + expected_error=bool(meta.get("expected_error", False)), + expected_error_code=meta.get("expected_error_code", ""), + conformance_level=meta.get("conformance_level", "full"), + status=test_status, + required_features=meta.get("required_features", []), + ) + ) + + return test_cases + + +def invoke_adapter( + adapter_path: Path, + model_path: Path, + query_path: Path, + dialect: str = "duckdb", + timeout: int = DEFAULT_ADAPTER_TIMEOUT, +) -> tuple[str, str, int]: + """Invoke the adapter subprocess. Returns (stdout, stderr, returncode). + + Raises :class:`AdapterTimeout` if the subprocess exceeds ``timeout`` + seconds, so callers can distinguish a timeout from an ordinary + non-zero exit. + """ + cmd = [ + sys.executable, + str(adapter_path), + "sql", + "--model", + str(model_path), + "--query-file", + str(query_path), + "--dialect", + dialect, + ] + + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + return proc.stdout, proc.stderr, proc.returncode + except subprocess.TimeoutExpired as exc: + raise AdapterTimeout(timeout) from exc + + +def run_test( + test: TestCase, + adapter_path: Path, + db: DBManager, + datasets_dir: Path, + timeout: int = DEFAULT_ADAPTER_TIMEOUT, +) -> TestResult: + """Run a single test case and return the result.""" + start = time.monotonic() + + try: + db.reset() + db.load_dataset(test.dataset, datasets_dir) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="dataset_load", + error_detail=str(e), + duration_ms=(time.monotonic() - start) * 1000, + ) + + try: + stdout, stderr, rc = invoke_adapter( + adapter_path, + test.model_path, + test.query_path, + timeout=timeout, + ) + except AdapterTimeout as exc: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="adapter_timeout", + error_detail=str(exc), + duration_ms=(time.monotonic() - start) * 1000, + ) + + if test.expected_error: + elapsed = (time.monotonic() - start) * 1000 + if rc != 0: + # Adapter correctly rejected — check error code if specified + if test.expected_error_code and test.expected_error_code not in stderr: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="wrong_error_code", + error_detail=( + f"Expected error code '{test.expected_error_code}' " + f"in stderr but got: {stderr.strip()[:200]}" + ), + duration_ms=elapsed, + ) + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.PASS, + spec_refs=test.spec_refs, + error_detail=f"Correctly rejected: {stderr.strip()[:200]}", + duration_ms=elapsed, + ) + else: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="expected_error_missing", + error_detail=( + "Expected adapter to reject this query with a non-zero " + "exit code, but it succeeded" + ), + generated_sql=stdout.strip(), + duration_ms=elapsed, + ) + + if rc != 0: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="adapter_error", + error_detail=stderr.strip() or f"exit code {rc}", + duration_ms=(time.monotonic() - start) * 1000, + ) + + generated_sql = stdout.strip() + + try: + generated_rows = db.execute_sql(generated_sql) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="generated_sql_error", + error_detail=f"Generated SQL failed: {e}", + generated_sql=generated_sql, + duration_ms=(time.monotonic() - start) * 1000, + ) + + gold_sql = test.gold_sql_path.read_text().strip() + try: + gold_rows = db.execute_sql(gold_sql) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="gold_sql_error", + error_detail=f"Gold SQL failed: {e}", + generated_sql=generated_sql, + duration_ms=(time.monotonic() - start) * 1000, + ) + + is_ordered = test.has_order_by + match, detail = compare_results( + generated_rows, + gold_rows, + ordered=is_ordered, + ) + + elapsed = (time.monotonic() - start) * 1000 + + if match: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.PASS, + spec_refs=test.spec_refs, + generated_sql=generated_sql, + generated_rows=generated_rows, + gold_rows=gold_rows, + duration_ms=elapsed, + ) + else: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="result_mismatch", + error_detail=detail, + generated_sql=generated_sql, + generated_rows=generated_rows, + gold_rows=gold_rows, + duration_ms=elapsed, + ) + + +def list_tests( + tests_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + conformance_level: str | None = None, + include_planned: bool = False, +) -> None: + """Print discovered tests without running them.""" + tests = discover_tests( + tests_dir, + difficulty=difficulty, + area=area, + include_planned=include_planned, + ) + if conformance_level: + tests = [t for t in tests if t.conformance_level == conformance_level] + + print(f"{'ID':<60} {'Area':<25} {'Diff':<12} {'Level':<10} {'Status':<8} {'Err?'}") + print("-" * 125) + for t in tests: + err = "yes" if t.expected_error else "" + status = t.status if t.status != "active" else "" + print( + f"{t.test_id:<60} {t.area:<25} {t.difficulty:<12} {t.conformance_level:<10} {status:<8} {err}" + ) + print(f"\nTotal: {len(tests)} test(s)") + + +def run_suite( + adapter_path: Path, + tests_dir: Path, + datasets_dir: Path, + output_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + conformance_level: str | None = None, + include_planned: bool = False, + verbose: bool = False, + adapter_features: set[str] | None = None, + timeout: int = DEFAULT_ADAPTER_TIMEOUT, +) -> SuiteResult: + """Run the full test suite and generate reports.""" + tests = discover_tests( + tests_dir, + difficulty=difficulty, + area=area, + include_planned=include_planned, + ) + if conformance_level: + tests = [t for t in tests if t.conformance_level == conformance_level] + + skipped_by_feature: list[TestCase] = [] + runnable_tests: list[TestCase] = tests + if adapter_features is not None: + runnable_tests = [] + for t in tests: + if t.required_features and not set(t.required_features).issubset( + adapter_features + ): + skipped_by_feature.append(t) + continue + runnable_tests.append(t) + + if not runnable_tests and not skipped_by_feature: + print("No test cases found.", file=sys.stderr) + return SuiteResult(adapter=str(adapter_path)) + + print(f"Discovered {len(tests)} test(s)") + if skipped_by_feature: + print( + f" skipping {len(skipped_by_feature)} test(s) " + "with unsupported proposals" + ) + print(f"Adapter: {adapter_path}") + print(f"Datasets: {datasets_dir}") + print() + + db = DBManager() + suite = SuiteResult( + adapter=str(adapter_path.name), + adapter_features=( + frozenset(adapter_features) if adapter_features is not None else None + ), + ) + + for t in skipped_by_feature: + missing = sorted(set(t.required_features) - set(adapter_features or ())) + suite.results.append( + TestResult( + test_id=t.test_id, + area=t.area, + difficulty=t.difficulty, + status=TestStatus.SKIP, + spec_refs=t.spec_refs, + error_type="unsupported_proposal", + error_detail=f"required proposals not advertised: {', '.join(missing)}", + required_features=list(t.required_features), + ) + ) + + for i, test in enumerate(runnable_tests, 1): + label = f"[{i}/{len(runnable_tests)}] {test.test_id}" + result = run_test(test, adapter_path, db, datasets_dir, timeout=timeout) + result.required_features = list(test.required_features) + suite.results.append(result) + + if result.status == TestStatus.PASS: + if test.expected_error: + print(f" PASS {label} (expected error)") + else: + print(f" PASS {label}") + elif result.status == TestStatus.FAIL: + print(f" FAIL {label}") + if verbose: + print(f" {result.error_detail}") + elif result.status == TestStatus.ERROR: + print(f" ERR {label}") + if verbose: + print(f" [{result.error_type}] {result.error_detail}") + else: + print(f" SKIP {label}") + + db.close() + + csv_path, md_path = write_reports(suite, output_dir) + print(format_summary_console(suite)) + print("\nReports written to:") + print(f" {csv_path}") + print(f" {md_path}") + + return suite + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="harness.runner", + description="OSI Compliance Test Suite Runner", + ) + parser.add_argument( + "--adapter", + help="Path to the adapter script (e.g., adapters/python_adapter.py)", + ) + parser.add_argument( + "--tests", + required=True, + help="Path to the tests directory", + ) + parser.add_argument( + "--datasets", + help="Path to the datasets directory", + ) + parser.add_argument( + "--output", + default="results/latest", + help=( + "Output directory for reports (default: results/latest). " + "Per-run artifacts (failures.csv, summary.md) go here. " + "The curated baseline at results/REPORT.md is committed and " + "must not be overwritten — choose a subdirectory of results/ " + "(e.g. results//) or another path entirely." + ), + ) + parser.add_argument( + "--difficulty", + choices=["easy", "moderate", "hard", "conversion"], + help="Filter tests by difficulty", + ) + parser.add_argument( + "--area", + help="Filter tests by area (e.g., grain_and_lod, filters)", + ) + parser.add_argument( + "--conformance-level", + help="Filter tests by conformance level (e.g. foundation_v0_1). " + "Valid values are the levels declared in the suite's " + "conformance.yaml.", + ) + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_ADAPTER_TIMEOUT, + help=f"Per-test adapter timeout in seconds (default: " + f"{DEFAULT_ADAPTER_TIMEOUT}). Exceeding it records the test as an " + "error with type 'adapter_timeout'.", + ) + parser.add_argument( + "--include-planned", + action="store_true", + help="Include tests with status: planned (normally skipped)", + ) + parser.add_argument( + "--list", + action="store_true", + dest="list_only", + help="List discovered tests without running them", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Show error details for failures", + ) + parser.add_argument( + "--adapter-features", + "--proposals", + nargs="*", + default=None, + dest="adapter_features", + help="Proposal IDs the adapter implements (from proposals.yaml). " + "Tests whose required_features aren't a subset of this set are " + "recorded as SKIP. Alias: --proposals.", + ) + + args = parser.parse_args() + + if args.list_only: + list_tests( + Path(args.tests), + difficulty=args.difficulty, + area=args.area, + conformance_level=args.conformance_level, + include_planned=args.include_planned, + ) + return 0 + + if not args.adapter: + parser.error("--adapter is required when running tests") + if not args.datasets: + parser.error("--datasets is required when running tests") + + if args.conformance_level: + levels = load_conformance_levels(Path(args.tests)) + if levels and args.conformance_level not in levels: + parser.error( + f"--conformance-level {args.conformance_level!r} is not " + f"defined in {CONFORMANCE_FILE_NAME}; known levels: " + f"{', '.join(sorted(levels))}" + ) + + feat = set(args.adapter_features) if args.adapter_features is not None else None + suite = run_suite( + adapter_path=Path(args.adapter), + tests_dir=Path(args.tests), + datasets_dir=Path(args.datasets), + output_dir=Path(args.output), + difficulty=args.difficulty, + area=args.area, + conformance_level=args.conformance_level, + include_planned=args.include_planned, + verbose=args.verbose, + adapter_features=feat, + timeout=args.timeout, + ) + + if suite.failed + suite.errors > 0: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compliance/harness/src/harness/tests/__init__.py b/compliance/harness/src/harness/tests/__init__.py new file mode 100644 index 00000000..a595c844 --- /dev/null +++ b/compliance/harness/src/harness/tests/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + diff --git a/compliance/harness/src/harness/tests/test_db_manager.py b/compliance/harness/src/harness/tests/test_db_manager.py new file mode 100644 index 00000000..a9c7d9e7 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_db_manager.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for DuckDB dataset loading, in particular sqlglot-based +statement splitting that a naive ``split(";")`` would get wrong.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness.db_manager import DBManager + +# A schema that breaks naive splitting: a semicolon inside a string +# literal, a ``--`` comment trailing a statement, a ``/* */`` block +# comment, and a lone comment line between statements. +TRICKY_SCHEMA = """\ +-- header comment +CREATE TABLE t (id INTEGER, note VARCHAR); -- trailing comment +/* block comment; with a semicolon */ +INSERT INTO t VALUES (1, 'a; not a separator'), (2, 'plain'); +-- lone comment between statements +INSERT INTO t VALUES (3, 'c'); +""" + + +@pytest.fixture +def db(): + manager = DBManager() + manager.connect() + yield manager + manager.close() + + +def _write_dataset(tmp_path: Path, name: str, schema: str) -> Path: + dataset_dir = tmp_path / name + dataset_dir.mkdir(parents=True) + (dataset_dir / "schema.sql").write_text(schema) + return tmp_path + + +def test_load_dataset_handles_semicolons_and_comments( + tmp_path: Path, db: DBManager +) -> None: + datasets_dir = _write_dataset(tmp_path, "d_tricky", TRICKY_SCHEMA) + + db.load_dataset("d_tricky", datasets_dir) + + rows = db.execute_sql("SELECT id, note FROM t ORDER BY id") + assert rows == [ + {"id": 1, "note": "a; not a separator"}, + {"id": 2, "note": "plain"}, + {"id": 3, "note": "c"}, + ] + + +def test_load_dataset_is_idempotent(tmp_path: Path, db: DBManager) -> None: + datasets_dir = _write_dataset( + tmp_path, + "d_once", + "CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (1);", + ) + + db.load_dataset("d_once", datasets_dir) + # A second load must not re-run the schema (which would error on the + # duplicate CREATE) — it's cached per connection. + db.load_dataset("d_once", datasets_dir) + + assert db.execute_sql("SELECT COUNT(*) AS n FROM t") == [{"n": 1}] + + +def test_load_dataset_missing_schema_raises(tmp_path: Path, db: DBManager) -> None: + with pytest.raises(FileNotFoundError): + db.load_dataset("does_not_exist", tmp_path) diff --git a/compliance/harness/src/harness/tests/test_proposals_check.py b/compliance/harness/src/harness/tests/test_proposals_check.py new file mode 100644 index 00000000..39b1d7a1 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_proposals_check.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for ``harness.proposals_check``. + +The check runs in CI and guards that every ``required_features`` entry in +test metadata is backed by a real proposal in ``proposals.yaml``. These +tests use a throwaway fixture tree rather than the live suite, so they +stay deterministic as tests come and go. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness import proposals_check + + +@pytest.fixture +def make_tree(tmp_path: Path): + """Build a minimal suite layout under ``tmp_path`` and return a builder.""" + + def _build(proposals_yaml: str, metadata: dict[str, str]) -> Path: + (tmp_path / "proposals.yaml").write_text(proposals_yaml) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + for rel, body in metadata.items(): + target = tests_dir / rel / "metadata.yaml" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + return tmp_path + + return _build + + +VALID_PROPOSALS = """\ +proposals: + - id: dataset_filters + status: proposed + title: Dataset filters + - id: non_equijoin + status: proposed + title: Non-equijoin + - id: pervasive_scope + status: proposed + title: Pervasive scope +""" + + +def test_clean_tree_returns_zero(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + { + "a/test_ok": "required_features: [dataset_filters]\n", + "b/test_multi": "required_features: [dataset_filters, non_equijoin]\n", + "c/test_no_features": "name: plain\n", + }, + ) + assert proposals_check.main([str(root)]) == 0 + assert "OK:" in capsys.readouterr().out + + +def test_unknown_feature_returns_one(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_bad": "required_features: [made_up_feature]\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + err = capsys.readouterr().err + assert "made_up_feature" in err + assert "x/test_bad" in err + + +def test_typo_is_caught(make_tree) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_typo": "required_features: [dataset_filterz]\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + + +def test_non_list_features_is_reported(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_wrong_type": "required_features: dataset_filters\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + assert "not-a-list" in capsys.readouterr().err + + +def test_duplicate_proposal_id_fails_fast(make_tree, capsys) -> None: + root = make_tree( + """\ +proposals: + - id: dupe + status: proposed + - id: dupe + status: proposed +""", + {}, + ) + assert proposals_check.main([str(root)]) == 2 + assert "duplicate" in capsys.readouterr().err + + +def test_invalid_status_fails_fast(make_tree, capsys) -> None: + root = make_tree( + """\ +proposals: + - id: foo + status: maybe +""", + {}, + ) + assert proposals_check.main([str(root)]) == 2 + assert "invalid status" in capsys.readouterr().err + + +def test_missing_top_level_key_fails_fast(make_tree, capsys) -> None: + root = make_tree("proposalz:\n - id: foo\n", {}) + assert proposals_check.main([str(root)]) == 2 + assert "top-level 'proposals:'" in capsys.readouterr().err + + +def test_live_registry_validates() -> None: + """The real ``proposals.yaml`` in the repo passes the check. + + Post-migration, ``proposals.yaml`` and the test corpus live under + ``compliance/foundation/``. We resolve the path relative to + this file so the test stays correct regardless of where the + harness package itself happens to be installed. + """ + foundation_root = ( + Path(__file__).resolve().parents[4] / "foundation" + ) + assert (foundation_root / "proposals.yaml").exists(), ( + f"proposals.yaml not found at {foundation_root} — the suite " + "layout under compliance/foundation/ has changed." + ) + assert proposals_check.main([str(foundation_root)]) == 0 diff --git a/compliance/harness/src/harness/tests/test_registry_yaml.py b/compliance/harness/src/harness/tests/test_registry_yaml.py new file mode 100644 index 00000000..89050f35 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_registry_yaml.py @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cleanliness gate for the compliance registry YAML files. + +``decisions.yaml`` is the source of truth that ties Appendix B decisions +in the Foundation spec to runnable witness tests. ``proposals.yaml`` +plays the same role for §10 deferred features. Both files have already +broken at least once because an unquoted ``:`` inside a title was +parsed by YAML as a mapping value (Phase 4 compliance review, finding +B1). This test pins both files so a future edit cannot reintroduce +that class of bug. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +_FOUNDATION_DIR = ( + Path(__file__).resolve().parents[4] / "foundation" +) + + +@pytest.mark.parametrize( + "yaml_path", + [ + _FOUNDATION_DIR / "decisions.yaml", + _FOUNDATION_DIR / "proposals.yaml", + _FOUNDATION_DIR / "conformance.yaml", + ], + ids=lambda p: p.name, +) +def test_registry_yaml_is_parseable(yaml_path: Path) -> None: + """Every registry YAML file must load as a top-level mapping. + + A future edit that introduces an unquoted ``:`` or a stray ``-`` + breaks the entire coverage-by-decision report — silently in CI + unless this test exists. The assertion is intentionally weak (only + ``isinstance(..., dict)``) because the harness loaders enforce the + shape; here we just want the bare YAML parse to succeed. + """ + assert yaml_path.exists(), f"registry file missing: {yaml_path}" + raw = yaml_path.read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + assert isinstance(loaded, dict), ( + f"{yaml_path.name} did not parse as a top-level YAML mapping; " + "every registry file is required to be a mapping with either a " + "``decisions`` or ``proposals`` top-level key." + ) + + +# Decision IDs that are intentionally absent from decisions.yaml because +# the spec has demoted them. Each entry must come with a one-line +# reference so a future reviewer can verify the demotion is still +# accurate. Update this set in lockstep with the spec. +_INTENTIONALLY_ABSENT_DECISIONS: dict[str, str] = { + # D-013 is reserved in the spec but has no Appendix-B row (the + # number is intentionally skipped — see Proposed_OSI_Semantics.md). + "D-013": "Reserved in Appendix B (number skipped).", + # D-015 is struck in Appendix B (~~D-015~~ Deferred — moved to a + # separate proposal). The compilation-strategy equivalence for + # field-level cross-grain aggregation is moot at the Foundation + # level because field-level aggregation itself is deferred per + # D-003. The Foundation surface is exercised by the D-003 + # rejection witness; the strategy equivalence returns alongside + # §10's grain-aware-functions proposal. + "D-015": "Struck in Appendix B; depends on the deferred D-003.", + # D-017 is deferred — semi-join filtering moved to a follow-up + # proposal. The negative test for EXISTS_IN is a rejection test. + "D-017": "Deferred — Proposed_OSI_Semantics.md table row marked", +} + + +def test_decisions_yaml_has_all_decision_ids() -> None: + """Every active Appendix-B decision must appear exactly once. + + Pinning the ID set here surfaces both: + + - a duplicate ID (``yaml.safe_load`` collapses duplicate keys + silently inside a mapping; we count list entries instead). + - a missing decision (Phase 4 compliance review I7 — several + decisions had no row at all). + + Decisions that the spec has intentionally demoted live in + :data:`_INTENTIONALLY_ABSENT_DECISIONS` with a rationale. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + decisions = loaded.get("decisions", []) + ids = [d["id"] for d in decisions] + + duplicates = {x for x in ids if ids.count(x) > 1} + assert not duplicates, ( + f"decisions.yaml has duplicate ids: {sorted(duplicates)}. Each " + "decision must appear exactly once." + ) + + expected = {f"D-{i:03d}" for i in range(1, 34)} + present = set(ids) + missing = expected - present - set(_INTENTIONALLY_ABSENT_DECISIONS) + assert not missing, ( + f"decisions.yaml is missing rows for: {sorted(missing)}. Every " + "active Appendix-B decision must have a registry row, even if " + "its ``tests:`` list is empty pending witness work. If a " + "decision was intentionally demoted, add it to " + "_INTENTIONALLY_ABSENT_DECISIONS with a rationale." + ) + + # Inverse check: an entry in _INTENTIONALLY_ABSENT_DECISIONS that + # actually appears in decisions.yaml is also drift. + accidental = present & set(_INTENTIONALLY_ABSENT_DECISIONS) + assert not accidental, ( + f"decisions.yaml has rows for IDs marked as intentionally " + f"absent: {sorted(accidental)}. Update " + "_INTENTIONALLY_ABSENT_DECISIONS or remove the row." + ) + + +def test_decisions_yaml_paths_exist_on_disk() -> None: + """Every ``tests:`` path in decisions.yaml must point to a real test. + + Phase 4 review B2 — pre-migration paths still littered the file + and the coverage-by-decision report was reading a fictional disk. + A path that doesn't resolve to a directory with a ``metadata.yaml`` + is a regression: either the test was renamed, moved, or deleted + and decisions.yaml wasn't updated. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + decisions = loaded.get("decisions", []) + + missing: list[tuple[str, str]] = [] + for row in decisions: + for test_rel in row.get("tests") or (): + test_dir = _FOUNDATION_DIR / test_rel + if not (test_dir / "metadata.yaml").exists(): + missing.append((row["id"], test_rel)) + + assert not missing, ( + "decisions.yaml references paths that don't exist on disk:\n" + + "\n".join(f" {d}: {p}" for d, p in missing) + + "\nRegenerate the tests: lists from disk metadata, or move " + "the renamed test back." + ) + + +def test_every_disk_test_pins_a_known_decision() -> None: + """The inverse: every metadata.yaml's ``decision`` must be in + decisions.yaml (or :data:`_INTENTIONALLY_ABSENT_DECISIONS`). + + Closes the second half of the drift gap — without this, a test + can pin a decision that no longer has a registry row and the + coverage report silently misses it. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + known = {row["id"] for row in loaded.get("decisions", [])} + known.update(_INTENTIONALLY_ABSENT_DECISIONS) + + unknown_pins: list[tuple[Path, str]] = [] + tests_dir = _FOUNDATION_DIR / "tests" + for meta_path in sorted(tests_dir.rglob("metadata.yaml")): + metadata = yaml.safe_load(meta_path.read_text()) + decision = metadata.get("decision") or metadata.get("decisions") + decisions = ( + [decision] + if isinstance(decision, str) + else (decision or []) + ) + for d in decisions: + if d not in known: + unknown_pins.append((meta_path, d)) + + assert not unknown_pins, ( + "Test metadata pins decisions that don't appear in " + "decisions.yaml:\n" + + "\n".join( + f" {p.relative_to(_FOUNDATION_DIR)}: {d}" + for p, d in unknown_pins + ) + ) diff --git a/compliance/harness/src/harness/tests/test_reporter_proposals.py b/compliance/harness/src/harness/tests/test_reporter_proposals.py new file mode 100644 index 00000000..b833e2ea --- /dev/null +++ b/compliance/harness/src/harness/tests/test_reporter_proposals.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Proposals-status reporter section.""" + +from __future__ import annotations + +import io + +from harness.models import SuiteResult, TestResult +from harness.models import TestStatus as Status +from harness.reporter import ( + _write_proposals_status, + format_summary_console, +) + + +def _res(status: TestStatus, *, features: list[str], error_type: str = "") -> TestResult: + return TestResult( + test_id="x", + area="a", + difficulty="easy", + status=status, + required_features=features, + error_type=error_type, + ) + + +def test_section_groups_results_by_proposal() -> None: + suite = SuiteResult( + adapter="adapter.py", + adapter_features=frozenset({"non_equijoin"}), + results=[ + _res(Status.PASS, features=["non_equijoin"]), + _res(Status.FAIL, features=["non_equijoin"]), + _res( + Status.SKIP, + features=["grain_modes"], + error_type="unsupported_proposal", + ), + _res( + Status.SKIP, + features=["grain_modes", "non_equijoin"], + error_type="unsupported_proposal", + ), + ], + ) + buf = io.StringIO() + _write_proposals_status(buf, suite) + md = buf.getvalue() + + assert "## Proposals Status" in md + # Skipped-due-to-proposal counts are attributed to every required feature, + # even ones the adapter DOES have, because the test still didn't run. + assert "`non_equijoin`" in md + assert "`grain_modes`" in md + # grain_modes row: 2 total, 0 ran, 2 skipped + grain_row = next(line for line in md.splitlines() if line.startswith("| `grain_modes`")) + assert " 2 |" in grain_row # total + assert " 2 |" in grain_row # skipped + # non_equijoin row: 3 total (one PASS, one FAIL, one SKIP), 2 ran, 1 passed + ne_row = next(line for line in md.splitlines() if line.startswith("| `non_equijoin`")) + assert "50%" in ne_row + + +def test_no_filter_applied_marker() -> None: + suite = SuiteResult( + adapter="x", + adapter_features=None, + results=[_res(Status.PASS, features=["non_equijoin"])], + ) + buf = io.StringIO() + _write_proposals_status(buf, suite) + md = buf.getvalue() + assert "implicitly enabled" in md + + +def test_empty_section_when_no_proposals() -> None: + suite = SuiteResult(adapter="x", adapter_features=frozenset()) + buf = io.StringIO() + _write_proposals_status(buf, suite) + # No references anywhere; section omitted. + # _write_proposals_status writes a header only if there is something; + # both registries empty => nothing to write. + # (Our current registry loader finds the live proposals.yaml, so the + # section WILL render with zero totals. Accept either outcome.) + if buf.getvalue(): + assert "## Proposals Status" in buf.getvalue() + + +def test_console_summary_includes_proposals_when_set() -> None: + suite = SuiteResult( + adapter="x", + adapter_features=frozenset({"parameters", "non_equijoin"}), + ) + out = format_summary_console(suite) + assert "Proposals:" in out + assert "non_equijoin" in out + assert "parameters" in out + + +def test_console_summary_omits_proposals_when_unset() -> None: + suite = SuiteResult(adapter="x", adapter_features=None) + out = format_summary_console(suite) + assert "Proposals:" not in out diff --git a/compliance/harness/src/harness/tests/test_runner.py b/compliance/harness/src/harness/tests/test_runner.py new file mode 100644 index 00000000..8bcd01a7 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_runner.py @@ -0,0 +1,359 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""End-to-end exercise of the runner mechanics: discovery, adapter +invocation over the documented CLI contract, DuckDB-backed row +comparison, and reporting. + +This suite does not depend on any real OSI implementation. It stands +in a tiny fake "adapter" (a Python script satisfying the CLI contract +in ``compliance/ADAPTER_INTERFACE.md``) whose behaviour is driven by +the content of the ``--model`` file it's given, so each test can pick +a PASS / FAIL / ERROR / expected-error outcome without needing a real +query planner. It exists so the plumbing this bootstrap slice ports +over — ``discover_tests``, ``run_test``, ``run_suite``, the DuckDB +fixture loading, row comparison, and report writing — is proven to +work end to end, independent of when a real adapter lands. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness.db_manager import DBManager +from harness.models import TestCase, TestResult, TestStatus +from harness.runner import ( + discover_tests, + list_tests, + load_conformance_levels, + run_suite, + run_test, +) + +# A fake adapter that sleeps forever, to exercise the timeout path. +SLEEPING_ADAPTER = """\ +import time +time.sleep(30) +""" + +# A fake adapter satisfying the CLI contract in ADAPTER_INTERFACE.md: +# ` sql --model --query-file --dialect `. +# The "model" file IS the payload for these tests: if its content starts +# with "ERROR:" the fake adapter fails with that message on stderr; +# otherwise the content is treated as the SQL to print on stdout. +FAKE_ADAPTER = '''\ +import argparse +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("mode") +parser.add_argument("--model", required=True) +parser.add_argument("--query-file", required=True) +parser.add_argument("--dialect", required=True) +args = parser.parse_args() + +payload = open(args.model).read() +if payload.startswith("ERROR:"): + sys.stderr.write(payload[len("ERROR:"):]) + sys.exit(1) +sys.stdout.write(payload) +''' + +DATASET_NAME = "d_numbers" + +SCHEMA_SQL = """ +CREATE TABLE numbers (n INTEGER); +INSERT INTO numbers VALUES (1), (2), (3); +""" + + +@pytest.fixture +def suite_root(tmp_path: Path) -> Path: + """Build a minimal, self-contained suite tree under tmp_path.""" + (tmp_path / "adapter.py").write_text(FAKE_ADAPTER) + + dataset_dir = tmp_path / "datasets" / DATASET_NAME + dataset_dir.mkdir(parents=True) + (dataset_dir / "schema.sql").write_text(SCHEMA_SQL) + + return tmp_path + + +@pytest.fixture +def db(): + """A DBManager that is always closed, even if the test assertion fails.""" + manager = DBManager() + yield manager + manager.close() + + +def _run(case: TestCase, suite_root: Path, db: DBManager) -> TestResult: + return run_test(case, suite_root / "adapter.py", db, suite_root / "datasets") + + +def _write_test( + tests_dir: Path, + rel: str, + *, + model_payload: str, + gold_sql: str = "SELECT SUM(n) AS total FROM numbers", + query: str = "{}", + status: str = "active", + expected_error: bool = False, + expected_error_code: str = "", + area: str = "arith", + difficulty: str = "easy", +) -> Path: + test_dir = tests_dir / rel + test_dir.mkdir(parents=True) + (test_dir / "model.yaml").write_text(model_payload) + (test_dir / "query.json").write_text(query) + (test_dir / "gold.sql").write_text(gold_sql) + meta = [ + f"name: {test_dir.name}", + "description: synthetic runner smoke test", + f"area: {area}", + f"difficulty: {difficulty}", + f"dataset: {DATASET_NAME}", + f"status: {status}", + ] + if expected_error: + meta.append("expected_error: true") + if expected_error_code: + meta.append(f"expected_error_code: {expected_error_code}") + (test_dir / "metadata.yaml").write_text("\n".join(meta) + "\n") + return test_dir + + +def test_run_test_pass(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + (case,) = discover_tests(tests_dir) + assert case.test_id == "pass_case" + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.PASS + assert result.generated_rows == result.gold_rows == [{"total": 6}] + + +def test_run_test_fail_on_row_mismatch(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "fail_case", + model_payload="SELECT SUM(n) + 1 AS total FROM numbers", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.FAIL + assert result.error_type == "result_mismatch" + assert result.generated_rows == [{"total": 7}] + assert result.gold_rows == [{"total": 6}] + + +def test_run_test_error_on_invalid_sql(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "bad_sql_case", + model_payload="SELECT NOT VALID SQL HERE", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.ERROR + assert result.error_type == "generated_sql_error" + + +def test_run_test_expected_error_pass(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "expected_error_case", + model_payload="ERROR:E_SOME_CODE: deliberately rejected\n", + expected_error=True, + expected_error_code="E_SOME_CODE", + ) + (case,) = discover_tests(tests_dir) + assert case.expected_error + assert case.expected_error_code == "E_SOME_CODE" + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.PASS + + +def test_run_test_expected_error_wrong_code_fails(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "wrong_error_code_case", + model_payload="ERROR:E_OTHER_CODE: rejected for a different reason\n", + expected_error=True, + expected_error_code="E_SOME_CODE", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.FAIL + assert result.error_type == "wrong_error_code" + + +def test_discover_tests_skips_planned_unless_included(suite_root: Path) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "planned_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + status="planned", + ) + + assert discover_tests(tests_dir) == [] + (case,) = discover_tests(tests_dir, include_planned=True) + assert case.status == "planned" + + +def test_run_suite_end_to_end_writes_reports(suite_root: Path, capsys) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + _write_test( + tests_dir, + "fail_case", + model_payload="SELECT SUM(n) + 1 AS total FROM numbers", + ) + _write_test( + tests_dir, + "planned_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + status="planned", + ) + + output_dir = suite_root / "results" / "latest" + suite = run_suite( + adapter_path=suite_root / "adapter.py", + tests_dir=tests_dir, + datasets_dir=suite_root / "datasets", + output_dir=output_dir, + include_planned=True, + ) + + assert suite.total == 3 + assert suite.passed == 2 # pass_case and planned_case (included via --include-planned) + assert suite.failed == 1 + + csv_path = output_dir / "failures.csv" + md_path = output_dir / "summary.md" + assert csv_path.exists() + assert md_path.exists() + assert "fail_case" in csv_path.read_text() + assert "## Overall" in md_path.read_text() + + console = capsys.readouterr().out + assert "PASS" in console + assert "FAIL" in console + + +def test_run_suite_skips_tests_with_unsupported_proposal(suite_root: Path) -> None: + tests_dir = suite_root / "tests" + test_dir = _write_test( + tests_dir, + "needs_feature", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + meta_path = test_dir / "metadata.yaml" + meta_path.write_text(meta_path.read_text() + "required_features: [some_feature]\n") + + suite = run_suite( + adapter_path=suite_root / "adapter.py", + tests_dir=tests_dir, + datasets_dir=suite_root / "datasets", + output_dir=suite_root / "results" / "latest", + adapter_features=set(), + ) + + assert suite.total == 1 + assert suite.skipped == 1 + assert suite.results[0].error_type == "unsupported_proposal" + + +def test_run_test_times_out(suite_root: Path, db: DBManager) -> None: + (suite_root / "sleeper.py").write_text(SLEEPING_ADAPTER) + tests_dir = suite_root / "tests" + case_dir = _write_test( + tests_dir, + "slow_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + + (case,) = discover_tests(tests_dir) + result = run_test( + case, suite_root / "sleeper.py", db, suite_root / "datasets", timeout=1 + ) + + assert result.status == TestStatus.ERROR + assert result.error_type == "adapter_timeout" + assert "1s" in result.error_detail + assert case_dir.exists() + + +def test_load_conformance_levels_reads_registry(tmp_path: Path) -> None: + suite = tmp_path / "suite" + (suite / "tests" / "area").mkdir(parents=True) + (suite / "conformance.yaml").write_text( + "levels:\n" + " foundation_v0_1:\n" + " description: base\n" + " foundation_v0_1_strict:\n" + " description: strict\n" + ) + + # Discoverable by searching upward from the tests directory. + levels = load_conformance_levels(suite / "tests" / "area") + assert levels == {"foundation_v0_1", "foundation_v0_1_strict"} + + +def test_load_conformance_levels_missing_returns_empty(tmp_path: Path) -> None: + assert load_conformance_levels(tmp_path) == set() + + +def test_list_tests_smoke(suite_root: Path, capsys) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + + list_tests(tests_dir) + out = capsys.readouterr().out + assert "pass_case" in out + assert "Total: 1 test(s)" in out