diff --git a/CUBE_IMPORT_SPEC.md b/CUBE_IMPORT_SPEC.md new file mode 100644 index 00000000..0e1b7c3f --- /dev/null +++ b/CUBE_IMPORT_SPEC.md @@ -0,0 +1,539 @@ +# Spec: Cube → SLayer ingestion (DEV-1608) + +Convert Cube (Cube.js / Cube.dev) data-model YAML into persisted `SlayerModel`s, +mirroring the existing `import-dbt` importer (`slayer/dbt/`). Two stages: + +- **Stage 1 (this PR)** — everything that maps cleanly without the Tesseract + engine: cubes, measures, dimensions, joins, segments, `extends`, and **views** + (via facade models). Plus any Tesseract feature that turns out to be an easy + win (assessment below: effectively none — see §9). +- **Stage 2 (follow-up issue/PR)** — the Tesseract-only feature set and the + hard edges Stage 1 routes to the report. Fully designed in §9, not built here. + +Conversion is **fully offline**: types come from Cube's declared dimension / +measure types; no database connection is required or used. Everything that does +not convert is captured in a **structured report** (Pydantic object + JSON file). + +--- + +## 1. Package layout (mirrors `slayer/dbt/`) + +```text +slayer/cube/ + __init__.py + models.py # Pydantic shapes for parsed Cube YAML + parser.py # walk dir, collect .yml/.yaml, Jinja-detect, parse cubes:/views: + refs.py # {CUBE}/{member}/{cube.member} → SLayer SQL / DSL translator + extends.py # extends-graph resolution + member flattening + converter.py # CubeToSlayerConverter → CubeConversionResult + report.py # CubeConversionReport / CubeConversionIssue / categories +``` + +CLI: `import-cube` subcommand in `slayer/cli.py` + `_run_import_cube`. + +The converter never touches the DB and never needs an engine. Persistence is +`run_sync(storage.save_model(model))`, exactly as `_run_import_dbt` does for +table-backed models. + +--- + +## 2. Parser (`parser.py`) + +- Recursively collect `*.yml` / `*.yaml` (skip hidden dirs, `target`, etc.) — + reuse the dbt parser's `_collect_yaml_paths` shape. +- **Jinja detection (Codex #6 — order resolved):** Cube's own SQL-ref syntax + uses **single** braces (`{CUBE}`, `{cube.member}`) so it never collides with + Jinja's double-brace / brace-percent. **Try YAML-parsing first.** + - If the file parses as YAML: scan only the **templatable fields** of each + member (`sql`, `filter`, `case` predicates, join `sql`) for `{{`/`{%`. A + member with a Jinja marker is skipped + reported (`requires_templating`), + keeping the rest of the cube. (Plain YAML containing one templated SQL member + is thus mostly imported — the earlier "scan raw text, skip whole file" rule + is replaced by this.) + - File-level skip applies **only** when the file fails to parse as YAML + *because of* templating directives (e.g. `{% for %}` generating list items) — + then skip the whole file + `requires_templating` report. +- Parse the two top-level keys: `cubes:` (list) and `views:` (list). Tolerate + single-object forms (wrap in a list), matching the dbt parser's leniency. +- Per-item `model_validate` into the `models.py` shapes; on `ValidationError`, + log + emit a `parse_error` report issue and continue (never abort the run for + one bad file). + +### Parsed Cube shapes (`models.py`, Pydantic v2, no dataclasses) + +```text +CubeMeasure: name, type, sql?, title?, description?, public?=True, meta?, + format?, filters?: list[{sql}], drill_members?, rolling_window?, + multi_stage?=False, time_shift?, grain?, filter? (Tesseract), + case? (Tesseract) +CubeDimension: name, sql?, type='string', title?, description?, public?=True, + meta?, format?, primary_key?=False, sub_query?=False, case?, + granularities?, latitude?, longitude? (geo) +CubeJoin: name (target cube), relationship, sql (ON clause) +CubeSegment: name, sql, title?, description?, public?=True, meta? +CubeCube: name, sql_table?, sql?, sql_alias?, extends?, data_source?, + title?, description?, public?=True, meta?, refresh_key?, + calendar?, measures?, dimensions?, joins?, segments?, + hierarchies?, pre_aggregations?, access_policy? +CubeViewCubeRef: join_path, includes?(list|'*'), excludes?, prefix?=False, + alias?, per-member overrides (alias/title/description/format/meta) +CubeView: name, cubes: list[CubeViewCubeRef], extends?, title?, description?, + public?=True, meta?, folders?, default_filters?, access_policy? +CubeProject: cubes: list[CubeCube], views: list[CubeView] +``` + +Unknown keys are tolerated (Cube evolves); they are ignored unless listed in the +"unmapped infra" set (§7), which is stashed in `meta`. + +--- + +## 3. Reference translator (`refs.py`) — the core correctness surface + +Cube `sql`/`filter` strings use Cube's curly reference syntax. Translate to +SLayer SQL (Mode A, for `Column.sql` / `Column.filter` / model `filters`) or +SLayer DSL (Mode B, for calculated-measure `ModelMeasure.formula`). These are +**not Jinja** and are translated, not skipped. + +| Cube ref | Meaning | SLayer rewrite | +|---|---|---| +| `{CUBE}.col` | own table column | bare `col` (SLayer auto-qualifies) | +| `{CUBE}` (bare) | own table alias | drop / context-specific (see joins) | +| `{member}` (same cube) | sibling member on this cube | bare `member` — SLayer inlines sibling derived columns recursively | +| `{other_cube.member}` | member on a joined cube | Mode A: `other_cube.member` (single hop) or `a__b.member` (multi-hop); SLayer inlines derived joined columns | +| `{measure}` inside a calc measure | another measure (post-agg) | Mode B: bare `measure` (resolves as `ModelMeasure` name) | + +Mechanics: +- A single-pass regex over the string, **skipping SQL string literals** + (reuse the `_STRING_LITERAL_RE` idea from `core/models.py`). +- Distinguish `{X}` (one dotless token → same-cube member or `CUBE`), + `{X.Y}` (cube.member), and `{CUBE}` / `{CUBE.col}` specially. +- Multi-hop `{a.b.c}` → SLayer multi-dot `a.b.c`, which `Column.sql`'s own + `_fix_multidot_sql` validator then converts to `a__b.c` (we lean on the + existing model-side normalization rather than duplicating it). +- **Boundary / report rule (user-established):** after translation, if the + result is not valid for its target mode, do **not** emit a broken member — + route it to the report: + - `Column.sql` / `Column.filter` / model `filters` are validated by Pydantic + construction (Mode A `parse_sql_predicate`). A `ValidationError` → catch → + `complex_sql` report issue, member dropped. + - Calculated-measure `ModelMeasure.formula` is Mode B. If the translated + formula is clean DSL (arithmetic / `||` / the `SCALAR_PASSTHROUGH` scalar + set / measure refs) it converts; otherwise (`CASE WHEN`, raw SQL funcs Mode B + rejects, unresolved refs) → `complex_measure` report issue, measure dropped. + +--- + +## 4. Cube → model conversion (`converter.py`) + +Top loop: **for each cube → one table-owning model; for each view → one facade +model** (§6). Both emit ordinary `SlayerModel`s. Order: resolve `extends` first +(§5), then convert cubes, then views (views need cube models to exist for +member/measure lookup and fan-out detection). + +### 4.1 Cube → SlayerModel + +| Cube | SLayer | Notes | +|---|---|---| +| `name` | `name` | | +| `sql_table` | `sql_table` | verbatim (Cube uses `schema.table`) | +| `sql` | `sql` | `{CUBE}`/`{member}` translated; if it references measures or is otherwise un-rewritable → `complex_sql` report and the cube is dropped (`no_source`), since a cube with no usable source can't be emitted (see the source rule below) | +| `data_source` (per-cube) | — | converter scopes ALL models under the single `--datasource`; per-cube `data_source` → `unmapped_infra` report + stashed in `meta.cube_unmapped.data_source` | +| `title` | — | no model title field; → `meta.cube_title` (info-level), not dropped silently | +| `description` | `description` | | +| `public: false` | `hidden: true` | | +| `meta` (incl `ai_context`) | `meta` | merged under the model's `meta` (preserved verbatim) | +| `sql_alias`, `refresh_key`, `calendar`, `hierarchies`, `pre_aggregations`, `access_policy` | — | §7 report + `meta.cube_unmapped.*` | +| `measures` | `columns` + `measures` | §4.2 | +| `dimensions` | `columns` | §4.3 | +| `joins` | `joins` | §4.4 | +| `segments` | `columns` (boolean) | §4.5 | + +A cube must end with exactly one source. Normal case: `sql_table`. If only `sql` +is given and it translates cleanly → `sql`. If neither is usable, emit nothing +for that cube and report `no_source` (error severity). + +### 4.2 Measures — Column + ModelMeasure split (same idiom as dbt) + +Aggregating measures (`type` ∈ `count`, `count_distinct`, `count_distinct_approx`, +`sum`, `avg`, `min`, `max`): + +- The `sql` expression → a `Column` (`DataType.DOUBLE`, `NumberFormat(FLOAT)` + default unless `format` maps — §8). Bare-identifier `sql` → Column named after + the column; non-trivial expression → Column named `_col`. Collisions + resolved by `_col` suffix (reuse dbt's logic). +- **Column dedup key must include the filter + window state** (Codex #4). The + dbt converter groups measures by `sql` expression alone (safe because dbt + measures carry no per-measure column state). Cube measures put `filters` on the + emitted `Column.filter`, so two measures sharing one `sql` but differing in + `filters` (or `rolling_window`) **must** become distinct columns — the dedup + key is `(translated_sql, translated_filter, window_spec)`, not `sql` alone. + Otherwise a filter would bleed across measures. +- The measure → a `ModelMeasure` whose formula is `:`. +- `type: count` with no `sql` → `ModelMeasure(formula="*:count")` (COUNT(*)), + no Column needed. `count` **with** `sql` → `:count`. +- Aggregation name map: `count_distinct`→`count_distinct`, + `count_distinct_approx`→`count_distinct` (+ `lossy_mapping` info report: SLayer + has no approximate distinct), others 1:1. + +Calculated measures (`type` ∈ `number`, `string`, `time`, `boolean`) — these are +post-aggregation expressions referencing other measures (e.g. +`sql: "{revenue} / {count}"`): + +- → a `ModelMeasure` whose formula is the Mode-B-translated expression, + `type` set to the mapped `DataType`. Convert if clean DSL; else `complex_measure` + report (§3 boundary). + +Per-measure extras: +- `filters: [{sql}, …]` (conditional aggregation) → the Column carries a `filter` + (the AND of the translated predicates). Same Column+ModelMeasure split; the + filter lives on the Column (dbt simple-filtered-metric idiom). +- `format` → §8. +- `title`→`ModelMeasure.label`, `description`→`ModelMeasure.description`, + `meta`→`ModelMeasure.meta`. +- `rolling_window` with a **finite `trailing`** and no `leading`/`offset` → + windowed aggregation `:(window='')` (Cube `1 month`→`1m`, + `7 day`→`7d`, etc.). `unbounded` / `leading` / `offset` / `rolling_window` on a + non-sum/avg agg → `unsupported_rolling_window` report, measure falls back to the + plain aggregation (still emitted) with a warning. +- `drill_members` → §7 report + `meta.cube_unmapped` on the ModelMeasure. +- `multi_stage`, `time_shift`, `grain`, `filter` (Tesseract), `case` (Tesseract) + → §9 (Stage 2). Measure is emitted as its plain aggregation if one exists, + else routed to report as `deferred_stage2`. + +### 4.3 Dimensions → Column + +| Cube dim | SLayer Column | Notes | +|---|---|---| +| `type: string` | `type=TEXT` | | +| `type: number` | `type=DOUBLE` | Cube doesn't distinguish int/float; refine later via `slayer ingest` | +| `type: boolean` | `type=BOOLEAN` | | +| `type: time` | `type=TIMESTAMP` | | +| `sql` | `Column.sql` (translated); omitted when it's just `{CUBE}.` | | +| `primary_key: true` | `Column.primary_key=True` | PK columns auto-restricted to count/count_distinct | +| `case:` (CASE-WHEN dim) | `Column.sql` built as `CASE WHEN … THEN … ELSE … END` from `when[].{sql,label}` + `else` | **Stage 1** — `case` *dimension* is not Tesseract | +| `title`→`label`, `description`, `meta`, `format` | direct | | +| `type: geo` (+ `latitude`/`longitude`) | — | §7 report + `meta.cube_unmapped.geo`; not split into lat/long columns in Stage 1 | +| `sub_query: true` | — | §7 report + `meta.cube_unmapped`; correlated per-row measure has no SLayer equivalent | +| `granularities:` (custom) | — | base time Column still emitted; custom grains → §7 report (SLayer granularity is query-time enum) | +| `type: switch` | — | §9 (Tesseract) | +| `links:`, `order:` | — | presentation; §7 report | + +### 4.4 Joins + +`CubeJoin.sql` is an ON clause like `{CUBE}.customer_id = {customers.id}`. + +- Parse the ON into equality column pairs. Conjunctions (`A=B AND C=D`) → + multiple `join_pairs` (SLayer supports composite keys). The qualifier matching + `{CUBE}` is the source column; the qualifier matching `{.…}` is the + target column. +- **Resolve each ON side to a physical column name** (Codex #2 — verified: the + SQL generator emits raw `alias.src = alias.tgt` from `join_pairs`; it does NOT + expand `Column.sql`). `{customers.id}` means the `id` *member*, which may have + `sql: "{CUBE}.customer_id"`. So follow each ON ref to its member's `sql`: if it + is a bare physical identifier, use it; if the member's `sql` is a non-trivial + expression (function, arithmetic, CASE), the column pair can't be expressed in + `join_pairs` → `unsupported_join` report, join dropped. Same for a `{CUBE}.col` + side that resolves to a derived dimension. +- `target_model` = the join `name` (the target cube). +- `relationship` (`many_to_one` / `one_to_many` / `one_to_one`, plus legacy + `belongs_to`/`has_many`/`has_one`) is **not stored on `ModelJoin`** (SLayer + joins are all LEFT). It IS recorded internally and used for view fan-out + detection (§6). Emitted `ModelJoin.join_type = LEFT`. +- Non-equi ON (ranges, function calls, inequalities), or an ON that doesn't + resolve to clean column pairs → `unsupported_join` report; the join is dropped + (both cubes still exist as models, just not auto-joined). + +### 4.5 Segments → boolean derived Column + +Each segment `{name, sql}` → `Column(name=, sql=, +type=BOOLEAN)`. Filterable (`name = true`) and group-able. `title`/`description`/ +`meta` carried onto the Column. Also recorded in the report (`segment_as_column`, +info). Name-collision with an existing column/measure → `_seg` suffix + warning. + +--- + +### 4.6 Namespace allocation & emit-time safety (Codex #5, #7) + +The core `SlayerModel` validators reject duplicate column names, duplicate +measure names, **any column↔measure overlap**, and `ModelMeasure` names that +shadow a built-in transform (`cumsum`, `rank`, `change`, `lag`, …). A naive +converter that lets these collide makes **whole-model construction throw**, +losing the entire model. So: + +- **Preflight namespace allocator** per emitted model/view: allocate column and + measure names against a shared seen-set (columns + measures share one + namespace). Dimension/segment/measure/entity name clashes are resolved by a + deterministic suffix (`_col` / `_seg`) **before** constructing the model. +- A Cube measure/metric name that shadows a SLayer transform, or a collision + that can't be safely renamed, is routed to the report (don't emit it) rather + than thrown. +- **Offline validation helper** (Codex #7 — verified: `Column.sql` is *not* + SQL-validated at construction; only `Column.filter` / `SlayerModel.filters` + parse a predicate, and `ModelMeasure.formula` only rejects raw `OVER`). After + building each model, run an explicit offline pass: sqlglot-parse every + translated `Column.sql`, `parse_sql_predicate` every filter, and formula-parse + every `ModelMeasure.formula`. A parse failure routes that member to the report + (`complex_sql` / `complex_measure`) and drops it — so a structurally-broken + member never persists to fail later at enrichment. + +## 5. `extends` (flatten) — `extends.py` + +> **Approach for this PR: flatten.** Native persisted model inheritance +> (making `ModelExtension` a saveable source mode) is tracked separately in +> **DEV-1610** and is deliberately *not* a dependency of DEV-1608. When DEV-1610 +> lands, `extends.py` swaps from flattening to emitting +> `SlayerModel(extends=ModelExtension(...))` — a ~10-line change. Flattening is +> faithful to Cube's own compile-time materialization of extended cubes, so it +> is not a stopgap-quality result. + +- Build the extends graph across all cubes. Resolve transitively (multi-level), + detect cycles → `extends_cycle` report (error) and skip the cycle members. +- Flatten: a child inherits the parent's measures / dimensions / joins / + segments; **child members win** on name conflict. Child's own `sql_table`/`sql` + override the parent's source. +- Every cube is still emitted as its own model (hidden iff `public: false`), so + an abstract base (`public: false`, only extended) becomes a **hidden** model + AND its members are flattened into children. Lossless, mirrors Cube's + `public: false`, matches SLayer's hidden-model convention. +- Views can `extends` other views — same flattening over view member lists. + +--- + +## 6. Views → facade models — the key structural mapping + +A Cube view owns no table; it re-exports members from cubes along a `join_path`. +SLayer has no view type, so a view becomes a **thin regular `SlayerModel` +anchored on the join_path root cube's table**: + +- **Source mode mirrors the root cube's emitted model** (Codex #3): copy + `sql_table` *or* `sql` — whichever the root cube produced. If the root cube was + not emitted (e.g. it failed conversion), drop + report the view + (`ambiguous_view_root` / `disconnected_view`). Never hard-code `sql_table`. +- `joins` = the joins implied by the view's `join_path`s (reuse the root cube's + existing join definitions by walking the path; each hop must correspond to a + declared cube join). +- Each included **dimension** → a derived `Column` whose `sql` references the + joined cube's column: `Column(name=, sql="customers.name")` + (single-dot Mode A joined ref; multi-hop uses `a__b.col`). Root-cube dimensions + reference their own column. +- Each included **measure** → a `ModelMeasure`, but it **must reference the + underlying `Column`, not the Cube measure name** (Codex #1 — verified: + `query_engine.py:2345` resolves cross-model aggs via + `target_model.get_column(name)`, never `get_measure`). So for a Cube measure + `revenue {type: sum, sql: amount}` whose emitted underlying column is `amount` + (or `_col` for an expression): + - **root-cube** measure → the facade also **carries the underlying `Column`** + (its physical/derived expression), and the `ModelMeasure` is a local + `amount:sum` (or `:`). A root-cube measure can't be re-exported by + bare name because the facade is a *separate* model that doesn't own the root + cube's measures. + - **joined-cube** measure → cross-model `customers.amount:sum` + (`joinpath.:`); the underlying column already lives on + the joined cube's emitted model, so no copy is needed. + - **filtered** Cube measure → the underlying column carries the `filter` + (root: copied onto the facade; joined: already on the joined model), so + `customers.:` still applies the CASE-WHEN correctly. + - **`count`** → `joinpath.*:count` (or local `*:count`); `customers.*:count` + resolves via the `measure_name == "*"` path. + - **calculated** (`type: number/…`, references multiple measures) re-export → + inline the cross-model **column** refs if the result is clean Mode B + (`customers.amount:sum / customers.*:count`); otherwise → `complex_measure` + report, that measure dropped from the view. +- The converter therefore threads, from cube-measure conversion, the + `(underlying_column_name, aggregation, filter?)` triple for every measure so + the facade builder can synthesize the correct `.:` formula. +- `prefix: true` → exported names are `_`. Per-member `alias` → + the exported name. `title`/`description`/`format`/`meta` overrides → the + Column/ModelMeasure fields. +- `default_filters` (`{member, operator, values, unless?}`) → model `filters` + (Mode A SQL predicates built from operator+values; `member` resolved to its + SQL column / joined ref). Operators that don't translate to plain SQL → + `unsupported_default_filter` report, that filter dropped. +- `excludes` / `includes: '*'` honored when selecting members. +- `meta: {cube_kind: "view"}` stamped so the report and future round-trips can + identify facade models. + +**Common case built in Stage 1.** Routed to the report (not guessed) when: +- the view's members span cubes **not on one connected join tree** rooted at the + join_path root (`disconnected_view`), +- the join_path root is **ambiguous** / not derivable (`ambiguous_view_root`), +- a hop in the path is `one_to_many` / `many_to_one`-reversed such that the + facade would **fan out** the root and double-count a root measure + (`view_fanout_risk` — detected via the recorded `relationship`), +- `folders` present → parked in `meta.cube_unmapped.folders` + `folders_unmapped` + report (no SLayer hierarchy concept), +- any per-member override or operator that doesn't map → reported, that member / + filter skipped, rest of the view still emitted. + +--- + +## 7. "No SLayer home" features — report + stash in meta + +For every Cube feature with no SLayer equivalent, emit a structured report issue +AND preserve the raw Cube fragment under a namespaced key on the owning entity's +`meta`: `meta.cube_unmapped.`. Set covers (per §4/§6): + +`pre_aggregations`, `refresh_key`, `calendar`, `hierarchies`, `drill_members`, +`access_policy`, `sql_alias`, per-cube `data_source`, `geo` dims (+lat/long), +`sub_query` dims, custom `granularities`, dimension `links`/`order`, view +`folders`, `multi_stage`/`time_shift`/`grain`/Tesseract bits (§9). + +Genuinely-semantic metadata is **not** in this bucket — `title`→`label`, +`description`, `meta`/`ai_context`, `format` always carry over directly to the +proper SLayer field. + +--- + +## 8. Format mapping (`format` → `NumberFormat`) + +`NumberFormat` has `type ∈ {percent, currency, integer, float}`, `precision`, +`symbol` (currency only). Map Cube formats: + +- `percent` → `PERCENT` +- `currency` (+ `currency_symbol` if present) → `CURRENCY` (symbol) +- `number` / numeric d3-ish → `FLOAT` (with `precision` if a `_N` suffix is + parseable) +- `accounting`, `abbr`, arbitrary d3-format strings, `imageUrl`/`link`/`id` → + `unsupported_format` report; format dropped (the field/measure still emitted). +- **`NumberFormat.symbol` guard** (Codex #8 — verified `format.py:40`: `symbol` + is rejected unless `type == CURRENCY`, and is auto-defaulted to `$` for + currency). Never pass `symbol` for non-currency mappings. A Cube format payload + carrying a symbol-like field on a non-currency type, or an otherwise invalid + format, is reported + dropped rather than allowed to raise at model + construction. + +--- + +## 9. Stage 2 (Tesseract) — designed here, built in the follow-up issue + +`CUBEJS_TESSERACT_SQL_PLANNER`-only features. **Assessment: none are easy wins** +— each lacks a clean SLayer mapping, so all are deferred. Stage 1 routes any cube +using them to the report as `deferred_stage2` (the cube's non-Tesseract members +still convert). + +| Tesseract feature | Why no clean SLayer map | Proposed Stage-2 approach | +|---|---|---| +| `switch` dimension | Query-time selectable dimension; SLayer dimensions are static | No direct map. Possibly enumerate switch cases into N separate columns + a report note; needs design. | +| `number_agg` measure | Aggregates an expression that itself contains aggregations (multi-stage) | Multi-stage `source_queries` model: inner stage materializes the inner aggregation, outer stage re-aggregates. Needs the query-backed-model builder. | +| `case` measure | Conditional measure keyed on a `switch` dimension | Depends on `switch`; without it, a pure `CASE WHEN` over a condition → a filtered Column (the one borderline "maybe easy" — evaluate during impl, default to defer). | +| measure `filter` (`exclude`/`keep_only`/`mode`) | Grain manipulation at aggregation time | No SLayer grain-override; candidate for a multi-stage rewrite. | +| `multi_stage` + `time_shift` + `grain` (non-Tesseract but same family) | Cube's measure-level time-shift/grain grammar ≠ SLayer's query-time `time_shift`/transform model | Map finite `rolling_window` trailing in Stage 1 (§4.2); defer `time_shift`/`grain` to the multi-stage rewrite. | + +Plus the Stage-1 "hard edges" routed to the report: disconnected/ambiguous/ +fan-out-risk views, non-equi joins, complex SQL/measures, custom granularities, +geo split, sub_query dims. The follow-up issue tackles these alongside Tesseract. + +--- + +## 10. Structured report (`report.py`) + +```python +class CubeIssueCategory(str, Enum): # requires_templating, parse_error, + complex_sql, complex_measure, lossy_mapping, unsupported_join, + unsupported_rolling_window, unsupported_format, unsupported_default_filter, + segment_as_column, unmapped_infra, geo_unmapped, subquery_unmapped, + granularity_unmapped, disconnected_view, ambiguous_view_root, view_fanout_risk, + folders_unmapped, extends_cycle, no_source, deferred_stage2 + +class CubeConversionIssue(BaseModel): + category: CubeIssueCategory + severity: Literal["info","warning","error"] + cube: str | None; view: str | None; member: str | None + message: str + raw: str | None # raw Cube fragment when useful + +class CubeConversionReport(BaseModel): + issues: list[CubeConversionIssue] + model_count: int; hidden_count: int; view_count: int + # counts derived; helpers to filter by category/severity + +class CubeConversionResult(BaseModel): # converter return + models: list[SlayerModel] + report: CubeConversionReport +``` + +(No `Dict`-typed LLM-output fields; this is internal, so plain fields are fine. +No dataclasses.) + +--- + +## 11. CLI + +```text +slayer import-cube --datasource NAME [--storage PATH] + [--report PATH] [--include-hidden] +``` + +- Recursively parse, convert, `storage.save_model` each model, print a console + summary (imported models with column/measure counts + `[hidden]`, then issues + grouped by severity), and **always write the JSON report** to + `/cube_import_report.json` (override with `--report PATH`). +- `_run_import_cube` mirrors `_run_import_dbt` structurally. +- `--datasource` is just the SLayer datasource name to file models under; it need + not exist or be reachable (offline). (`--include-hidden` reserved for parity; + cubes are already emitted hidden when `public: false`, so it mainly governs + whether hidden models print — keep minimal.) +- The `slayer/cube/` converter API stays importable for programmatic use. + +--- + +## 12. Tests (TDD — full suite first, per `feedback_tdd_style.md`) + +Mirror `tests/test_dbt_*`. New files: + +- `tests/test_cube_parser.py` — YAML collection; Jinja file-skip + member-skip + + report; single-object tolerance; malformed-file `parse_error` continue. +- `tests/test_cube_refs.py` — `{CUBE}.col`→`col`; `{member}`→bare; `{cube.member}` + → joined ref; multi-hop; string-literal skipping; calc-measure Mode-B + translation; un-rewritable → report boundary. +- `tests/test_cube_converter.py` — cube→model 1:1; measure Column+ModelMeasure + split (count/count_distinct/approx/sum/avg/min/max); `*:count`; calc measures; + measure `filters`→Column.filter; finite rolling_window→`window=`; + dimensions (string/number/bool/time, `case` dim, primary_key); joins (single + + composite + non-equi→report); segments→boolean column; format mapping; + unmapped infra→report+meta stash. +- `tests/test_cube_extends.py` — single + multi-level flatten; child-wins; abstract + base emitted hidden; cycle→report. +- `tests/test_cube_views.py` — facade model: dims→derived columns, measures→ + local/cross-model ModelMeasures, prefix/alias/overrides, default_filters→model + filters, excludes/`*`; disconnected/ambiguous/fanout/folders→report. +- `tests/test_cube_report.py` — categories, severities, counts, JSON round-trip. +- `tests/test_cube_cli.py` (or fold into converter) — `import-cube` writes models + + JSON report; offline (no datasource needed); console summary. +- `tests/fixtures/cube_project/` — hand-written sample `.yml` (cubes, a view, + extends, segments, a Jinja file, a Tesseract cube) covering the above. +- **`tests/test_cube_smoke.py` — enrich/execute converted models, not just + assert converter output** (Codex test-gap). Build a tiny SQLite datasource + + converted models and actually run queries: (a) a view cross-model measure that + references the underlying joined column (`customers.amount:sum`), (b) a view + rooted on a `sql`-backed cube, (c) a multi-hop facade dimension, (d) two + filtered same-`sql` measures returning different values, (e) `show_sql` on a + facade measure. These catch the §6/§4.4 mapping breaks at the SQL layer. +- **Negative validator-boundary construction tests** (Codex test-gap): a converter + input that would yield column↔measure namespace overlap, a measure named after + a transform (`cumsum`), a Cube name containing `.`/`:`, an ON that yields empty + `join_pairs`, a facade whose root cube wasn't emitted, and a non-currency + format carrying a symbol — assert each is routed to the **report** (model still + emitted where possible), NOT raised as an unhandled `ValidationError`. + +Run the full non-integration suite after implementation; fix all failures. + +--- + +## 13. Docs (update on user-facing change) + +- New `docs/cube/cube_import.md` (mirror `docs/dbt/dbt_import.md`): mapping + tables, the non-mapping catalog, the report, CLI usage. Link from `mkdocs.yml`. +- `CLAUDE.md` — short note under an importer/CLI section. +- `.claude/skills/` — mention `import-cube` where `import-dbt` is referenced. + +--- + +## 14. Explicit non-goals (Stage 1) + +- No live DB connection / type refinement / sample profiling (run `slayer ingest` + afterward). +- No Jinja/Python template rendering. +- No Tesseract features built (designed in §9). +- No MCP/REST surface (CLI + importable API only), matching `import-dbt`. +- No round-trip SLayer→Cube export. diff --git a/docs/cube/cube_import.md b/docs/cube/cube_import.md new file mode 100644 index 00000000..b7be9edd --- /dev/null +++ b/docs/cube/cube_import.md @@ -0,0 +1,138 @@ +# Importing Cube definitions + +SLayer can import [Cube](https://cube.dev) (Cube.js / Cube.dev) YAML data models — +cubes and views — and convert them to SLayer models. The conversion is **fully +offline**: data types come from Cube's declared dimension / measure types, so no +database connection is required. Everything that can't be mapped cleanly is +captured in a structured JSON report rather than silently dropped. + +## Quick start + +```bash +slayer import-cube ./cube_project --datasource my_postgres --storage ./slayer_data +``` + +This recursively reads every `.yml`/`.yaml` file under the path, extracts +`cubes:` and `views:`, writes SLayer model files to the storage directory, and +writes `cube_import_report.json` next to it. + +`--datasource` is just the SLayer datasource name to file the models under — it +does not need to exist or be reachable. After importing, run `slayer ingest` +against a live connection to profile sample values and refine numeric types. + +## What gets converted + +### Cubes → models + +Each cube becomes one `SlayerModel` anchored on its `sql_table` (or `sql`). + +| Cube | SLayer | +|------|--------| +| `name` | `name` | +| `sql_table` / `sql` | `sql_table` / `sql` (with `{CUBE}`/`{member}` refs translated) | +| `description` | `description` | +| `public: false` | `hidden: true` | +| `meta` (incl. `ai_context`) | `meta` | +| `title` | `meta.cube_title` | + +### Measures → columns + measures + +Cube bakes the aggregation into each measure; SLayer separates the row-level +expression (a `Column`) from the named aggregation (a `ModelMeasure`). + +```yaml +# Cube +measures: + - { name: total_revenue, type: sum, sql: "{CUBE}.amount" } +# SLayer +columns: + - { name: amount, type: DOUBLE } +measures: + - { name: total_revenue, formula: "amount:sum" } +``` + +- `count` with no `sql` → `*:count`; `count_distinct_approx` → `count_distinct`. +- Conditional `filters:` become a `CASE WHEN` on the column's `filter`. Two + measures over the same expression but different filters get distinct columns. +- A finite trailing `rolling_window` becomes a windowed aggregation + (`amount:sum(window='30d')`). +- Calculated measures (`type: number/string/time/boolean`) referencing other + measures become a `ModelMeasure` formula (`{revenue} / {count}` → `revenue / count`). +- `format` maps to `NumberFormat` (`percent`, `currency`, `number`). + +### Dimensions → columns + +`string`→`TEXT`, `number`→`DOUBLE`, `boolean`→`BOOLEAN`, `time`→`TIMESTAMP`. +`primary_key: true` carries over. A `case:` dimension becomes a `CASE WHEN` +column. + +### Joins + +A join's ON clause (`{CUBE}.customer_id = {customers.id}`) becomes +`join_pairs`; member references resolve to their physical columns. Composite +(`AND`-joined) keys are supported. All joins emit as `LEFT`. + +### Segments → boolean columns + +Each segment becomes a boolean column carrying the predicate, so it stays +filterable (`completed = true`) and group-able. + +### Views → facade models + +A Cube view (which owns no table) becomes a thin model anchored on its +`join_path` root cube: included dimensions become derived columns +(`sql: "customers.name"`), and included measures become local or cross-model +`ModelMeasure`s that reference the measure's **underlying column** — a joined +measure `revenue` with `sql: {CUBE}.amount` becomes `customers.amount:sum`, not +`customers.revenue:sum`. `prefix: true` prepends the cube name, and +`default_filters` become model filters. + +### `extends` + +Cube inheritance is **flattened** at import time — a child inherits the parent's +members (child wins on conflicts), and abstract bases (`public: false`) are +emitted as hidden models. + +## What does not map (reported) + +These are recorded in `cube_import_report.json` and, where useful, preserved +under `meta.cube_unmapped.`: + +- Caching / infra: `pre_aggregations`, `refresh_key`, `calendar`, `sql_alias`. +- Presentation: `hierarchies`, `drill_members`, folders, dimension `links`/`order`. +- Security: `access_policy`. +- No SLayer equivalent: `geo` dimensions, `sub_query` dimensions, custom + `granularities`, per-cube `data_source`. +- Non-equality / non-column join ON clauses (the join is dropped). +- Files or members using Jinja templating (`{{ }}` / `{% %}`) — skipped, since + conversion is offline and does not render templates. + +### Tesseract features (deferred) + +Features that require the Tesseract SQL planner — `switch` dimensions, +`number_agg` measures, `case` measures, and the measure `filter` grain control — +have no clean SLayer mapping yet and are reported as `deferred_stage2`. The +cube's other members still convert. + +## The report + +`CubeConversionResult` carries the emitted `models` and a `CubeConversionReport` +of categorized issues (each with a category, severity, the owning cube/view/member, +a message, and the raw Cube fragment when useful). The CLI always writes it to +`cube_import_report.json` (override with `--report PATH`) and prints a summary +grouped by severity. + +## CLI reference + +```text +slayer import-cube [options] + +Arguments: + cube_project_path Path to the Cube project (or its model directory) + +Options: + --datasource NAME SLayer datasource name for the imported models (required) + --storage PATH Storage directory / .db file (default: platform path) + --report PATH JSON report path (default: /cube_import_report.json) + --include-hidden Also print hidden (public: false) models in the summary +``` diff --git a/mkdocs.yml b/mkdocs.yml index 954cd7ec..7372eeca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,6 +40,8 @@ nav: - dbt: - SLayer vs dbt: dbt/slayer_vs_dbt.md - Importing dbt definitions: dbt/dbt_import.md + - Cube: + - Importing Cube definitions: cube/cube_import.md - Tutorials: - Make it dynamic: examples/01_dynamic/dynamic.md - SQL vs DSL: diff --git a/slayer/cli.py b/slayer/cli.py index f6758018..c32620f4 100644 --- a/slayer/cli.py +++ b/slayer/cli.py @@ -330,6 +330,23 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli ) _add_storage_arg(import_dbt_parser) + # ── import-cube ─────────────────────────────────────────────────── + import_cube_parser = subparsers.add_parser( + "import-cube", + help="Import Cube (Cube.js / Cube.dev) YAML data models into SLayer models", + epilog="""\ +examples: + slayer import-cube ./cube_project --datasource my_postgres + slayer import-cube ./cube_project/model --datasource my_postgres --report ./report.json +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + import_cube_parser.add_argument("cube_project_path", help="Path to the Cube project (or its model directory)") + import_cube_parser.add_argument("--datasource", required=True, help="SLayer datasource name to file the imported models under") + import_cube_parser.add_argument("--report", default=None, help="Path for the JSON conversion report (default: /cube_import_report.json)") + import_cube_parser.add_argument("--include-hidden", action="store_true", help="Also print hidden (public: false) models in the summary") + _add_storage_arg(import_cube_parser) + # ── models ──────────────────────────────────────────────────────── models_parser = subparsers.add_parser( "models", @@ -761,6 +778,8 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli _run_validate_models(args) elif args.command == "import-dbt": _run_import_dbt(args) + elif args.command == "import-cube": + _run_import_cube(args) elif args.command == "models": _run_models(args) elif args.command == "datasources": @@ -1441,6 +1460,68 @@ def _run_import_dbt(args): ) +def _run_import_cube(args): + from slayer.cube.converter import CubeToSlayerConverter + from slayer.cube.parser import parse_cube_project + + storage = _resolve_storage(args) + project, parse_issues = parse_cube_project(args.cube_project_path) + + if not project.cubes and not project.views: + print(f"No cubes or views found in {args.cube_project_path}") + sys.exit(1) + + result = CubeToSlayerConverter( + project=project, data_source=args.datasource, parse_issues=parse_issues, + ).convert() + from slayer.cube.report import CubeConversionIssue, CubeIssueCategory + for model in result.models: + try: + run_sync(storage.save_model(model)) + except Exception as exc: # noqa: BLE001 — one bad model shouldn't abort the import + result.report.add(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="error", + cube=model.name, + message=f"Failed to save model '{model.name}': {exc}")) + + _print_cube_import_summary(result, include_hidden=args.include_hidden) + report_path = _write_cube_report(result, args) + print( + f"\nDone: {result.report.model_count} models " + f"({result.report.hidden_count} hidden, {result.report.view_count} views), " + f"{len(result.report.issues)} report issues. Report: {report_path}" + ) + + +def _print_cube_import_summary(result, *, include_hidden): + for model in result.models: + if model.hidden and not include_hidden: + continue + suffix = " [hidden]" if model.hidden else "" + kind = " (view)" if (model.meta or {}).get("cube_kind") == "view" else "" + print( + f"Imported model: {model.name}{kind}{suffix} " + f"({len(model.columns)} columns, {len(model.measures)} measures)" + ) + for severity in ("error", "warning", "info"): + for issue in result.report.by_severity(severity): + print(f" {severity.upper()} [{issue.category.value}/{issue.context}]: {issue.message}") + + +def _write_cube_report(result, args) -> str: + import os + + storage_base = args.storage or args.models_dir or _STORAGE_DEFAULT + if storage_base.endswith(".db"): + storage_base = os.path.dirname(storage_base) or "." + report_path = args.report or os.path.join(storage_base, "cube_import_report.json") + # Report path is an intended, user-specified CLI output location. + os.makedirs(os.path.dirname(report_path) or ".", exist_ok=True) # NOSONAR(S8707) + with open(report_path, "w", encoding="utf-8") as fh: # NOSONAR(S8707) + fh.write(result.report.model_dump_json(indent=2)) + return report_path + + def _run_models(args): import yaml diff --git a/slayer/cube/__init__.py b/slayer/cube/__init__.py new file mode 100644 index 00000000..f5adfeaa --- /dev/null +++ b/slayer/cube/__init__.py @@ -0,0 +1,3 @@ +"""Cube (Cube.js / Cube.dev) data-model ingestion — parse Cube YAML and +convert to SLayer models. Mirrors ``slayer/dbt/``. See DEV-1608. +""" diff --git a/slayer/cube/converter.py b/slayer/cube/converter.py new file mode 100644 index 00000000..fd10b346 --- /dev/null +++ b/slayer/cube/converter.py @@ -0,0 +1,710 @@ +"""Convert a parsed CubeProject into SLayer models. + +Mirrors ``slayer/dbt/converter.py``. For each cube → one table-owning model; +for each view → one facade model. Everything that can't map cleanly is recorded +on the ``CubeConversionReport``. See DEV-1608. +""" + +import logging +import re + +import sqlglot +from pydantic import BaseModel + +from slayer.core.enums import DataType, JoinType +from slayer.core.format import NumberFormat, NumberFormatType +from slayer.core.formula import ALL_TRANSFORMS, parse_formula +from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel +from slayer.cube.extends import flatten_cube_extends, flatten_view_extends +from slayer.cube.models import CubeCube, CubeDimension, CubeMeasure, CubeView +from slayer.cube.refs import is_bare_identifier, parse_join_on, translate_cube_refs +from slayer.cube.report import ( + CubeConversionIssue, + CubeConversionReport, + CubeConversionResult, + CubeIssueCategory, +) + +logger = logging.getLogger(__name__) + +_AGG_TYPES = {"sum", "avg", "min", "max", "count", "count_distinct", "count_distinct_approx"} +_CALC_TYPES = {"number", "string", "time", "boolean"} +_DEFERRED_MEASURE_TYPES = {"number_agg"} +_DIM_TYPE_MAP = { + "string": DataType.TEXT, "number": DataType.DOUBLE, + "boolean": DataType.BOOLEAN, "time": DataType.TIMESTAMP, +} +_FORMAT_MAP = { + "percent": NumberFormatType.PERCENT, "currency": NumberFormatType.CURRENCY, + "number": NumberFormatType.FLOAT, +} +_DURATION_UNITS = {"day": "d", "month": "m", "week": "w", "year": "y", + "hour": "h", "minute": "min", "second": "s"} +_STAR_COUNT = "*:count" +_CUBE_INFRA_FIELDS = ("refresh_key", "calendar", "hierarchies", "access_policy", "sql_alias") + + +def _sql_str_literal(value) -> str: + """Render a value as a single-quoted SQL string literal, escaping quotes.""" + return "'" + str(value).replace("'", "''") + "'" + + +class _MeasureInfo(BaseModel): + """How a converted cube measure can be re-aggregated by a view facade.""" + kind: str # "agg" | "calc" | "star_count" + underlying_col: str | None = None + agg: str | None = None + emitted_name: str | None = None # the emitted ModelMeasure name (for pruning) + + +class _Names: + """Shared column+measure namespace allocator.""" + + def __init__(self) -> None: + self.used: set[str] = set() + + def take(self, base: str, suffix: str = "_col") -> str: + name = base + while name in self.used: + name = name + suffix + self.used.add(name) + return name + + def reserve(self, name: str) -> None: + self.used.add(name) + + +def _map_format(fmt, report: CubeConversionReport, *, cube: str, member: str) -> NumberFormat | None: + if fmt is None: + return None + ftype = fmt.get("type") if isinstance(fmt, dict) else fmt + nf_type = _FORMAT_MAP.get(ftype) if isinstance(ftype, str) else None + if nf_type is None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_FORMAT, severity="info", + cube=cube, member=member, message=f"Unsupported format '{fmt}'; dropped.", + )) + return None + kwargs = {"type": nf_type} + if nf_type == NumberFormatType.CURRENCY and isinstance(fmt, dict) and fmt.get("currency_symbol"): + kwargs["symbol"] = fmt["currency_symbol"] # symbol ONLY for currency (Codex #8) + try: + return NumberFormat(**kwargs) + except Exception: # noqa: BLE001 + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_FORMAT, severity="info", + cube=cube, member=member, message=f"Invalid format '{fmt}'; dropped.", + )) + return None + + +def _window_from_rolling(rolling, report, *, cube, member) -> str | None: + trailing = rolling.get("trailing") + if rolling.get("leading") or rolling.get("offset") or trailing in (None, "unbounded"): + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_ROLLING_WINDOW, severity="warning", + cube=cube, member=member, + message="Only finite trailing rolling_window maps; fell back to plain aggregation.", + )) + return None + m = re.match(r"(\d+)\s+(\w+)", str(trailing)) + unit = _DURATION_UNITS.get(m.group(2).rstrip("s")) if m else None + if not m or unit is None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_ROLLING_WINDOW, severity="warning", + cube=cube, member=member, message=f"Unparseable rolling_window '{trailing}'.", + )) + return None + return f"{m.group(1)}{unit}" + + +class CubeToSlayerConverter: + """Convert a CubeProject into SLayer models + a structured report.""" + + def __init__(self, project, data_source: str, parse_issues=None) -> None: + self.project = project + self.data_source = data_source + self.parse_issues = parse_issues or [] + self._cubes: dict[str, CubeCube] = {} + self._models: dict[str, SlayerModel] = {} + # model name → {measure name → _MeasureInfo} + self._measure_info: dict[str, dict[str, _MeasureInfo]] = {} + + # ── pipeline ─────────────────────────────────────────────────────────── + + def convert(self) -> CubeConversionResult: + report = CubeConversionReport(issues=list(self.parse_issues)) + + cubes, cube_issues = flatten_cube_extends(self.project.cubes) + report.issues.extend(cube_issues) + views, view_issues = flatten_view_extends(self.project.views) + report.issues.extend(view_issues) + + self._cubes = {c.name: c for c in cubes} + models: list[SlayerModel] = [] + for cube in cubes: + model = self._convert_cube(cube, report) + if model is not None: + models.append(model) + self._models[model.name] = model + + for view in views: + model = self._convert_view(view, report) + if model is not None: + models.append(model) + self._models[model.name] = model + + report.model_count = len(models) + report.hidden_count = sum(1 for m in models if m.hidden) + report.view_count = sum( + 1 for m in models if (m.meta or {}).get("cube_kind") == "view") + return CubeConversionResult(models=models, report=report) + + # ── cube → model ─────────────────────────────────────────────────────── + + def _convert_cube(self, cube: CubeCube, report: CubeConversionReport) -> SlayerModel | None: + source = self._cube_source(cube, report) + if source is None: + return None + + meta: dict = dict(cube.meta or {}) + if cube.title: + meta["cube_title"] = cube.title + unmapped: dict = {} + if cube.data_source: + unmapped["data_source"] = cube.data_source + if cube.pre_aggregations: + unmapped["pre_aggregations"] = cube.pre_aggregations + for field in _CUBE_INFRA_FIELDS: + val = getattr(cube, field, None) + if val is not None: + unmapped[field] = val + for key in unmapped: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNMAPPED_INFRA, severity="warning", + cube=cube.name, message=f"'{key}' has no SLayer equivalent; stashed in meta.", + raw=str(unmapped[key]), + )) + + names = _Names() + columns: list[Column] = [] + measures: list[ModelMeasure] = [] + info: dict[str, _MeasureInfo] = {} + + for dim in cube.dimensions: + self._convert_dimension(cube, dim, columns, names, unmapped, report) + dedup: dict[tuple, str] = {} + for meas in cube.measures: + self._convert_measure(cube, meas, columns, measures, names, dedup, info, report) + for seg in cube.segments: + self._convert_segment(cube, seg, columns, names, report) + + joins = self._convert_joins(cube, report) + columns, measures = self._validate_offline(cube.name, columns, measures, report) + self._dedisambiguate_namespace(columns, measures, report, cube=cube.name) + # Keep _measure_info in sync with what actually survived validation, so + # view facades never re-export a measure the model no longer has. + surviving = {m.name for m in measures} + info = {k: v for k, v in info.items() if v.emitted_name in surviving} + + if unmapped: + meta["cube_unmapped"] = unmapped + + try: + model = SlayerModel( + name=cube.name, data_source=self.data_source, + hidden=not cube.public, description=cube.description, + meta=meta or None, columns=columns, measures=measures, joins=joins, + **source, + ) + # Illegal model name etc. → report, don't crash the run. + except Exception as exc: # noqa: BLE001 + report.add(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="error", + cube=cube.name, message=f"Could not build model: {exc}", + )) + return None + self._measure_info[cube.name] = info + return model + + def _cube_source(self, cube: CubeCube, report) -> dict | None: + if cube.sql_table: + return {"sql_table": cube.sql_table} + if cube.sql: + translated = translate_cube_refs(cube.sql, mode="sql", cube=cube.name) + try: + sqlglot.parse_one(translated) + except Exception: # noqa: BLE001 + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_SQL, severity="error", + cube=cube.name, message="Cube 'sql' could not be translated; cube dropped.", + )) + return None + return {"sql": translated} + report.add(CubeConversionIssue( + category=CubeIssueCategory.NO_SOURCE, severity="error", + cube=cube.name, message="Cube has no sql_table/sql source; dropped.", + )) + return None + + # ── dimensions ───────────────────────────────────────────────────────── + + def _convert_dimension(self, cube, dim: CubeDimension, columns, names, unmapped, report) -> None: + if dim.type == "switch": + report.add(CubeConversionIssue( + category=CubeIssueCategory.DEFERRED_STAGE2, severity="warning", + cube=cube.name, member=dim.name, + message="`switch` dimension is a Tesseract feature (Stage 2); skipped.", + )) + return + if dim.type == "geo": + unmapped.setdefault("geo", []).append( + {"name": dim.name, "latitude": dim.latitude, "longitude": dim.longitude}) + report.add(CubeConversionIssue( + category=CubeIssueCategory.GEO_UNMAPPED, severity="warning", + cube=cube.name, member=dim.name, message="geo dimension has no SLayer type; stashed.")) + return + if dim.sub_query: + unmapped.setdefault("sub_query", []).append(dim.name) + report.add(CubeConversionIssue( + category=CubeIssueCategory.SUBQUERY_UNMAPPED, severity="warning", + cube=cube.name, member=dim.name, message="sub_query dimension has no SLayer equivalent.")) + return + if dim.granularities: + report.add(CubeConversionIssue( + category=CubeIssueCategory.GRANULARITY_UNMAPPED, severity="info", + cube=cube.name, member=dim.name, + message="Custom granularities are query-time in SLayer; base column kept.")) + + if dim.case: + sql = self._build_case_sql(cube, dim.case) + elif dim.sql: + translated = translate_cube_refs(dim.sql, mode="sql", cube=cube.name) + sql = None if translated == dim.name else translated + else: + sql = None + + name = names.take(dim.name) + columns.append(Column( + name=name, sql=sql, type=_DIM_TYPE_MAP.get(dim.type, DataType.TEXT), + primary_key=dim.primary_key, hidden=not dim.public, + label=dim.title, description=dim.description, meta=dim.meta, + format=_map_format(dim.format, report, cube=cube.name, member=dim.name), + )) + + def _build_case_sql(self, cube, case: dict) -> str: + parts = ["CASE"] + for when in case.get("when", []): + cond = translate_cube_refs(when.get("sql", ""), mode="sql", cube=cube.name) + parts.append(f"WHEN {cond} THEN {_sql_str_literal(when.get('label', ''))}") + if case.get("else"): + parts.append(f"ELSE {_sql_str_literal(case['else'].get('label', ''))}") + parts.append("END") + return " ".join(parts) + + # ── measures ─────────────────────────────────────────────────────────── + + def _convert_measure(self, cube, meas: CubeMeasure, columns, measures, names, dedup, info, report) -> None: + if meas.type in _DEFERRED_MEASURE_TYPES or meas.case is not None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.DEFERRED_STAGE2, severity="warning", + cube=cube.name, member=meas.name, + message=f"Measure type/shape '{meas.type}' is a Tesseract feature (Stage 2); skipped.")) + return + if meas.drill_members: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNMAPPED_INFRA, severity="info", + cube=cube.name, member=meas.name, message="drill_members has no SLayer equivalent.")) + if meas.multi_stage or meas.time_shift or meas.grain or meas.filter is not None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.DEFERRED_STAGE2, severity="info", + cube=cube.name, member=meas.name, + message="multi_stage/time_shift/grain/filter on measure deferred to Stage 2; " + "emitted as plain aggregation if possible.")) + + if meas.name in ALL_TRANSFORMS: + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", + cube=cube.name, member=meas.name, + message=f"Measure name '{meas.name}' shadows a SLayer transform; skipped.")) + return + + # Reserve the measure's name BEFORE creating its underlying column, so a + # bare-identifier column yields (`rate` → `rate_col`) and the measure keeps + # the Cube name (dbt-importer idiom). Collision with a dimension column + # falls through to the `_measure` suffix. + final_name = names.take(meas.name, suffix="_measure") + if meas.type in _CALC_TYPES and meas.sql: + self._convert_calc_measure(cube, meas, measures, final_name, info, report) + return + self._convert_agg_measure(cube, meas, columns, measures, names, dedup, info, report, final_name) + + def _convert_agg_measure(self, cube, meas, columns, measures, names, dedup, info, report, final_name) -> None: + agg = "count_distinct" if meas.type == "count_distinct_approx" else meas.type + if meas.type == "count_distinct_approx": + report.add(CubeConversionIssue( + category=CubeIssueCategory.LOSSY_MAPPING, severity="info", + cube=cube.name, member=meas.name, + message="count_distinct_approx → exact count_distinct (SLayer has no approx).")) + + window = _window_from_rolling(meas.rolling_window, report, cube=cube.name, member=meas.name) \ + if meas.rolling_window else None + + if meas.type == "count" and not meas.sql: + formula = _STAR_COUNT + (f"(window='{window}')" if window else "") + if self._emit_measure(measures, names, final_name, formula, meas, report, cube.name): + info[meas.name] = _MeasureInfo(kind="star_count", emitted_name=final_name) + return + + translated = translate_cube_refs(meas.sql, mode="sql", cube=cube.name) + filter_pred = self._measure_filter(cube, meas) + col_name = self._get_or_create_column( + meas, translated, filter_pred, columns, names, dedup, report, cube) + formula = f"{col_name}:{agg}" + (f"(window='{window}')" if window else "") + if self._emit_measure(measures, names, final_name, formula, meas, report, cube.name): + info[meas.name] = _MeasureInfo( + kind="agg", underlying_col=col_name, agg=agg, emitted_name=final_name) + + def _convert_calc_measure(self, cube, meas, measures, final_name, info, report) -> None: + formula = translate_cube_refs(meas.sql, mode="dsl", cube=cube.name) + if self._emit_measure(measures, None, final_name, formula, meas, report, cube.name, + result_type=_DIM_TYPE_MAP.get(meas.type)): + info[meas.name] = _MeasureInfo(kind="calc", emitted_name=final_name) + + def _measure_filter(self, cube, meas) -> str | None: + if not meas.filters: + return None + preds = [translate_cube_refs(f.sql, mode="sql", cube=cube.name) for f in meas.filters] + return " AND ".join(preds) if preds else None + + def _get_or_create_column(self, meas, translated_sql, filter_pred, columns, names, dedup, report, cube) -> str: + key = (translated_sql, filter_pred) + if key in dedup: + return dedup[key] + base = translated_sql if is_bare_identifier(translated_sql) else f"{meas.name}_col" + col_name = names.take(base) + columns.append(Column( + name=col_name, + sql=None if col_name == translated_sql else translated_sql, + type=DataType.DOUBLE, filter=filter_pred, + format=_map_format(meas.format, report, cube=cube.name, member=meas.name), + )) + dedup[key] = col_name + return col_name + + def _emit_measure(self, measures, names, final_name, formula, meas, report, cube_name, *, result_type=None) -> bool: + try: + measures.append(ModelMeasure( + name=final_name, formula=formula, label=meas.title, + description=meas.description, type=result_type, meta=meas.meta)) + return True + except Exception as exc: # noqa: BLE001 + if names is not None: + names.used.discard(final_name) + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", + cube=cube_name, member=meas.name, + message=f"Measure '{meas.name}' could not be built: {exc}")) + return False + + # ── segments ─────────────────────────────────────────────────────────── + + def _convert_segment(self, cube, seg, columns, names, report) -> None: + name = names.take(seg.name, suffix="_seg") + columns.append(Column( + name=name, sql=translate_cube_refs(seg.sql, mode="sql", cube=cube.name), + type=DataType.BOOLEAN, hidden=not seg.public, + label=seg.title, description=seg.description, meta=seg.meta)) + report.add(CubeConversionIssue( + category=CubeIssueCategory.SEGMENT_AS_COLUMN, severity="info", + cube=cube.name, member=seg.name, + message=f"Segment '{seg.name}' mapped to a boolean column.")) + + # ── joins ────────────────────────────────────────────────────────────── + + def _convert_joins(self, cube, report) -> list[ModelJoin]: + joins: list[ModelJoin] = [] + for cj in cube.joins: + if cj.name not in self._cubes: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_JOIN, severity="warning", + cube=cube.name, member=cj.name, + message=f"Join target cube '{cj.name}' is not available; dropped.")) + continue + pairs = parse_join_on(cj.sql, source_cube=cube.name, target_cube=cj.name) + resolved = self._resolve_join_pairs(cube, cj, pairs) if pairs else None + if not resolved: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_JOIN, severity="warning", + cube=cube.name, member=cj.name, + message=f"Join ON '{cj.sql}' is not an equality of physical columns; dropped.")) + continue + joins.append(ModelJoin(target_model=cj.name, join_pairs=resolved, join_type=JoinType.LEFT)) + return joins + + def _resolve_join_pairs(self, cube, cj, pairs) -> list[list[str]] | None: + target = self._cubes.get(cj.name) + out: list[list[str]] = [] + for src_member, tgt_member in pairs: + src = self._physical_col(cube, src_member) + tgt = self._physical_col(target, tgt_member) if target else tgt_member + if src is None or tgt is None: + return None + out.append([src, tgt]) + return out + + def _physical_col(self, cube, member: str) -> str | None: + if cube is None: + return member + dim = next((d for d in cube.dimensions if d.name == member), None) + if dim is None or dim.sql is None: + return member + translated = translate_cube_refs(dim.sql, mode="sql", cube=cube.name) + return translated.strip() if is_bare_identifier(translated) else None + + # ── offline validation + namespace safety ────────────────────────────── + + def _validate_offline(self, cube_name, columns, measures, report): + good_cols = [] + dropped: set[str] = set() + for col in columns: + if col.sql is None: + good_cols.append(col) + continue + try: + sqlglot.parse_one(col.sql) + good_cols.append(col) + except Exception: # noqa: BLE001 + dropped.add(col.name) + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_SQL, severity="warning", + cube=cube_name, member=col.name, + message=f"Column sql does not parse; dropped: {col.sql!r}")) + col_names = {c.name for c in good_cols} + known = col_names | {m.name for m in measures if m.name} + good_measures = [] + for m in measures: + ref_col = m.formula.split(":")[0].strip() + if ref_col in dropped: + continue + if not self._formula_parses(m.formula, known): + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", + cube=cube_name, member=m.name, + message=f"Measure formula does not parse; dropped: {m.formula!r}")) + continue + good_measures.append(m) + return good_cols, good_measures + + def _formula_parses(self, formula: str, known_names: set[str]) -> bool: + nm = dict.fromkeys(known_names, _STAR_COUNT) + try: + parse_formula(formula, named_measures=nm or None) + return True + except Exception: # noqa: BLE001 + return False + + def _dedisambiguate_namespace(self, columns, measures, report, *, cube) -> None: + # The allocator already guarantees uniqueness; this is a defensive check. + col_names = {c.name for c in columns} + for m in measures: + if m.name in col_names: # pragma: no cover — allocator prevents this + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="info", + cube=cube, member=m.name, message="measure/column name overlap auto-resolved.")) + + # ── views → facade models ────────────────────────────────────────────── + + def _convert_view(self, view: CubeView, report) -> SlayerModel | None: + if not view.cubes: + report.add(CubeConversionIssue( + category=CubeIssueCategory.AMBIGUOUS_VIEW_ROOT, severity="warning", + view=view.name, message="View has no cubes; skipped.")) + return None + root_cube_name = view.cubes[0].join_path.split(".")[0] + root_cube = self._cubes.get(root_cube_name) + root_model = self._models.get(root_cube_name) + if root_cube is None or root_model is None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.AMBIGUOUS_VIEW_ROOT, severity="warning", + view=view.name, + message=f"View root cube '{root_cube_name}' was not emitted; view dropped.")) + return None + + source = {"sql_table": root_model.sql_table} if root_model.sql_table else {"sql": root_model.sql} + meta = {"cube_kind": "view"} + unmapped: dict = {} + if view.folders: + unmapped["folders"] = view.folders + report.add(CubeConversionIssue( + category=CubeIssueCategory.FOLDERS_UNMAPPED, severity="info", + view=view.name, message="Folders have no SLayer hierarchy; stashed in meta.")) + + names = _Names() + columns: list[Column] = [] + measures: list[ModelMeasure] = [] + join_targets: set[str] = set() + + for ref in view.cubes: + self._convert_view_ref(view, ref, root_cube_name, root_model, + columns, measures, names, join_targets, report) + + joins = [j for j in root_model.joins if j.target_model in join_targets] + filters = self._view_default_filters(view, root_cube_name, report) + if unmapped: + meta["cube_unmapped"] = unmapped + + try: + return SlayerModel( + name=view.name, data_source=self.data_source, + hidden=not view.public, description=view.description, meta=meta, + columns=columns, measures=measures, joins=joins, filters=filters, + **source, + ) + except Exception as exc: # noqa: BLE001 + report.add(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="error", + view=view.name, message=f"Could not build facade model: {exc}")) + return None + + def _convert_view_ref(self, view, ref, root_cube_name, root_model, + columns, measures, names, join_targets, report) -> None: + path = ref.join_path.split(".") + cube_name = path[-1] + cube = self._cubes.get(cube_name) + cube_model = self._models.get(cube_name) + is_root = (len(path) == 1 and cube_name == root_cube_name) + + if cube is None or cube_model is None: + report.add(CubeConversionIssue( + category=CubeIssueCategory.DISCONNECTED_VIEW, severity="warning", + view=view.name, member=ref.join_path, + message=f"View member cube '{cube_name}' not available; skipped.")) + return + + if not is_root: + join = next((j for j in root_cube.joins if j.name == cube_name), None) \ + if (root_cube := self._cubes.get(root_cube_name)) else None + if join is None or not any(j.target_model == cube_name for j in root_model.joins): + report.add(CubeConversionIssue( + category=CubeIssueCategory.DISCONNECTED_VIEW, severity="warning", + view=view.name, member=ref.join_path, + message=f"'{cube_name}' is not joined to root '{root_cube_name}'; skipped.")) + return + if join.relationship in ("one_to_many", "has_many"): + report.add(CubeConversionIssue( + category=CubeIssueCategory.VIEW_FANOUT_RISK, severity="warning", + view=view.name, member=ref.join_path, + message=f"Join to '{cube_name}' is {join.relationship}; root measures may fan out.")) + join_targets.add(cube_name) + + prefix = f"{ref.alias or cube_name}_" if ref.prefix else "" + dim_names, meas_names = self._selected_members(cube, ref, view, report) + + for dname in dim_names: + self._facade_dimension(cube, cube_model, dname, prefix, is_root, + columns, names) + for mname in meas_names: + self._facade_measure(view, cube_name, cube_model, mname, prefix, is_root, + columns, measures, names, report) + + def _selected_members(self, cube, ref, view, report) -> tuple[list[str], list[str]]: + dims = [d.name for d in cube.dimensions if d.type not in ("geo", "switch") and not d.sub_query] + meas = [m.name for m in cube.measures] + exclude = set(ref.excludes or []) + if ref.includes in (None, "*"): + # `includes: "*"` must not re-export members the cube marked private. + private = {d.name for d in cube.dimensions if not d.public} \ + | {m.name for m in cube.measures if not m.public} + chosen = [n for n in dims + meas if n not in exclude and n not in private] + else: + chosen = self._include_names(ref.includes, exclude, view, report) + chosen_set = set(chosen) + return ([d for d in dims if d in chosen_set], [m for m in meas if m in chosen_set]) + + def _include_names(self, includes, exclude, view, report) -> list[str]: + """Extract member names from an ``includes`` list. Cube's per-member + override object form (``{name, format, meta, …}``) is accepted so it + doesn't crash the view; the overrides are reported as unsupported.""" + names: list[str] = [] + for entry in includes: + if isinstance(entry, dict): + name = entry.get("name") + if any(k != "name" for k in entry): + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNMAPPED_INFRA, severity="info", + view=view.name, member=name, + message=f"Per-member override on '{name}' is not applied (Stage 1).")) + else: + name = entry + if name and name not in exclude: + names.append(name) + return names + + def _facade_dimension(self, cube, cube_model, dname, prefix, is_root, columns, names) -> None: + col = cube_model.get_column(dname) + if col is None: + return + if is_root: + sql = col.sql if col.sql else col.name + else: + sql = f"{cube.name}.{col.name}" + exported = names.take(f"{prefix}{dname}") + columns.append(Column( + name=exported, sql=sql, type=col.type, label=col.label, + description=col.description, format=col.format)) + + def _facade_measure(self, view, cube_name, cube_model, mname, prefix, is_root, + columns, measures, names, report) -> None: + info = self._measure_info.get(cube_name, {}).get(mname) + if info is None: + return + exported = names.take(f"{prefix}{mname}", suffix="_measure") + if info.kind == "star_count": + formula = _STAR_COUNT if is_root else f"{cube_name}.{_STAR_COUNT}" + elif info.kind == "agg": + base = f"{info.underlying_col}:{info.agg}" + if is_root: + src_col = cube_model.get_column(info.underlying_col) + if src_col is not None and not any(c.name == info.underlying_col for c in columns): + columns.append(src_col.model_copy()) # carry the underlying column onto the facade + formula = base + else: + formula = f"{cube_name}.{info.underlying_col}:{info.agg}" + else: + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="info", + view=view.name, member=mname, + message=f"Calculated measure '{mname}' re-export not supported in Stage 1.")) + return + try: + measures.append(ModelMeasure(name=exported, formula=formula)) + except Exception as exc: # noqa: BLE001 + report.add(CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", + view=view.name, member=mname, message=f"Facade measure failed: {exc}")) + + def _view_default_filters(self, view, root_cube_name, report) -> list[str]: + filters: list[str] = [] + for df in view.default_filters or []: + member = df.get("member", "") + op = df.get("operator") + values = df.get("values") or [] + col = self._resolve_view_member(member, root_cube_name) + if op == "equals" and len(values) == 1: + filters.append(f"{col} = {_sql_str_literal(values[0])}") + elif op in ("equals", "in") and values: + vlist = ", ".join(_sql_str_literal(v) for v in values) + filters.append(f"{col} IN ({vlist})") + else: + report.add(CubeConversionIssue( + category=CubeIssueCategory.UNSUPPORTED_DEFAULT_FILTER, severity="info", + view=view.name, member=member, + message=f"default_filter operator '{op}' not mapped; dropped.")) + return filters + + def _resolve_view_member(self, member: str, root_cube_name: str) -> str: + parts = member.split(".") + if parts and parts[0] == root_cube_name: + parts = parts[1:] + return ".".join(parts) if parts else member diff --git a/slayer/cube/extends.py b/slayer/cube/extends.py new file mode 100644 index 00000000..43e634db --- /dev/null +++ b/slayer/cube/extends.py @@ -0,0 +1,126 @@ +"""Resolve Cube `extends` by flattening base members into children. + +DEV-1608 §5. Native persisted inheritance is tracked separately in DEV-1610; +this module flattens (faithful to Cube's own compile-time materialization). +""" + +from slayer.cube.models import CubeCube, CubeView +from slayer.cube.report import CubeConversionIssue, CubeIssueCategory + + +def _merge_by_name(parent_items: list, child_items: list) -> list: + """Merge two member lists by ``.name``; child wins on conflict, parent order + preserved, child-new appended.""" + merged: dict[str, object] = {item.name: item for item in parent_items} + for item in child_items: + merged[item.name] = item + return list(merged.values()) + + +def _merge_cube(parent: CubeCube, child: CubeCube) -> CubeCube: + if child.sql_table or child.sql: + sql_table, sql = child.sql_table, child.sql + else: + sql_table, sql = parent.sql_table, parent.sql + return child.model_copy(update={ + "extends": None, + "sql_table": sql_table, + "sql": sql, + "dimensions": _merge_by_name(parent.dimensions, child.dimensions), + "measures": _merge_by_name(parent.measures, child.measures), + "joins": _merge_by_name(parent.joins, child.joins), + "segments": _merge_by_name(parent.segments, child.segments), + }) + + +def _find_cyclic(by_name: dict) -> set[str]: + """Return every node that lies on an ``extends`` cycle. + + Detecting the full cycle up front (rather than only the closing frame) means + *no* node on a cycle inherits — matching the "flattened without inheritance" + contract. Nodes that merely *extend into* a cycle are not cyclic; they + inherit the cyclic node's own (un-merged) members. + """ + cyclic: set[str] = set() + for start in by_name: + path: list[str] = [] + seen: set[str] = set() + cur = start + while cur in by_name: + nxt = by_name[cur].extends + if not nxt or nxt not in by_name: + break + if nxt in seen or nxt == cur: + if nxt in path: + cyclic.update(path[path.index(nxt):]) + cyclic.add(nxt) + cyclic.add(cur) + break + path.append(cur) + seen.add(cur) + cur = nxt + return cyclic + + +def flatten_cube_extends( + cubes: list[CubeCube], +) -> tuple[list[CubeCube], list[CubeConversionIssue]]: + """Flatten the cube `extends` graph (child wins; multi-level transitive; + cycles reported). Every cube is still returned (hidden iff ``public: false`` + is handled downstream by the converter).""" + by_name = {c.name: c for c in cubes} + cyclic = _find_cyclic(by_name) + issues = [CubeConversionIssue( + category=CubeIssueCategory.EXTENDS_CYCLE, severity="error", cube=name, + message=f"Cube '{name}' is in an extends cycle; flattened without inheritance.") + for name in sorted(cyclic)] + resolved: dict[str, CubeCube] = {} + + def resolve(name: str) -> CubeCube: + if name in resolved: + return resolved[name] + cube = by_name[name] + if name in cyclic or not cube.extends or cube.extends not in by_name: + resolved[name] = cube + return cube + merged = _merge_cube(resolve(cube.extends), cube) + resolved[name] = merged + return merged + + return [resolve(c.name) for c in cubes], issues + + +def _merge_view(parent: CubeView, child: CubeView) -> CubeView: + return child.model_copy(update={ + "extends": None, + "cubes": list(parent.cubes) + list(child.cubes), + "default_filters": ((parent.default_filters or []) + (child.default_filters or [])) or None, + "folders": ((parent.folders or []) + (child.folders or [])) or None, + }) + + +def flatten_view_extends( + views: list[CubeView], +) -> tuple[list[CubeView], list[CubeConversionIssue]]: + """Flatten the view `extends` graph by concatenating member-contributing + cube refs (the converter dedups members).""" + by_name = {v.name: v for v in views} + cyclic = _find_cyclic(by_name) + issues = [CubeConversionIssue( + category=CubeIssueCategory.EXTENDS_CYCLE, severity="error", view=name, + message=f"View '{name}' is in an extends cycle; flattened without inheritance.") + for name in sorted(cyclic)] + resolved: dict[str, CubeView] = {} + + def resolve(name: str) -> CubeView: + if name in resolved: + return resolved[name] + view = by_name[name] + if name in cyclic or not view.extends or view.extends not in by_name: + resolved[name] = view + return view + merged = _merge_view(resolve(view.extends), view) + resolved[name] = merged + return merged + + return [resolve(v.name) for v in views], issues diff --git a/slayer/cube/models.py b/slayer/cube/models.py new file mode 100644 index 00000000..e26bc7d3 --- /dev/null +++ b/slayer/cube/models.py @@ -0,0 +1,136 @@ +"""Pydantic v2 models for parsed Cube (Cube.js / Cube.dev) YAML objects. + +Lightweight representations of Cube's ``cubes:`` and ``views:`` YAML. We don't +depend on any Cube runtime — these shapes are populated directly from the YAML +by ``slayer.cube.parser`` and consumed by ``slayer.cube.converter``. + +Unknown keys are tolerated (Cube's schema evolves); Pydantic's default +``extra="ignore"`` drops them, except for the fields explicitly captured below +so the converter can either map them or stash the raw value under +``meta.cube_unmapped.`` (see the spec on DEV-1608, §7). +""" + +from typing import Any + +from pydantic import BaseModel, Field + + +class CubeMeasureFilter(BaseModel): + """One entry of a Cube measure ``filters:`` list — a conditional-aggregation + predicate (``{sql: "..."}``).""" + + sql: str + + +class CubeMeasure(BaseModel): + name: str + type: str # count, count_distinct, count_distinct_approx, sum, avg, min, max, + # number, string, time, boolean, number_agg (Tesseract) + sql: str | None = None + title: str | None = None + description: str | None = None + public: bool = True + meta: dict[str, Any] | None = None + format: Any | None = None # str ("percent"/"currency"/…) or object + filters: list[CubeMeasureFilter] = Field(default_factory=list) + drill_members: list[str] = Field(default_factory=list) + rolling_window: dict[str, Any] | None = None + # Multi-stage / Tesseract-leaning fields (mostly Stage-2 — see §9). + multi_stage: bool = False + time_shift: Any | None = None + grain: Any | None = None + filter: Any | None = None # Tesseract grain filter (exclude/keep_only/mode) + case: Any | None = None # Tesseract conditional measure (switch-keyed) + + +class CubeDimension(BaseModel): + name: str + sql: str | None = None + type: str = "string" # string, number, boolean, time, geo, switch (Tesseract) + title: str | None = None + description: str | None = None + public: bool = True + meta: dict[str, Any] | None = None + format: Any | None = None + primary_key: bool = False + sub_query: bool = False + case: dict[str, Any] | None = None # CASE-WHEN dimension (Stage 1) + granularities: list[dict[str, Any]] | None = None # custom time granularities + latitude: dict[str, Any] | None = None # geo + longitude: dict[str, Any] | None = None # geo + links: Any | None = None # presentation + order: str | None = None # presentation + + +class CubeJoin(BaseModel): + name: str # target cube name + relationship: str = "many_to_one" + sql: str # ON clause, e.g. "{CUBE}.customer_id = {customers.id}" + + +class CubeSegment(BaseModel): + name: str + sql: str + title: str | None = None + description: str | None = None + public: bool = True + meta: dict[str, Any] | None = None + + +class CubeCube(BaseModel): + name: str + sql_table: str | None = None + sql: str | None = None + sql_alias: str | None = None + extends: str | None = None + data_source: str | None = None + title: str | None = None + description: str | None = None + public: bool = True + meta: dict[str, Any] | None = None + refresh_key: dict[str, Any] | None = None + calendar: bool | None = None + measures: list[CubeMeasure] = Field(default_factory=list) + dimensions: list[CubeDimension] = Field(default_factory=list) + joins: list[CubeJoin] = Field(default_factory=list) + segments: list[CubeSegment] = Field(default_factory=list) + hierarchies: list[dict[str, Any]] | None = None + pre_aggregations: list[dict[str, Any]] | None = None + access_policy: list[dict[str, Any]] | None = None + + +class CubeViewCubeRef(BaseModel): + """One entry of a view's ``cubes:`` list — a cube reached via ``join_path`` + contributing a set of members to the view. + + ``includes`` is a list of member names, the string ``"*"``, or Cube's + per-member override object form (``[{name, alias, title, format, meta}, …]``). + The converter extracts the member names and reports per-member overrides as + unsupported (Stage 1) rather than silently dropping them. + """ + + join_path: str + includes: list[str | dict[str, Any]] | str | None = None + excludes: list[str] = Field(default_factory=list) + prefix: bool = False + alias: str | None = None # renames the cube for member prefixing + + +class CubeView(BaseModel): + name: str + cubes: list[CubeViewCubeRef] = Field(default_factory=list) + extends: str | None = None + title: str | None = None + description: str | None = None + public: bool = True + meta: dict[str, Any] | None = None + folders: list[dict[str, Any]] | None = None + default_filters: list[dict[str, Any]] | None = None + access_policy: list[dict[str, Any]] | None = None + + +class CubeProject(BaseModel): + """Aggregated result of parsing all YAML files in a Cube project.""" + + cubes: list[CubeCube] = Field(default_factory=list) + views: list[CubeView] = Field(default_factory=list) diff --git a/slayer/cube/parser.py b/slayer/cube/parser.py new file mode 100644 index 00000000..70c85cb2 --- /dev/null +++ b/slayer/cube/parser.py @@ -0,0 +1,158 @@ +"""Parse a Cube project directory into a CubeProject. + +Walks the directory for ``*.yml`` / ``*.yaml``, extracts top-level ``cubes:`` +and ``views:``, and surfaces Jinja-templated files/members + malformed files as +report issues. See DEV-1608 §2. +""" + +import logging +import os + +import yaml + +from slayer.cube.models import CubeCube, CubeProject, CubeView +from slayer.cube.refs import contains_jinja +from slayer.cube.report import CubeConversionIssue, CubeIssueCategory + +logger = logging.getLogger(__name__) + + +def _collect_yaml_paths(directory: str) -> list[str]: + paths: list[str] = [] + for root, dirs, files in os.walk(directory): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "target"] + for filename in sorted(files): + if filename.startswith("."): + continue + if filename.endswith((".yaml", ".yml")): + paths.append(os.path.join(root, filename)) + return paths + + +def _str_has_jinja(value) -> bool: + return isinstance(value, str) and contains_jinja(value) + + +def _member_has_jinja(item) -> bool: + """True if any templatable field of a member (``sql`` / ``filter`` / + measure ``filters`` / ``case`` predicates) contains Jinja.""" + if not isinstance(item, dict): + return False + if _str_has_jinja(item.get("sql")) or _str_has_jinja(item.get("filter")): + return True + if any(_str_has_jinja((f or {}).get("sql")) for f in item.get("filters") or []): + return True + case = item.get("case") + if isinstance(case, dict): + return any(_str_has_jinja((w or {}).get("sql")) for w in case.get("when") or []) + return False + + +def _filter_jinja(items: list, *, cube_name, path, issues, has_jinja) -> list: + kept = [] + for item in items: + if has_jinja(item): + issues.append(CubeConversionIssue( + category=CubeIssueCategory.REQUIRES_TEMPLATING, severity="warning", + cube=cube_name, + member=item.get("name") if isinstance(item, dict) else None, + message=f"Member in '{path}' uses Jinja templating; skipped.", + )) + else: + kept.append(item) + return kept + + +def _strip_jinja_members(raw_cube: dict, issues: list, path: str) -> None: + """Drop members whose templatable fields contain Jinja, reporting each.""" + cube_name = raw_cube.get("name") + for key in ("dimensions", "measures", "segments"): + items = raw_cube.get(key) + if isinstance(items, list): + raw_cube[key] = _filter_jinja( + items, cube_name=cube_name, path=path, issues=issues, + has_jinja=_member_has_jinja) + joins = raw_cube.get("joins") + if isinstance(joins, list): + raw_cube["joins"] = _filter_jinja( + joins, cube_name=cube_name, path=path, issues=issues, + has_jinja=lambda j: _str_has_jinja((j or {}).get("sql"))) + + +def _as_list(value) -> list: + if value is None: + return [] + return value if isinstance(value, list) else [value] + + +def _load_yaml(path: str, issues: list): + try: + with open(path, encoding="utf-8") as fh: + raw_text = fh.read() + except OSError as exc: + issues.append(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="warning", + message=f"File '{path}' could not be read: {exc}", + )) + return None + try: + return yaml.safe_load(raw_text) + except yaml.YAMLError: + templated = contains_jinja(raw_text) + category = (CubeIssueCategory.REQUIRES_TEMPLATING if templated + else CubeIssueCategory.PARSE_ERROR) + issues.append(CubeConversionIssue( + category=category, severity="warning", + message=f"File '{path}' could not be parsed as YAML" + + (" (Jinja templating)." if templated else "."), + )) + return None + + +def _parse_cubes(data: dict, path: str, cubes: list, issues: list) -> None: + for raw_cube in _as_list(data.get("cubes")): + if not isinstance(raw_cube, dict): + continue + _strip_jinja_members(raw_cube, issues, path) + try: + cubes.append(CubeCube.model_validate(raw_cube)) + except Exception as exc: # noqa: BLE001 — keep parsing the rest + issues.append(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="warning", + cube=raw_cube.get("name"), + message=f"Failed to parse cube in '{path}': {exc}", + )) + + +def _parse_views(data: dict, path: str, views: list, issues: list) -> None: + for raw_view in _as_list(data.get("views")): + if not isinstance(raw_view, dict): + continue + try: + views.append(CubeView.model_validate(raw_view)) + except Exception as exc: # noqa: BLE001 — keep parsing the rest + issues.append(CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, severity="warning", + view=raw_view.get("name"), + message=f"Failed to parse view in '{path}': {exc}", + )) + + +def parse_cube_project(project_path: str) -> tuple[CubeProject, list[CubeConversionIssue]]: + """Parse a Cube project directory. + + Returns the parsed ``CubeProject`` plus parse-time issues + (``requires_templating`` for Jinja, ``parse_error`` for malformed files). + """ + cubes: list[CubeCube] = [] + views: list[CubeView] = [] + issues: list[CubeConversionIssue] = [] + + for path in _collect_yaml_paths(project_path): + data = _load_yaml(path, issues) + if not isinstance(data, dict): + continue + _parse_cubes(data, path, cubes, issues) + _parse_views(data, path, views, issues) + + return CubeProject(cubes=cubes, views=views), issues diff --git a/slayer/cube/refs.py b/slayer/cube/refs.py new file mode 100644 index 00000000..a149380b --- /dev/null +++ b/slayer/cube/refs.py @@ -0,0 +1,129 @@ +"""Translate Cube curly references to SLayer SQL (Mode A) / DSL (Mode B). + +Cube `sql`/`filter` strings use `{CUBE}`, `{member}`, `{cube.member}` — single +braces, distinct from Jinja's `{{ }}` / `{% %}` (which are detected and skipped +upstream). See DEV-1608 §3. +""" + +import re + +_JINJA_RE = re.compile(r"\{\{|\{%|%\}|\}\}") +_LITERAL_RE = re.compile(r"'(?:''|[^'])*'") # SQL string literal, doubled-quote aware +_REF_RE = re.compile(r"\{([^{}]+)\}") + +# Operand forms accepted inside a join ON clause. +_OPERAND_BRACE_DOT = re.compile(r"^\{([A-Za-z_]\w*)\}\.(\w+)$") # {CUBE}.col +_OPERAND_BRACED = re.compile(r"^\{([^{}]+)\}$") # {cube.col} +# `\bAND\b` (no surrounding `\s+` quantifiers) avoids the polynomial-backtracking +# shape Sonar S5852 flags; operands are whitespace-stripped after the split. +_AND_SPLIT = re.compile(r"\bAND\b", re.IGNORECASE) +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_]\w*$") + + +def contains_jinja(text: str) -> bool: + """True if ``text`` contains Jinja markers (`{{ }}` or `{% %}`). + + Cube's own `{CUBE}` / `{cube.member}` single-brace refs are NOT Jinja. + """ + return bool(_JINJA_RE.search(text)) + + +def translate_cube_refs(text: str, *, mode: str, cube: str | None = None) -> str: + """Translate Cube curly refs in ``text`` (string literals are left intact). + + - ``{CUBE}.col`` → ``col`` (SLayer auto-qualifies bare names) + - bare ``{CUBE}`` → the cube name (table reference) + - ``{member}`` → ``member`` (same-cube sibling) + - ``{cube.member}`` / ``{a.b.c}`` → ``cube.member`` / ``a.b.c`` (dotted) + + ``mode`` (``"sql"``/``"dsl"``) documents the intended target layer for the + caller; it is validated but does not change the syntactic rewrite. + """ + if mode not in ("sql", "dsl"): + raise ValueError(f"mode must be 'sql' or 'dsl', got {mode!r}") + literal_spans = [m.span() for m in _LITERAL_RE.finditer(text)] + + def _in_literal(pos: int) -> bool: + return any(s <= pos < e for s, e in literal_spans) + + out: list[str] = [] + last = 0 + for m in _REF_RE.finditer(text): + if m.start() < last or _in_literal(m.start()): + continue + out.append(text[last:m.start()]) + next_char = text[m.end()] if m.end() < len(text) else "" + replacement, extra = _resolve_ref(m.group(1).strip(), next_char, cube) + out.append(replacement) + last = m.end() + extra + out.append(text[last:]) + return "".join(out) + + +def _resolve_ref(inner: str, next_char: str, cube: str | None) -> tuple[str, int]: + """Resolve one ``{ref}`` to ``(replacement, extra_chars_consumed)``.""" + if inner == "CUBE": + if next_char == ".": + return "", 1 # `{CUBE}.col` → drop the `{CUBE}` and the following dot + return cube or "", 0 + return inner, 0 # `{member}` / `{a.b}` → inner verbatim + + +def _operand_ref(operand: str) -> tuple[str, str] | None: + """Parse one side of an equality into ``(qualifier, column)``. + + Returns ``None`` for anything that isn't a bare Cube column reference + (function calls, arithmetic, literals) — those can't go in ``join_pairs``. + """ + operand = operand.strip() + m = _OPERAND_BRACE_DOT.match(operand) + if m: + return m.group(1), m.group(2) + m = _OPERAND_BRACED.match(operand) + if m: + parts = m.group(1).strip().split(".") + if len(parts) >= 2: + return parts[0], ".".join(parts[1:]) + return None + + +def parse_join_on(on_sql: str, *, source_cube: str, target_cube: str) -> list[list[str]] | None: + """Parse a Cube join ON clause into SLayer ``join_pairs``. + + Returns ``[[src_col, tgt_col], ...]`` for an equality (or AND-conjunction of + equalities); ``None`` for any non-equality / non-column ON. The column names + are the *member* names as written — the converter resolves them to physical + columns (and drops the join if a member's sql is non-trivial). + """ + pairs: list[list[str]] = [] + for part in _AND_SPLIT.split(on_sql.strip()): + pair = _equality_pair(part, source_cube=source_cube, target_cube=target_cube) + if pair is None: + return None + pairs.append(pair) + return pairs or None + + +def _equality_pair(part: str, *, source_cube: str, target_cube: str) -> list[str] | None: + """Parse one ``A = B`` equality into ``[src_col, tgt_col]`` or ``None``.""" + if part.count("=") != 1 or any(op in part for op in ("<", ">", "!")): + return None + left, right = part.split("=") + lhs = _operand_ref(left) + rhs = _operand_ref(right) + if lhs is None or rhs is None: + return None + src_col = tgt_col = None + for qualifier, col in (lhs, rhs): + if qualifier in ("CUBE", source_cube): + src_col = col + elif qualifier == target_cube: + tgt_col = col + if src_col is None or tgt_col is None: + return None + return [src_col, tgt_col] + + +def is_bare_identifier(sql: str) -> bool: + """True if ``sql`` is a single bare column identifier (usable in join_pairs).""" + return bool(_IDENTIFIER_RE.match(sql.strip())) diff --git a/slayer/cube/report.py b/slayer/cube/report.py new file mode 100644 index 00000000..449a7aea --- /dev/null +++ b/slayer/cube/report.py @@ -0,0 +1,82 @@ +"""Structured report for Cube → SLayer conversion (DEV-1608, §10). + +Everything the converter cannot map cleanly is recorded as a +``CubeConversionIssue`` rather than silently dropped or raised. The full +``CubeConversionResult`` (models + report) is returned by the converter and the +report is also written to JSON by the CLI. +""" + +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, Field + +from slayer.core.models import SlayerModel + + +class CubeIssueCategory(str, Enum): + REQUIRES_TEMPLATING = "requires_templating" + PARSE_ERROR = "parse_error" + COMPLEX_SQL = "complex_sql" + COMPLEX_MEASURE = "complex_measure" + LOSSY_MAPPING = "lossy_mapping" + UNSUPPORTED_JOIN = "unsupported_join" + UNSUPPORTED_ROLLING_WINDOW = "unsupported_rolling_window" + UNSUPPORTED_FORMAT = "unsupported_format" + UNSUPPORTED_DEFAULT_FILTER = "unsupported_default_filter" + SEGMENT_AS_COLUMN = "segment_as_column" + UNMAPPED_INFRA = "unmapped_infra" + GEO_UNMAPPED = "geo_unmapped" + SUBQUERY_UNMAPPED = "subquery_unmapped" + GRANULARITY_UNMAPPED = "granularity_unmapped" + DISCONNECTED_VIEW = "disconnected_view" + AMBIGUOUS_VIEW_ROOT = "ambiguous_view_root" + VIEW_FANOUT_RISK = "view_fanout_risk" + FOLDERS_UNMAPPED = "folders_unmapped" + EXTENDS_CYCLE = "extends_cycle" + NO_SOURCE = "no_source" + DEFERRED_STAGE2 = "deferred_stage2" + + +Severity = Literal["info", "warning", "error"] + + +class CubeConversionIssue(BaseModel): + category: CubeIssueCategory + severity: Severity = "warning" + cube: str | None = None + view: str | None = None + member: str | None = None + message: str + raw: str | None = None # raw Cube fragment when useful + + @property + def context(self) -> str: + return self.cube or self.view or self.member or "general" + + +class CubeConversionReport(BaseModel): + issues: list[CubeConversionIssue] = Field(default_factory=list) + model_count: int = 0 + hidden_count: int = 0 + view_count: int = 0 + + def add(self, issue: CubeConversionIssue) -> None: + self.issues.append(issue) + + def by_category(self, category: CubeIssueCategory) -> list[CubeConversionIssue]: + return [i for i in self.issues if i.category == category] + + def by_severity(self, severity: Severity) -> list[CubeConversionIssue]: + return [i for i in self.issues if i.severity == severity] + + @property + def has_errors(self) -> bool: + return any(i.severity == "error" for i in self.issues) + + +class CubeConversionResult(BaseModel): + """Return value of ``CubeToSlayerConverter.convert``.""" + + models: list[SlayerModel] = Field(default_factory=list) + report: CubeConversionReport = Field(default_factory=CubeConversionReport) diff --git a/tests/fixtures/cube_project/model/cubes/customers.yml b/tests/fixtures/cube_project/model/cubes/customers.yml new file mode 100644 index 00000000..2b24fb1b --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/customers.yml @@ -0,0 +1,20 @@ +cubes: + - name: customers + sql_table: public.customers + measures: + - name: count + type: count + - name: lifetime_value + type: sum + sql: "{CUBE}.ltv" + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + primary_key: true + - name: name + sql: "{CUBE}.name" + type: string + - name: region + sql: "{CUBE}.region" + type: string diff --git a/tests/fixtures/cube_project/model/cubes/events.yml b/tests/fixtures/cube_project/model/cubes/events.yml new file mode 100644 index 00000000..3a4da16d --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/events.yml @@ -0,0 +1,22 @@ +cubes: + - name: base_events + sql_table: public.events + public: false + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + primary_key: true + - name: event_type + sql: "{CUBE}.event_type" + type: string + measures: + - name: count + type: count + - name: clicks + extends: base_events + sql_table: public.clicks + measures: + - name: click_value + type: sum + sql: "{CUBE}.value" diff --git a/tests/fixtures/cube_project/model/cubes/for_loop.yml b/tests/fixtures/cube_project/model/cubes/for_loop.yml new file mode 100644 index 00000000..66db2846 --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/for_loop.yml @@ -0,0 +1,5 @@ +cubes: +{% for t in tables %} + - name: "{{ t }}" + sql_table: "public.{{ t }}" +{% endfor %} diff --git a/tests/fixtures/cube_project/model/cubes/malformed.yml b/tests/fixtures/cube_project/model/cubes/malformed.yml new file mode 100644 index 00000000..962caa3f --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/malformed.yml @@ -0,0 +1,5 @@ +cubes: + - sql_table: public.orphan + measures: + - name: count + type: count diff --git a/tests/fixtures/cube_project/model/cubes/orders.yml b/tests/fixtures/cube_project/model/cubes/orders.yml new file mode 100644 index 00000000..155616f3 --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/orders.yml @@ -0,0 +1,39 @@ +cubes: + - name: orders + sql_table: public.orders + description: Customer orders + joins: + - name: customers + relationship: many_to_one + sql: "{CUBE}.customer_id = {customers.id}" + measures: + - name: count + type: count + - name: total_revenue + type: sum + sql: "{CUBE}.amount" + title: Total Revenue + format: currency + - name: completed_revenue + type: sum + sql: "{CUBE}.amount" + filters: + - sql: "{CUBE}.status = 'completed'" + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + primary_key: true + - name: status + sql: "{CUBE}.status" + type: string + - name: created_at + sql: "{CUBE}.created_at" + type: time + segments: + - name: completed + sql: "{CUBE}.status = 'completed'" + pre_aggregations: + - name: main + measures: + - orders.count diff --git a/tests/fixtures/cube_project/model/cubes/templated_member.yml b/tests/fixtures/cube_project/model/cubes/templated_member.yml new file mode 100644 index 00000000..a7ca706e --- /dev/null +++ b/tests/fixtures/cube_project/model/cubes/templated_member.yml @@ -0,0 +1,10 @@ +cubes: + - name: tenant_scoped + sql_table: public.events + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + - name: tenant + sql: "{{ user_attr('tenant') }}" + type: string diff --git a/tests/fixtures/cube_project/model/views/orders_overview.yml b/tests/fixtures/cube_project/model/views/orders_overview.yml new file mode 100644 index 00000000..808d8900 --- /dev/null +++ b/tests/fixtures/cube_project/model/views/orders_overview.yml @@ -0,0 +1,19 @@ +views: + - name: orders_overview + cubes: + - join_path: orders + includes: + - count + - total_revenue + - status + - join_path: orders.customers + prefix: true + includes: + - name + - region + - lifetime_value + default_filters: + - member: orders.status + operator: equals + values: + - completed diff --git a/tests/test_cube_boundaries.py b/tests/test_cube_boundaries.py new file mode 100644 index 00000000..bffb671b --- /dev/null +++ b/tests/test_cube_boundaries.py @@ -0,0 +1,112 @@ +"""Negative validator-boundary tests (Codex test-gap, DEV-1608 §4.6). + +A naive converter would let collisions / reserved names / broken SQL throw a +`ValidationError` and lose the whole model. The converter must instead route +them to the report and emit what it safely can — `convert()` never raises. +""" + +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.models import ( + CubeCube, + CubeDimension, + CubeJoin, + CubeMeasure, + CubeProject, +) + +DS = "test_ds" + + +def _convert(project: CubeProject): + result = CubeToSlayerConverter(project=project, data_source=DS).convert() + return {m.name: m for m in result.models}, result.report + + +def test_measure_named_after_transform_is_routed_to_report(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="cumsum", type="sum", sql="{CUBE}.amount")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + # Must not raise; the reserved-name measure is dropped (or safely renamed). + models, report = _convert(project) + assert "orders" in models + assert report.issues # something was reported about it + assert models["orders"].get_measure("cumsum") is None + + +def test_dimension_measure_name_overlap_does_not_raise(): + """A dimension and a measure that would share a name (SLayer forbids the + overlap) must be disambiguated, not crash the model.""" + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="revenue", type="sum", sql="{CUBE}.amount")], + dimensions=[CubeDimension(name="revenue", sql="{CUBE}.revenue_flag", + type="string")], + )]) + models, _ = _convert(project) + orders = models["orders"] + col_names = {c.name for c in orders.columns} + measure_names = {m.name for m in orders.measures} + assert not (col_names & measure_names) # disjoint after disambiguation + + +def test_whole_run_survives_one_broken_cube(): + """A structurally-broken cube must not abort conversion of the others.""" + project = CubeProject(cubes=[ + CubeCube(name="good", sql_table="public.good", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="bad"), # no source + ]) + models, report = _convert(project) + assert "good" in models + assert report.issues + + +def test_structurally_broken_column_sql_dropped_via_offline_validation(): + """Codex #7: a translated Column.sql that doesn't parse as SQL must be caught + by the offline validation pass and dropped (complex_sql), not persisted to + blow up later at enrichment.""" + from slayer.cube.report import CubeIssueCategory + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[ + CubeDimension(name="id", sql="{CUBE}.id", type="number"), + CubeDimension(name="broken", sql="{CUBE}.amount + )", type="number"), + ], + )]) + models, report = _convert(project) + assert models["orders"].get_column("broken") is None + assert models["orders"].get_column("id") is not None + assert any(i.category == CubeIssueCategory.COMPLEX_SQL for i in report.issues) + + +def test_cube_name_with_illegal_chars_is_reported_not_fatal(): + """A Cube name containing `.`/`:` (rejected by SlayerModel.name) must be + routed to the report, not crash the run.""" + project = CubeProject(cubes=[ + CubeCube(name="weird.name", sql_table="public.x"), + CubeCube(name="ok", sql_table="public.ok", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + ]) + models, report = _convert(project) + assert "weird.name" not in models + assert "ok" in models + assert report.issues + + +def test_join_on_without_equality_yields_no_empty_join_pairs(): + """An ON with no equality must drop the join (never construct ModelJoin with + empty join_pairs, which the validator rejects).""" + from slayer.cube.report import CubeIssueCategory + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.customer_id")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="customers", sql_table="public.customers", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + ]) + models, report = _convert(project) + assert models["orders"].joins == [] + assert any(i.category == CubeIssueCategory.UNSUPPORTED_JOIN for i in report.issues) diff --git a/tests/test_cube_cli.py b/tests/test_cube_cli.py new file mode 100644 index 00000000..49c5b2d6 --- /dev/null +++ b/tests/test_cube_cli.py @@ -0,0 +1,100 @@ +"""Tests for the `slayer import-cube` CLI (slayer/cli.py). + +DEV-1608 §11. Offline: writes SLayer models to storage + a JSON report. No +datasource connection required. +""" + +import asyncio +import json +import os +import sys +from contextlib import contextmanager + +from slayer.cli import main as cli_main +from slayer.storage.yaml_storage import YAMLStorage + +FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "cube_project") + + +@contextmanager +def _argv(*argv: str): + original = sys.argv + sys.argv = ["slayer", *argv] + try: + yield + finally: + sys.argv = original + + +def _run(*argv: str) -> int: + with _argv(*argv): + try: + cli_main() + except SystemExit as exc: # NOSONAR — capture CLI exit code + return int(exc.code or 0) + return 0 + + +def test_import_cube_writes_models_and_report(tmp_path): + storage_dir = tmp_path / "store" + code = _run( + "import-cube", FIXTURE, + "--datasource", "cube_ds", + "--storage", str(storage_dir), + ) + assert code == 0 + + # Models persisted, offline (no datasource ever created/connected). + storage = YAMLStorage(base_dir=str(storage_dir)) + names = asyncio.new_event_loop().run_until_complete(storage.list_models()) + assert "orders" in names + assert "customers" in names + assert "orders_overview" in names + + # JSON report written next to storage. + report_path = storage_dir / "cube_import_report.json" + assert report_path.exists() + report = json.loads(report_path.read_text()) + assert "issues" in report + assert report["model_count"] >= 3 + + +def test_import_cube_survives_save_failure(tmp_path, monkeypatch): + # A save_model failure on one model must not abort the run — the report is + # still written ("report, don't crash"). + import slayer.storage.yaml_storage as ys + + async def _boom(self, model): + raise RuntimeError("save exploded") + + monkeypatch.setattr(ys.YAMLStorage, "save_model", _boom) + storage_dir = tmp_path / "store" + code = _run("import-cube", FIXTURE, "--datasource", "cube_ds", "--storage", str(storage_dir)) + assert code == 0 + assert (storage_dir / "cube_import_report.json").exists() + + +def test_import_cube_report_honors_models_dir(tmp_path): + # With --models-dir (and no --storage) the report lands next to the models, + # matching _resolve_storage's resolution chain. + models_dir = tmp_path / "mstore" + code = _run( + "import-cube", FIXTURE, + "--datasource", "cube_ds", + "--models-dir", str(models_dir), + ) + assert code == 0 + assert (models_dir / "cube_import_report.json").exists() + + +def test_import_cube_report_path_override(tmp_path): + storage_dir = tmp_path / "store" + report_path = tmp_path / "custom_report.json" + code = _run( + "import-cube", FIXTURE, + "--datasource", "cube_ds", + "--storage", str(storage_dir), + "--report", str(report_path), + ) + assert code == 0 + assert report_path.exists() diff --git a/tests/test_cube_converter.py b/tests/test_cube_converter.py new file mode 100644 index 00000000..b23113c8 --- /dev/null +++ b/tests/test_cube_converter.py @@ -0,0 +1,553 @@ +"""Tests for the Cube → SLayer converter (slayer/cube/converter.py). + +DEV-1608 §4. Projects are built in Python (parser is tested separately) so these +pin the mapping semantics directly. +""" + +import pytest + +from slayer.core.enums import DataType, JoinType +from slayer.core.format import NumberFormatType +from slayer.core.models import SlayerModel +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.models import ( + CubeCube, + CubeDimension, + CubeJoin, + CubeMeasure, + CubeMeasureFilter, + CubeProject, + CubeSegment, +) +from slayer.cube.report import CubeIssueCategory + +DS = "test_ds" + + +def _convert(project: CubeProject) -> tuple[dict[str, SlayerModel], object]: + result = CubeToSlayerConverter(project=project, data_source=DS).convert() + return {m.name: m for m in result.models}, result.report + + +def _measure_column(model: SlayerModel, measure_name: str): + """Return the Column a `:` measure formula references.""" + m = model.get_measure(measure_name) + assert m is not None, f"measure {measure_name} missing on {model.name}" + col_ref = m.formula.split(":")[0].strip() + return model.get_column(col_ref) + + +# ── 4.1 cube → model ─────────────────────────────────────────────────────── + +def test_cube_becomes_table_backed_model(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", description="Customer orders", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", primary_key=True)], + )]) + models, _ = _convert(project) + orders = models["orders"] + assert orders.sql_table == "public.orders" + assert orders.data_source == DS + assert orders.description == "Customer orders" + assert orders.get_column("id").primary_key is True + assert orders.get_column("id").type == DataType.DOUBLE + + +def test_public_false_dimension_is_hidden(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[ + CubeDimension(name="id", sql="{CUBE}.id", type="number"), + CubeDimension(name="secret", sql="{CUBE}.secret", type="string", public=False), + ], + )]) + models, _ = _convert(project) + assert models["orders"].get_column("secret").hidden is True + assert models["orders"].get_column("id").hidden is False + + +def test_public_false_cube_is_hidden_and_title_goes_to_meta(): + project = CubeProject(cubes=[CubeCube( + name="internal", sql_table="public.internal", public=False, title="Internal", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + assert models["internal"].hidden is True + assert models["internal"].meta["cube_title"] == "Internal" + + +def test_per_cube_data_source_reported_and_stashed(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", data_source="warehouse_b", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + # All models scoped under the single --datasource, NOT the per-cube one. + assert models["orders"].data_source == DS + assert models["orders"].meta["cube_unmapped"]["data_source"] == "warehouse_b" + assert any(i.category == CubeIssueCategory.UNMAPPED_INFRA for i in report.issues) + + +# ── 4.2 measures ─────────────────────────────────────────────────────────── + +def test_count_measure_no_sql_becomes_star_count(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="count", type="count")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + assert models["orders"].get_measure("count").formula == "*:count" + + +def test_sum_measure_splits_column_and_modelmeasure_with_currency_format(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure( + name="total_revenue", type="sum", sql="{CUBE}.amount", + title="Total Revenue", format="currency")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + orders = models["orders"] + m = orders.get_measure("total_revenue") + assert m.formula.endswith(":sum") + assert m.label == "Total Revenue" + col = _measure_column(orders, "total_revenue") + assert col.type == DataType.DOUBLE + assert col.format.type == NumberFormatType.CURRENCY + + +def test_filtered_measures_same_sql_get_distinct_columns(): + """Codex #4: two measures over the same `sql` but different `filters` must + not collapse — otherwise the filter bleeds across both.""" + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[ + CubeMeasure(name="total_revenue", type="sum", sql="{CUBE}.amount"), + CubeMeasure(name="completed_revenue", type="sum", sql="{CUBE}.amount", + filters=[CubeMeasureFilter(sql="{CUBE}.status = 'completed'")]), + ], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + orders = models["orders"] + unfiltered = _measure_column(orders, "total_revenue") + filtered = _measure_column(orders, "completed_revenue") + assert unfiltered.name != filtered.name + assert unfiltered.filter is None + assert filtered.filter == "status = 'completed'" + + +def test_count_distinct_approx_maps_to_count_distinct_with_lossy_report(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="uniq_users", type="count_distinct_approx", + sql="{CUBE}.user_id")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].get_measure("uniq_users").formula.endswith(":count_distinct") + assert any(i.category == CubeIssueCategory.LOSSY_MAPPING for i in report.issues) + + +def test_calculated_number_measure_becomes_dsl_formula(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[ + CubeMeasure(name="total_revenue", type="sum", sql="{CUBE}.amount"), + CubeMeasure(name="count", type="count"), + CubeMeasure(name="aov", type="number", sql="{total_revenue} / {count}"), + ], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + assert models["orders"].get_measure("aov").formula == "total_revenue / count" + + +def test_calculated_measure_with_case_when_is_reported_not_emitted(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[ + CubeMeasure(name="count", type="count"), + CubeMeasure(name="tier", type="string", + sql="CASE WHEN {count} > 100 THEN 'high' ELSE 'low' END"), + ], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].get_measure("tier") is None + assert any(i.category == CubeIssueCategory.COMPLEX_MEASURE for i in report.issues) + + +def test_finite_rolling_window_becomes_windowed_aggregation(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="revenue_30d", type="sum", sql="{CUBE}.amount", + rolling_window={"trailing": "30 day"})], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, _ = _convert(project) + assert models["orders"].get_measure("revenue_30d").formula == "amount:sum(window='30d')" + + +def test_unbounded_rolling_window_falls_back_and_reports(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + measures=[CubeMeasure(name="revenue_total", type="sum", sql="{CUBE}.amount", + rolling_window={"trailing": "unbounded"})], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].get_measure("revenue_total").formula == "amount:sum" + assert any(i.category == CubeIssueCategory.UNSUPPORTED_ROLLING_WINDOW + for i in report.issues) + + +# ── 4.3 dimensions ───────────────────────────────────────────────────────── + +@pytest.mark.parametrize("cube_type,expected", [ + ("string", DataType.TEXT), + ("number", DataType.DOUBLE), + ("boolean", DataType.BOOLEAN), + ("time", DataType.TIMESTAMP), +]) +def test_dimension_type_mapping(cube_type, expected): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="d", sql="{CUBE}.d", type=cube_type)], + )]) + models, _ = _convert(project) + assert models["orders"].get_column("d").type == expected + + +def test_dimension_sql_omitted_when_just_cube_dot_name(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="status", sql="{CUBE}.status", type="string")], + )]) + models, _ = _convert(project) + assert models["orders"].get_column("status").sql is None + + +def test_case_dimension_becomes_case_when_column(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="size_bucket", type="string", case={ + "when": [{"sql": "{CUBE}.size < 10", "label": "small"}], + "else": {"label": "big"}, + })], + )]) + models, _ = _convert(project) + sql = models["orders"].get_column("size_bucket").sql + assert "CASE WHEN" in sql and "'small'" in sql and "'big'" in sql + assert "{CUBE}" not in sql + + +def test_geo_dimension_reported_not_emitted(): + project = CubeProject(cubes=[CubeCube( + name="stores", sql_table="public.stores", + dimensions=[CubeDimension(name="location", type="geo", + latitude={"sql": "{CUBE}.lat"}, + longitude={"sql": "{CUBE}.lng"})], + )]) + models, report = _convert(project) + assert models["stores"].get_column("location") is None + assert models["stores"].meta["cube_unmapped"]["geo"] + assert any(i.category == CubeIssueCategory.GEO_UNMAPPED for i in report.issues) + + +def test_subquery_dimension_reported_not_emitted(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="cust_ltv", type="number", sub_query=True, + sql="{customers.lifetime_value}")], + )]) + models, report = _convert(project) + assert models["orders"].get_column("cust_ltv") is None + assert any(i.category == CubeIssueCategory.SUBQUERY_UNMAPPED for i in report.issues) + + +def test_custom_granularities_emit_base_column_and_report(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="created_at", sql="{CUBE}.created_at", + type="time", + granularities=[{"name": "fiscal_year", + "interval": "1 year", + "offset": "3 months"}])], + )]) + models, report = _convert(project) + assert models["orders"].get_column("created_at").type == DataType.TIMESTAMP + assert any(i.category == CubeIssueCategory.GRANULARITY_UNMAPPED for i in report.issues) + + +# ── 4.4 joins ────────────────────────────────────────────────────────────── + +def test_join_becomes_left_modeljoin_with_pairs(): + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.customer_id = {customers.id}")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="customers", sql_table="public.customers", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", + primary_key=True)]), + ]) + models, _ = _convert(project) + joins = models["orders"].joins + assert len(joins) == 1 + assert joins[0].target_model == "customers" + assert joins[0].join_pairs == [["customer_id", "id"]] + assert joins[0].join_type == JoinType.LEFT + + +def test_join_to_missing_target_cube_reported(): + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="ghost", relationship="many_to_one", + sql="{CUBE}.ghost_id = {ghost.id}")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + ]) + models, report = _convert(project) + assert models["orders"].joins == [] + assert any(i.category == CubeIssueCategory.UNSUPPORTED_JOIN for i in report.issues) + + +def test_case_dimension_label_escapes_quotes(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="owner", type="string", case={ + "when": [{"sql": "{CUBE}.x = 1", "label": "Bob's"}], + "else": {"label": "n/a"}, + })], + )]) + models, _ = _convert(project) + sql = models["orders"].get_column("owner").sql + assert "'Bob''s'" in sql # apostrophe doubled, not a broken literal + + +def test_non_equi_join_reported_and_dropped(): + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="windows", relationship="many_to_one", + sql="{CUBE}.ts > {windows.start}")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="windows", sql_table="public.windows", + dimensions=[CubeDimension(name="start", sql="{CUBE}.start", type="time")]), + ]) + models, report = _convert(project) + assert models["orders"].joins == [] + assert any(i.category == CubeIssueCategory.UNSUPPORTED_JOIN for i in report.issues) + + +# ── 4.5 segments ─────────────────────────────────────────────────────────── + +def test_segment_becomes_boolean_column(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + segments=[CubeSegment(name="completed", sql="{CUBE}.status = 'completed'")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + col = models["orders"].get_column("completed") + assert col.type == DataType.BOOLEAN + assert col.sql == "status = 'completed'" + assert any(i.category == CubeIssueCategory.SEGMENT_AS_COLUMN for i in report.issues) + + +# ── 7. unmapped infra ────────────────────────────────────────────────────── + +def test_pre_aggregations_reported_and_stashed_in_meta(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + pre_aggregations=[{"name": "main", "measures": ["CUBE.count"]}], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].meta["cube_unmapped"]["pre_aggregations"] + assert any(i.category == CubeIssueCategory.UNMAPPED_INFRA for i in report.issues) + + +def test_cube_with_no_source_is_dropped_and_reported(): + project = CubeProject(cubes=[CubeCube(name="bad")]) + models, report = _convert(project) + assert "bad" not in models + assert any(i.category == CubeIssueCategory.NO_SOURCE for i in report.issues) + + +# ── 4.2 measures — more aggregation kinds ────────────────────────────────── + +def _orders_with(measures): + return CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", measures=measures, + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + + +def test_count_with_sql_counts_column(): + models, _ = _convert(_orders_with( + [CubeMeasure(name="paid_count", type="count", sql="{CUBE}.paid_id")])) + assert models["orders"].get_measure("paid_count").formula.endswith(":count") + assert models["orders"].get_measure("paid_count").formula != "*:count" + + +def test_count_distinct_exact(): + models, report = _convert(_orders_with( + [CubeMeasure(name="uniq", type="count_distinct", sql="{CUBE}.user_id")])) + assert models["orders"].get_measure("uniq").formula.endswith(":count_distinct") + assert not any(i.category == CubeIssueCategory.LOSSY_MAPPING for i in report.issues) + + +@pytest.mark.parametrize("agg", ["avg", "min", "max"]) +def test_simple_aggregation_passthrough(agg): + models, _ = _convert(_orders_with( + [CubeMeasure(name=f"m_{agg}", type=agg, sql="{CUBE}.amount")])) + assert models["orders"].get_measure(f"m_{agg}").formula.endswith(f":{agg}") + + +@pytest.mark.parametrize("cube_type,expected", [ + ("string", DataType.TEXT), + ("time", DataType.TIMESTAMP), + ("boolean", DataType.BOOLEAN), +]) +def test_calculated_measure_result_type_is_set(cube_type, expected): + models, _ = _convert(_orders_with([ + CubeMeasure(name="count", type="count"), + CubeMeasure(name="derived", type=cube_type, sql="{count} + 1"), + ])) + m = models["orders"].get_measure("derived") + assert m.formula == "count + 1" + assert m.type == expected + + +@pytest.mark.parametrize("rolling", [ + {"trailing": "30 day", "offset": "start"}, + {"leading": "1 month"}, +]) +def test_rolling_window_leading_or_offset_unsupported(rolling): + models, report = _convert(_orders_with( + [CubeMeasure(name="r", type="sum", sql="{CUBE}.amount", rolling_window=rolling)])) + assert models["orders"].get_measure("r").formula == "amount:sum" + assert any(i.category == CubeIssueCategory.UNSUPPORTED_ROLLING_WINDOW + for i in report.issues) + + +def test_window_is_part_of_dedup_key(): + """Codex #4 window half: same sql + same (no) filter but different + rolling_window → distinct columns, not a collapsed one.""" + models, _ = _convert(_orders_with([ + CubeMeasure(name="rev", type="sum", sql="{CUBE}.amount"), + CubeMeasure(name="rev_30d", type="sum", sql="{CUBE}.amount", + rolling_window={"trailing": "30 day"}), + ])) + orders = models["orders"] + assert orders.get_measure("rev").formula == "amount:sum" + assert orders.get_measure("rev_30d").formula == "amount:sum(window='30d')" + + +# ── 4.4 joins — physical-column resolution (Codex #2) ────────────────────── + +def test_join_member_resolves_to_physical_column(): + """`{customers.id}` where the `id` member's sql is `{CUBE}.cust_pk` must emit + the physical column `cust_pk`, not the member name `id`.""" + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.customer_id = {customers.id}")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="customers", sql_table="public.customers", + dimensions=[CubeDimension(name="id", sql="{CUBE}.cust_pk", + type="number", primary_key=True)]), + ]) + models, _ = _convert(project) + assert models["orders"].joins[0].join_pairs == [["customer_id", "cust_pk"]] + + +def test_join_with_nontrivial_member_sql_is_unsupported(): + project = CubeProject(cubes=[ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.email = {customers.email}")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="customers", sql_table="public.customers", + dimensions=[CubeDimension(name="email", sql="LOWER({CUBE}.email)", + type="string")]), + ]) + models, report = _convert(project) + assert models["orders"].joins == [] + assert any(i.category == CubeIssueCategory.UNSUPPORTED_JOIN for i in report.issues) + + +# ── 8. format mapping (Codex #8) ─────────────────────────────────────────── + +def test_percent_format_maps_to_percent(): + models, _ = _convert(_orders_with( + [CubeMeasure(name="rate", type="avg", sql="{CUBE}.rate", format="percent")])) + col = _measure_column(models["orders"], "rate") + assert col.format.type == NumberFormatType.PERCENT + + +@pytest.mark.parametrize("fmt", ["accounting", "abbr", "0.00%", "imageUrl"]) +def test_unsupported_format_reported_and_dropped(fmt): + models, report = _convert(_orders_with( + [CubeMeasure(name="m", type="sum", sql="{CUBE}.amount", format=fmt)])) + # measure still emitted; format dropped (defaults to FLOAT). + assert models["orders"].get_measure("m") is not None + assert any(i.category == CubeIssueCategory.UNSUPPORTED_FORMAT for i in report.issues) + + +def test_non_currency_format_never_carries_symbol(): + """Codex #8: a percent format with a stray symbol field must not pass + `symbol` to NumberFormat (which would raise).""" + models, _ = _convert(_orders_with([CubeMeasure( + name="rate", type="avg", sql="{CUBE}.rate", + format={"type": "percent", "currency_symbol": "$"})])) + col = _measure_column(models["orders"], "rate") + assert col.format.type == NumberFormatType.PERCENT + assert col.format.symbol is None + + +# ── 7. unmapped-infra meta stash (matrix) ────────────────────────────────── + +@pytest.mark.parametrize("field,value", [ + ("refresh_key", {"every": "1 hour"}), + ("calendar", True), + ("hierarchies", [{"name": "geo", "levels": ["country"]}]), + ("access_policy", [{"role": "admin"}]), + ("sql_alias", "ord"), +]) +def test_unmapped_cube_infra_stashed_and_reported(field, value): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", **{field: value}, + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].meta["cube_unmapped"][field] is not None + assert any(i.category == CubeIssueCategory.UNMAPPED_INFRA for i in report.issues) + + +def test_drill_members_reported(): + _, report = _convert(_orders_with( + [CubeMeasure(name="count", type="count", drill_members=["id", "status"])])) + assert any(i.category == CubeIssueCategory.UNMAPPED_INFRA for i in report.issues) + + +# ── 9. Stage-2 / Tesseract deferral ──────────────────────────────────────── + +def test_switch_dimension_deferred(): + project = CubeProject(cubes=[CubeCube( + name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="selector", type="switch"), + CubeDimension(name="id", sql="{CUBE}.id", type="number")], + )]) + models, report = _convert(project) + assert models["orders"].get_column("selector") is None + assert any(i.category == CubeIssueCategory.DEFERRED_STAGE2 for i in report.issues) + + +def test_number_agg_measure_deferred(): + models, report = _convert(_orders_with( + [CubeMeasure(name="na", type="number_agg", sql="{CUBE}.amount")])) + assert models["orders"].get_measure("na") is None + assert any(i.category == CubeIssueCategory.DEFERRED_STAGE2 for i in report.issues) diff --git a/tests/test_cube_extends.py b/tests/test_cube_extends.py new file mode 100644 index 00000000..3e6c219b --- /dev/null +++ b/tests/test_cube_extends.py @@ -0,0 +1,103 @@ +"""Tests for Cube `extends` flattening (slayer/cube/extends.py + converter). + +DEV-1608 §5. Flatten base members into children (child wins); abstract bases +emitted hidden; cycles reported. +""" + +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.models import CubeCube, CubeDimension, CubeMeasure, CubeProject +from slayer.cube.report import CubeIssueCategory + +DS = "test_ds" + + +def _convert(project: CubeProject): + result = CubeToSlayerConverter(project=project, data_source=DS).convert() + return {m.name: m for m in result.models}, result.report + + +def test_child_inherits_base_members(): + project = CubeProject(cubes=[ + CubeCube(name="base_events", sql_table="public.events", public=False, + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", + primary_key=True), + CubeDimension(name="event_type", sql="{CUBE}.event_type", + type="string")], + measures=[CubeMeasure(name="count", type="count")]), + CubeCube(name="clicks", extends="base_events", sql_table="public.clicks", + measures=[CubeMeasure(name="click_value", type="sum", + sql="{CUBE}.value")]), + ]) + models, _ = _convert(project) + clicks = models["clicks"] + # Inherited dimensions + measure, plus the child's own measure. + assert clicks.get_column("id") is not None + assert clicks.get_column("event_type") is not None + assert clicks.get_measure("count") is not None + assert clicks.get_measure("click_value") is not None + assert clicks.sql_table == "public.clicks" # child source wins + + +def test_abstract_base_emitted_hidden(): + project = CubeProject(cubes=[ + CubeCube(name="base_events", sql_table="public.events", public=False, + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="clicks", extends="base_events", sql_table="public.clicks"), + ]) + models, _ = _convert(project) + assert models["base_events"].hidden is True + assert models["clicks"].hidden is False + + +def test_child_overrides_inherited_member(): + project = CubeProject(cubes=[ + CubeCube(name="base", sql_table="public.base", public=False, + dimensions=[CubeDimension(name="label", sql="{CUBE}.old_label", + type="string")]), + CubeCube(name="child", extends="base", sql_table="public.child", + dimensions=[CubeDimension(name="label", sql="{CUBE}.new_label", + type="string")]), + ]) + models, _ = _convert(project) + # Child wins on name conflict. + assert models["child"].get_column("label").sql == "new_label" + + +def test_multi_level_extends_chain(): + project = CubeProject(cubes=[ + CubeCube(name="a", sql_table="public.a", public=False, + dimensions=[CubeDimension(name="x", sql="{CUBE}.x", type="number")]), + CubeCube(name="b", extends="a", sql_table="public.b", public=False, + dimensions=[CubeDimension(name="y", sql="{CUBE}.y", type="number")]), + CubeCube(name="c", extends="b", sql_table="public.c", + dimensions=[CubeDimension(name="z", sql="{CUBE}.z", type="number")]), + ]) + models, _ = _convert(project) + c = models["c"] + assert c.get_column("x") is not None # from a, transitively + assert c.get_column("y") is not None # from b + assert c.get_column("z") is not None # own + + +def test_extends_cycle_is_reported(): + project = CubeProject(cubes=[ + CubeCube(name="a", extends="b", sql_table="public.a"), + CubeCube(name="b", extends="a", sql_table="public.b"), + ]) + _, report = _convert(project) + assert any(i.category == CubeIssueCategory.EXTENDS_CYCLE for i in report.issues) + + +def test_extends_cycle_flattens_without_inheritance(): + """Every node on the cycle keeps only its own members — no ancestor merges + a partially-resolved cyclic node.""" + project = CubeProject(cubes=[ + CubeCube(name="a", extends="b", sql_table="public.a", + dimensions=[CubeDimension(name="a_col", sql="{CUBE}.a_col", type="number")]), + CubeCube(name="b", extends="a", sql_table="public.b", + dimensions=[CubeDimension(name="b_col", sql="{CUBE}.b_col", type="number")]), + ]) + models, _ = _convert(project) + assert models["a"].get_column("a_col") is not None + assert models["a"].get_column("b_col") is None # did NOT inherit across the cycle + assert models["b"].get_column("a_col") is None diff --git a/tests/test_cube_parser.py b/tests/test_cube_parser.py new file mode 100644 index 00000000..1f3548dc --- /dev/null +++ b/tests/test_cube_parser.py @@ -0,0 +1,128 @@ +"""Tests for the Cube project parser (slayer/cube/parser.py). + +DEV-1608 §2. Walk a directory, parse cubes:/views:, skip + report Jinja +(file-level and member-level) and malformed files without aborting the run. +""" + +import os +import textwrap + +from slayer.cube.parser import parse_cube_project +from slayer.cube.report import CubeIssueCategory + +FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "cube_project") + + +def test_parses_cubes_and_views(): + project, _issues = parse_cube_project(FIXTURE) + cube_names = {c.name for c in project.cubes} + view_names = {v.name for v in project.views} + assert {"orders", "customers", "base_events", "clicks"} <= cube_names + assert "orders_overview" in view_names + + +def test_orders_cube_fields_populated(): + project, _ = parse_cube_project(FIXTURE) + orders = next(c for c in project.cubes if c.name == "orders") + assert orders.sql_table == "public.orders" + assert {m.name for m in orders.measures} >= {"count", "total_revenue", "completed_revenue"} + assert orders.joins[0].name == "customers" + assert orders.pre_aggregations # captured for unmapped-infra stashing + + +def test_file_level_jinja_is_skipped_and_reported(): + _project, issues = parse_cube_project(FIXTURE) + assert any(i.category == CubeIssueCategory.REQUIRES_TEMPLATING for i in issues) + + +def test_member_level_jinja_skips_member_keeps_cube(): + project, _issues = parse_cube_project(FIXTURE) + tenant = next((c for c in project.cubes if c.name == "tenant_scoped"), None) + assert tenant is not None + dim_names = {d.name for d in tenant.dimensions} + assert "id" in dim_names # plain member kept + assert "tenant" not in dim_names # templated member dropped + + +def test_malformed_cube_is_reported_not_fatal(): + project, issues = parse_cube_project(FIXTURE) + # The nameless cube in malformed.yml is dropped... + assert all(c.name != "orphan" for c in project.cubes) + # ...via a parse_error, and the rest of the project still parsed. + assert any(i.category == CubeIssueCategory.PARSE_ERROR for i in issues) + assert any(c.name == "orders" for c in project.cubes) + + +def test_single_object_cubes_block_tolerated(tmp_path): + (tmp_path / "single.yml").write_text(textwrap.dedent(""" + cubes: + name: solo + sql_table: public.solo + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + """)) + project, _ = parse_cube_project(str(tmp_path)) + assert any(c.name == "solo" for c in project.cubes) + + +def test_single_object_views_block_tolerated(tmp_path): + (tmp_path / "v.yml").write_text(textwrap.dedent(""" + views: + name: solo_view + cubes: + - join_path: orders + includes: ["status"] + """)) + project, _ = parse_cube_project(str(tmp_path)) + assert any(v.name == "solo_view" for v in project.views) + + +def test_member_level_jinja_in_measure_sql_skips_measure(tmp_path): + (tmp_path / "m.yml").write_text(textwrap.dedent(""" + cubes: + - name: orders + sql_table: public.orders + measures: + - name: plain + type: count + - name: templated + type: sum + sql: "{{ user_attr('scale') }} * {CUBE}.amount" + dimensions: + - name: id + sql: "{CUBE}.id" + type: number + """)) + project, issues = parse_cube_project(str(tmp_path)) + orders = next(c for c in project.cubes if c.name == "orders") + names = {m.name for m in orders.measures} + assert "plain" in names + assert "templated" not in names + assert any(i.category == CubeIssueCategory.REQUIRES_TEMPLATING for i in issues) + + +def test_hidden_dirs_and_target_skipped(tmp_path): + (tmp_path / ".hidden").mkdir() + (tmp_path / ".hidden" / "x.yml").write_text( + "cubes:\n - name: ghost\n sql_table: public.ghost\n") + project, _ = parse_cube_project(str(tmp_path)) + assert all(c.name != "ghost" for c in project.cubes) + + +def test_unreadable_file_is_reported_not_fatal(tmp_path): + # A broken symlink with a .yml extension raises OSError on open — it must be + # reported like a malformed file, not abort the whole import. + (tmp_path / "broken.yml").symlink_to(tmp_path / "nonexistent.yml") + (tmp_path / "good.yml").write_text( + "cubes:\n - name: ok\n sql_table: public.ok\n") + project, issues = parse_cube_project(str(tmp_path)) + assert any(c.name == "ok" for c in project.cubes) + assert any(i.category == CubeIssueCategory.PARSE_ERROR for i in issues) + + +def test_empty_dir_yields_empty_project(tmp_path): + project, _ = parse_cube_project(str(tmp_path)) + assert project.cubes == [] + assert project.views == [] diff --git a/tests/test_cube_refs.py b/tests/test_cube_refs.py new file mode 100644 index 00000000..b132a86b --- /dev/null +++ b/tests/test_cube_refs.py @@ -0,0 +1,118 @@ +"""Tests for the Cube curly-reference translator (slayer/cube/refs.py). + +DEV-1608 §3 / §4.4. `{CUBE}` / `{member}` / `{cube.member}` are Cube's +single-brace SQL-ref syntax — distinct from Jinja's `{{ }}` / `{% %}`. +""" + + +import pytest + +from slayer.cube.refs import contains_jinja, parse_join_on, translate_cube_refs + + +def test_translate_rejects_invalid_mode(): + with pytest.raises(ValueError): + translate_cube_refs("{CUBE}.x", mode="bogus", cube="orders") + + +# ── Jinja detection ──────────────────────────────────────────────────────── + +def test_contains_jinja_double_brace(): + assert contains_jinja("{{ env_var('SCHEMA') }}.events") + + +def test_contains_jinja_block(): + assert contains_jinja("{% for t in tables %}") + + +def test_single_brace_cube_ref_is_not_jinja(): + assert not contains_jinja("{CUBE}.amount") + assert not contains_jinja("{customers.id}") + assert not contains_jinja("SUM({CUBE}.amount)") + + +# ── Mode A (SQL) translation ─────────────────────────────────────────────── + +def test_cube_dot_col_to_bare(): + assert translate_cube_refs("{CUBE}.amount", mode="sql", cube="orders") == "amount" + + +def test_same_cube_member_to_bare(): + out = translate_cube_refs( + "{first_name} || ' ' || {last_name}", mode="sql", cube="people" + ) + assert out == "first_name || ' ' || last_name" + + +def test_cross_cube_member_single_hop(): + assert ( + translate_cube_refs("{customers.name}", mode="sql", cube="orders") + == "customers.name" + ) + + +def test_cross_cube_member_multi_hop(): + # `{a.b.c}` → SLayer multi-dot `a.b.c` (model-side `_fix_multidot_sql` later + # rewrites it to `a__b.c`). + assert ( + translate_cube_refs("{customers.regions.name}", mode="sql", cube="orders") + == "customers.regions.name" + ) + + +def test_translation_skips_doubled_quote_literal(): + # A `{CUBE}` inside a literal containing an escaped (doubled) quote must be + # left alone — the literal regex has to be doubled-quote aware. + out = translate_cube_refs("status = 'can''t {CUBE}'", mode="sql", cube="orders") + assert out == "status = 'can''t {CUBE}'" + + +def test_translation_skips_string_literals(): + # A `{CUBE}` inside a SQL string literal must be left untouched. + out = translate_cube_refs( + "CASE WHEN {CUBE}.status = '{CUBE}' THEN 1 END", mode="sql", cube="orders" + ) + assert out == "CASE WHEN status = '{CUBE}' THEN 1 END" + + +# ── Mode B (DSL) translation ─────────────────────────────────────────────── + +def test_dsl_measure_refs_to_bare_names(): + out = translate_cube_refs("{revenue} / {count}", mode="dsl", cube="orders") + assert out == "revenue / count" + + +# ── Join ON parsing ──────────────────────────────────────────────────────── + +def test_parse_join_on_simple_equality(): + pairs = parse_join_on( + "{CUBE}.customer_id = {customers.id}", + source_cube="orders", + target_cube="customers", + ) + assert pairs == [["customer_id", "id"]] + + +def test_parse_join_on_composite_key(): + pairs = parse_join_on( + "{CUBE}.a = {t.x} AND {CUBE}.b = {t.y}", + source_cube="o", + target_cube="t", + ) + assert pairs == [["a", "x"], ["b", "y"]] + + +def test_parse_join_on_non_equi_returns_none(): + assert ( + parse_join_on("{CUBE}.ts > {t.start}", source_cube="o", target_cube="t") + is None + ) + + +def test_parse_join_on_function_call_returns_none(): + assert ( + parse_join_on( + "LOWER({CUBE}.email) = {t.email}", source_cube="o", target_cube="t" + ) + is None + ) diff --git a/tests/test_cube_report.py b/tests/test_cube_report.py new file mode 100644 index 00000000..dec32079 --- /dev/null +++ b/tests/test_cube_report.py @@ -0,0 +1,99 @@ +"""Tests for the Cube conversion report shapes (slayer/cube/report.py). + +DEV-1608 §10. +""" + +import json + +from slayer.cube.report import ( + CubeConversionIssue, + CubeConversionReport, + CubeConversionResult, + CubeIssueCategory, +) + + +def test_issue_context_prefers_cube_then_view_then_member(): + assert CubeConversionIssue( + category=CubeIssueCategory.NO_SOURCE, message="x", cube="orders" + ).context == "orders" + assert CubeConversionIssue( + category=CubeIssueCategory.DISCONNECTED_VIEW, message="x", view="ov" + ).context == "ov" + assert CubeConversionIssue( + category=CubeIssueCategory.COMPLEX_MEASURE, message="x", member="aov" + ).context == "aov" + assert CubeConversionIssue( + category=CubeIssueCategory.PARSE_ERROR, message="x" + ).context == "general" + + +def test_report_filters_by_category_and_severity(): + report = CubeConversionReport() + report.add(CubeConversionIssue( + category=CubeIssueCategory.NO_SOURCE, severity="error", message="a")) + report.add(CubeConversionIssue( + category=CubeIssueCategory.LOSSY_MAPPING, severity="info", message="b")) + report.add(CubeConversionIssue( + category=CubeIssueCategory.LOSSY_MAPPING, severity="info", message="c")) + + assert len(report.by_category(CubeIssueCategory.LOSSY_MAPPING)) == 2 + assert len(report.by_severity("info")) == 2 + assert len(report.by_severity("error")) == 1 + assert report.has_errors + + +def test_report_has_no_errors_when_all_info_or_warning(): + report = CubeConversionReport() + report.add(CubeConversionIssue( + category=CubeIssueCategory.SEGMENT_AS_COLUMN, severity="info", message="x")) + assert not report.has_errors + + +def test_result_json_round_trips(): + result = CubeConversionResult( + report=CubeConversionReport( + issues=[CubeConversionIssue( + category=CubeIssueCategory.UNMAPPED_INFRA, + severity="warning", + cube="orders", + message="pre_aggregations dropped", + raw="name: main", + )], + model_count=3, + hidden_count=1, + view_count=1, + ) + ) + blob = result.model_dump_json() + parsed = json.loads(blob) + assert parsed["report"]["model_count"] == 3 + assert parsed["report"]["issues"][0]["category"] == "unmapped_infra" + + # Re-validate from the JSON to confirm the categories survive the round-trip. + restored = CubeConversionResult.model_validate_json(blob) + assert restored.report.issues[0].category == CubeIssueCategory.UNMAPPED_INFRA + assert restored.report.hidden_count == 1 + # `raw` fragment is preserved. + assert restored.report.issues[0].raw == "name: main" + + +def test_converter_derives_report_counts(): + """The converter must populate model/hidden/view counts on the report.""" + from slayer.cube.converter import CubeToSlayerConverter + from slayer.cube.models import CubeCube, CubeDimension, CubeProject, CubeView, CubeViewCubeRef + + project = CubeProject( + cubes=[ + CubeCube(name="orders", sql_table="public.orders", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + CubeCube(name="internal", sql_table="public.internal", public=False, + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + ], + views=[CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["id"])])], + ) + result = CubeToSlayerConverter(project=project, data_source="ds").convert() + assert result.report.model_count == len(result.models) + assert result.report.hidden_count == 1 + assert result.report.view_count == 1 diff --git a/tests/test_cube_smoke.py b/tests/test_cube_smoke.py new file mode 100644 index 00000000..c32eb4c7 --- /dev/null +++ b/tests/test_cube_smoke.py @@ -0,0 +1,120 @@ +"""Smoke tests: enrich/generate SQL against converted Cube models. + +DEV-1608 §12 (Codex test-gap). These don't just assert converter *output* — they +push the converted models through the engine's enrichment + SQL generation, which +is where the §6/§4.4 mapping breaks actually surface (e.g. a facade measure that +referenced a Cube measure *name* instead of the underlying Column would raise +"Column '' not found" at enrichment). +""" + + +from slayer.core.query import ColumnRef, SlayerQuery +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.models import ( + CubeCube, + CubeDimension, + CubeJoin, + CubeMeasure, + CubeMeasureFilter, + CubeProject, + CubeView, + CubeViewCubeRef, +) +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.sql.generator import SQLGenerator +from slayer.storage.yaml_storage import YAMLStorage + +DS = "cube_ds" + + +def _orders_customers_view_project() -> CubeProject: + return CubeProject( + cubes=[ + CubeCube( + name="orders", sql_table="public.orders", + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.customer_id = {customers.id}")], + measures=[ + CubeMeasure(name="count", type="count"), + CubeMeasure(name="total_revenue", type="sum", sql="{CUBE}.amount"), + CubeMeasure(name="completed_revenue", type="sum", sql="{CUBE}.amount", + filters=[CubeMeasureFilter(sql="{CUBE}.status = 'completed'")]), + ], + dimensions=[ + CubeDimension(name="id", sql="{CUBE}.id", type="number", primary_key=True), + CubeDimension(name="status", sql="{CUBE}.status", type="string"), + CubeDimension(name="customer_id", sql="{CUBE}.customer_id", type="number"), + ], + ), + CubeCube( + name="customers", sql_table="public.customers", + measures=[CubeMeasure(name="lifetime_value", type="sum", sql="{CUBE}.ltv")], + dimensions=[ + CubeDimension(name="id", sql="{CUBE}.id", type="number", primary_key=True), + CubeDimension(name="name", sql="{CUBE}.name", type="string"), + ], + ), + ], + views=[CubeView(name="orders_overview", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["count", "total_revenue", + "completed_revenue", "status"]), + CubeViewCubeRef(join_path="orders.customers", prefix=True, + includes=["name", "lifetime_value"]), + ])], + ) + + +async def _save_converted(tmp_path) -> tuple[SlayerQueryEngine, dict]: + result = CubeToSlayerConverter( + project=_orders_customers_view_project(), data_source=DS).convert() + storage = YAMLStorage(base_dir=str(tmp_path)) + for model in result.models: + await storage.save_model(model) + engine = SlayerQueryEngine(storage=storage) + return engine, {m.name: m for m in result.models} + + +async def _gen_sql(engine: SlayerQueryEngine, query: SlayerQuery, model) -> str: + enriched = await engine._enrich(query=query, model=model) + return SQLGenerator(dialect="sqlite").generate(enriched=enriched) + + +async def test_view_cross_model_measure_resolves_to_underlying_column(tmp_path): + """The facade measure `customers_lifetime_value` must resolve through the + join to the underlying `ltv` column — this is the Codex #1 regression.""" + engine, models = await _save_converted(tmp_path) + view = models["orders_overview"] + query = SlayerQuery( + source_model="orders_overview", + dimensions=[ColumnRef(name="status")], + measures=[{"formula": "customers_lifetime_value"}], + ) + sql = await _gen_sql(engine, query, view) # must not raise "Column not found" + assert "ltv" in sql.lower() + assert "sum" in sql.lower() + + +async def test_view_status_dimension_generates(tmp_path): + engine, models = await _save_converted(tmp_path) + view = models["orders_overview"] + query = SlayerQuery( + source_model="orders_overview", + dimensions=[ColumnRef(name="status")], + measures=[{"formula": "total_revenue"}], + ) + sql = await _gen_sql(engine, query, view) + assert "status" in sql.lower() + + +async def test_filtered_and_unfiltered_measures_emit_distinct_sql(tmp_path): + """Codex #4 at the SQL layer: total_revenue (plain SUM) and + completed_revenue (SUM over CASE WHEN) must both appear, distinctly.""" + engine, models = await _save_converted(tmp_path) + orders = models["orders"] + query = SlayerQuery( + source_model="orders", + measures=[{"formula": "total_revenue"}, {"formula": "completed_revenue"}], + ) + sql = await _gen_sql(engine, query, orders) + assert sql.lower().count("sum(") >= 2 + assert "case" in sql.lower() # the filtered one wraps amount in CASE WHEN diff --git a/tests/test_cube_views.py b/tests/test_cube_views.py new file mode 100644 index 00000000..468af6e7 --- /dev/null +++ b/tests/test_cube_views.py @@ -0,0 +1,279 @@ +"""Tests for Cube views → SLayer facade models (slayer/cube/converter.py). + +DEV-1608 §6 + Codex #1/#3 corrections: facade measures reference the underlying +Column (not the Cube measure name); facade source mirrors the root cube's mode. +""" + +from slayer.core.models import SlayerModel +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.models import ( + CubeCube, + CubeDimension, + CubeJoin, + CubeMeasure, + CubeProject, + CubeView, + CubeViewCubeRef, +) +from slayer.cube.report import CubeIssueCategory + +DS = "test_ds" + + +def _orders_customers_cubes(*, orders_sql_mode: bool = False) -> list[CubeCube]: + orders_kwargs = ( + {"sql": "SELECT * FROM public.orders"} if orders_sql_mode + else {"sql_table": "public.orders"} + ) + return [ + CubeCube( + name="orders", **orders_kwargs, + joins=[CubeJoin(name="customers", relationship="many_to_one", + sql="{CUBE}.customer_id = {customers.id}")], + measures=[CubeMeasure(name="count", type="count"), + CubeMeasure(name="total_revenue", type="sum", sql="{CUBE}.amount")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", + primary_key=True), + CubeDimension(name="status", sql="{CUBE}.status", type="string")], + ), + CubeCube( + name="customers", sql_table="public.customers", + measures=[CubeMeasure(name="lifetime_value", type="sum", sql="{CUBE}.ltv")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", + primary_key=True), + CubeDimension(name="name", sql="{CUBE}.name", type="string"), + CubeDimension(name="region", sql="{CUBE}.region", type="string")], + ), + ] + + +def _view() -> CubeView: + return CubeView(name="orders_overview", cubes=[ + CubeViewCubeRef(join_path="orders", + includes=["count", "total_revenue", "status"]), + CubeViewCubeRef(join_path="orders.customers", prefix=True, + includes=["name", "region", "lifetime_value"]), + ]) + + +def _convert(project: CubeProject) -> tuple[dict[str, SlayerModel], object]: + result = CubeToSlayerConverter(project=project, data_source=DS).convert() + return {m.name: m for m in result.models}, result.report + + +def test_view_facade_model_basic_shape(): + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + assert view.sql_table == "public.orders" # rooted on orders + assert view.meta["cube_kind"] == "view" + # join to customers present on the facade + assert any(j.target_model == "customers" for j in view.joins) + + +def test_view_root_dimension_is_local_derived_column(): + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + assert view.get_column("status") is not None + + +def test_view_prefixed_joined_dimension_references_joined_column(): + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + # prefix: true → "_" (Cube prepends the cube name verbatim). + col = view.get_column("customers_name") + assert col is not None + assert col.sql == "customers.name" + + +def test_view_root_measure_carries_underlying_column(): + """Codex #1: a root-cube measure re-export needs the underlying column on + the facade, referenced by `:` (NOT the measure name).""" + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + m = view.get_measure("total_revenue") + assert m is not None + col_ref = m.formula.split(":")[0].strip() + assert view.get_column(col_ref) is not None # underlying column copied onto facade + assert m.formula.endswith(":sum") + + +def test_view_joined_measure_is_cross_model_underlying_column_ref(): + """Codex #1: joined-cube measure → `customers.:` + (the underlying column `ltv`, never the measure name `lifetime_value`).""" + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + m = view.get_measure("customers_lifetime_value") # prefix: "_" + assert m is not None + assert m.formula == "customers.ltv:sum" + + +def test_view_count_measure_maps_to_star_count(): + project = CubeProject(cubes=_orders_customers_cubes(), views=[_view()]) + models, _ = _convert(project) + assert models["orders_overview"].get_measure("count").formula == "*:count" + + +def test_view_default_filters_become_model_filters(): + view = _view() + view.default_filters = [{"member": "orders.status", "operator": "equals", + "values": ["completed"]}] + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, _ = _convert(project) + filters = " ".join(models["orders_overview"].filters) + assert "completed" in filters + + +def test_view_on_sql_backed_root_mirrors_sql_source(): + """Codex #3: facade mirrors the root cube's source mode, not always sql_table.""" + project = CubeProject(cubes=_orders_customers_cubes(orders_sql_mode=True), + views=[_view()]) + models, _ = _convert(project) + view = models["orders_overview"] + assert view.sql_table is None + assert view.sql == "SELECT * FROM public.orders" + + +def test_view_folders_parked_in_meta_and_reported(): + view = _view() + view.folders = [{"name": "Revenue", "includes": ["total_revenue"]}] + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, report = _convert(project) + assert models["orders_overview"].meta["cube_unmapped"]["folders"] + assert any(i.category == CubeIssueCategory.FOLDERS_UNMAPPED for i in report.issues) + + +def test_view_excludes_drops_member(): + view = CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes="*", excludes=["status"]), + ]) + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, _ = _convert(project) + assert models["ov"].get_column("status") is None + assert models["ov"].get_measure("total_revenue") is not None + + +def test_view_includes_star_takes_all_members(): + view = CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes="*"), + ]) + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, _ = _convert(project) + ov = models["ov"] + assert ov.get_column("status") is not None + assert ov.get_measure("total_revenue") is not None + assert ov.get_measure("count") is not None + + +def test_view_disconnected_members_reported(): + cubes = _orders_customers_cubes() + cubes.append(CubeCube(name="weather", sql_table="public.weather", + dimensions=[CubeDimension(name="temp", sql="{CUBE}.temp", + type="number")])) + view = CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["status"]), + CubeViewCubeRef(join_path="weather", includes=["temp"]), # not joined to orders + ]) + project = CubeProject(cubes=cubes, views=[view]) + _models, report = _convert(project) + assert any(i.category == CubeIssueCategory.DISCONNECTED_VIEW for i in report.issues) + + +def test_view_fanout_risk_reported(): + cubes = [ + CubeCube(name="orders", sql_table="public.orders", + joins=[CubeJoin(name="line_items", relationship="one_to_many", + sql="{CUBE}.id = {line_items.order_id}")], + measures=[CubeMeasure(name="total_revenue", type="sum", sql="{CUBE}.amount")], + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number", + primary_key=True)]), + CubeCube(name="line_items", sql_table="public.line_items", + dimensions=[CubeDimension(name="order_id", sql="{CUBE}.order_id", + type="number"), + CubeDimension(name="sku", sql="{CUBE}.sku", type="string")]), + ] + view = CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["total_revenue"]), + CubeViewCubeRef(join_path="orders.line_items", includes=["sku"]), + ]) + project = CubeProject(cubes=cubes, views=[view]) + _models, report = _convert(project) + assert any(i.category == CubeIssueCategory.VIEW_FANOUT_RISK for i in report.issues) + + +def test_view_extends_flattens_member_lists(): + base = CubeView(name="base_view", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["status"])]) + child = CubeView(name="child_view", extends="base_view", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["total_revenue"])]) + project = CubeProject(cubes=_orders_customers_cubes(), views=[base, child]) + models, _ = _convert(project) + child_model = models["child_view"] + assert child_model.get_column("status") is not None # inherited + assert child_model.get_measure("total_revenue") is not None # own + + +def test_view_star_skips_private_member(): + cubes = _orders_customers_cubes() + # mark a root-cube dimension private + cubes[0].dimensions.append(CubeDimension(name="internal_flag", sql="{CUBE}.flag", + type="string", public=False)) + view = CubeView(name="ov", cubes=[CubeViewCubeRef(join_path="orders", includes="*")]) + project = CubeProject(cubes=cubes, views=[view]) + models, _ = _convert(project) + assert models["ov"].get_column("internal_flag") is None + assert models["ov"].get_column("status") is not None + + +def test_view_default_filter_escapes_single_quotes(): + view = _view() + view.default_filters = [{"member": "orders.status", "operator": "equals", + "values": ["O'Reilly"]}] + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, _ = _convert(project) + filters = " ".join(models["orders_overview"].filters) + assert "O''Reilly" in filters # single quote doubled, not "O'Reilly'" + + +def test_view_object_form_includes_reports_override(): + view = CubeView(name="ov", cubes=[CubeViewCubeRef( + join_path="orders", + includes=[{"name": "status", "format": "upper"}, "total_revenue"])]) + project = CubeProject(cubes=_orders_customers_cubes(), views=[view]) + models, report = _convert(project) + assert models["ov"].get_column("status") is not None # name still extracted + assert models["ov"].get_measure("total_revenue") is not None + assert any(i.category == CubeIssueCategory.UNMAPPED_INFRA for i in report.issues) + + +def test_view_calc_measure_reexport_reported(): + cubes = _orders_customers_cubes() + cubes[0].measures.append(CubeMeasure(name="aov", type="number", + sql="{total_revenue} / {count}")) + view = CubeView(name="ov", cubes=[CubeViewCubeRef(join_path="orders", includes=["aov"])]) + project = CubeProject(cubes=cubes, views=[view]) + models, report = _convert(project) + # A calculated measure can't be re-exported as a facade cross-model measure + # in Stage 1 → reported, not silently skipped. + assert models["ov"].get_measure("aov") is None + assert any(i.category == CubeIssueCategory.COMPLEX_MEASURE for i in report.issues) + + +def test_view_dropped_when_root_cube_not_emitted(): + # orders has no source → not emitted; a view rooted on it can't be built. + cubes = [ + CubeCube(name="orders"), # no source + CubeCube(name="customers", sql_table="public.customers", + dimensions=[CubeDimension(name="id", sql="{CUBE}.id", type="number")]), + ] + view = CubeView(name="ov", cubes=[ + CubeViewCubeRef(join_path="orders", includes=["id"])]) + project = CubeProject(cubes=cubes, views=[view]) + models, report = _convert(project) + assert "ov" not in models + assert any(i.category == CubeIssueCategory.AMBIGUOUS_VIEW_ROOT + for i in report.issues)