diff --git a/.opencode/skills/dbt-analyze/SKILL.md b/.opencode/skills/dbt-analyze/SKILL.md new file mode 100644 index 0000000000..7aa5ba2f5d --- /dev/null +++ b/.opencode/skills/dbt-analyze/SKILL.md @@ -0,0 +1,121 @@ +--- +name: dbt-analyze +description: Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt. +--- + +# dbt Impact Analysis + +## Requirements +**Agent:** any (read-only analysis) +**Tools used:** bash (runs `altimate-dbt` commands), read, glob, dbt_manifest, lineage_check, sql_analyze + +## When to Use This Skill + +**Use when the user wants to:** +- Understand what breaks if they change a model +- Evaluate downstream impact before shipping +- Find all consumers of a model or column +- Assess risk of a refactoring + +**Do NOT use for:** +- Creating or fixing models → use `dbt-develop` or `dbt-troubleshoot` +- Adding tests → use `dbt-test` + +## Workflow + +### 1. Identify the Changed Model + +Accept from the user, or auto-detect: +```bash +# From git diff +git diff --name-only | grep '\.sql$' + +# Or user provides a model name +altimate-dbt compile --model # verify it exists +``` + +### 2. Map the Dependency Graph + +```bash +altimate-dbt children --model # direct downstream +altimate-dbt parents --model # what feeds it +``` + +For the full downstream tree, recursively call `children` on each downstream model. + +### 3. Run Column-Level Lineage + +Use the `lineage_check` tool on the changed model's SQL to understand: +- Which source columns flow to which output columns +- Which columns were added, removed, or renamed + +### 4. Cross-Reference with Downstream + +For each downstream model: +1. Read its SQL +2. Check if it references any changed/removed columns +3. Classify impact: + +| Classification | Meaning | Action | +|---------------|---------|--------| +| **BREAKING** | Removed/renamed column used downstream | Must fix before shipping | +| **SAFE** | Added column, no downstream reference | Ship freely | +| **UNKNOWN** | Can't determine (dynamic SQL, macros) | Manual review needed | + +### 5. Generate Impact Report + +``` +Impact Analysis: stg_orders +════════════════════════════ + +Changed Model: stg_orders (materialized: view) + Columns: 5 → 6 (+1 added) + Removed: total_amount (renamed to order_total) + +Downstream Impact (3 models): + + Depth 1: + [BREAKING] int_order_metrics + Uses: total_amount → COLUMN RENAMED + Fix: Update column reference to order_total + + [SAFE] int_order_summary + No references to changed columns + + Depth 2: + [BREAKING] mart_revenue + Uses: total_amount via int_order_metrics → CASCADING + Fix: Verify after fixing int_order_metrics + +Tests at Risk: 4 + - not_null_stg_orders_order_total + - unique_int_order_metrics_order_id + +Summary: 2 BREAKING, 1 SAFE + Recommended: Fix int_order_metrics first, then: + altimate-dbt build --model stg_orders --downstream +``` + +## Without Manifest (SQL-Only Mode) + +If no manifest is available: +1. Run `lineage_check` on the changed SQL +2. Show column-level data flow +3. Note: downstream impact requires a manifest +4. Suggest: `altimate-dbt build-project` to generate one + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Only checking direct children | Always trace the FULL downstream tree recursively | +| Ignoring test impacts | Check which tests reference changed columns | +| Shipping without building downstream | Always `altimate-dbt build --model --downstream` | +| Not considering renamed columns | A rename is a break + add — downstream still references the old name | + +## Reference Guides + +| Guide | Use When | +|-------|----------| +| [references/altimate-dbt-commands.md](references/altimate-dbt-commands.md) | Need the full CLI reference | +| [references/lineage-interpretation.md](references/lineage-interpretation.md) | Understanding lineage output | diff --git a/.opencode/skills/dbt-analyze/references/altimate-dbt-commands.md b/.opencode/skills/dbt-analyze/references/altimate-dbt-commands.md new file mode 100644 index 0000000000..6dbb8e9731 --- /dev/null +++ b/.opencode/skills/dbt-analyze/references/altimate-dbt-commands.md @@ -0,0 +1,66 @@ +# altimate-dbt Command Reference + +All dbt operations use the `altimate-dbt` CLI. Output is JSON to stdout; logs go to stderr. + +```bash +altimate-dbt [args...] +altimate-dbt [args...] --format text # Human-readable output +``` + +## First-Time Setup + +```bash +altimate-dbt init # Auto-detect project root +altimate-dbt init --project-root /path # Explicit root +altimate-dbt init --python-path /path # Override Python +altimate-dbt doctor # Verify setup +altimate-dbt info # Project name, adapter, root +``` + +## Build & Run + +```bash +altimate-dbt build --model [--downstream] # compile + run + test +altimate-dbt run --model [--downstream] # materialize only +altimate-dbt test --model # run tests only +altimate-dbt build-project # full project build +``` + +## Compile + +```bash +altimate-dbt compile --model +altimate-dbt compile-query --query "SELECT * FROM {{ ref('stg_orders') }}" [--model ] +``` + +## Execute SQL + +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('orders') }}" --limit 100 +``` + +## Schema & DAG + +```bash +altimate-dbt columns --model # column names and types +altimate-dbt columns-source --source --table # source table columns +altimate-dbt column-values --model --column # sample values +altimate-dbt children --model # downstream models +altimate-dbt parents --model # upstream models +``` + +## Packages + +```bash +altimate-dbt deps # install packages.yml +altimate-dbt add-packages --packages dbt-utils,dbt-expectations +``` + +## Error Handling + +All errors return JSON with `error` and `fix` fields: +```json +{ "error": "dbt-core is not installed", "fix": "Install it: python3 -m pip install dbt-core" } +``` + +Run `altimate-dbt doctor` as the first diagnostic step for any failure. diff --git a/.opencode/skills/dbt-analyze/references/lineage-interpretation.md b/.opencode/skills/dbt-analyze/references/lineage-interpretation.md new file mode 100644 index 0000000000..a27c145fdb --- /dev/null +++ b/.opencode/skills/dbt-analyze/references/lineage-interpretation.md @@ -0,0 +1,58 @@ +# Lineage Interpretation Guide + +## Understanding Column-Level Lineage + +Column-level lineage traces how data flows from source columns through transformations to output columns. + +### Direct Lineage +``` +source.customers.name → stg_customers.customer_name → dim_customers.full_name +``` +Column was renamed at each step. A change to the source column affects all downstream. + +### Aggregation Lineage +``` +source.orders.amount → (SUM) → fct_daily_revenue.total_revenue +``` +Multiple source rows feed into one output value. The column type changes from row-level to aggregate. + +### Conditional Lineage +``` +source.orders.status → (CASE WHEN) → fct_orders.is_completed +``` +The source column feeds a derived boolean. The relationship is logical, not direct. + +## Impact Classification + +### BREAKING Changes +- **Column removed**: Downstream models referencing it will fail +- **Column renamed**: Same as removed — downstream still uses the old name +- **Type changed**: May cause cast errors or silent data loss downstream +- **Logic changed**: Downstream aggregations/filters may produce wrong results + +### SAFE Changes +- **Column added**: No downstream model can reference what didn't exist +- **Description changed**: No runtime impact +- **Test added/modified**: No impact on model data + +### REQUIRES REVIEW +- **Filter changed**: May change which rows appear → downstream counts change +- **JOIN type changed**: LEFT→INNER drops rows, INNER→LEFT adds NULLs +- **Materialization changed**: view→table has no logical impact but affects freshness + +## Reading the DAG + +```bash +altimate-dbt parents --model # what this model depends on +altimate-dbt children --model # what depends on this model +``` + +A model with many children has high blast radius. A model with many parents has high complexity. + +## Depth Matters + +- **Depth 1**: Direct consumers — highest risk, most likely to break +- **Depth 2+**: Cascading impact — will break IF depth 1 breaks +- **Depth 3+**: Usually only affected by breaking column removals/renames + +Focus investigation on depth 1 first. diff --git a/.opencode/skills/dbt-develop/SKILL.md b/.opencode/skills/dbt-develop/SKILL.md new file mode 100644 index 0000000000..b323a2409f --- /dev/null +++ b/.opencode/skills/dbt-develop/SKILL.md @@ -0,0 +1,147 @@ +--- +name: dbt-develop +description: Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt. +--- + +# dbt Model Development + +## Requirements +**Agent:** builder or migrator (requires file write access) +**Tools used:** bash (runs `altimate-dbt` commands), read, glob, write, edit + +## When to Use This Skill + +**Use when the user wants to:** +- Create a new dbt model (staging, intermediate, mart, OBT) +- Add or modify SQL logic in an existing model +- Generate sources.yml or schema.yml from warehouse metadata +- Reorganize models into layers (staging/intermediate/mart or bronze/silver/gold) +- Convert a model to incremental materialization +- Scaffold a new dbt project structure + +**Do NOT use for:** +- Adding tests to models → use `dbt-test` +- Writing model/column descriptions → use `dbt-docs` +- Debugging build failures → use `dbt-troubleshoot` +- Analyzing change impact → use `dbt-analyze` + +## Core Workflow: Plan → Discover → Write → Validate + +### 1. Plan — Understand Before Writing + +Before writing any SQL: +- Read the task requirements carefully +- Identify which layer this model belongs to (staging, intermediate, mart) +- Check existing models for naming conventions and patterns +- **Check dependencies:** If `packages.yml` exists, check for `dbt_packages/` or `package-lock.yml`. Only run `dbt deps` if packages are declared but not yet installed. + +```bash +altimate-dbt info # project name, adapter type +altimate-dbt parents --model # understand what feeds this model +altimate-dbt children --model # understand what consumes it +``` + +### 2. Discover — Understand the Data Before Writing + +**Never write SQL without deeply understanding your data first.** The #1 cause of wrong results is writing SQL blind — assuming grain, relationships, column names, or values without checking. + +**Step 2a: Read all documentation and schema definitions** +- Read `sources.yml`, `schema.yml`, and any YAML files that describe the source/parent models +- These contain column descriptions, data types, tests, and business context +- Pay special attention to: primary keys, unique constraints, relationships between tables, and what each column represents + +**Step 2b: Understand the grain of each parent model/source** +- What does one row represent? (one customer? one event? one day per customer?) +- What are the primary/unique keys? +- This is critical for JOINs — joining on the wrong grain causes fan-out (too many rows) or missing rows + +```bash +altimate-dbt columns --model # existing model columns +altimate-dbt columns-source --source --table # source table columns +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('model') }}" --limit 1 +altimate-dbt execute --query "SELECT * FROM {{ ref('model') }}" --limit 5 +altimate-dbt column-values --model --column # sample values for key columns +``` + +**Step 2c: Query the actual data to verify your understanding** +- Check row counts, NULLs, date ranges, cardinality of key columns +- Verify foreign key relationships actually hold (do all IDs in child exist in parent?) +- Check for duplicates in what you think are unique keys + +**Step 2d: Read existing models that your new model will reference** +- Read the actual SQL of parent models — understand their logic, filters, and transformations +- Read 2-3 existing models in the same directory to match patterns and conventions + +```bash +glob models/**/*.sql # find all model files +read # understand existing patterns and logic +``` + +### 3. Write — Follow Layer Patterns + +See [references/layer-patterns.md](references/layer-patterns.md) for staging/intermediate/mart templates. +See [references/medallion-architecture.md](references/medallion-architecture.md) for bronze/silver/gold patterns. +See [references/incremental-strategies.md](references/incremental-strategies.md) for incremental materialization. +See [references/yaml-generation.md](references/yaml-generation.md) for sources.yml and schema.yml. + +### 4. Validate — Build, Verify, Check Impact + +Never stop at writing the SQL. Always validate: + +**Build it:** +```bash +altimate-dbt compile --model # catch Jinja errors +altimate-dbt build --model # materialize + run tests +``` + +**Verify the output:** +```bash +altimate-dbt columns --model # confirm expected columns exist +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('') }}" --limit 1 +altimate-dbt execute --query "SELECT * FROM {{ ref('') }}" --limit 10 # spot-check values +``` +- Do the columns match what schema.yml or the task expects? +- Does the row count make sense? (no fan-out from bad joins, no missing rows from wrong filters) +- Are values correct? (spot-check NULLs, aggregations, date ranges) + +**Check SQL quality** (on the compiled SQL from `altimate-dbt compile`): +- `sql_analyze` — catches anti-patterns (SELECT *, cartesian products, missing filters) +- `altimate_core_validate` — validates syntax and schema references +- `altimate_core_column_lineage` — traces how source columns flow to output columns. Use this to verify your SELECT is pulling the right columns from the right sources, especially for complex JOINs or multi-CTE models. + +**Check downstream impact** (when modifying an existing model): +```bash +altimate-dbt children --model # who depends on this? +altimate-dbt build --model --downstream # rebuild downstream to catch breakage +``` +Use `altimate-dbt children` and `altimate-dbt parents` to verify the DAG is intact when changes could affect downstream models. + +## Iron Rules + +1. **Never write SQL without reading the source columns first.** Use `altimate-dbt columns` or `altimate-dbt columns-source`. +2. **Never stop at compile.** Always `altimate-dbt build` to catch runtime errors. +3. **Match existing patterns.** Read 2-3 existing models in the same directory before writing. +4. **One model, one purpose.** A staging model should not contain business logic. An intermediate model should not be materialized as a table unless it has consumers. +5. **Fix ALL errors, not just yours.** After creating/modifying models, run a full `dbt build`. If ANY model fails — even pre-existing ones you didn't touch — fix them. Your job is to leave the project in a fully working state. + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Writing SQL without checking column names | Run `altimate-dbt columns` or `altimate-dbt columns-source` first | +| Stopping at `compile` — "it compiled, ship it" | Always `altimate-dbt build` to materialize and run tests | +| Hardcoding table references instead of `{{ ref() }}` | Always use `{{ ref('model') }}` or `{{ source('src', 'table') }}` | +| Creating a staging model with JOINs | Staging = 1:1 with source. JOINs belong in intermediate or mart | +| Not checking existing naming conventions | Read existing models in the same directory first | +| Using `SELECT *` in final models | Explicitly list columns for clarity and contract stability | + +## Reference Guides + +| Guide | Use When | +|-------|----------| +| [references/altimate-dbt-commands.md](references/altimate-dbt-commands.md) | Need the full CLI reference | +| [references/layer-patterns.md](references/layer-patterns.md) | Creating staging, intermediate, or mart models | +| [references/medallion-architecture.md](references/medallion-architecture.md) | Organizing into bronze/silver/gold layers | +| [references/incremental-strategies.md](references/incremental-strategies.md) | Converting to incremental materialization | +| [references/yaml-generation.md](references/yaml-generation.md) | Generating sources.yml or schema.yml | +| [references/common-mistakes.md](references/common-mistakes.md) | Extended anti-patterns catalog | diff --git a/.opencode/skills/dbt-develop/references/altimate-dbt-commands.md b/.opencode/skills/dbt-develop/references/altimate-dbt-commands.md new file mode 100644 index 0000000000..6dbb8e9731 --- /dev/null +++ b/.opencode/skills/dbt-develop/references/altimate-dbt-commands.md @@ -0,0 +1,66 @@ +# altimate-dbt Command Reference + +All dbt operations use the `altimate-dbt` CLI. Output is JSON to stdout; logs go to stderr. + +```bash +altimate-dbt [args...] +altimate-dbt [args...] --format text # Human-readable output +``` + +## First-Time Setup + +```bash +altimate-dbt init # Auto-detect project root +altimate-dbt init --project-root /path # Explicit root +altimate-dbt init --python-path /path # Override Python +altimate-dbt doctor # Verify setup +altimate-dbt info # Project name, adapter, root +``` + +## Build & Run + +```bash +altimate-dbt build --model [--downstream] # compile + run + test +altimate-dbt run --model [--downstream] # materialize only +altimate-dbt test --model # run tests only +altimate-dbt build-project # full project build +``` + +## Compile + +```bash +altimate-dbt compile --model +altimate-dbt compile-query --query "SELECT * FROM {{ ref('stg_orders') }}" [--model ] +``` + +## Execute SQL + +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('orders') }}" --limit 100 +``` + +## Schema & DAG + +```bash +altimate-dbt columns --model # column names and types +altimate-dbt columns-source --source --table # source table columns +altimate-dbt column-values --model --column # sample values +altimate-dbt children --model # downstream models +altimate-dbt parents --model # upstream models +``` + +## Packages + +```bash +altimate-dbt deps # install packages.yml +altimate-dbt add-packages --packages dbt-utils,dbt-expectations +``` + +## Error Handling + +All errors return JSON with `error` and `fix` fields: +```json +{ "error": "dbt-core is not installed", "fix": "Install it: python3 -m pip install dbt-core" } +``` + +Run `altimate-dbt doctor` as the first diagnostic step for any failure. diff --git a/.opencode/skills/dbt-develop/references/common-mistakes.md b/.opencode/skills/dbt-develop/references/common-mistakes.md new file mode 100644 index 0000000000..9b398606e3 --- /dev/null +++ b/.opencode/skills/dbt-develop/references/common-mistakes.md @@ -0,0 +1,49 @@ +# Common dbt Development Mistakes + +## SQL Mistakes + +| Mistake | Why It's Wrong | Fix | +|---------|---------------|-----| +| `SELECT *` in mart/fact models | Breaks contracts when upstream adds columns | Explicitly list all columns | +| Hardcoded table names | Breaks when schema/database changes | Use `{{ ref() }}` and `{{ source() }}` | +| Business logic in staging models | Staging = 1:1 source mirror | Move logic to intermediate layer | +| JOINs in staging models | Same as above | Move JOINs to intermediate layer | +| `LEFT JOIN` when `INNER JOIN` is correct | Returns NULLs for unmatched rows | Think about what NULL means for your use case | +| Missing `GROUP BY` columns | Query fails or returns wrong results | Every non-aggregate column must be in GROUP BY | +| Window functions without `PARTITION BY` | Aggregates across entire table | Add appropriate partitioning | + +## Project Structure Mistakes + +| Mistake | Fix | +|---------|-----| +| Models in wrong layer directory | staging/ for source mirrors, intermediate/ for transforms, marts/ for business tables | +| No schema.yml for new models | Always create companion YAML with at minimum `unique` + `not_null` on primary key | +| Naming doesn't match convention | Check existing models: `stg_`, `int_`, `fct_`, `dim_` prefixes | +| Missing `{{ config() }}` block | Every model should declare materialization explicitly or inherit from dbt_project.yml | + +## Incremental Mistakes + +| Mistake | Fix | +|---------|-----| +| No `unique_key` on merge strategy | Causes duplicates. Set `unique_key` to your primary key | +| Using `created_at` for mutable records | Use `updated_at` — `created_at` misses updates | +| No lookback window | Use `max(ts) - interval '1 hour'` to catch late-arriving data | +| Forgetting `is_incremental()` returns false on first run | The `WHERE` clause only applies after first run | + +## Validation Mistakes + +| Mistake | Fix | +|---------|-----| +| "It compiled, ship it" | Compilation only checks Jinja syntax. Always `altimate-dbt build` | +| Not spot-checking output data | Run `altimate-dbt execute --query "SELECT * FROM {{ ref('model') }}" --limit 10` | +| Not checking row counts | Compare source vs output: `SELECT count(*) FROM {{ ref('model') }}` | +| Skipping downstream builds | Use `altimate-dbt build --model --downstream` | + +## Rationalizations to Resist + +| You're Thinking... | Reality | +|--------------------|---------| +| "I'll add tests later" | You won't. Add them now. | +| "SELECT * is fine for now" | It will break when upstream changes. List columns explicitly. | +| "This model is temporary" | Nothing is more permanent than a temporary solution. | +| "The data looks right at a glance" | Run the build. Check the tests. Spot-check edge cases. | diff --git a/.opencode/skills/dbt-develop/references/incremental-strategies.md b/.opencode/skills/dbt-develop/references/incremental-strategies.md new file mode 100644 index 0000000000..aa5077455d --- /dev/null +++ b/.opencode/skills/dbt-develop/references/incremental-strategies.md @@ -0,0 +1,118 @@ +# Incremental Materialization Strategies + +## When to Use Incremental + +Use incremental when: +- Table has millions+ rows +- Source data is append-only or has reliable `updated_at` timestamps +- Full refreshes take too long or cost too much + +Do NOT use incremental when: +- Table is small (< 1M rows) +- Source data doesn't have reliable timestamps +- Logic requires full-table window functions + +## Strategy Decision Tree + +``` +Is the data append-only (events, logs)? + YES → Append strategy + NO → Can rows be updated? + YES → Does your warehouse support MERGE? + YES → Merge/Upsert strategy + NO → Delete+Insert strategy + NO → Is data date-partitioned? + YES → Insert Overwrite strategy + NO → Append with dedup +``` + +## Append (Event Logs) + +```sql +{{ config( + materialized='incremental', + on_schema_change='append_new_columns' +) }} + +select + event_id, + event_type, + created_at +from {{ ref('stg_events') }} + +{% if is_incremental() %} +where created_at > (select max(created_at) from {{ this }}) +{% endif %} +``` + +## Merge/Upsert (Mutable Records) + +```sql +{{ config( + materialized='incremental', + unique_key='order_id', + merge_update_columns=['status', 'updated_at', 'amount'], + on_schema_change='sync_all_columns' +) }} + +select + order_id, + status, + amount, + created_at, + updated_at +from {{ ref('stg_orders') }} + +{% if is_incremental() %} +where updated_at > (select max(updated_at) from {{ this }}) +{% endif %} +``` + +## Insert Overwrite (Partitioned) + +```sql +{{ config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by={'field': 'event_date', 'data_type': 'date'}, + on_schema_change='fail' +) }} + +select + date_trunc('day', created_at) as event_date, + count(*) as event_count +from {{ ref('stg_events') }} + +{% if is_incremental() %} +where date_trunc('day', created_at) >= (select max(event_date) - interval '3 days' from {{ this }}) +{% endif %} + +group by 1 +``` + +## Common Pitfalls + +| Issue | Problem | Fix | +|-------|---------|-----| +| Missing `unique_key` | Duplicates on re-run | Add `unique_key` matching the primary key | +| Wrong timestamp column | Missed updates | Use `updated_at` not `created_at` for mutable data | +| No lookback window | Late-arriving data missed | `max(ts) - interval '1 hour'` instead of strict `>` | +| `on_schema_change='fail'` | Breaks on column additions | Use `'append_new_columns'` or `'sync_all_columns'` | +| Full refresh needed | Schema drift accumulated | `altimate-dbt run --model ` with `--full-refresh` flag | + +## Official Documentation + +For the latest syntax and adapter-specific options, refer to: +- **dbt incremental models**: https://docs.getdbt.com/docs/build/incremental-models +- **Incremental strategies by adapter**: https://docs.getdbt.com/docs/build/incremental-strategy +- **Configuring incremental models**: https://docs.getdbt.com/reference/resource-configs/materialized#incremental + +## Warehouse Support + +| Warehouse | Default Strategy | Merge | Partition | Notes | +|-----------|-----------------|-------|-----------|-------| +| Snowflake | `merge` | Yes | Cluster keys | Best incremental support | +| BigQuery | `merge` | Yes | `partition_by` | Requires partition for insert_overwrite | +| PostgreSQL | `append` | No | No | Use delete+insert pattern | +| DuckDB | `append` | Partial | No | Limited incremental support | +| Redshift | `append` | No | dist/sort keys | Use delete+insert pattern | diff --git a/.opencode/skills/dbt-develop/references/layer-patterns.md b/.opencode/skills/dbt-develop/references/layer-patterns.md new file mode 100644 index 0000000000..879af0ecd8 --- /dev/null +++ b/.opencode/skills/dbt-develop/references/layer-patterns.md @@ -0,0 +1,158 @@ +# dbt Layer Patterns + +## Staging (`stg_`) + +**Purpose**: 1:1 with source table. Rename, cast, no joins, no business logic. +**Materialization**: `view` +**Naming**: `stg___.sql` +**Location**: `models/staging//` + +```sql +with source as ( + select * from {{ source('source_name', 'table_name') }} +), + +renamed as ( + select + -- Primary key + column_id as table_id, + + -- Dimensions + column_name, + + -- Timestamps + created_at, + updated_at + + from source +) + +select * from renamed +``` + +**Rules**: +- One CTE named `source`, one named `renamed` (or `cast`, `cleaned`) +- Only type casting, renaming, deduplication +- No joins, no filters (except dedup), no business logic + +## Intermediate (`int_`) + +**Purpose**: Business logic, joins, transformations between staging and marts. +**Materialization**: `ephemeral` or `view` +**Naming**: `int___.sql` (e.g., `int_orders__joined`, `int_payments__pivoted`) +**Location**: `models/intermediate/` + +```sql +with orders as ( + select * from {{ ref('stg_source__orders') }} +), + +customers as ( + select * from {{ ref('stg_source__customers') }} +), + +joined as ( + select + orders.order_id, + orders.customer_id, + customers.customer_name, + orders.order_date, + orders.amount + from orders + left join customers on orders.customer_id = customers.customer_id +) + +select * from joined +``` + +**Rules**: +- Cross-source joins allowed +- Business logic transformations +- Not exposed to end users directly +- Name the verb: `__joined`, `__pivoted`, `__filtered`, `__aggregated` + +## Mart: Facts (`fct_`) + +**Purpose**: Business events. Immutable, timestamped, narrow. +**Materialization**: `table` or `incremental` +**Naming**: `fct_.sql` +**Location**: `models/marts//` + +```sql +with final as ( + select + order_id, + customer_id, + order_date, + amount, + discount_amount, + amount - discount_amount as net_amount + from {{ ref('int_orders__joined') }} +) + +select * from final +``` + +## Mart: Dimensions (`dim_`) + +**Purpose**: Descriptive attributes. Slowly changing, wide. +**Materialization**: `table` +**Naming**: `dim_.sql` + +```sql +with final as ( + select + customer_id, + customer_name, + email, + segment, + first_order_date, + most_recent_order_date, + lifetime_order_count + from {{ ref('int_customers__aggregated') }} +) + +select * from final +``` + +## One Big Table (`obt_`) + +**Purpose**: Denormalized wide table combining fact + dimensions for BI consumption. +**Materialization**: `table` +**Naming**: `obt_.sql` + +```sql +with facts as ( + select * from {{ ref('fct_orders') }} +), + +customers as ( + select * from {{ ref('dim_customers') }} +), + +dates as ( + select * from {{ ref('dim_dates') }} +), + +final as ( + select + facts.*, + customers.customer_name, + customers.segment, + dates.day_of_week, + dates.month_name, + dates.is_weekend + from facts + left join customers on facts.customer_id = customers.customer_id + left join dates on facts.order_date = dates.date_day +) + +select * from final +``` + +## CTE Style Guide + +- Name CTEs after what they contain, not what they do: `orders` not `get_orders` +- Use `final` as the last CTE name +- One CTE per source model (via `ref()` or `source()`) +- End every model with `select * from final` diff --git a/.opencode/skills/dbt-develop/references/medallion-architecture.md b/.opencode/skills/dbt-develop/references/medallion-architecture.md new file mode 100644 index 0000000000..dbf365bdc4 --- /dev/null +++ b/.opencode/skills/dbt-develop/references/medallion-architecture.md @@ -0,0 +1,125 @@ +# Medallion Architecture (Bronze / Silver / Gold) + +An alternative to staging/intermediate/mart layering. Common in Databricks and lakehouse environments. + +## Layer Mapping + +| Medallion | Traditional dbt | Purpose | +|-----------|----------------|---------| +| Bronze | Staging (`stg_`) | Raw ingestion, minimal transform | +| Silver | Intermediate (`int_`) | Cleaned, conformed, joined | +| Gold | Marts (`fct_`/`dim_`) | Business-ready aggregations | + +## Directory Structure + +``` +models/ + bronze/ + source_system/ + _source_system__sources.yml + brz_source_system__table.sql + silver/ + domain/ + slv_domain__entity.sql + gold/ + domain/ + fct_metric.sql + dim_entity.sql +``` + +## Bronze (Raw) + +```sql +-- brz_stripe__payments.sql +{{ config(materialized='view') }} + +with source as ( + select * from {{ source('stripe', 'payments') }} +), + +cast as ( + select + cast(id as varchar) as payment_id, + cast(amount as integer) as amount_cents, + cast(created as timestamp) as created_at, + _loaded_at + from source +) + +select * from cast +``` + +**Rules**: 1:1 with source, only cast/rename, no joins, `view` materialization. + +## Silver (Cleaned) + +```sql +-- slv_finance__orders_enriched.sql +{{ config(materialized='table') }} + +with orders as ( + select * from {{ ref('brz_stripe__payments') }} +), + +customers as ( + select * from {{ ref('brz_crm__customers') }} +), + +enriched as ( + select + o.payment_id, + o.amount_cents / 100.0 as amount_dollars, + c.customer_name, + c.segment, + o.created_at + from orders o + left join customers c on o.customer_id = c.customer_id + where o.created_at is not null +) + +select * from enriched +``` + +**Rules**: Cross-source joins, business logic, quality filters, `table` materialization. + +## Gold (Business-Ready) + +```sql +-- fct_daily_revenue.sql +{{ config(materialized='table') }} + +with daily as ( + select + date_trunc('day', created_at) as revenue_date, + segment, + count(*) as order_count, + sum(amount_dollars) as gross_revenue + from {{ ref('slv_finance__orders_enriched') }} + group by 1, 2 +) + +select * from daily +``` + +**Rules**: Aggregations, metrics, KPIs, `table` or `incremental` materialization. + +## dbt_project.yml Config + +```yaml +models: + my_project: + bronze: + +materialized: view + silver: + +materialized: table + gold: + +materialized: table +``` + +## When to Use Medallion vs Traditional + +| Use Medallion When | Use Traditional When | +|-------------------|---------------------| +| Databricks/lakehouse environment | dbt Cloud or Snowflake-centric | +| Team already uses bronze/silver/gold terminology | Team uses staging/intermediate/mart | +| Data platform team maintains bronze layer | Analytics engineers own the full stack | diff --git a/.opencode/skills/dbt-develop/references/yaml-generation.md b/.opencode/skills/dbt-develop/references/yaml-generation.md new file mode 100644 index 0000000000..c2b8fe1a48 --- /dev/null +++ b/.opencode/skills/dbt-develop/references/yaml-generation.md @@ -0,0 +1,90 @@ +# YAML Config Generation + +## sources.yml — Define Raw Data Sources + +Generate from warehouse metadata using: +```bash +altimate-dbt columns-source --source --table +``` + +Template: +```yaml +version: 2 + +sources: + - name: raw_stripe + description: Raw Stripe payment data + database: raw + schema: stripe + tables: + - name: payments + description: All payment transactions + columns: + - name: payment_id + description: Primary key + tests: + - unique + - not_null + - name: amount + description: Payment amount in cents + - name: created_at + description: Payment creation timestamp + tests: + - not_null +``` + +## schema.yml — Model Documentation and Tests + +Generate after building a model using: +```bash +altimate-dbt columns --model +``` + +Template: +```yaml +version: 2 + +models: + - name: stg_stripe__payments + description: Staged Stripe payments with renamed columns and type casts + columns: + - name: payment_id + description: Primary key from source + tests: + - unique + - not_null + - name: amount_dollars + description: Payment amount converted to dollars +``` + +## Column Pattern Heuristics + +When auto-generating YAML, infer descriptions and tests from column names: + +| Pattern | Description Template | Auto-Tests | +|---------|---------------------|------------| +| `*_id` | "Foreign key to {table}" or "Primary key" | `unique`, `not_null` | +| `*_at`, `*_date`, `*_timestamp` | "Timestamp of {event}" | `not_null` | +| `*_amount`, `*_price`, `*_cost` | "Monetary value" | `not_null` | +| `is_*`, `has_*` | "Boolean flag for {condition}" | `accepted_values: [true, false]` | +| `*_type`, `*_status`, `*_category` | "Categorical" | `accepted_values` (if inferable) | +| `*_count`, `*_total`, `*_sum` | "Aggregated count/total" | — | +| `*_name`, `*_title`, `*_label` | "Human-readable name" | — | + +## File Naming Conventions + +| Convention | Example | +|-----------|---------| +| Sources | `___sources.yml` | +| Model schema | `___models.yml` or `schema.yml` | +| Properties | `___models.yml` | + +Match whatever convention the project already uses. Check existing `.yml` files in the same directory. + +## Merging with Existing YAML + +When YAML files already exist: +1. Read the existing file first +2. Add new entries without duplicating existing ones +3. Preserve human-written descriptions +4. Use `edit` tool (not `write`) to merge diff --git a/.opencode/skills/dbt-docs/SKILL.md b/.opencode/skills/dbt-docs/SKILL.md index 14b16702b5..e34ee516a2 100644 --- a/.opencode/skills/dbt-docs/SKILL.md +++ b/.opencode/skills/dbt-docs/SKILL.md @@ -1,85 +1,99 @@ --- name: dbt-docs -description: Generate or improve dbt model documentation — column descriptions, model descriptions, and doc blocks. +description: Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt. --- -# Generate dbt Documentation +# dbt Documentation ## Requirements **Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, schema_inspect, dbt_manifest, edit, write +**Tools used:** bash (runs `altimate-dbt` commands), read, glob, write, edit -> **When to use this vs other skills:** Use /dbt-docs to add or improve descriptions in existing schema.yml. Use /yaml-config to create schema.yml from scratch. Use /generate-tests to add test scaffolding. +## When to Use This Skill -Generate comprehensive documentation for dbt models by analyzing SQL logic, schema metadata, and existing docs. +**Use when the user wants to:** +- Add or improve model descriptions in schema.yml +- Write column-level descriptions with business context +- Create shared doc blocks for reusable definitions +- Improve dbt docs site content + +**Do NOT use for:** +- Adding tests → use `dbt-test` +- Creating new models → use `dbt-develop` +- Generating sources.yml from scratch → use `dbt-develop` ## Workflow -1. **Find the target model** — Use `glob` to locate the model SQL and any existing schema YAML -2. **Read the model SQL** — Understand the transformations, business logic, and column derivations -3. **Read existing docs** — Check for existing `schema.yml`, `___models.yml`, and `docs/` blocks -4. **Inspect schema** — Use `schema_inspect` to get column types and nullability -5. **Read upstream models** — Use `dbt_manifest` to find dependencies, then `read` upstream SQL to understand data flow -6. **Generate documentation**: - -### Model-Level Description -Write a clear, concise description that covers: -- **What** this model represents (business entity) -- **Why** it exists (use case) -- **How** it's built (key transformations, joins, filters) -- **When** it refreshes (materialization strategy) - -Example: +### 1. Understand the Model + +```bash +altimate-dbt columns --model # what columns exist +altimate-dbt parents --model # what feeds this model +altimate-dbt children --model # who consumes it +altimate-dbt compile --model # see the rendered SQL +``` + +Read the model SQL to understand the transformations: +```bash +glob models/**/.sql +read +``` + +### 2. Read Existing Documentation + +Check what's already documented: +```bash +glob models/**/*schema*.yml models/**/*_models.yml +read +``` + +### 3. Write Documentation + +See [references/documentation-standards.md](references/documentation-standards.md) for quality guidelines. + +#### Model-Level Description +Cover: **What** (business entity), **Why** (use case), **How** (key transforms), **When** (materialization). + ```yaml - name: fct_daily_revenue description: > Daily revenue aggregation by product category. Joins staged orders with product dimensions and calculates gross/net revenue. Materialized as - incremental with a unique key on (date_day, category_id). Used by the + incremental with unique key on (date_day, category_id). Used by the finance team for daily P&L reporting. ``` -### Column-Level Descriptions -For each column, describe: -- What the column represents in business terms -- How it's derived (if calculated/transformed) -- Any important caveats (nullability, edge cases) +#### Column-Level Description +Describe business meaning, derivation formula, and caveats: -Example: ```yaml columns: - name: net_revenue description: > Total revenue minus refunds and discounts for the day. - Calculated as: gross_revenue - refund_amount - discount_amount. + Formula: gross_revenue - refund_amount - discount_amount. Can be negative if refunds exceed sales. ``` -### Doc Blocks (for shared definitions) -If a definition is reused across models, generate a doc block: +### 4. Validate -```markdown -{% docs customer_id %} -Unique identifier for a customer. Sourced from the `customers` table -in the raw Stripe schema. Used as the primary join key across all -customer-related models. -{% enddocs %} +```bash +altimate-dbt compile --model # ensure YAML is valid ``` -7. **Write output** — Use `edit` to update existing YAML or `write` to create new files - -## Quality Checklist -- Every column has a description (no empty descriptions) -- Descriptions use business terms, not technical jargon -- Calculated columns explain their formula -- Primary keys are identified -- Foreign key relationships are documented -- Edge cases and null handling are noted +## Common Mistakes -## Usage +| Mistake | Fix | +|---------|-----| +| Restating the column name as the description | `"order_id: The order ID"` → describe business meaning | +| Empty descriptions | Every column should have a description. If unsure, describe the source. | +| Not reading the SQL before documenting | Read the model to understand derivation logic | +| Duplicating descriptions across models | Use doc blocks for shared definitions | +| Writing implementation details instead of business context | Describe what it means to the business, not how it's computed | -- `/dbt-docs models/marts/fct_daily_revenue.sql` -- `/dbt-docs stg_stripe__payments` -- `/dbt-docs --all models/staging/stripe/` — Document all models in a directory +## Reference Guides -Use the tools: `glob`, `read`, `schema_inspect`, `dbt_manifest`, `edit`, `write`. +| Guide | Use When | +|-------|----------| +| [references/altimate-dbt-commands.md](references/altimate-dbt-commands.md) | Need the full CLI reference | +| [references/documentation-standards.md](references/documentation-standards.md) | Writing high-quality descriptions | diff --git a/.opencode/skills/dbt-docs/references/altimate-dbt-commands.md b/.opencode/skills/dbt-docs/references/altimate-dbt-commands.md new file mode 100644 index 0000000000..6dbb8e9731 --- /dev/null +++ b/.opencode/skills/dbt-docs/references/altimate-dbt-commands.md @@ -0,0 +1,66 @@ +# altimate-dbt Command Reference + +All dbt operations use the `altimate-dbt` CLI. Output is JSON to stdout; logs go to stderr. + +```bash +altimate-dbt [args...] +altimate-dbt [args...] --format text # Human-readable output +``` + +## First-Time Setup + +```bash +altimate-dbt init # Auto-detect project root +altimate-dbt init --project-root /path # Explicit root +altimate-dbt init --python-path /path # Override Python +altimate-dbt doctor # Verify setup +altimate-dbt info # Project name, adapter, root +``` + +## Build & Run + +```bash +altimate-dbt build --model [--downstream] # compile + run + test +altimate-dbt run --model [--downstream] # materialize only +altimate-dbt test --model # run tests only +altimate-dbt build-project # full project build +``` + +## Compile + +```bash +altimate-dbt compile --model +altimate-dbt compile-query --query "SELECT * FROM {{ ref('stg_orders') }}" [--model ] +``` + +## Execute SQL + +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('orders') }}" --limit 100 +``` + +## Schema & DAG + +```bash +altimate-dbt columns --model # column names and types +altimate-dbt columns-source --source --table # source table columns +altimate-dbt column-values --model --column # sample values +altimate-dbt children --model # downstream models +altimate-dbt parents --model # upstream models +``` + +## Packages + +```bash +altimate-dbt deps # install packages.yml +altimate-dbt add-packages --packages dbt-utils,dbt-expectations +``` + +## Error Handling + +All errors return JSON with `error` and `fix` fields: +```json +{ "error": "dbt-core is not installed", "fix": "Install it: python3 -m pip install dbt-core" } +``` + +Run `altimate-dbt doctor` as the first diagnostic step for any failure. diff --git a/.opencode/skills/dbt-docs/references/documentation-standards.md b/.opencode/skills/dbt-docs/references/documentation-standards.md new file mode 100644 index 0000000000..b99b459e15 --- /dev/null +++ b/.opencode/skills/dbt-docs/references/documentation-standards.md @@ -0,0 +1,94 @@ +# Documentation Standards + +## Model Descriptions + +A good model description answers four questions: + +1. **What**: What business entity or metric does this represent? +2. **Why**: Who uses it and for what purpose? +3. **How**: Key transformations (joins, filters, aggregations) +4. **Grain**: What does one row represent? + +### Good Example +```yaml +description: > + Daily revenue by product category. One row per category per day. + Joins staged orders with product dimensions, calculates gross and net revenue. + Used by the finance team for P&L reporting and the marketing team for + campaign attribution. +``` + +### Bad Example +```yaml +description: "This model contains daily revenue data." +``` + +## Column Descriptions + +### Primary Keys +State that it's a primary key and where it originates: +```yaml +- name: order_id + description: "Primary key. Unique identifier for each order, sourced from the orders table in Stripe." +``` + +### Foreign Keys +State what it references: +```yaml +- name: customer_id + description: "Foreign key to dim_customers. The customer who placed this order." +``` + +### Calculated Columns +Include the formula: +```yaml +- name: net_revenue + description: "Gross revenue minus refunds and discounts. Formula: gross_amount - refund_amount - discount_amount. Can be negative." +``` + +### Status/Category Columns +List the possible values: +```yaml +- name: order_status + description: "Current order status. Values: pending, shipped, delivered, cancelled, refunded." +``` + +### Timestamps +State what event they represent: +```yaml +- name: shipped_at + description: "Timestamp when the order was shipped. NULL if not yet shipped. UTC timezone." +``` + +## Doc Blocks (Shared Definitions) + +For definitions reused across multiple models, create doc blocks: + +```markdown +-- docs/shared_definitions.md +{% docs customer_id %} +Unique identifier for a customer. Sourced from the customers table +in the CRM system. Used as the primary join key across all +customer-related models. +{% enddocs %} +``` + +Reference in YAML: +```yaml +- name: customer_id + description: "{{ doc('customer_id') }}" +``` + +## Anti-Patterns + +| Anti-Pattern | Why It's Bad | Better Approach | +|-------------|-------------|-----------------| +| `"The order ID"` | Just restates the column name | Describe origin and business meaning | +| `"See code"` | Defeats the purpose of docs | Summarize the logic | +| `""` (empty) | Missing documentation | Write something, even if brief | +| Technical jargon only | Business users can't understand | Lead with business meaning, add technical detail after | + +## Official Documentation + +- **dbt documentation**: https://docs.getdbt.com/docs/build/documentation +- **Doc blocks**: https://docs.getdbt.com/docs/build/documentation#using-docs-blocks diff --git a/.opencode/skills/dbt-test/SKILL.md b/.opencode/skills/dbt-test/SKILL.md new file mode 100644 index 0000000000..5787de839b --- /dev/null +++ b/.opencode/skills/dbt-test/SKILL.md @@ -0,0 +1,107 @@ +--- +name: dbt-test +description: Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt. +--- + +# dbt Testing + +## Requirements +**Agent:** builder or migrator (requires file write access) +**Tools used:** bash (runs `altimate-dbt` commands), read, glob, write, edit + +## When to Use This Skill + +**Use when the user wants to:** +- Add tests to a model's schema.yml (unique, not_null, relationships, accepted_values) +- Write dbt unit tests (mock inputs → expected outputs) +- Create custom generic or singular tests +- Debug why a test is failing +- Practice test-driven development in dbt + +**Do NOT use for:** +- Creating or modifying model SQL → use `dbt-develop` +- Writing model descriptions → use `dbt-docs` +- Debugging build/compilation errors → use `dbt-troubleshoot` + +## The Iron Rule + +**Never modify a test to make it pass without understanding why it's failing.** + +A failing test is information. It means either: +1. The data has a real quality issue (fix the data or the model) +2. The test expectation is wrong (update the test with justification) +3. The model logic is wrong (fix the model) + +Option 2 requires explicit user confirmation. Do not silently weaken tests. + +## Schema Test Workflow + +### 1. Discover Columns + +```bash +altimate-dbt columns --model +altimate-dbt column-values --model --column +``` + +### 2. Read Existing Tests + +```bash +glob models/**/*schema*.yml models/**/*_models.yml +read +``` + +### 3. Generate Tests + +Apply test rules based on column patterns — see [references/schema-test-patterns.md](references/schema-test-patterns.md). + +### 4. Write YAML + +Merge into existing schema.yml (don't duplicate). Use `edit` for existing files, `write` for new ones. + +### 5. Run Tests + +```bash +altimate-dbt test --model # run tests for this model +altimate-dbt build --model # build + test together +``` + +## Unit Test Workflow + +See [references/unit-test-guide.md](references/unit-test-guide.md) for the full unit test framework. + +### Quick Pattern + +```yaml +unit_tests: + - name: test_order_total_calculation + model: fct_orders + given: + - input: ref('stg_orders') + rows: + - { order_id: 1, quantity: 3, unit_price: 10.00 } + - { order_id: 2, quantity: 1, unit_price: 25.00 } + expect: + rows: + - { order_id: 1, order_total: 30.00 } + - { order_id: 2, order_total: 25.00 } +``` + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Testing every column with `not_null` | Only test columns that should never be null. Think about what NULL means. | +| Missing `unique` test on primary keys | Every primary key needs `unique` + `not_null` | +| `accepted_values` with incomplete list | Use `altimate-dbt column-values` to discover real values first | +| Modifying a test to make it pass | Understand WHY it fails first. The test might be right. | +| No `relationships` test on foreign keys | Add `relationships: {to: ref('parent'), field: parent_id}` | +| Unit testing trivial logic | Don't unit test `SELECT a, b FROM source`. Test calculations and business logic. | + +## Reference Guides + +| Guide | Use When | +|-------|----------| +| [references/altimate-dbt-commands.md](references/altimate-dbt-commands.md) | Need the full CLI reference | +| [references/schema-test-patterns.md](references/schema-test-patterns.md) | Adding schema.yml tests by column pattern | +| [references/unit-test-guide.md](references/unit-test-guide.md) | Writing dbt unit tests | +| [references/custom-tests.md](references/custom-tests.md) | Creating generic or singular tests | diff --git a/.opencode/skills/dbt-test/references/altimate-dbt-commands.md b/.opencode/skills/dbt-test/references/altimate-dbt-commands.md new file mode 100644 index 0000000000..6dbb8e9731 --- /dev/null +++ b/.opencode/skills/dbt-test/references/altimate-dbt-commands.md @@ -0,0 +1,66 @@ +# altimate-dbt Command Reference + +All dbt operations use the `altimate-dbt` CLI. Output is JSON to stdout; logs go to stderr. + +```bash +altimate-dbt [args...] +altimate-dbt [args...] --format text # Human-readable output +``` + +## First-Time Setup + +```bash +altimate-dbt init # Auto-detect project root +altimate-dbt init --project-root /path # Explicit root +altimate-dbt init --python-path /path # Override Python +altimate-dbt doctor # Verify setup +altimate-dbt info # Project name, adapter, root +``` + +## Build & Run + +```bash +altimate-dbt build --model [--downstream] # compile + run + test +altimate-dbt run --model [--downstream] # materialize only +altimate-dbt test --model # run tests only +altimate-dbt build-project # full project build +``` + +## Compile + +```bash +altimate-dbt compile --model +altimate-dbt compile-query --query "SELECT * FROM {{ ref('stg_orders') }}" [--model ] +``` + +## Execute SQL + +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('orders') }}" --limit 100 +``` + +## Schema & DAG + +```bash +altimate-dbt columns --model # column names and types +altimate-dbt columns-source --source --table # source table columns +altimate-dbt column-values --model --column # sample values +altimate-dbt children --model # downstream models +altimate-dbt parents --model # upstream models +``` + +## Packages + +```bash +altimate-dbt deps # install packages.yml +altimate-dbt add-packages --packages dbt-utils,dbt-expectations +``` + +## Error Handling + +All errors return JSON with `error` and `fix` fields: +```json +{ "error": "dbt-core is not installed", "fix": "Install it: python3 -m pip install dbt-core" } +``` + +Run `altimate-dbt doctor` as the first diagnostic step for any failure. diff --git a/.opencode/skills/dbt-test/references/custom-tests.md b/.opencode/skills/dbt-test/references/custom-tests.md new file mode 100644 index 0000000000..2539c5c888 --- /dev/null +++ b/.opencode/skills/dbt-test/references/custom-tests.md @@ -0,0 +1,59 @@ +# Custom dbt Tests + +## Generic Tests (Reusable) + +Generic tests are macros in `tests/generic/` or `macros/` that accept parameters: + +```sql +-- tests/generic/test_positive_values.sql +{% test positive_values(model, column_name) %} + +select * +from {{ model }} +where {{ column_name }} < 0 + +{% endtest %} +``` + +Usage in schema.yml: +```yaml +columns: + - name: amount + tests: + - positive_values +``` + +## Singular Tests (One-Off) + +Singular tests are standalone SQL files in `tests/` that return failing rows: + +```sql +-- tests/assert_no_orphan_orders.sql +-- Orders must have a matching customer +select o.order_id +from {{ ref('fct_orders') }} o +left join {{ ref('dim_customers') }} c on o.customer_id = c.customer_id +where c.customer_id is null +``` + +A passing test returns zero rows. + +## When to Use Which + +| Type | Use When | +|------|----------| +| Schema tests (built-in) | `unique`, `not_null`, `accepted_values`, `relationships` | +| dbt-utils tests | `expression_is_true`, `unique_combination_of_columns` | +| Generic tests (custom) | Reusable validation logic across multiple models | +| Singular tests | One-off business rules, cross-model assertions | +| Unit tests | Testing calculation logic with mocked inputs | + +## Naming Conventions + +- Generic tests: `test_.sql` in `tests/generic/` +- Singular tests: `assert_.sql` in `tests/` + +## Official Documentation + +- **Data tests**: https://docs.getdbt.com/docs/build/data-tests +- **Custom generic tests**: https://docs.getdbt.com/docs/build/data-tests#generic-data-tests diff --git a/.opencode/skills/dbt-test/references/schema-test-patterns.md b/.opencode/skills/dbt-test/references/schema-test-patterns.md new file mode 100644 index 0000000000..d4961a0d83 --- /dev/null +++ b/.opencode/skills/dbt-test/references/schema-test-patterns.md @@ -0,0 +1,103 @@ +# Schema Test Patterns + +## Test Generation Rules + +Apply tests based on column name patterns: + +| Column Pattern | Tests | +|---|---| +| `*_id` (primary key) | `unique`, `not_null` | +| `*_id` (foreign key) | `not_null`, `relationships: {to: ref('parent'), field: id}` | +| `status`, `type`, `category` | `accepted_values` (discover values with `altimate-dbt column-values`) | +| `*_at`, `*_date`, `*_timestamp` | `not_null` (if event timestamp that should always exist) | +| `is_*`, `has_*` (booleans) | `accepted_values: [true, false]` | +| Columns in JOIN conditions | `not_null` (nulls cause dropped rows in INNER JOIN) | + +## Test Priority Framework + +Not every column needs every test. Prioritize: + +### Tier 1: Always Test (Primary Keys) +```yaml +- name: order_id + tests: + - unique + - not_null +``` + +### Tier 2: Test Foreign Keys +```yaml +- name: customer_id + tests: + - not_null + - relationships: + to: ref('dim_customers') + field: customer_id +``` + +### Tier 3: Test Business-Critical Columns +```yaml +- name: order_status + tests: + - not_null + - accepted_values: + values: ['pending', 'shipped', 'delivered', 'cancelled'] +``` + +### Tier 4: Test Derived Columns (When Logic is Complex) +```yaml +- name: net_revenue + tests: + - not_null + - dbt_utils.expression_is_true: + expression: ">= 0" +``` + +## Discovering Values for accepted_values + +Before adding `accepted_values`, discover what actually exists: + +```bash +altimate-dbt column-values --model --column status +``` + +This prevents false test failures from values you didn't know about. + +## Cost-Conscious Testing + +For large tables, add `where` clauses to expensive tests: + +```yaml +- name: order_id + tests: + - unique: + config: + where: "order_date >= current_date - interval '30 days'" +``` + +## dbt-utils Test Extensions + +Common additional tests from `dbt-utils`: + +```yaml +# Ensure expression evaluates to true for all rows +- dbt_utils.expression_is_true: + expression: "amount >= 0" + +# Ensure no duplicate combinations +- dbt_utils.unique_combination_of_columns: + combination_of_columns: + - date_day + - customer_id + +# Ensure referential integrity across multiple columns +- dbt_utils.relationships_where: + to: ref('dim_products') + field: product_id + from_condition: "product_id is not null" +``` + +## Official Documentation + +- **dbt tests**: https://docs.getdbt.com/docs/build/data-tests +- **dbt-utils tests**: https://github.com/dbt-labs/dbt-utils#generic-tests diff --git a/.opencode/skills/dbt-test/references/unit-test-guide.md b/.opencode/skills/dbt-test/references/unit-test-guide.md new file mode 100644 index 0000000000..e2e9374132 --- /dev/null +++ b/.opencode/skills/dbt-test/references/unit-test-guide.md @@ -0,0 +1,121 @@ +# dbt Unit Tests + +Unit tests validate model logic by mocking inputs and asserting expected outputs. Available in dbt-core 1.8+. + +## When to Unit Test + +**DO unit test:** +- Complex calculations (revenue attribution, MRR changes, scoring) +- Business logic with edge cases (null handling, date boundaries, status transitions) +- Models with conditional logic (`CASE WHEN`, `IFF`, `COALESCE`) +- Aggregations where correctness is critical + +**Do NOT unit test:** +- Simple staging models (just rename/cast) +- Pass-through models with no logic +- Built-in dbt functions + +## Basic Structure + +Unit tests live in schema.yml (or a dedicated `_unit_tests.yml` file): + +```yaml +unit_tests: + - name: test_net_revenue_calculation + description: Verify net_revenue = gross - refunds - discounts + model: fct_daily_revenue + given: + - input: ref('stg_orders') + rows: + - { order_id: 1, gross_amount: 100.00, refund_amount: 10.00, discount_amount: 5.00 } + - { order_id: 2, gross_amount: 50.00, refund_amount: 0.00, discount_amount: 0.00 } + expect: + rows: + - { order_id: 1, net_revenue: 85.00 } + - { order_id: 2, net_revenue: 50.00 } +``` + +## Mocking Multiple Inputs + +```yaml +unit_tests: + - name: test_customer_lifetime_value + model: dim_customers + given: + - input: ref('stg_customers') + rows: + - { customer_id: 1, name: "Alice" } + - input: ref('stg_orders') + rows: + - { order_id: 1, customer_id: 1, amount: 50.00 } + - { order_id: 2, customer_id: 1, amount: 75.00 } + expect: + rows: + - { customer_id: 1, lifetime_value: 125.00 } +``` + +## Testing Edge Cases + +```yaml +unit_tests: + - name: test_handles_null_discounts + model: fct_orders + given: + - input: ref('stg_orders') + rows: + - { order_id: 1, amount: 100.00, discount: null } + expect: + rows: + - { order_id: 1, net_amount: 100.00 } + + - name: test_handles_zero_quantity + model: fct_orders + given: + - input: ref('stg_orders') + rows: + - { order_id: 1, quantity: 0, unit_price: 10.00 } + expect: + rows: + - { order_id: 1, order_total: 0.00 } +``` + +## Overriding Macros and Vars + +```yaml +unit_tests: + - name: test_with_specific_date + model: fct_daily_metrics + overrides: + vars: + run_date: "2024-01-15" + macros: + - name: current_timestamp + result: "2024-01-15 00:00:00" + given: + - input: ref('stg_events') + rows: + - { event_id: 1, event_date: "2024-01-15" } + expect: + rows: + - { event_date: "2024-01-15", event_count: 1 } +``` + +## Running Unit Tests + +```bash +altimate-dbt test --model # runs all tests including unit tests +altimate-dbt build --model # build + test +``` + +## Test-Driven Development Pattern + +1. Write the unit test YAML first (expected inputs → outputs) +2. Run it — it should fail (model doesn't exist yet or logic is missing) +3. Write/fix the model SQL +4. Run again — it should pass +5. Add schema tests for data quality + +## Official Documentation + +- **dbt unit tests**: https://docs.getdbt.com/docs/build/unit-tests +- **Unit test YAML spec**: https://docs.getdbt.com/reference/resource-properties/unit-tests diff --git a/.opencode/skills/dbt-troubleshoot/SKILL.md b/.opencode/skills/dbt-troubleshoot/SKILL.md new file mode 100644 index 0000000000..c8333a2160 --- /dev/null +++ b/.opencode/skills/dbt-troubleshoot/SKILL.md @@ -0,0 +1,174 @@ +--- +name: dbt-troubleshoot +description: Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt. +--- + +# dbt Troubleshooting + +## Requirements +**Agent:** any (read-only diagnosis), builder (if applying fixes) +**Tools used:** bash (runs `altimate-dbt` commands), read, glob, edit, altimate_core_semantics, altimate_core_column_lineage, altimate_core_correct + +## When to Use This Skill + +**Use when:** +- A dbt model fails to compile or build +- Tests are failing +- Model produces wrong or unexpected data +- Builds are slow or timing out +- User shares an error message from dbt + +**Do NOT use for:** +- Creating new models → use `dbt-develop` +- Adding tests → use `dbt-test` +- Analyzing change impact → use `dbt-analyze` + +## Iron Rules + +1. **Never modify a test to make it pass without understanding why it's failing.** +2. **Fix ALL errors, not just the reported one.** After fixing the specific issue, run a full `dbt build`. If other models fail — even ones not mentioned in the error report — fix them too. Your job is to leave the project in a fully working state. Never dismiss errors as "pre-existing" or "out of scope". + +## Diagnostic Workflow + +### Step 1: Health Check + +```bash +altimate-dbt doctor +altimate-dbt info +``` + +If `doctor` fails, fix the environment first. Common issues: +- Python not found → reinstall or set `--python-path` +- dbt-core not installed → `pip install dbt-core` +- No `dbt_project.yml` → wrong directory +- Missing packages → if `packages.yml` exists but `dbt_packages/` doesn't, run `dbt deps` + +### Step 2: Classify the Error + +| Error Type | Symptom | Jump To | +|-----------|---------|---------| +| Compilation Error | Jinja/YAML parse failure | [references/compilation-errors.md](references/compilation-errors.md) | +| Runtime/Database Error | SQL execution failure | [references/runtime-errors.md](references/runtime-errors.md) | +| Test Failure | Tests return failing rows | [references/test-failures.md](references/test-failures.md) | +| Wrong Data | Model builds but data is incorrect | Step 3 below | + +### Step 3: Isolate the Problem + +```bash +# Compile only — catches Jinja errors without hitting the database +altimate-dbt compile --model + +# If compile succeeds, try building +altimate-dbt build --model + +# Probe the data directly +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('') }}" --limit 1 +altimate-dbt execute --query "SELECT * FROM {{ ref('') }}" --limit 5 +``` + +### Step 3b: Offline SQL Analysis + +Before hitting the database, analyze the compiled SQL offline: + +```bash +# Check for semantic issues (wrong joins, cartesian products, NULL comparisons) +altimate_core_semantics --sql + +# Trace column lineage to find where wrong data originates +altimate_core_column_lineage --sql + +# Auto-suggest fixes for SQL errors +altimate_core_correct --sql +``` + +Common findings: +- **Wrong join type**: `INNER JOIN` dropping rows that should appear → switch to `LEFT JOIN` +- **Fan-out**: One-to-many join inflating row counts → add deduplication or aggregate +- **Column mismatch**: Output columns don't match schema.yml definition → reorder SELECT +- **NULL comparison**: Using `= NULL` instead of `IS NULL` → silent data loss + +### Step 3c: Wrong Data Diagnosis — Deep Data Exploration + +When a model builds but produces wrong results, the bug is almost always in the data assumptions, not the SQL syntax. **You must explore the actual data to find it.** + +```bash +# 1. Check the output for unexpected NULLs +altimate-dbt execute --query "SELECT count(*) as total, count() as non_null, count(*) - count() as nulls FROM {{ ref('') }}" --limit 1 + +# 2. Check value ranges — are metrics within expected bounds? +altimate-dbt execute --query "SELECT min(), max(), avg() FROM {{ ref('') }}" --limit 1 + +# 3. Check distinct values for key columns — do they look right? +altimate-dbt execute --query "SELECT , count(*) FROM {{ ref('') }} GROUP BY 1 ORDER BY 2 DESC" --limit 20 + +# 4. Compare row counts between model output and parent tables +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('') }}" --limit 1 +``` + +**Common wrong-data root causes:** +- **Fan-out from joins**: If row count is higher than expected, a join key isn't unique — check with `SELECT key, count(*) ... GROUP BY 1 HAVING count(*) > 1` +- **Missing rows from INNER JOIN**: If row count is lower than expected, switch to LEFT JOIN and check for NULL join keys +- **Date spine issues**: If using `current_date` or `dbt_utils.date_spine`, output changes daily — check min/max dates + +### Step 4: Check Upstream + +Most errors cascade from upstream models: + +```bash +altimate-dbt parents --model +``` + +Read the parent models. Build them individually. **Query the parent data** — don't assume it's correct: +```bash +altimate-dbt execute --query "SELECT count(*), count(DISTINCT ) FROM {{ ref('') }}" --limit 1 +altimate-dbt execute --query "SELECT * FROM {{ ref('') }}" --limit 5 +``` + +### Step 5: Fix and Verify + +After applying a fix: + +```bash +altimate-dbt build --model --downstream +``` + +Always build with `--downstream` to catch cascading impacts. + +**Then verify the fix with data queries** — don't just trust the build: +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('') }}" --limit 1 +altimate-dbt execute --query "SELECT * FROM {{ ref('') }}" --limit 10 +# Check the specific metric/column that was wrong: +altimate-dbt execute --query "SELECT min(), max(), count(*) - count() as nulls FROM {{ ref('') }}" --limit 1 +``` + +## Rationalizations to Resist + +| You're Thinking... | Reality | +|--------------------|---------| +| "Just make the test pass" | The test is telling you something. Investigate first. | +| "Let me delete this test" | Ask WHY it exists before removing it. | +| "It works on my machine" | Check the adapter, Python version, and profile config. | +| "I'll fix it later" | Later never comes. Fix it now. | + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Changing tests before understanding failures | Read the error. Query the data. Understand the root cause. | +| Fixing symptoms instead of root cause | Trace the problem upstream. The bug is often 2 models back. | +| Not checking upstream models | Run `altimate-dbt parents` and build parents individually | +| Ignoring warnings | Warnings often become errors. Fix them proactively. | +| Not running offline SQL analysis | Use `altimate_core_semantics` before building to catch join issues | +| Column names/order don't match schema | Use `altimate_core_column_lineage` to verify output columns match schema.yml | +| Not querying the actual data when debugging wrong results | Always run data exploration queries — check NULLs, value ranges, distinct values | +| Trusting build success as proof of correctness | Build only checks syntax and constraints — wrong values pass silently | + +## Reference Guides + +| Guide | Use When | +|-------|----------| +| [references/altimate-dbt-commands.md](references/altimate-dbt-commands.md) | Need the full CLI reference | +| [references/compilation-errors.md](references/compilation-errors.md) | Jinja, YAML, or parse errors | +| [references/runtime-errors.md](references/runtime-errors.md) | Database execution errors | +| [references/test-failures.md](references/test-failures.md) | Understanding and fixing test failures | diff --git a/.opencode/skills/dbt-troubleshoot/references/altimate-dbt-commands.md b/.opencode/skills/dbt-troubleshoot/references/altimate-dbt-commands.md new file mode 100644 index 0000000000..6dbb8e9731 --- /dev/null +++ b/.opencode/skills/dbt-troubleshoot/references/altimate-dbt-commands.md @@ -0,0 +1,66 @@ +# altimate-dbt Command Reference + +All dbt operations use the `altimate-dbt` CLI. Output is JSON to stdout; logs go to stderr. + +```bash +altimate-dbt [args...] +altimate-dbt [args...] --format text # Human-readable output +``` + +## First-Time Setup + +```bash +altimate-dbt init # Auto-detect project root +altimate-dbt init --project-root /path # Explicit root +altimate-dbt init --python-path /path # Override Python +altimate-dbt doctor # Verify setup +altimate-dbt info # Project name, adapter, root +``` + +## Build & Run + +```bash +altimate-dbt build --model [--downstream] # compile + run + test +altimate-dbt run --model [--downstream] # materialize only +altimate-dbt test --model # run tests only +altimate-dbt build-project # full project build +``` + +## Compile + +```bash +altimate-dbt compile --model +altimate-dbt compile-query --query "SELECT * FROM {{ ref('stg_orders') }}" [--model ] +``` + +## Execute SQL + +```bash +altimate-dbt execute --query "SELECT count(*) FROM {{ ref('orders') }}" --limit 100 +``` + +## Schema & DAG + +```bash +altimate-dbt columns --model # column names and types +altimate-dbt columns-source --source --table # source table columns +altimate-dbt column-values --model --column # sample values +altimate-dbt children --model # downstream models +altimate-dbt parents --model # upstream models +``` + +## Packages + +```bash +altimate-dbt deps # install packages.yml +altimate-dbt add-packages --packages dbt-utils,dbt-expectations +``` + +## Error Handling + +All errors return JSON with `error` and `fix` fields: +```json +{ "error": "dbt-core is not installed", "fix": "Install it: python3 -m pip install dbt-core" } +``` + +Run `altimate-dbt doctor` as the first diagnostic step for any failure. diff --git a/.opencode/skills/dbt-troubleshoot/references/compilation-errors.md b/.opencode/skills/dbt-troubleshoot/references/compilation-errors.md new file mode 100644 index 0000000000..152ee07f8f --- /dev/null +++ b/.opencode/skills/dbt-troubleshoot/references/compilation-errors.md @@ -0,0 +1,57 @@ +# Compilation Errors + +Compilation errors happen before SQL hits the database. They're Jinja, YAML, or reference problems. + +## Diagnosis + +```bash +altimate-dbt compile --model +``` + +## Common Compilation Errors + +### `Compilation Error: Model 'model.project.name' depends on a node named 'missing_model'` + +**Cause**: `{{ ref('missing_model') }}` references a model that doesn't exist. + +**Fix**: +1. Check the spelling: `glob models/**/*missing_model*` +2. Check if it's in a package: `glob dbt_packages/**/*missing_model*` +3. If it should be a source: use `{{ source('src', 'table') }}` instead + +### `Compilation Error: 'source_name' is undefined` + +**Cause**: Source not defined in any `sources.yml`. + +**Fix**: Create or update `sources.yml` with the source definition. + +### `Parsing Error in YAML` + +**Cause**: Invalid YAML syntax (bad indentation, missing colons, unquoted special characters). + +**Fix**: Check indentation (must be spaces, not tabs). Ensure strings with special characters are quoted. + +### `Compilation Error: Jinja template not found` + +**Cause**: Missing macro or wrong macro path. + +**Fix**: +1. Check `macros/` directory +2. Check `dbt_packages/` for package macros +3. Verify `packages.yml` is installed: `altimate-dbt deps` + +### `dbt_utils is undefined` + +**Cause**: Package not installed. + +**Fix**: +```bash +altimate-dbt deps +``` + +## General Approach + +1. Read the full error message — it usually tells you exactly which file and line +2. Open that file and read the surrounding context +3. Check for typos in `ref()` and `source()` calls +4. Verify all packages are installed with `altimate-dbt deps` diff --git a/.opencode/skills/dbt-troubleshoot/references/runtime-errors.md b/.opencode/skills/dbt-troubleshoot/references/runtime-errors.md new file mode 100644 index 0000000000..0b0996b258 --- /dev/null +++ b/.opencode/skills/dbt-troubleshoot/references/runtime-errors.md @@ -0,0 +1,71 @@ +# Runtime / Database Errors + +Runtime errors happen when compiled SQL fails to execute against the database. + +## Diagnosis + +```bash +# First compile to rule out Jinja issues +altimate-dbt compile --model + +# Then try to build +altimate-dbt build --model + +# Probe the data directly +altimate-dbt execute --query "" --limit 10 +``` + +## Common Runtime Errors + +### `Database Error: column "x" does not exist` + +**Cause**: Model references a column that doesn't exist in the source/upstream model. + +**Fix**: +```bash +altimate-dbt columns --model # check what columns actually exist +``` +Update the column name in the SQL. + +### `Database Error: relation "schema.table" does not exist` + +**Cause**: The upstream model hasn't been built yet, or the schema doesn't exist. + +**Fix**: +```bash +altimate-dbt build --model # build the dependency first +``` + +### `Database Error: division by zero` + +**Cause**: Dividing by a column that contains zeros. + +**Fix**: Add a `NULLIF(denominator, 0)` or `CASE WHEN denominator = 0 THEN NULL ELSE ...` guard. + +### `Database Error: ambiguous column reference` + +**Cause**: Column name exists in multiple tables in a JOIN. + +**Fix**: Qualify with table alias: `orders.customer_id` instead of `customer_id`. + +### `Database Error: type mismatch` + +**Cause**: Comparing or operating on incompatible types (string vs integer, date vs timestamp). + +**Fix**: Add explicit `CAST()` to align types. + +### `Timeout` or `Memory Exceeded` + +**Cause**: Query is too expensive — full table scan, massive JOIN, or no partition pruning. + +**Fix**: +1. Check if model should be incremental +2. Add `WHERE` filters to limit data +3. Check JOIN keys — are they indexed/clustered? + +## General Approach + +1. Read the compiled SQL: `altimate-dbt compile --model ` +2. Try running a simplified version of the query directly +3. Check upstream columns: `altimate-dbt columns --model ` +4. Add diagnostic queries to understand the data shape diff --git a/.opencode/skills/dbt-troubleshoot/references/test-failures.md b/.opencode/skills/dbt-troubleshoot/references/test-failures.md new file mode 100644 index 0000000000..52761adec2 --- /dev/null +++ b/.opencode/skills/dbt-troubleshoot/references/test-failures.md @@ -0,0 +1,95 @@ +# Test Failures + +Test failures mean the data violates an expected constraint. The test is usually right — investigate before changing it. + +## Diagnosis + +```bash +altimate-dbt test --model +``` + +## Common Test Failures + +### `unique` test fails + +**Meaning**: Duplicate values exist in the column. + +**Investigate**: +```bash +altimate-dbt execute --query " + SELECT , count(*) as cnt + FROM {{ ref('') }} + GROUP BY 1 + HAVING count(*) > 1 + ORDER BY cnt DESC +" --limit 10 +``` + +**Common causes**: +- Missing deduplication in staging model +- Incorrect JOIN producing row multiplication (LEFT JOIN with 1:many relationship) +- Incorrect `GROUP BY` (missing a dimension) + +### `not_null` test fails + +**Meaning**: NULL values exist where they shouldn't. + +**Investigate**: +```bash +altimate-dbt execute --query " + SELECT * FROM {{ ref('') }} + WHERE IS NULL +" --limit 5 +``` + +**Common causes**: +- LEFT JOIN where INNER JOIN was intended (unmatched rows become NULL) +- Source data has genuine NULLs — may need `COALESCE()` or filter +- Wrong column referenced in the model SQL + +### `accepted_values` test fails + +**Meaning**: Values exist that weren't in the expected list. + +**Investigate**: +```bash +altimate-dbt column-values --model --column +``` + +**Common causes**: +- New value appeared in source data (update the accepted list) +- Data quality issue upstream (fix the source or add a filter) +- Test list is incomplete (add the missing values) + +### `relationships` test fails + +**Meaning**: Foreign key references a value that doesn't exist in the parent table. + +**Investigate**: +```bash +altimate-dbt execute --query " + SELECT child., count(*) + FROM {{ ref('') }} child + LEFT JOIN {{ ref('') }} parent ON child. = parent. + WHERE parent. IS NULL + GROUP BY 1 +" --limit 10 +``` + +**Common causes**: +- Parent table hasn't been rebuilt with latest data +- Orphan records in source data +- Type mismatch between FK and PK (e.g., string vs integer) + +## The Decision Framework + +When a test fails: + +1. **Understand**: Query the failing rows. Why do they exist? +2. **Classify**: Is it a data issue, a model logic bug, or a test definition problem? +3. **Fix the right thing**: + - Data issue → fix upstream or add a filter/coalesce + - Logic bug → fix the model SQL + - Test is wrong → update the test (with explicit justification to the user) + +**Never silently weaken a test.** If you need to change a test, explain why to the user. diff --git a/.opencode/skills/generate-tests/SKILL.md b/.opencode/skills/generate-tests/SKILL.md deleted file mode 100644 index ec6bcb9eea..0000000000 --- a/.opencode/skills/generate-tests/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: generate-tests -description: Generate dbt tests for a model by inspecting its schema and SQL, producing schema.yml test definitions. ---- - -# Generate dbt Tests - -## Requirements -**Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, schema_inspect, write, edit - -> **When to use this vs other skills:** Use /generate-tests for automated test scaffolding based on column patterns. Use /yaml-config for generating full schema.yml from scratch. Use /dbt-docs for adding descriptions to existing YAML. - -Generate comprehensive dbt test definitions for a model. This skill inspects the model's schema, reads its SQL, and produces appropriate tests. - -## Workflow - -1. **Find the model file** — Use `glob` to locate the model SQL file -2. **Read the model SQL** — Understand the transformations, joins, and column expressions -3. **Inspect the schema** — Use `schema_inspect` to get column names, types, and constraints if a warehouse connection is available. If not, infer columns from the SQL. -4. **Read existing schema.yml** — Use `glob` and `read` to find and load any existing `schema.yml` or `_schema.yml` in the same directory -5. **Generate tests** based on column patterns: - -### Test Generation Rules - -| Column Pattern | Tests to Generate | -|---|---| -| `*_id` columns | `unique`, `not_null`, `relationships` (if source table is identifiable) | -| `status`, `type`, `category` columns | `accepted_values` (infer values from SQL if possible, otherwise leave as placeholder) | -| Date/timestamp columns | `not_null` | -| Boolean columns | `accepted_values: [true, false]` | -| Columns in PRIMARY KEY | `unique`, `not_null` | -| Columns marked NOT NULL in schema | `not_null` | -| All columns | Consider `not_null` if they appear in JOIN conditions or WHERE filters | - -### Output Format - -Generate a YAML block that can be merged into the model's `schema.yml`: - -```yaml -models: - - name: model_name - columns: - - name: column_name - tests: - - unique - - not_null - - relationships: - to: ref('source_model') - field: id -``` - -6. **Write or patch the schema.yml** — If a schema.yml exists, merge the new tests into it (don't duplicate existing tests). If none exists, create one in the same directory as the model. - -## Usage - -The user invokes this skill with a model name or path: -- `/generate-tests models/staging/stg_orders.sql` -- `/generate-tests stg_orders` - -Use the tools: `glob`, `read`, `schema_inspect` (if warehouse available), `write` or `edit`. diff --git a/.opencode/skills/impact-analysis/SKILL.md b/.opencode/skills/impact-analysis/SKILL.md deleted file mode 100644 index a899ef42d5..0000000000 --- a/.opencode/skills/impact-analysis/SKILL.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -name: impact-analysis -description: Analyze the downstream impact of changes to a dbt model by combining column-level lineage with the dbt dependency graph. ---- - -# Impact Analysis - -## Requirements -**Agent:** any (read-only analysis) -**Tools used:** dbt_manifest, lineage_check, sql_analyze, glob, bash, read - -Determine which downstream models, tests, and dashboards are affected when a dbt model changes. - -## Workflow - -1. **Identify the changed model** — Either: - - Accept a model name or file path from the user - - Detect changed `.sql` files via `git diff --name-only` using `bash` - -2. **Load the dbt manifest** — Call `dbt_manifest` with the project's `target/manifest.json` path. - - If the user specifies a manifest path, use that - - Otherwise search for `target/manifest.json` or `manifest.json` using `glob` - -3. **Find the changed model in the manifest** — Match by model name or file path. - Extract: `unique_id`, `depends_on`, `columns`, `materialized` - -4. **Build the downstream dependency graph** — From the manifest: - - Find all models whose `depends_on` includes the changed model's `unique_id` - - Recursively expand to get the full downstream tree (depth-first) - - Track depth level for each downstream model - -5. **Run column-level lineage** — Call `lineage_check` on the changed model's SQL to get: - - Which source columns flow to which output columns - - Which columns were added, removed, or renamed (if comparing old vs new) - -6. **Cross-reference lineage with downstream models** — For each downstream model: - - Check if it references any of the changed columns - - Run `lineage_check` on the downstream model's SQL if available - - Classify impact: BREAKING (removed/renamed column used downstream), SAFE (added column, no downstream reference), UNKNOWN (can't determine) - -7. **Generate the impact report**: - -``` -Impact Analysis: stg_orders -════════════════════════════ - -Changed Model: stg_orders (materialized: view) - Source columns: 5 → 6 (+1 added) - Removed columns: none - Modified columns: order_total (renamed from total_amount) - -Downstream Impact (3 models affected): - - Depth 1: - [BREAKING] int_order_metrics - References: order_total (was total_amount) — COLUMN RENAMED - Action needed: Update column reference - - [SAFE] int_order_summary - No references to changed columns - - Depth 2: - [BREAKING] mart_revenue - References: order_total via int_order_metrics — CASCADING BREAK - Action needed: Verify after fixing int_order_metrics - -Tests at Risk: 4 - - not_null_stg_orders_order_total - - unique_int_order_metrics_order_id - - accepted_values_stg_orders_status - - relationships_int_order_metrics_order_id - -Summary: 2 BREAKING, 1 SAFE, 0 UNKNOWN - Recommended: Fix int_order_metrics first, then run `dbt test -s stg_orders+` -``` - -## Without Manifest (SQL-only mode) - -If no dbt manifest is available, fall back to SQL-only analysis: -1. Run `lineage_check` on the changed SQL -2. Show the column-level data flow -3. Note that downstream impact cannot be determined without a manifest -4. Suggest running `dbt docs generate` to create a manifest - -## Tools Used - -- `dbt_manifest` — Load the dbt dependency graph -- `lineage_check` — Column-level lineage for each model -- `sql_analyze` — Check for anti-patterns in changed SQL -- `glob` — Find manifest and SQL files -- `bash` — Git operations for detecting changes -- `read` — Read SQL files from disk - -## Usage Examples - -- `/impact-analysis stg_orders` — Analyze impact of changes to stg_orders -- `/impact-analysis models/staging/stg_orders.sql` — Analyze by file path -- `/impact-analysis` — Auto-detect changed models from git diff diff --git a/.opencode/skills/incremental-logic/SKILL.md b/.opencode/skills/incremental-logic/SKILL.md deleted file mode 100644 index 8d6fa53633..0000000000 --- a/.opencode/skills/incremental-logic/SKILL.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: incremental-logic -description: Add or fix incremental materialization logic in dbt models — is_incremental(), unique keys, merge strategies. ---- - -# Incremental Logic Assistant - -## Requirements -**Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, sql_analyze, lineage_check, schema_inspect, edit, write - -Help convert batch models to incremental or fix existing incremental logic. Covers `is_incremental()` patterns, merge strategies, and common pitfalls. - -## Workflow - -1. **Read the model** — Use `glob` and `read` to find and understand the current model SQL -2. **Analyze the query** — Use `sql_analyze` to check for anti-patterns and `lineage_check` to understand column flow -3. **Inspect the schema** — Use `schema_inspect` to understand column types, especially timestamp columns -4. **Determine the strategy** — Choose the right incremental approach based on the data pattern -5. **Generate the incremental version** — Rewrite the model with proper `is_incremental()` logic -6. **Update config** — Add `unique_key`, `on_schema_change`, and strategy settings - -## Incremental Strategies - -### Append-Only (Event Logs) -Best for: immutable event streams where rows are never updated. -```sql -{{ config( - materialized='incremental', - on_schema_change='append_new_columns' -) }} - -select - event_id, - event_type, - payload, - created_at -from {{ ref('stg_events') }} - -{% if is_incremental() %} -where created_at > (select max(created_at) from {{ this }}) -{% endif %} -``` - -### Merge/Upsert (Mutable Records) -Best for: records that can be updated (orders, customers). -```sql -{{ config( - materialized='incremental', - unique_key='order_id', - merge_update_columns=['status', 'updated_at', 'amount'], - on_schema_change='sync_all_columns' -) }} - -select - order_id, - status, - amount, - created_at, - updated_at -from {{ ref('stg_orders') }} - -{% if is_incremental() %} -where updated_at > (select max(updated_at) from {{ this }}) -{% endif %} -``` - -### Insert Overwrite (Partitioned) -Best for: date-partitioned fact tables, full partition replacement. -```sql -{{ config( - materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by={'field': 'event_date', 'data_type': 'date'}, - on_schema_change='fail' -) }} - -select - date_trunc('day', created_at) as event_date, - count(*) as event_count -from {{ ref('stg_events') }} - -{% if is_incremental() %} -where date_trunc('day', created_at) >= (select max(event_date) - interval '3 days' from {{ this }}) -{% endif %} - -group by 1 -``` - -### Microbatch (Snowflake) -Best for: large tables needing controlled batch sizes. -```sql -{{ config( - materialized='incremental', - incremental_strategy='microbatch', - event_time='created_at', - begin='2024-01-01', - batch_size='day' -) }} - -select * from {{ ref('stg_events') }} -``` - -## Common Pitfalls - -| Issue | Problem | Fix | -|-------|---------|-----| -| Missing `unique_key` | Duplicates on re-run | Add `unique_key` matching the primary key | -| Wrong timestamp column | Missed updates | Use `updated_at` (not `created_at`) for mutable data | -| No lookback window | Late-arriving data missed | Use `max(ts) - interval '1 hour'` instead of strict `>` | -| `on_schema_change='fail'` | Breaks on column additions | Use `'append_new_columns'` or `'sync_all_columns'` | -| Full refresh needed | Schema drift accumulated | Run `dbt run --full-refresh -s model_name` | -| Stale `{{ this }}` | First run fails | `is_incremental()` returns false on first run — outer query runs unfiltered | - -## Dialect-Specific Notes - -| Warehouse | Default Strategy | Partition Support | Merge Support | -|-----------|-----------------|-------------------|---------------| -| Snowflake | `merge` | Yes (cluster keys) | Yes | -| BigQuery | `merge` | Yes (partition_by) | Yes | -| PostgreSQL | `append` | No native | No (use delete+insert) | -| DuckDB | `append` | No native | Partial | -| Redshift | `append` | Yes (dist/sort keys) | No (use delete+insert) | - -## Usage - -- `/incremental-logic models/marts/fct_orders.sql` — Convert to incremental -- `/incremental-logic fix models/marts/fct_orders.sql` — Fix existing incremental logic -- `/incremental-logic strategy orders` — Recommend best strategy for a table - -Use the tools: `glob`, `read`, `sql_analyze`, `lineage_check`, `schema_inspect`, `edit`, `write`. diff --git a/.opencode/skills/medallion-patterns/SKILL.md b/.opencode/skills/medallion-patterns/SKILL.md deleted file mode 100644 index eeada69105..0000000000 --- a/.opencode/skills/medallion-patterns/SKILL.md +++ /dev/null @@ -1,183 +0,0 @@ ---- -name: medallion-patterns -description: Apply medallion architecture (bronze/silver/gold) patterns to organize dbt models into clean data layers. ---- - -# Medallion Architecture Patterns - -## Requirements -**Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, dbt_manifest, dbt_run, write, edit - -Guide and scaffold dbt projects following the medallion (bronze/silver/gold) architecture pattern. - -## Workflow - -1. **Audit current structure** — Use `glob` to scan the project's `models/` directory and `dbt_manifest` to understand existing organization -2. **Identify the current state** — Classify existing models into medallion layers based on naming and content -3. **Recommend reorganization** — Suggest how to reorganize models into proper layers -4. **Scaffold missing layers** — Create directory structure and template models for gaps - -## Medallion Layer Definitions - -### Bronze (Raw / Staging) -**Purpose**: Ingest raw data with minimal transformation. Preserve source fidelity. - -``` -models/ - bronze/ - source_system/ - _source_system__sources.yml - brz_source_system__table.sql -``` - -**Pattern**: -```sql --- brz_stripe__payments.sql -{{ config(materialized='view') }} - -with source as ( - select * from {{ source('stripe', 'payments') }} -), - -cast as ( - select - cast(id as varchar) as payment_id, - cast(amount as integer) as amount_cents, - cast(created as timestamp) as created_at, - _loaded_at -- preserve load metadata - from source -) - -select * from cast -``` - -**Rules**: -- 1:1 mapping with source tables -- Only type casting, renaming, deduplication -- No joins, no business logic -- Materialized as `view` or `ephemeral` -- Naming: `brz___
` - -### Silver (Cleaned / Intermediate) -**Purpose**: Business-conformant data. Joins, deduplication, standardization. - -``` -models/ - silver/ - domain/ - slv_domain__entity.sql -``` - -**Pattern**: -```sql --- slv_finance__orders_enriched.sql -{{ config(materialized='table') }} - -with orders as ( - select * from {{ ref('brz_stripe__payments') }} -), - -customers as ( - select * from {{ ref('brz_crm__customers') }} -), - -enriched as ( - select - o.payment_id, - o.amount_cents / 100.0 as amount_dollars, - c.customer_name, - c.segment, - o.created_at - from orders o - left join customers c on o.customer_id = c.customer_id - where o.created_at is not null -- quality filter -) - -select * from enriched -``` - -**Rules**: -- Cross-source joins allowed -- Business logic transformations -- Data quality filters (remove nulls, duplicates) -- Standardized naming conventions -- Materialized as `table` or `incremental` -- Naming: `slv___` - -### Gold (Business / Marts) -**Purpose**: Business-ready aggregations and metrics. Direct consumption by BI/analytics. - -``` -models/ - gold/ - domain/ - fct_metric.sql - dim_entity.sql -``` - -**Pattern**: -```sql --- fct_daily_revenue.sql -{{ config( - materialized='incremental', - unique_key='revenue_date', - on_schema_change='append_new_columns' -) }} - -with orders as ( - select * from {{ ref('slv_finance__orders_enriched') }} - {% if is_incremental() %} - where created_at > (select max(revenue_date) from {{ this }}) - {% endif %} -), - -daily as ( - select - date_trunc('day', created_at) as revenue_date, - segment, - count(*) as order_count, - sum(amount_dollars) as gross_revenue - from orders - group by 1, 2 -) - -select * from daily -``` - -**Rules**: -- Aggregations, metrics, KPIs -- Wide denormalized tables for BI -- Fact tables (`fct_`) and dimension tables (`dim_`) -- Materialized as `table` or `incremental` -- Naming: `fct_` or `dim_` - -## Migration Checklist - -When reorganizing an existing project: - -1. Create the directory structure (`bronze/`, `silver/`, `gold/`) -2. Map existing `stg_` models → `bronze/` layer -3. Map existing `int_` models → `silver/` layer -4. Map existing `fct_`/`dim_` models → `gold/` layer -5. Update `dbt_project.yml` with layer-specific materializations: -```yaml -models: - my_project: - bronze: - +materialized: view - silver: - +materialized: table - gold: - +materialized: incremental -``` -6. Update all `ref()` calls to match new model names -7. Run `dbt build` to verify no breakages - -## Usage - -- `/medallion-patterns audit` — Analyze current project structure -- `/medallion-patterns scaffold stripe` — Create bronze/silver/gold for a new source -- `/medallion-patterns migrate` — Plan migration of existing stg/int/mart to medallion - -Use the tools: `glob`, `read`, `dbt_manifest`, `dbt_run`, `write`, `edit`. diff --git a/.opencode/skills/model-scaffold/SKILL.md b/.opencode/skills/model-scaffold/SKILL.md deleted file mode 100644 index f5bc6254a7..0000000000 --- a/.opencode/skills/model-scaffold/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: model-scaffold -description: Scaffold a new dbt model following staging/intermediate/mart patterns with proper naming, materialization, and structure. ---- - -# Scaffold dbt Model - -## Requirements -**Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, dbt_manifest, schema_inspect, schema_search, write - -Generate a new dbt model file following established data modeling patterns. Supports staging, intermediate, and mart layer scaffolding. - -## Workflow - -1. **Determine layer** — Ask or infer whether this is a staging, intermediate, or mart model -2. **Read the dbt project** — Use `glob` to find `dbt_project.yml` and understand the project structure (model paths, naming conventions) -3. **Read the manifest** — If available, use `dbt_manifest` to understand existing models, sources, and dependencies -4. **Inspect source schema** — Use `schema_inspect` or `schema_search` to discover source table columns and types -5. **Generate the model SQL** based on the layer pattern: - -### Layer Patterns - -#### Staging (`stg_`) -```sql -with source as ( - select * from {{ source('source_name', 'table_name') }} -), - -renamed as ( - select - -- Primary key - column_id as table_id, - - -- Dimensions - column_name, - - -- Timestamps - created_at, - updated_at - - from source -) - -select * from renamed -``` -- **Materialization**: `view` (lightweight, always fresh) -- **Naming**: `stg___
.sql` -- **Location**: `models/staging//` -- **Purpose**: 1:1 with source table, rename columns, cast types, no joins - -#### Intermediate (`int_`) -```sql -with orders as ( - select * from {{ ref('stg_source__orders') }} -), - -customers as ( - select * from {{ ref('stg_source__customers') }} -), - -joined as ( - select - orders.order_id, - orders.customer_id, - customers.customer_name, - orders.order_date, - orders.amount - from orders - left join customers on orders.customer_id = customers.customer_id -) - -select * from joined -``` -- **Materialization**: `ephemeral` or `view` -- **Naming**: `int___.sql` (e.g., `int_orders__joined`) -- **Location**: `models/intermediate/` -- **Purpose**: Joins, filters, business logic transformations - -#### Mart (`fct_` / `dim_`) -```sql -with final as ( - select - order_id, - customer_id, - order_date, - amount, - -- Derived metrics - sum(amount) over (partition by customer_id) as customer_lifetime_value - from {{ ref('int_orders__joined') }} -) - -select * from final -``` -- **Materialization**: `table` or `incremental` -- **Naming**: `fct_.sql` (facts) or `dim_.sql` (dimensions) -- **Location**: `models/marts//` -- **Purpose**: Business-facing, wide tables, aggregations, final metrics - -6. **Generate schema.yml** — Create a companion `___models.yml` with column descriptions and basic tests -7. **Write the files** — Use `write` to create the SQL model and schema YAML - -## Usage - -- `/model-scaffold staging orders from raw.public.orders` -- `/model-scaffold mart fct_daily_revenue` -- `/model-scaffold intermediate int_orders__enriched` - -Use the tools: `glob`, `read`, `dbt_manifest`, `schema_inspect`, `schema_search`, `write`. diff --git a/.opencode/skills/yaml-config/SKILL.md b/.opencode/skills/yaml-config/SKILL.md deleted file mode 100644 index 0d72cc87f4..0000000000 --- a/.opencode/skills/yaml-config/SKILL.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -name: yaml-config -description: Generate dbt YAML configuration files — sources.yml, schema.yml, properties.yml — from warehouse schema or existing models. ---- - -# Generate dbt YAML Config - -## Requirements -**Agent:** builder or migrator (requires file write access) -**Tools used:** glob, read, schema_inspect, schema_search, dbt_manifest, write, edit - -> **When to use this vs other skills:** Use /yaml-config to generate sources.yml or schema.yml from warehouse metadata. Use /generate-tests to add test definitions. Use /dbt-docs to enrich existing YAML with descriptions. - -Generate or update dbt YAML configuration files by inspecting warehouse schemas and existing models. - -## Workflow - -1. **Determine config type** — sources.yml, schema.yml, or properties.yml -2. **Read existing configs** — Use `glob` to find existing YAML files in the project and `read` to understand current state -3. **Inspect warehouse schema** — Use `schema_inspect` and `schema_search` to discover tables and columns -4. **Read the manifest** — If available, use `dbt_manifest` to find existing model definitions -5. **Generate the YAML** based on the config type: - -### Config Types - -#### sources.yml — Define raw data sources -```yaml -version: 2 - -sources: - - name: raw_stripe - description: Raw Stripe payment data - database: raw - schema: stripe - tables: - - name: payments - description: All payment transactions - columns: - - name: payment_id - description: Primary key - tests: - - unique - - not_null - - name: amount - description: Payment amount in cents - - name: created_at - description: Payment creation timestamp - tests: - - not_null -``` - -#### schema.yml — Model documentation and tests -```yaml -version: 2 - -models: - - name: stg_stripe__payments - description: Staged Stripe payments with renamed columns and type casts - columns: - - name: payment_id - description: Primary key from source - tests: - - unique - - not_null - - name: amount_dollars - description: Payment amount converted to dollars -``` - -#### properties.yml — Model-level config -```yaml -version: 2 - -models: - - name: fct_daily_revenue - config: - materialized: incremental - unique_key: date_day - on_schema_change: append_new_columns - columns: - - name: date_day - description: The calendar date -``` - -6. **Merge with existing** — If YAML files already exist, merge new entries without duplicating existing definitions. Preserve human-written descriptions. -7. **Write the output** — Use `write` or `edit` to save the YAML file - -### Column Pattern Heuristics - -When generating column descriptions and tests automatically: - -| Pattern | Description Template | Auto-Tests | -|---------|---------------------|------------| -| `*_id` | "Foreign key to {table}" or "Primary key" | `unique`, `not_null` | -| `*_at`, `*_date`, `*_timestamp` | "Timestamp of {event}" | `not_null` | -| `*_amount`, `*_price`, `*_cost` | "Monetary value in {currency}" | `not_null` | -| `is_*`, `has_*` | "Boolean flag for {condition}" | `accepted_values: [true, false]` | -| `*_type`, `*_status`, `*_category` | "Categorical: {values}" | `accepted_values` (if inferable) | -| `*_count`, `*_total`, `*_sum` | "Aggregated count/total" | — | -| `*_name`, `*_title`, `*_label` | "Human-readable name" | — | - -## Usage - -- `/yaml-config sources raw.stripe` — Generate sources.yml from warehouse schema -- `/yaml-config schema stg_stripe__payments` — Generate schema.yml for a model -- `/yaml-config properties fct_daily_revenue` — Generate properties.yml with config - -Use the tools: `glob`, `read`, `schema_inspect`, `schema_search`, `dbt_manifest`, `write`, `edit`. diff --git a/bun.lock b/bun.lock index 84f2722d8c..3d5f89e968 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,21 @@ "turbo": "2.8.13", }, }, + "packages/dbt-tools": { + "name": "@altimateai/dbt-tools", + "version": "0.1.0", + "bin": { + "altimate-dbt": "./bin/altimate-dbt", + }, + "dependencies": { + "@altimateai/dbt-integration": "^0.2.2", + }, + "devDependencies": { + "@tsconfig/bun": "catalog:", + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "packages/opencode": { "name": "@altimateai/altimate-code", "version": "1.2.20", @@ -306,6 +321,22 @@ "@altimateai/altimate-code": ["@altimateai/altimate-code@workspace:packages/opencode"], + "@altimateai/altimate-core": ["@altimateai/altimate-core@0.1.6", "", { "optionalDependencies": { "@altimateai/altimate-core-darwin-arm64": "0.1.6", "@altimateai/altimate-core-darwin-x64": "0.1.6", "@altimateai/altimate-core-linux-arm64-gnu": "0.1.6", "@altimateai/altimate-core-linux-x64-gnu": "0.1.6", "@altimateai/altimate-core-win32-x64-msvc": "0.1.6" } }, "sha512-Kl0hjT88Q56AdGxKJyCcPElxcpZYDYmLhDHK7ZeZIn2oVaXyynExLcIHn+HktUe9USuWtba3tZA/52jJsMyrGg=="], + + "@altimateai/altimate-core-darwin-arm64": ["@altimateai/altimate-core-darwin-arm64@0.1.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-lcndcluAsWMdI0fq2xwgukxFBwoISqKeLWBMjRAhIlCU0qVO+sR/UGUj2FiKZHIWmwgHAou3V5K2fKoYMh9PdQ=="], + + "@altimateai/altimate-core-darwin-x64": ["@altimateai/altimate-core-darwin-x64@0.1.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-pYD0Yj/j7SKrU5BY3utPGM4ImoZcV/6yJ7cTFGQXONj3Ikmoo1FdUZR5/CgZE7CCYAa0T9pjOfxB1rLD1B1fxQ=="], + + "@altimateai/altimate-core-linux-arm64-gnu": ["@altimateai/altimate-core-linux-arm64-gnu@0.1.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-VDxrk3z30cWJfIudpiGbAz+PFpYB2OE+XSyOPFN+GI7K9NOd5RtC7e9pG50nMF55Wz5R5Yp0ywa/dL+QpUp8SA=="], + + "@altimateai/altimate-core-linux-x64-gnu": ["@altimateai/altimate-core-linux-x64-gnu@0.1.6", "", { "os": "linux", "cpu": "x64" }, "sha512-wXWYNxGhUBBaf2b6dsvuD1ZuiTLQZbKg4/fckrQELuW+BrsN1+nzna+0IZ9FTATYfF1OatpqVun9QyBkvlQn+Q=="], + + "@altimateai/altimate-core-win32-x64-msvc": ["@altimateai/altimate-core-win32-x64-msvc@0.1.6", "", { "os": "win32", "cpu": "x64" }, "sha512-6Sbneg0DLHMmo1lDVd9oDgGtqPJpDUXZvXwAbGb7eoh+vUmXMxABA43//hBbwkMVsWKClKjv1KXSKp44shrUiw=="], + + "@altimateai/dbt-integration": ["@altimateai/dbt-integration@0.2.9", "", { "dependencies": { "@altimateai/altimate-core": "0.1.6", "node-abort-controller": "^3.1.1", "node-fetch": "^3.3.2", "python-bridge": "^1.1.0", "semver": "^7.6.3", "yaml": "^2.5.0" }, "peerDependencies": { "patch-package": "^8.0.0" } }, "sha512-L+sazdclVNVPuRrSRq/0dGfyNEOHHGKqOCGEkZiXFbaW9hRGRqk+9LgmOUwyDq2VA79qvduOehe7+Uk0Oo3sow=="], + + "@altimateai/dbt-tools": ["@altimateai/dbt-tools@workspace:packages/dbt-tools"], + "@ampproject/remapping": ["@ampproject/remapping@2.3.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw=="], "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.71.2", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ=="], @@ -948,6 +979,8 @@ "@webgpu/types": ["@webgpu/types@0.1.69", "", {}, "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ=="], + "@yarnpkg/lockfile": ["@yarnpkg/lockfile@1.1.0", "", {}, "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="], + "@zip.js/zip.js": ["@zip.js/zip.js@2.7.62", "", {}, "sha512-OaLvZ8j4gCkLn048ypkZu29KX30r8/OfFF2w4Jo5WXFr+J04J+lzJ5TKZBVgFXhlvSkqNFQdfnY1Q8TMTCyBVA=="], "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], @@ -1004,6 +1037,8 @@ "bl": ["bl@6.1.6", "", { "dependencies": { "@types/readable-stream": "^4.0.0", "buffer": "^6.0.3", "inherits": "^2.0.4", "readable-stream": "^4.2.0" } }, "sha512-jLsPgN/YSvPUg9UX0Kd73CXpm2Psg9FxMeCSXnk3WBO3CMT10JMwijubhGfHCnFu6TPn1ei3b975dxv7K2pWVg=="], + "bluebird": ["bluebird@3.7.2", "", {}, "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="], + "bmp-ts": ["bmp-ts@1.0.9", "", {}, "sha512-cTEHk2jLrPyi+12M3dhpEbnnPOsaZuq7C45ylbbQIiWgDFZq4UVYPEY5mlqjvsj/6gJv9qX5sa+ebDzLXT28Vw=="], "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], @@ -1044,6 +1079,8 @@ "c12": ["c12@3.3.3", "", { "dependencies": { "chokidar": "^5.0.0", "confbox": "^0.2.2", "defu": "^6.1.4", "dotenv": "^17.2.3", "exsolve": "^1.0.8", "giget": "^2.0.0", "jiti": "^2.6.1", "ohash": "^2.0.11", "pathe": "^2.0.3", "perfect-debounce": "^2.0.0", "pkg-types": "^2.3.0", "rc9": "^2.1.2" }, "peerDependencies": { "magicast": "*" }, "optionalPeers": ["magicast"] }, "sha512-750hTRvgBy5kcMNPdh95Qo+XUBeGo8C7nsKSmedDmaQI+E0r82DwHeM6vBewDe4rGFbnxoa4V9pw+sPh5+Iz8Q=="], + "call-bind": ["call-bind@1.0.8", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", "get-intrinsic": "^1.2.4", "set-function-length": "^1.2.2" } }, "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww=="], + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], @@ -1052,12 +1089,16 @@ "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], + "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "character-entities-html4": ["character-entities-html4@2.1.0", "", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="], "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], + "ci-info": ["ci-info@3.9.0", "", {}, "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ=="], + "citty": ["citty@0.1.6", "", { "dependencies": { "consola": "^3.2.3" } }, "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ=="], "cli-spinners": ["cli-spinners@3.4.0", "", {}, "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw=="], @@ -1108,6 +1149,8 @@ "default-browser-id": ["default-browser-id@5.0.1", "", {}, "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q=="], + "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], + "define-lazy-prop": ["define-lazy-prop@3.0.0", "", {}, "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg=="], "defu": ["defu@6.1.4", "", {}, "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg=="], @@ -1234,6 +1277,8 @@ "find-up": ["find-up@3.0.0", "", { "dependencies": { "locate-path": "^3.0.0" } }, "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg=="], + "find-yarn-workspace-root": ["find-yarn-workspace-root@2.0.0", "", { "dependencies": { "micromatch": "^4.0.2" } }, "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ=="], + "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], @@ -1242,6 +1287,8 @@ "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + "fs-extra": ["fs-extra@10.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ=="], + "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -1276,6 +1323,8 @@ "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "graphql": ["graphql@16.12.0", "", {}, "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ=="], "graphql-request": ["graphql-request@6.1.0", "", { "dependencies": { "@graphql-typed-document-node/core": "^3.2.0", "cross-fetch": "^3.1.5" }, "peerDependencies": { "graphql": "14 - 16" } }, "sha512-p+XPfS4q7aIpKVcgmnZKhMNqhltk20hfXtkaIkTfjjmiKMJ5xrt5c743cL03y/K7y1rg3WrIC49xGiEQ4mxdNw=="], @@ -1284,6 +1333,10 @@ "gtoken": ["gtoken@8.0.0", "", { "dependencies": { "gaxios": "^7.0.0", "jws": "^4.0.0" } }, "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw=="], + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], + "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], @@ -1352,6 +1405,8 @@ "is64bit": ["is64bit@2.0.0", "", { "dependencies": { "system-architecture": "^0.1.0" } }, "sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw=="], + "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], + "isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], @@ -1388,10 +1443,16 @@ "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="], + "json-stable-stringify": ["json-stable-stringify@1.3.0", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "isarray": "^2.0.5", "jsonify": "^0.0.1", "object-keys": "^1.1.1" } }, "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg=="], + "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], "jsonc-parser": ["jsonc-parser@3.3.1", "", {}, "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ=="], + "jsonfile": ["jsonfile@6.2.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg=="], + + "jsonify": ["jsonify@0.0.1", "", {}, "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg=="], + "jsonwebtoken": ["jsonwebtoken@9.0.3", "", { "dependencies": { "jws": "^4.0.1", "lodash.includes": "^4.3.0", "lodash.isboolean": "^3.0.3", "lodash.isinteger": "^4.0.4", "lodash.isnumber": "^3.0.3", "lodash.isplainobject": "^4.0.6", "lodash.isstring": "^4.0.1", "lodash.once": "^4.0.0", "ms": "^2.1.1", "semver": "^7.5.4" } }, "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g=="], "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], @@ -1400,6 +1461,8 @@ "kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="], + "klaw-sync": ["klaw-sync@6.0.0", "", { "dependencies": { "graceful-fs": "^4.1.11" } }, "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ=="], + "kubernetes-types": ["kubernetes-types@1.30.0", "", {}, "sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q=="], "light-my-request": ["light-my-request@6.6.0", "", { "dependencies": { "cookie": "^1.0.1", "process-warning": "^4.0.0", "set-cookie-parser": "^2.6.0" } }, "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A=="], @@ -1464,6 +1527,8 @@ "minimatch": ["minimatch@10.0.3", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw=="], + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + "minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -1484,6 +1549,8 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], + "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], @@ -1506,6 +1573,8 @@ "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], + "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], + "ohash": ["ohash@2.0.11", "", {}, "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ=="], "omggif": ["omggif@1.0.10", "", {}, "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw=="], @@ -1552,6 +1621,8 @@ "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="], + "patch-package": ["patch-package@8.0.1", "", { "dependencies": { "@yarnpkg/lockfile": "^1.1.0", "chalk": "^4.1.2", "ci-info": "^3.7.0", "cross-spawn": "^7.0.3", "find-yarn-workspace-root": "^2.0.0", "fs-extra": "^10.0.0", "json-stable-stringify": "^1.0.2", "klaw-sync": "^6.0.0", "minimist": "^1.2.6", "open": "^7.4.2", "semver": "^7.5.3", "slash": "^2.0.0", "tmp": "^0.2.4", "yaml": "^2.2.2" }, "bin": { "patch-package": "index.js" } }, "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw=="], + "path-exists": ["path-exists@3.0.0", "", {}, "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], @@ -1604,6 +1675,8 @@ "pure-rand": ["pure-rand@8.1.0", "", {}, "sha512-53B3MB8wetRdD6JZ4W/0gDKaOvKwuXrEmV1auQc0hASWge8rieKV4PCCVNVbJ+i24miiubb4c/B+dg8Ho0ikYw=="], + "python-bridge": ["python-bridge@1.1.0", "", { "dependencies": { "bluebird": "^3.5.0" } }, "sha512-qjQ0QB8p9cn/XDeILQH0aP307hV58lrmv0Opjyub68Um7FHdF+ZXlTqyxNkKaXOFk2QSkScoPWwn7U9GGnrkeQ=="], + "qs": ["qs@6.15.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ=="], "quansync": ["quansync@0.2.11", "", {}, "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA=="], @@ -1684,6 +1757,8 @@ "set-cookie-parser": ["set-cookie-parser@2.7.2", "", {}, "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw=="], + "set-function-length": ["set-function-length@1.2.2", "", { "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", "has-property-descriptors": "^1.0.2" } }, "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg=="], + "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], @@ -1706,6 +1781,8 @@ "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], + "slash": ["slash@2.0.0", "", {}, "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A=="], + "socket.io-client": ["socket.io-client@4.8.3", "", { "dependencies": { "@socket.io/component-emitter": "~3.1.0", "debug": "~4.4.1", "engine.io-client": "~6.6.1", "socket.io-parser": "~4.2.4" } }, "sha512-uP0bpjWrjQmUt5DTHq9RuoCBdFJF10cdX9X+a368j/Ft0wmaVgxlrjvK3kjvgCODOMMOz9lcaRzxmso0bTWZ/g=="], "socket.io-parser": ["socket.io-parser@4.2.5", "", { "dependencies": { "@socket.io/component-emitter": "~3.1.0", "debug": "~4.4.1" } }, "sha512-bPMmpy/5WWKHea5Y/jYAP6k74A+hvmRCQaJuJB6I/ML5JZq/KfNieUVo/3Mh7SAqn7TyFdIo6wqYHInG1MU1bQ=="], @@ -1746,6 +1823,8 @@ "strtok3": ["strtok3@6.3.0", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^4.1.0" } }, "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw=="], + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "supports-preserve-symlinks-flag": ["supports-preserve-symlinks-flag@1.0.0", "", {}, "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w=="], "system-architecture": ["system-architecture@0.1.0", "", {}, "sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA=="], @@ -1764,6 +1843,8 @@ "tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="], + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], "toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="], @@ -1826,6 +1907,8 @@ "universal-user-agent": ["universal-user-agent@7.0.3", "", {}, "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A=="], + "universalify": ["universalify@2.0.1", "", {}, "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw=="], + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], @@ -2050,6 +2133,8 @@ "c12/chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], + "chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "cross-fetch/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], "cross-spawn/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], @@ -2076,6 +2161,8 @@ "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + "patch-package/open": ["open@7.4.2", "", { "dependencies": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" } }, "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q=="], + "path-scurry/lru-cache": ["lru-cache@11.2.6", "", {}, "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ=="], "pixelmatch/pngjs": ["pngjs@6.0.0", "", {}, "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg=="], @@ -2176,6 +2263,10 @@ "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "patch-package/open/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], + + "patch-package/open/is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="], + "rimraf/glob/jackspeak": ["jackspeak@3.4.3", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" }, "optionalDependencies": { "@pkgjs/parseargs": "^0.11.0" } }, "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw=="], "rimraf/glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], diff --git a/package.json b/package.json index da07125ecf..483d820f69 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "packages/plugin", "packages/script", "packages/util", - "packages/sdk/js" + "packages/sdk/js", + "packages/dbt-tools" ], "catalog": { "@types/bun": "1.3.9", diff --git a/packages/dbt-tools/bin/altimate-dbt b/packages/dbt-tools/bin/altimate-dbt new file mode 100755 index 0000000000..489c4a0367 --- /dev/null +++ b/packages/dbt-tools/bin/altimate-dbt @@ -0,0 +1,2 @@ +#!/usr/bin/env node +import("../dist/index.js") diff --git a/packages/dbt-tools/package.json b/packages/dbt-tools/package.json new file mode 100644 index 0000000000..679d82045e --- /dev/null +++ b/packages/dbt-tools/package.json @@ -0,0 +1,22 @@ +{ + "name": "@altimateai/dbt-tools", + "version": "0.1.0", + "type": "module", + "private": false, + "bin": { + "altimate-dbt": "./bin/altimate-dbt" + }, + "scripts": { + "build": "bun build src/index.ts --outdir dist --target node --format esm && bun run script/copy-python.ts", + "typecheck": "tsc --noEmit", + "test": "bun test --timeout 30000" + }, + "dependencies": { + "@altimateai/dbt-integration": "^0.2.2" + }, + "devDependencies": { + "@tsconfig/bun": "catalog:", + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/packages/dbt-tools/script/copy-python.ts b/packages/dbt-tools/script/copy-python.ts new file mode 100644 index 0000000000..d34b07f855 --- /dev/null +++ b/packages/dbt-tools/script/copy-python.ts @@ -0,0 +1,9 @@ +import { cpSync } from "fs" +import { dirname, join } from "path" + +const resolved = require.resolve("@altimateai/dbt-integration") +const source = join(dirname(resolved), "altimate_python_packages") +const target = join(import.meta.dir, "..", "dist", "altimate_python_packages") + +cpSync(source, target, { recursive: true }) +console.log(`Copied altimate_python_packages → dist/`) diff --git a/packages/dbt-tools/src/adapter.ts b/packages/dbt-tools/src/adapter.ts new file mode 100644 index 0000000000..c9d262b10b --- /dev/null +++ b/packages/dbt-tools/src/adapter.ts @@ -0,0 +1,137 @@ +import type { Config } from "./config" +import { + DBTProjectIntegrationAdapter, + DEFAULT_CONFIGURATION_VALUES, + DBTCommandFactory, + DBTCommandExecutionInfrastructure, + PythonDBTCommandExecutionStrategy, + CLIDBTCommandExecutionStrategy, + ChildrenParentParser, + NodeParser, + MacroParser, + MetricParser, + GraphParser, + SourceParser, + TestParser, + ExposureParser, + FunctionParser, + DocParser, + ModelDepthParser, + AltimateHttpClient, + DbtIntegrationClient, + DBTCoreProjectIntegration, + DBTCloudProjectIntegration, + DBTCoreCommandProjectIntegration, + DBTFusionCommandProjectIntegration, + CommandProcessExecutionFactory, +} from "@altimateai/dbt-integration" +import type { + DBTConfiguration, + DBTTerminal, + DBTProjectIntegration, + DBTDiagnosticData, + DeferConfig, + RuntimePythonEnvironment, +} from "@altimateai/dbt-integration" + +function configuration(cfg: Config): DBTConfiguration { + return { + getDbtCustomRunnerImport: () => DEFAULT_CONFIGURATION_VALUES.dbtCustomRunnerImport, + getDbtIntegration: () => cfg.dbtIntegration ?? DEFAULT_CONFIGURATION_VALUES.dbtIntegration, + getRunModelCommandAdditionalParams: () => DEFAULT_CONFIGURATION_VALUES.runModelCommandAdditionalParams, + getBuildModelCommandAdditionalParams: () => DEFAULT_CONFIGURATION_VALUES.buildModelCommandAdditionalParams, + getTestModelCommandAdditionalParams: () => DEFAULT_CONFIGURATION_VALUES.testModelCommandAdditionalParams, + getQueryTemplate: () => DEFAULT_CONFIGURATION_VALUES.queryTemplate, + getQueryLimit: () => cfg.queryLimit ?? DEFAULT_CONFIGURATION_VALUES.queryLimit, + getEnableNotebooks: () => DEFAULT_CONFIGURATION_VALUES.enableNotebooks, + getDisableQueryHistory: () => DEFAULT_CONFIGURATION_VALUES.disableQueryHistory, + getInstallDepsOnProjectInitialization: () => DEFAULT_CONFIGURATION_VALUES.installDepsOnProjectInitialization, + getDisableDepthsCalculation: () => true, + getWorkingDirectory: () => cfg.projectRoot, + getAltimateUrl: () => "", + getIsLocalMode: () => true, + getAltimateInstanceName: () => undefined, + getAltimateAiKey: () => undefined, + } +} + +function terminal(): DBTTerminal { + return { + show: async () => {}, + log: (msg: string) => console.error("[dbt]", msg), + trace: () => {}, + debug: () => {}, + info: (_name: string, msg: string) => console.error("[dbt]", msg), + warn: (_name: string, msg: string) => console.error("[dbt:warn]", msg), + error: (_name: string, msg: string, e: unknown) => { + const err = e instanceof Error ? e.message : String(e) + console.error("[dbt:error]", msg, err) + }, + dispose: () => {}, + } +} + +function env(cfg: Config): RuntimePythonEnvironment { + const vars = { ...process.env } + return { + pythonPath: cfg.pythonPath, + getEnvironmentVariables: () => vars, + } +} + +export async function create(cfg: Config): Promise { + const config = configuration(cfg) + const term = terminal() + const factory = new DBTCommandFactory(config) + + const http = new AltimateHttpClient(term, config) + const client = new DbtIntegrationClient(http, term) + + const runtime = env(cfg) + const provider = { + getCurrentEnvironment: () => runtime, + onEnvironmentChanged: () => () => {}, + } + const exec = new CommandProcessExecutionFactory(term) + const infra = new DBTCommandExecutionInfrastructure(runtime, term) + const python = new PythonDBTCommandExecutionStrategy(exec, runtime, term, config) + const cli = (cwd: string, path: string) => new CLIDBTCommandExecutionStrategy(exec, runtime, term, cwd, path) + + const core = (root: string, diag: DBTDiagnosticData[], defer: DeferConfig, changed: () => void): DBTProjectIntegration => + new DBTCoreProjectIntegration(infra, runtime, provider, python, cli, term, config, client, root, diag, defer, changed) + + const cloud = (root: string, diag: DBTDiagnosticData[], defer: DeferConfig, changed: () => void): DBTProjectIntegration => + new DBTCloudProjectIntegration(infra, factory, cli, runtime, provider, term, root, diag, defer, changed) + + const command = (root: string, diag: DBTDiagnosticData[], defer: DeferConfig, changed: () => void): DBTProjectIntegration => + new DBTCoreCommandProjectIntegration(infra, runtime, provider, python, cli, term, config, client, root, diag, defer, changed) + + const fusion = (root: string, diag: DBTDiagnosticData[], defer: DeferConfig, changed: () => void): DBTProjectIntegration => + new DBTFusionCommandProjectIntegration(infra, factory, cli, runtime, provider, term, root, diag, defer, changed) + + const adapter = new DBTProjectIntegrationAdapter( + config, + factory, + core, + cloud, + fusion, + command, + cfg.projectRoot, + undefined, + new ChildrenParentParser(), + new NodeParser(term), + new MacroParser(term), + new MetricParser(term), + new GraphParser(term), + new SourceParser(term), + new TestParser(term), + new ExposureParser(term), + new FunctionParser(term), + new DocParser(term), + term, + new ModelDepthParser(term, client, config), + ) + + await adapter.initialize() + return adapter +} diff --git a/packages/dbt-tools/src/check.ts b/packages/dbt-tools/src/check.ts new file mode 100644 index 0000000000..4e5c4d2290 --- /dev/null +++ b/packages/dbt-tools/src/check.ts @@ -0,0 +1,62 @@ +import { existsSync } from "fs" +import { execFileSync } from "child_process" +import { join } from "path" +import type { Config } from "./config" + +type Status = { ok: true } | { ok: false; error: string; fix: string } + +function run(cmd: string, args: string[]): { ok: boolean; stdout: string } { + try { + const out = execFileSync(cmd, args, { encoding: "utf-8", timeout: 10000 }) + return { ok: true, stdout: out.trim() } + } catch { + return { ok: false, stdout: "" } + } +} + +async function python(path: string): Promise { + if (!path) return { ok: false, error: "No Python path configured", fix: "Run: altimate-dbt init --python-path /path/to/python3" } + if (!existsSync(path)) return { ok: false, error: `Python not found at: ${path}`, fix: "Run: altimate-dbt init --python-path $(which python3)" } + const result = run(path, ["--version"]) + if (!result.ok) return { ok: false, error: `Python at ${path} is not working`, fix: "Ensure Python 3.8+ is installed: brew install python3" } + return { ok: true } +} + +async function dbt(pythonPath: string): Promise { + const result = run(pythonPath, ["-c", "import dbt.version; print(dbt.version.installed)"]) + if (!result.ok) { + const pip = run(pythonPath, ["-m", "pip", "install", "dbt-core", "--dry-run"]) + const cmd = pip.ok ? `${pythonPath} -m pip install dbt-core` : "pip install dbt-core" + return { ok: false, error: "dbt-core is not installed in the configured Python environment", fix: `Install it: ${cmd}` } + } + return { ok: true } +} + +function project(root: string): Status { + if (!root) return { ok: false, error: "No project root configured", fix: "Run: altimate-dbt init --project-root /path/to/dbt/project" } + if (!existsSync(root)) return { ok: false, error: `Project directory not found: ${root}`, fix: "Check the path or re-run: altimate-dbt init" } + if (!existsSync(join(root, "dbt_project.yml"))) return { ok: false, error: `No dbt_project.yml in: ${root}`, fix: "Ensure this is a dbt project directory, then re-run: altimate-dbt init" } + return { ok: true } +} + +export async function all(cfg: Config) { + const checks = { + python: await python(cfg.pythonPath), + dbt: cfg.pythonPath && existsSync(cfg.pythonPath) ? await dbt(cfg.pythonPath) : { ok: false as const, error: "Skipped (Python not found)", fix: "Fix Python first" }, + project: project(cfg.projectRoot), + } + const passed = Object.values(checks).every((c) => c.ok) + return { passed, checks } +} + +export async function validate(cfg: Config): Promise { + const result = await all(cfg) + if (result.passed) return null + const errors = Object.entries(result.checks) + .filter(([, v]) => !v.ok) + .map(([k, v]) => { + const s = v as { ok: false; error: string; fix: string } + return ` ${k}: ${s.error}\n -> ${s.fix}` + }) + return `Prerequisites not met:\n${errors.join("\n")}` +} diff --git a/packages/dbt-tools/src/commands/build.ts b/packages/dbt-tools/src/commands/build.ts new file mode 100644 index 0000000000..07432ea4ad --- /dev/null +++ b/packages/dbt-tools/src/commands/build.ts @@ -0,0 +1,50 @@ +import type { DBTProjectIntegrationAdapter, CommandProcessResult } from "@altimateai/dbt-integration" + +export async function build(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + const downstream = args.includes("--downstream") + const result = await adapter.unsafeBuildModelImmediately({ + plusOperatorLeft: "", + modelName: model, + plusOperatorRight: downstream ? "+" : "", + }) + return format(result) +} + +export async function run(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + const downstream = args.includes("--downstream") + const result = await adapter.unsafeRunModelImmediately({ + plusOperatorLeft: "", + modelName: model, + plusOperatorRight: downstream ? "+" : "", + }) + return format(result) +} + +export async function test(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + const result = await adapter.unsafeRunModelTestImmediately(model) + return format(result) +} + +export async function project(adapter: DBTProjectIntegrationAdapter) { + const result = await adapter.unsafeBuildProjectImmediately() + return format(result) +} + +// TODO: dbt writes info/progress logs to stderr even on success — checking stderr +// alone causes false failures. CommandProcessResult has no exit_code field, so we +// can't distinguish real errors yet. Revisit when the type is extended. +function format(result?: CommandProcessResult) { + if (result?.stderr) return { error: result.stderr, stdout: result.stdout } + return { stdout: result?.stdout ?? "" } +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/columns.ts b/packages/dbt-tools/src/commands/columns.ts new file mode 100644 index 0000000000..ea03d5c1b7 --- /dev/null +++ b/packages/dbt-tools/src/commands/columns.ts @@ -0,0 +1,42 @@ +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" + +export async function columns(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + try { + const result = await adapter.getColumnsOfModel(model) + if (!result) return { error: `Model '${model}' not found in manifest. Try: altimate-dbt execute --query "SELECT * FROM ${model} LIMIT 1"` } + return result + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { error: `Failed to get columns for '${model}': ${msg}. Try: altimate-dbt execute --query "SELECT * FROM ${model} LIMIT 1"` } + } +} + +export async function source(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const name = flag(args, "source") + const table = flag(args, "table") + if (!name) return { error: "Missing --source" } + if (!table) return { error: "Missing --table" } + try { + const result = await adapter.getColumnsOfSource(name, table) + if (!result) return { error: `Source '${name}.${table}' not found. Try: altimate-dbt execute --query "SELECT * FROM ${table} LIMIT 1"` } + return result + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { error: `Failed to get columns for source '${name}.${table}': ${msg}` } + } +} + +export async function values(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + const col = flag(args, "column") + if (!model) return { error: "Missing --model" } + if (!col) return { error: "Missing --column" } + return adapter.getColumnValues(model, col) +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/compile.ts b/packages/dbt-tools/src/commands/compile.ts new file mode 100644 index 0000000000..b7f81068ca --- /dev/null +++ b/packages/dbt-tools/src/commands/compile.ts @@ -0,0 +1,21 @@ +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" + +export async function compile(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + const sql = await adapter.unsafeCompileNode(model) + return { sql } +} + +export async function query(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const sql = flag(args, "query") + if (!sql) return { error: "Missing --query" } + const model = flag(args, "model") + const result = await adapter.unsafeCompileQuery(sql, model) + return { sql: result } +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/deps.ts b/packages/dbt-tools/src/commands/deps.ts new file mode 100644 index 0000000000..feda088730 --- /dev/null +++ b/packages/dbt-tools/src/commands/deps.ts @@ -0,0 +1,26 @@ +import type { DBTProjectIntegrationAdapter, CommandProcessResult } from "@altimateai/dbt-integration" + +export async function deps(adapter: DBTProjectIntegrationAdapter) { + const result = await adapter.installDeps() + return format(result) +} + +export async function add(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const raw = flag(args, "packages") + if (!raw) return { error: "Missing --packages" } + const result = await adapter.installDbtPackages(raw.split(",")) + return format(result) +} + +// TODO: dbt writes info/progress logs to stderr even on success — checking stderr +// alone causes false failures. CommandProcessResult has no exit_code field, so we +// can't distinguish real errors yet. Revisit when the type is extended. +function format(result?: CommandProcessResult) { + if (result?.stderr) return { error: result.stderr, stdout: result.stdout } + return { stdout: result?.stdout ?? "" } +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/doctor.ts b/packages/dbt-tools/src/commands/doctor.ts new file mode 100644 index 0000000000..2d18e053d2 --- /dev/null +++ b/packages/dbt-tools/src/commands/doctor.ts @@ -0,0 +1,13 @@ +import type { Config } from "../config" +import { all } from "../check" + +export async function doctor(cfg: Config) { + const result = await all(cfg) + + const summary = Object.entries(result.checks).map(([name, status]) => { + if (status.ok) return { check: name, status: "ok" } + return { check: name, status: "fail", error: status.error, fix: status.fix } + }) + + return { passed: result.passed, checks: summary } +} diff --git a/packages/dbt-tools/src/commands/execute.ts b/packages/dbt-tools/src/commands/execute.ts new file mode 100644 index 0000000000..1c929ff279 --- /dev/null +++ b/packages/dbt-tools/src/commands/execute.ts @@ -0,0 +1,16 @@ +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" + +export async function execute(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const sql = flag(args, "query") + if (!sql) return { error: "Missing --query" } + const model = flag(args, "model") ?? "" + const raw = flag(args, "limit") + const limit = raw !== undefined ? parseInt(raw, 10) : undefined + if (limit !== undefined && !Number.isNaN(limit)) return adapter.immediatelyExecuteSQLWithLimit(sql, model, limit) + return adapter.immediatelyExecuteSQL(sql, model) +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/graph.ts b/packages/dbt-tools/src/commands/graph.ts new file mode 100644 index 0000000000..e4909b73ca --- /dev/null +++ b/packages/dbt-tools/src/commands/graph.ts @@ -0,0 +1,18 @@ +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" + +export function children(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + return adapter.getChildrenModels({ table: model }) +} + +export function parents(adapter: DBTProjectIntegrationAdapter, args: string[]) { + const model = flag(args, "model") + if (!model) return { error: "Missing --model" } + return adapter.getParentModels({ table: model }) +} + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} diff --git a/packages/dbt-tools/src/commands/info.ts b/packages/dbt-tools/src/commands/info.ts new file mode 100644 index 0000000000..df14440bfc --- /dev/null +++ b/packages/dbt-tools/src/commands/info.ts @@ -0,0 +1,5 @@ +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" + +export async function info(adapter: DBTProjectIntegrationAdapter) { + return adapter.getProjectInfo() +} diff --git a/packages/dbt-tools/src/commands/init.ts b/packages/dbt-tools/src/commands/init.ts new file mode 100644 index 0000000000..6b1de6283b --- /dev/null +++ b/packages/dbt-tools/src/commands/init.ts @@ -0,0 +1,54 @@ +import { join, resolve } from "path" +import { existsSync } from "fs" +import { execFileSync } from "child_process" +import { write, type Config } from "../config" +import { all } from "../check" + +function find(start: string): string | null { + let dir = resolve(start) + while (true) { + if (existsSync(join(dir, "dbt_project.yml"))) return dir + const parent = resolve(dir, "..") + if (parent === dir) return null + dir = parent + } +} + +function python(): string { + for (const cmd of ["python3", "python"]) { + try { + return execFileSync("which", [cmd], { encoding: "utf-8" }).trim() + } catch {} + } + return "python3" +} + +export async function init(args: string[]) { + const idx = args.indexOf("--project-root") + const root = idx >= 0 ? args[idx + 1] : undefined + const pidx = args.indexOf("--python-path") + const py = pidx >= 0 ? args[pidx + 1] : undefined + + const project = root ? resolve(root) : find(process.cwd()) + if (!project) return { error: "No dbt_project.yml found. Use --project-root to specify." } + if (!existsSync(join(project, "dbt_project.yml"))) return { error: `No dbt_project.yml in ${project}` } + + const cfg: Config = { + projectRoot: project, + pythonPath: py ?? python(), + dbtIntegration: "corecommand", + queryLimit: 500, + } + + await write(cfg) + + const health = await all(cfg) + if (!health.passed) { + const warnings = Object.entries(health.checks) + .filter(([, v]) => !v.ok) + .map(([k, v]) => ({ check: k, ...(v as { ok: false; error: string; fix: string }) })) + return { ok: true, config: cfg, warnings } + } + + return { ok: true, config: cfg } +} diff --git a/packages/dbt-tools/src/config.ts b/packages/dbt-tools/src/config.ts new file mode 100644 index 0000000000..07fdc0e3c9 --- /dev/null +++ b/packages/dbt-tools/src/config.ts @@ -0,0 +1,34 @@ +import { homedir } from "os" +import { join } from "path" +import { readFile, writeFile, mkdir } from "fs/promises" +import { existsSync } from "fs" + +type Config = { + projectRoot: string + pythonPath: string + dbtIntegration: string + queryLimit: number +} + +function configDir() { + return join(process.env.HOME || homedir(), ".altimate-code") +} + +function configPath() { + return join(configDir(), "dbt.json") +} + +async function read(): Promise { + const p = configPath() + if (!existsSync(p)) return null + const raw = await readFile(p, "utf-8") + return JSON.parse(raw) as Config +} + +async function write(cfg: Config) { + const d = configDir() + await mkdir(d, { recursive: true }) + await writeFile(join(d, "dbt.json"), JSON.stringify(cfg, null, 2)) +} + +export { read, write, configPath as path, type Config } diff --git a/packages/dbt-tools/src/index.ts b/packages/dbt-tools/src/index.ts new file mode 100644 index 0000000000..a38b320721 --- /dev/null +++ b/packages/dbt-tools/src/index.ts @@ -0,0 +1,196 @@ +import { join, resolve } from "path" +import { existsSync } from "fs" +import { read, type Config } from "./config" +import { init } from "./commands/init" +import { validate } from "./check" + +const USAGE = { + commands: { + init: "Auto-detect dbt project and write config", + doctor: "Check prerequisites (Python, dbt, project)", + info: "Get project info (paths, targets, version)", + compile: "Compile a model (Jinja to SQL) --model ", + "compile-query": "Compile a raw query --query [--model ]", + build: "Build a model --model [--downstream]", + run: "Run a model --model [--downstream]", + test: "Test a model --model ", + "build-project": "Build entire project", + execute: "Execute SQL --query [--model ] [--limit ]", + columns: "Get columns of model --model ", + "columns-source": "Get columns of source --source --table ", + "column-values": "Get column values --model --column ", + children: "Get downstream models --model ", + parents: "Get upstream models --model ", + deps: "Install dbt dependencies", + "add-packages": "Add dbt packages --packages pkg1,pkg2", + }, +} + +const cmd = process.argv[2] +const rest = process.argv.slice(3) + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} + +function diagnose(err: Error): { error: string; fix?: string } { + const msg = err.message || String(err) + if (msg.includes("private field") || msg.includes("#stdin") || msg.includes("#stdout")) { + return { + error: "Internal error: Bun runtime incompatibility with python-bridge. Please report this issue.", + fix: "Try updating: bun install, or file a bug at https://github.com/AltimateAI/altimate-code/issues", + } + } + if (msg.includes("IPC channel") || msg.includes("ERR_IPC_CHANNEL_CLOSED")) { + return { + error: "Failed to start Python bridge. The dbt Python process exited unexpectedly.", + fix: "Check your dbt installation: run `dbt debug` in your project directory. Then run: altimate-dbt doctor", + } + } + if (msg.includes("ENOENT") || msg.includes("spawn")) { + return { + error: `Could not start process: ${msg}`, + fix: "Ensure Python and dbt are installed and accessible. Run: altimate-dbt doctor", + } + } + if (msg.includes("ModuleNotFoundError") || msg.includes("No module named")) { + return { + error: `Missing Python package: ${msg}`, + fix: "Install dbt in your Python environment: pip install dbt-core", + } + } + if (msg.includes("Python process closed")) { + return { + error: `dbt Python process crashed: ${msg}`, + fix: "Verify your dbt project is valid: run `dbt debug` in your project directory", + } + } + return { error: msg, fix: "Run: altimate-dbt doctor" } +} + +function output(result: unknown) { + const fmt = rest.includes("--format") && rest[rest.indexOf("--format") + 1] === "text" + if (fmt && typeof result === "object" && result !== null) { + const obj = result as Record + if ("error" in obj) { + console.error(String(obj.error)) + if ("fix" in obj) console.error(String(obj.fix)) + process.exit(1) + } + if ("sql" in obj) { + console.log(String(obj.sql)) + return + } + if ("stdout" in obj) { + console.log(String(obj.stdout)) + return + } + } + console.log(JSON.stringify(result, null, 2)) + if (result && typeof result === "object" && "error" in result) process.exit(1) +} + +function bail(err: unknown): never { + const result = diagnose(err instanceof Error ? err : new Error(String(err))) + console.log(JSON.stringify(result, null, 2)) + process.exit(1) +} + +// Catch unhandled rejections (bluebird throws outside normal promise chains) +process.on("unhandledRejection", bail) +process.on("uncaughtException", bail) + +async function main() { + if (!cmd || cmd === "help" || cmd === "--help") return USAGE + + if (cmd === "init") return init(rest) + + const cfg = await read() + if (!cfg) return { error: "No config found. Run: altimate-dbt init" } + + // Override projectRoot: --project-dir flag > cwd auto-detect > config + const dirFlag = flag(rest, "project-dir") + if (dirFlag) { + cfg.projectRoot = resolve(dirFlag) + } else { + const cwdProject = join(process.cwd(), "dbt_project.yml") + if (existsSync(cwdProject) && resolve(process.cwd()) !== resolve(cfg.projectRoot)) { + cfg.projectRoot = resolve(process.cwd()) + } + } + + if (cmd === "doctor") return (await import("./commands/doctor")).doctor(cfg) + + // Validate prerequisites before loading the heavy adapter + const issue = await validate(cfg) + if (issue) return { error: issue } + + // Lazy import to avoid loading python-bridge until needed + let adapter + try { + const { create } = await import("./adapter") + adapter = await create(cfg) + } catch (err) { + return diagnose(err instanceof Error ? err : new Error(String(err))) + } + + let result: unknown + try { + switch (cmd) { + case "info": + result = await (await import("./commands/info")).info(adapter) + break + case "compile": + result = await (await import("./commands/compile")).compile(adapter, rest) + break + case "compile-query": + result = await (await import("./commands/compile")).query(adapter, rest) + break + case "build": + result = await (await import("./commands/build")).build(adapter, rest) + break + case "run": + result = await (await import("./commands/build")).run(adapter, rest) + break + case "test": + result = await (await import("./commands/build")).test(adapter, rest) + break + case "build-project": + result = await (await import("./commands/build")).project(adapter) + break + case "execute": + result = await (await import("./commands/execute")).execute(adapter, rest) + break + case "columns": + result = await (await import("./commands/columns")).columns(adapter, rest) + break + case "columns-source": + result = await (await import("./commands/columns")).source(adapter, rest) + break + case "column-values": + result = await (await import("./commands/columns")).values(adapter, rest) + break + case "children": + result = await (await import("./commands/graph")).children(adapter, rest) + break + case "parents": + result = await (await import("./commands/graph")).parents(adapter, rest) + break + case "deps": + result = await (await import("./commands/deps")).deps(adapter) + break + case "add-packages": + result = await (await import("./commands/deps")).add(adapter, rest) + break + default: + result = { error: `Unknown command: ${cmd}`, usage: USAGE } + } + } finally { + try { await adapter.dispose() } catch {} + } + + return result +} + +main().then(output).catch(bail) diff --git a/packages/dbt-tools/test/cli.test.ts b/packages/dbt-tools/test/cli.test.ts new file mode 100644 index 0000000000..9faa6dbab3 --- /dev/null +++ b/packages/dbt-tools/test/cli.test.ts @@ -0,0 +1,76 @@ +import { describe, test, expect } from "bun:test" +import { join } from "path" + +const entry = join(import.meta.dir, "../src/index.ts") + +describe("cli", () => { + test("no args prints usage", async () => { + const result = Bun.spawnSync(["bun", entry], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const out = result.stdout.toString() + const parsed = JSON.parse(out) + expect(parsed.commands).toBeDefined() + expect(parsed.commands.init).toBeDefined() + expect(parsed.commands.build).toBeDefined() + }) + + test("help prints usage", async () => { + const result = Bun.spawnSync(["bun", entry, "help"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const out = result.stdout.toString() + const parsed = JSON.parse(out) + expect(parsed.commands).toBeDefined() + }) + + test("unknown command without config produces config error", async () => { + const result = Bun.spawnSync(["bun", entry, "nonexistent"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const out = result.stdout.toString() + const parsed = JSON.parse(out) + expect(parsed.error).toContain("altimate-dbt init") + }) + + test("missing config produces init hint", async () => { + const result = Bun.spawnSync(["bun", entry, "info"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const out = result.stdout.toString() + const parsed = JSON.parse(out) + expect(parsed.error).toContain("altimate-dbt init") + }) + + test("init without dbt_project.yml produces error", async () => { + const result = Bun.spawnSync(["bun", entry, "init", "--project-root", "/tmp/nonexistent-dbt-project"], { + env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" }, + }) + const out = result.stdout.toString() + const parsed = JSON.parse(out) + expect(parsed.error).toBeDefined() + }) + + test("usage includes doctor command", async () => { + const result = Bun.spawnSync(["bun", entry], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const parsed = JSON.parse(result.stdout.toString()) + expect(parsed.commands.doctor).toBeDefined() + }) + + test("doctor without config produces init hint", async () => { + const result = Bun.spawnSync(["bun", entry, "doctor"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const parsed = JSON.parse(result.stdout.toString()) + expect(parsed.error).toContain("altimate-dbt init") + }) + + test("info with bad prerequisites shows actionable error", async () => { + const { mkdtempSync, writeFileSync } = await import("fs") + const { join } = await import("path") + const home = mkdtempSync("/tmp/dbt-tools-err-") + const dir = join(home, ".altimate-code") + const { mkdirSync } = await import("fs") + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, "dbt.json"), JSON.stringify({ + projectRoot: "/tmp/definitely-not-a-project", + pythonPath: "/usr/bin/python3", + dbtIntegration: "core", + queryLimit: 500, + })) + const result = Bun.spawnSync(["bun", entry, "info"], { env: { ...process.env, HOME: home } }) + const parsed = JSON.parse(result.stdout.toString()) + expect(parsed.error).toContain("Prerequisites not met") + expect(parsed.error).toContain("project") + }) +}) diff --git a/packages/dbt-tools/test/config.test.ts b/packages/dbt-tools/test/config.test.ts new file mode 100644 index 0000000000..2689375e26 --- /dev/null +++ b/packages/dbt-tools/test/config.test.ts @@ -0,0 +1,54 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test" +import { join } from "path" +import { mkdtemp, rm } from "fs/promises" +import { tmpdir, homedir } from "os" + +describe("config", () => { + let dir: string + let originalHome: string + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "dbt-tools-test-")) + originalHome = process.env.HOME! + process.env.HOME = dir + }) + + afterEach(async () => { + process.env.HOME = originalHome + await rm(dir, { recursive: true, force: true }) + }) + + test("read returns null for missing file", async () => { + const { read } = await import("../src/config") + const result = await read() + expect(result).toBeNull() + }) + + test("write and read round-trip", async () => { + const { read, write } = await import("../src/config") + const cfg = { + projectRoot: "/tmp/my-dbt-project", + pythonPath: "/usr/bin/python3", + dbtIntegration: "corecommand", + queryLimit: 500, + } + + await write(cfg) + const result = await read() + expect(result).toEqual(cfg) + }) + + test("write creates .altimate-code directory", async () => { + const { write } = await import("../src/config") + const { existsSync } = await import("fs") + const cfg = { + projectRoot: "/tmp/project", + pythonPath: "/usr/bin/python3", + dbtIntegration: "corecommand", + queryLimit: 1000, + } + + await write(cfg) + expect(existsSync(join(dir, ".altimate-code", "dbt.json"))).toBe(true) + }) +}) diff --git a/packages/dbt-tools/tsconfig.json b/packages/dbt-tools/tsconfig.json new file mode 100644 index 0000000000..1f888652e3 --- /dev/null +++ b/packages/dbt-tools/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@tsconfig/bun/tsconfig.json", + "compilerOptions": { + "strict": true, + "noUncheckedIndexedAccess": true + } +} diff --git a/packages/opencode/src/altimate/index.ts b/packages/opencode/src/altimate/index.ts index 0fbe5247c8..3dad78d862 100644 --- a/packages/opencode/src/altimate/index.ts +++ b/packages/opencode/src/altimate/index.ts @@ -45,7 +45,7 @@ export * from "./tools/altimate-core-validate" export * from "./tools/dbt-lineage" export * from "./tools/dbt-manifest" export * from "./tools/dbt-profiles" -export * from "./tools/dbt-run" + export * from "./tools/finops-analyze-credits" export * from "./tools/finops-expensive-queries" export * from "./tools/finops-query-history" diff --git a/packages/opencode/src/altimate/prompts/builder.txt b/packages/opencode/src/altimate/prompts/builder.txt index d330fef790..75a71a6ae0 100644 --- a/packages/opencode/src/altimate/prompts/builder.txt +++ b/packages/opencode/src/altimate/prompts/builder.txt @@ -1,5 +1,12 @@ You are altimate-code in builder mode — a data engineering agent specializing in dbt models, SQL, and data pipelines. +## Principles + +1. **Understand before writing** — Read existing code, schemas, and actual data before writing any SQL. Never write blind. +2. **Follow conventions** — Match the project's naming patterns, layer structure, and style. Read 2-3 similar files first. +3. **Validate the output** — A task isn't done until the output data looks right. Check row counts, sample values, and column names. +4. **Fix everything** — After finishing your changes, run a full project build (no `--select`). If ANY model fails — even ones you didn't touch — fix it. Leave the project fully green. + You have full read/write access to the project. You can: - Create and modify dbt models, SQL files, and YAML configs - Execute SQL against connected warehouses via `sql_execute` @@ -8,10 +15,26 @@ You have full read/write access to the project. You can: - Inspect database schemas via `schema_inspect` - Check column-level lineage via `lineage_check` - List and test warehouse connections via `warehouse_list` and `warehouse_test` -- Run dbt commands (run, test, compile) -- Generate dbt tests using the `/generate-tests` skill +- Run dbt commands via `altimate-dbt` (build, compile, columns, execute, graph, info) - Use all standard file tools (read, write, edit, bash, grep, glob) +## dbt Operations + +Use `altimate-dbt` instead of raw `dbt` commands. Key commands: + +``` +altimate-dbt build --model # Build + test a specific model +altimate-dbt execute --query "..." --limit N # Query the database +altimate-dbt columns --model # Inspect model columns +altimate-dbt info # Project metadata +``` + +**Never call raw `dbt` directly** (except `dbt deps` for package installation). Never connect to DuckDB or project databases directly via python — use `altimate-dbt execute`. + +**Before the first build**, if `packages.yml` exists but `dbt_packages/` does not, run `dbt deps` to install packages. + +**After finishing your model(s)**, run a full project build: `altimate-dbt build` (no `--model` flag). Fix every failure — even pre-existing ones. + When writing SQL: - Always run `sql_analyze` to check for anti-patterns before finalizing queries - Validate SQL with `sql_validate` before executing against a warehouse @@ -22,7 +45,6 @@ When writing SQL: When creating dbt models: - Follow the project's existing naming conventions - Place staging models in staging/, intermediate in intermediate/, marts in marts/ -- Use `/generate-tests` to auto-generate test definitions - Add tests for primary keys and not-null constraints - Update schema.yml files alongside model changes - Run `lineage_check` to verify column-level data flow @@ -53,6 +75,24 @@ After ANY dbt operation (build, run, test, model creation/modification): 4. **Test coverage**: Check that the model has not_null and unique tests on primary keys at minimum. If missing, suggest adding them. Do NOT consider a dbt task complete until steps 1-4 pass. A model that compiles but has anti-patterns or broken lineage is NOT done. +## Workflow + +1. **Explore**: Read existing models, schemas, and sample data before writing anything. +2. **Write**: Create models following project conventions. Use `altimate-dbt build --model ` to validate each model. +3. **Verify**: Check row counts and sample data with `altimate-dbt execute`. Work isn't done until the output data looks right. + +## Common Pitfalls + +- **Writing SQL without checking columns first** — Always inspect schemas and sample data before writing +- **Date spine models**: Derive date boundaries from `MIN(date)`/`MAX(date)` in source data, never use `current_date`. +- **Fan-out joins**: One-to-many joins inflate aggregates. Check grain before joining. +- **Missing packages**: If `packages.yml` exists, run `dbt deps` before building +- **NULL vs 0 confusion**: Do not add `coalesce(x, 0)` unless the task explicitly requires it. Preserve NULLs from source data. +- **Column casing**: Many warehouses are case-insensitive but return UPPER-case column names. Always check actual column names with `altimate-dbt columns` before writing SQL. +- **Stopping at compile**: Compile only checks Jinja syntax. Always follow up with `altimate-dbt build` to catch runtime SQL errors. +- **Skipping full project build**: After your model works, run `altimate-dbt build` (no flags) to catch any failures across the whole project. +- **Ignoring pre-existing failures**: If a model you didn't touch fails during full build, fix it anyway. The project must be fully green. + ## Self-Review Before Completion Before declaring any task complete, review your own work: @@ -71,17 +111,14 @@ Only after self-review passes should you present the result to the user. ## Available Skills You have access to these skills that users can invoke with /: -- /generate-tests — Generate dbt test definitions (not_null, unique, relationships) -- /lineage-diff — Compare column lineage between SQL versions +- /dbt-develop — Create dbt models following project conventions +- /dbt-test — Add tests and data quality validation +- /dbt-docs — Generate model and column descriptions +- /dbt-analyze — Lineage analysis and impact assessment +- /dbt-troubleshoot — Debug and fix dbt errors - /cost-report — Snowflake cost analysis with optimization suggestions - /sql-translate — Cross-dialect SQL translation with warnings - /query-optimize — Query optimization with anti-pattern detection -- /impact-analysis — Downstream impact analysis using lineage + manifest -- /model-scaffold — Scaffold staging/intermediate/mart dbt models -- /yaml-config — Generate sources.yml, schema.yml from warehouse schema -- /dbt-docs — Generate model and column descriptions -- /medallion-patterns — Bronze/silver/gold architecture patterns -- /incremental-logic — Incremental materialization strategies - /teach — Teach a pattern from an example file - /train — Learn standards from a document - /training-status — Show training dashboard diff --git a/packages/opencode/src/altimate/tools/dbt-run.ts b/packages/opencode/src/altimate/tools/dbt-run.ts deleted file mode 100644 index a8a150e176..0000000000 --- a/packages/opencode/src/altimate/tools/dbt-run.ts +++ /dev/null @@ -1,49 +0,0 @@ -import z from "zod" -import { Tool } from "../../tool/tool" -import { Bridge } from "../bridge/client" - -export const DbtRunTool = Tool.define("dbt_run", { - description: - "Run a dbt CLI command (run, test, build, compile, etc.). Executes dbt in the project directory and returns stdout/stderr.", - parameters: z.object({ - command: z.string().optional().default("run").describe("dbt command to run (run, test, build, compile, seed, snapshot)"), - select: z.string().optional().describe("dbt node selector (e.g. 'my_model', '+my_model', 'tag:daily')"), - args: z.array(z.string()).optional().default([]).describe("Additional CLI arguments"), - project_dir: z.string().optional().describe("Path to dbt project directory"), - }), - async execute(args, ctx) { - try { - const result = await Bridge.call("dbt.run", { - command: args.command, - select: args.select, - args: args.args, - project_dir: args.project_dir, - }) - - const success = result.exit_code === 0 - const lines: string[] = [] - - if (result.stdout) { - lines.push(result.stdout) - } - if (result.stderr) { - if (lines.length > 0) lines.push("") - lines.push("--- stderr ---") - lines.push(result.stderr) - } - - return { - title: `dbt ${args.command}: ${success ? "OK" : `FAIL (exit ${result.exit_code})`}`, - metadata: { exit_code: result.exit_code, success }, - output: lines.join("\n") || "(no output)", - } - } catch (e) { - const msg = e instanceof Error ? e.message : String(e) - return { - title: `dbt ${args.command}: ERROR`, - metadata: { exit_code: 1, success: false }, - output: `Failed to run dbt: ${msg}\n\nEnsure dbt-core is installed and the Python bridge is running.`, - } - } - }, -}) diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index f6020a0f76..6b4100404d 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -44,7 +44,7 @@ import { WarehouseTestTool } from "../altimate/tools/warehouse-test" import { WarehouseAddTool } from "../altimate/tools/warehouse-add" import { WarehouseRemoveTool } from "../altimate/tools/warehouse-remove" import { WarehouseDiscoverTool } from "../altimate/tools/warehouse-discover" -import { DbtRunTool } from "../altimate/tools/dbt-run" + import { DbtManifestTool } from "../altimate/tools/dbt-manifest" import { DbtProfilesTool } from "../altimate/tools/dbt-profiles" import { DbtLineageTool } from "../altimate/tools/dbt-lineage" @@ -218,7 +218,7 @@ export namespace ToolRegistry { WarehouseAddTool, WarehouseRemoveTool, WarehouseDiscoverTool, - DbtRunTool, + DbtManifestTool, DbtProfilesTool, DbtLineageTool,