From 7878eba6ff0dafa5b25cc02bc789ae88f09c6d25 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 16 Mar 2026 18:09:14 -0700 Subject: [PATCH] chore: open-source release cleanup - Remove internal planning docs (FEATURES_*.md, IMPLEMENTATION_PLAN.md, PLATFORM_ENGINEER_SIMULATION.md, STATS.md) - Remove internal scripts (beta.ts, duplicate-pr.ts, stats.ts) - Remove internal CI workflows (beta.yml, stats.yml) - Remove `.github/TEAM_MEMBERS` (internal employee list) - Remove `.zed/` (single dev's editor config) - Remove `docs/design/` (internal product strategy) - Move `RELEASING.md` to `docs/` - Fix `pr-management.yml`: remove broken `check-duplicates` job, keep `add-contributor-label` - Fix `opencode.yml`: replace self-hosted runner with `ubuntu-latest` - Standardize internal placeholder URLs to `altimate-code.internal` Closes #207 --- .github/CODEOWNERS | 5 - .github/TEAM_MEMBERS | 30 - .github/VOUCHED.td | 23 - .github/meta/commit.txt | 13 - .github/workflows/beta.yml | 39 -- .github/workflows/opencode.yml | 2 +- .github/workflows/pr-management.yml | 66 -- .github/workflows/stats.yml | 35 - .gitignore | 3 + .zed/settings.json | 9 - FEATURES_COMPARISON.md | 131 ---- FEATURES_MAIN.md | 513 --------------- FEATURES_RESTRUCTURE.md | 550 ---------------- IMPLEMENTATION_PLAN.md | 172 ----- LICENSE | 1 + PLATFORM_ENGINEER_SIMULATION.md | 604 ------------------ README.md | 2 - SECURITY.md | 2 +- STATS.md | 217 ------- RELEASING.md => docs/RELEASING.md | 0 docs/design/ai-teammate-repositioning.md | 566 ---------------- packages/opencode/src/cli/cmd/tui/thread.ts | 2 +- packages/opencode/src/cli/cmd/tui/worker.ts | 2 +- packages/opencode/src/config/config.ts | 1 + .../src/control-plane/adaptors/worktree.ts | 2 +- packages/opencode/src/tool/skill.ts | 2 - .../opencode/test/branding/branding.test.ts | 10 - script/beta.ts | 202 ------ script/duplicate-pr.ts | 79 --- script/stats.ts | 225 ------- 30 files changed, 10 insertions(+), 3498 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/TEAM_MEMBERS delete mode 100644 .github/VOUCHED.td delete mode 100644 .github/meta/commit.txt delete mode 100644 .github/workflows/beta.yml delete mode 100644 .github/workflows/stats.yml delete mode 100644 .zed/settings.json delete mode 100644 FEATURES_COMPARISON.md delete mode 100644 FEATURES_MAIN.md delete mode 100644 FEATURES_RESTRUCTURE.md delete mode 100644 IMPLEMENTATION_PLAN.md delete mode 100644 PLATFORM_ENGINEER_SIMULATION.md delete mode 100644 STATS.md rename RELEASING.md => docs/RELEASING.md (100%) delete mode 100644 docs/design/ai-teammate-repositioning.md delete mode 100755 script/beta.ts delete mode 100755 script/duplicate-pr.ts delete mode 100755 script/stats.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 3aeef82d62..0000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# web + desktop packages -packages/app/ @adamdotdevin -packages/tauri/ @adamdotdevin -packages/desktop/src-tauri/ @brendonovich -packages/desktop/ @adamdotdevin diff --git a/.github/TEAM_MEMBERS b/.github/TEAM_MEMBERS deleted file mode 100644 index 34b7b23648..0000000000 --- a/.github/TEAM_MEMBERS +++ /dev/null @@ -1,30 +0,0 @@ -aidtya -aloks98 -altimateanas -anandgupta42 -ankitksharma -anusha-sharma -arora-saurabh448 -dvanaken -frasermarlow -gaurpulkit -govindpawa -jontsai -kulvirgit -mdesmet -mhallida -ppradnesh -rakendd -ralphstodomingo -ravik-aai -robertmaybin -sahrizvi -sanjaykr5 -saravmajestic -sgvarsh -shreyastelkar -sourabhchrs93 -suryaiyer95 -tshreyas -vivekvenkatareddy -yukthagv diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td deleted file mode 100644 index 28535b5779..0000000000 --- a/.github/VOUCHED.td +++ /dev/null @@ -1,23 +0,0 @@ -# Vouched contributors for this project. -# -# See https://github.com/mitchellh/vouch for details. -# -# Syntax: -# - One handle per line (without @), sorted alphabetically. -# - Optional platform prefix: platform:username (e.g., github:user). -# - Denounce with minus prefix: -username or -platform:username. -# - Optional details after a space following the handle. -adamdotdevin --agusbasari29 AI PR slop -ariane-emory -edemaine --florianleibert -fwang -iamdavidhill -jayair -kitlangton -kommander -r44vc0rp -rekram1-node --spider-yamet clawdbot/llm psychosis, spam pinging the team -thdxr diff --git a/.github/meta/commit.txt b/.github/meta/commit.txt deleted file mode 100644 index fbbe81e31e..0000000000 --- a/.github/meta/commit.txt +++ /dev/null @@ -1,13 +0,0 @@ -fix: [AI-190] prevent tracing exporter timeout from leaking timers - -- Add `clearTimeout` in `.finally()` to `withTimeout` so the event loop - exits immediately after `endTrace()` instead of hanging for 5 seconds -- Log a `console.warn` when an exporter times out (uses the previously - unused `name` parameter for diagnostics) -- Align `HttpExporter` internal `AbortSignal.timeout` from 10s to 5s to - match the per-exporter wrapper timeout -- Clean up safety-net timer in adversarial test to prevent open handles - -Closes #190 - -Co-Authored-By: Claude Opus 4.6 (1M context) diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml deleted file mode 100644 index 2e61dc1529..0000000000 --- a/.github/workflows/beta.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: beta - -# Disabled — upstream beta branch syncing workflow, not needed for our fork. -# To re-enable, uncomment the triggers below. -on: - workflow_dispatch: - # schedule: - # - cron: "0 * * * *" - -jobs: - sync: - runs-on: blacksmith-4vcpu-ubuntu-2404 - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Bun - uses: ./.github/actions/setup-bun - - - name: Setup Git Committer - id: setup-git-committer - uses: ./.github/actions/setup-git-committer - with: - opencode-app-id: ${{ vars.OPENCODE_APP_ID }} - opencode-app-secret: ${{ secrets.OPENCODE_APP_SECRET }} - - - name: Install Altimate Code - run: bun i -g @altimateai/altimate-code - - - name: Sync beta branch - env: - GH_TOKEN: ${{ steps.setup-git-committer.outputs.token }} - OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} - run: bun script/beta.ts diff --git a/.github/workflows/opencode.yml b/.github/workflows/opencode.yml index d913d7270b..a8605a1597 100644 --- a/.github/workflows/opencode.yml +++ b/.github/workflows/opencode.yml @@ -13,7 +13,7 @@ jobs: startsWith(github.event.comment.body, '/altimate') || contains(github.event.comment.body, ' /ac') || startsWith(github.event.comment.body, '/ac') - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest permissions: id-token: write contents: read diff --git a/.github/workflows/pr-management.yml b/.github/workflows/pr-management.yml index 06eca8f3ae..073df4205a 100644 --- a/.github/workflows/pr-management.yml +++ b/.github/workflows/pr-management.yml @@ -5,72 +5,6 @@ on: types: [opened] jobs: - check-duplicates: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Check team membership - id: team-check - run: | - LOGIN=$(echo "${{ github.event.pull_request.user.login }}" | tr '[:upper:]' '[:lower:]') - if [ "$LOGIN" = "opencode-agent[bot]" ] || grep -qixF "$LOGIN" .github/TEAM_MEMBERS; then - echo "is_team=true" >> "$GITHUB_OUTPUT" - echo "Skipping: $LOGIN is a team member or bot" - else - echo "is_team=false" >> "$GITHUB_OUTPUT" - fi - - - name: Setup Bun - if: steps.team-check.outputs.is_team != 'true' - uses: ./.github/actions/setup-bun - - - name: Install dependencies - if: steps.team-check.outputs.is_team != 'true' - run: bun install - - - name: Install altimate-code - if: steps.team-check.outputs.is_team != 'true' - run: npm install -g @altimateai/altimate-code - - - name: Build prompt - if: steps.team-check.outputs.is_team != 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - { - echo "Check for duplicate PRs related to this new PR:" - echo "" - echo "CURRENT_PR_NUMBER: $PR_NUMBER" - echo "" - echo "Title: $(gh pr view "$PR_NUMBER" --json title --jq .title)" - echo "" - echo "Description:" - gh pr view "$PR_NUMBER" --json body --jq .body - } > pr_info.txt - - - name: Check for duplicate PRs - if: steps.team-check.outputs.is_team != 'true' - env: - OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - COMMENT=$(bun script/duplicate-pr.ts -f pr_info.txt "Check the attached file for PR details and search for duplicates") - - if [ "$COMMENT" != "No duplicate PRs found" ]; then - gh pr comment "$PR_NUMBER" --body "_The following comment was made by an LLM, it may be inaccurate:_ - - $COMMENT" - fi - add-contributor-label: runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml deleted file mode 100644 index deba206a52..0000000000 --- a/.github/workflows/stats.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: stats - -on: - schedule: - - cron: "0 12 * * *" # Run daily at 12:00 UTC - workflow_dispatch: # Allow manual trigger - -concurrency: ${{ github.workflow }}-${{ github.ref }} - -jobs: - stats: - if: github.repository == 'AltimateAI/altimate-code' - runs-on: blacksmith-4vcpu-ubuntu-2404 - permissions: - contents: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Bun - uses: ./.github/actions/setup-bun - - - name: Run stats script - run: bun script/stats.ts - - - name: Commit stats - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add STATS.md - git diff --staged --quiet || git commit -m "ignore: update download stats $(date -I)" - git push - env: - POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }} diff --git a/.gitignore b/.gitignore index 9de8d33939..889392f134 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ target .scripts .direnv/ +# Commit message scratch files +.github/meta/ + # Local dev files opencode-dev logs/ diff --git a/.zed/settings.json b/.zed/settings.json deleted file mode 100644 index a3a5e1e2b2..0000000000 --- a/.zed/settings.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "format_on_save": "on", - "formatter": { - "external": { - "command": "bunx", - "arguments": ["prettier", "--stdin-filepath", "{buffer_path}"] - } - } -} diff --git a/FEATURES_COMPARISON.md b/FEATURES_COMPARISON.md deleted file mode 100644 index e8ee9b7dc5..0000000000 --- a/FEATURES_COMPARISON.md +++ /dev/null @@ -1,131 +0,0 @@ -# Feature Comparison: origin/main vs restructure/main - -## Verdict: No features lost. All custom functionality is present on restructure/main. - ---- - -## Side-by-Side Summary - -| Category | origin/main | restructure/main | Status | -|----------|:-----------:|:-----------------:|:------:| -| Custom tools (TS files) | 68 | 68 | MATCH | -| Bridge (client/engine/protocol) | 3 | 3 | MATCH | -| Agent modes | 5 | 5 | MATCH | -| Agent prompts | 5 | 5 | MATCH | -| Telemetry | 1 | 1 | MATCH | -| Anthropic plugin | 1 | 1 | MATCH | -| Engine CLI command | 1 | 1 | MATCH | -| Skills | 11 | 11 | MATCH | -| Python engine (all modules) | 71 files | 71 files | MATCH (byte-identical) | -| Experiments | 2 benchmarks | 2 benchmarks | MATCH | -| CI/CD workflows | 4 | 4 | MATCH | -| Docs site | full | full | MATCH | -| ACP protocol | yes | yes | MATCH | -| Dockerfile | yes | yes | MATCH | -| Theme (altimate-code.json) | yes | yes | MATCH | -| PAID_CONTEXT_FEATURES.md | yes | yes | MATCH | -| Tests (bridge, ACP, telemetry, engine) | all | all | MATCH | -| Upstream merge tooling | n/a | yes | NEW (intentional) | - ---- - -## Detailed Feature-by-Feature Comparison - -### Tools — ALL 68 PRESENT - -| Tool Group | Files (main) | Files (restructure) | Status | -|-----------|:---:|:---:|:---:| -| SQL (10) | `src/tool/sql-*.ts` | `src/altimate/tools/sql-*.ts` | MOVED | -| Schema (7) | `src/tool/schema-*.ts` | `src/altimate/tools/schema-*.ts` | MOVED | -| Warehouse (5) | `src/tool/warehouse-*.ts` | `src/altimate/tools/warehouse-*.ts` | MOVED | -| dbt (4) | `src/tool/dbt-*.ts` | `src/altimate/tools/dbt-*.ts` | MOVED | -| FinOps (7) | `src/tool/finops-*.ts` | `src/altimate/tools/finops-*.ts` | MOVED | -| altimate-core (33) | `src/tool/altimate-core-*.ts` | `src/altimate/tools/altimate-core-*.ts` | MOVED | -| lineage-check (1) | `src/tool/lineage-check.ts` | `src/altimate/tools/lineage-check.ts` | MOVED | -| project-scan (1) | `src/tool/project-scan.ts` | `src/altimate/tools/project-scan.ts` | MOVED | - -### Bridge — ALL 3 PRESENT - -| File | main path | restructure path | Status | -|------|-----------|-------------------|--------| -| client.ts | `src/bridge/client.ts` | `src/altimate/bridge/client.ts` | MOVED | -| engine.ts | `src/bridge/engine.ts` | `src/altimate/bridge/engine.ts` | MOVED | -| protocol.ts | `src/bridge/protocol.ts` | `src/altimate/bridge/protocol.ts` | MOVED | - -### Agent Modes — ALL 5 PRESENT - -| Agent | main prompt | restructure prompt | Status | -|-------|-------------|-------------------|--------| -| builder | `src/agent/prompt/builder.txt` | `src/altimate/prompts/builder.txt` | MOVED | -| analyst | `src/agent/prompt/analyst.txt` | `src/altimate/prompts/analyst.txt` | MOVED | -| executive | `src/agent/prompt/executive.txt` | `src/altimate/prompts/executive.txt` | MOVED | -| migrator | `src/agent/prompt/migrator.txt` | `src/altimate/prompts/migrator.txt` | MOVED | -| validator | `src/agent/prompt/validator.txt` | `src/altimate/prompts/validator.txt` | MOVED | - -Agent registration in `agent.ts` — verified via `altimate_change` markers (3 blocks, all closed). - -### Telemetry — PRESENT -- main: `src/telemetry/index.ts` (full implementation inline) -- restructure: `src/altimate/telemetry/index.ts` (implementation) + `src/telemetry/index.ts` (re-export via marker) - -### Plugin — PRESENT -- main: `src/plugin/anthropic.ts` -- restructure: `src/altimate/plugin/anthropic.ts` - -### CLI — PRESENT -- main: `src/cli/cmd/engine.ts` -- restructure: `src/altimate/cli/engine.ts` - -### Skills — ALL 11 PRESENT -cost-report, dbt-docs, generate-tests, impact-analysis, incremental-logic, lineage-diff, medallion-patterns, model-scaffold, query-optimize, sql-translate, yaml-config - -### Python Engine — 71/71 BYTE-IDENTICAL -All Python files verified as exact copies. - -### Config Modifications (altimate_change markers) — ALL 9 FILES VERIFIED -21 marker blocks across 9 files, all properly closed: -- `src/tool/registry.ts` (2 blocks) -- `src/agent/agent.ts` (3 blocks) -- `src/config/config.ts` (1 block) -- `src/config/paths.ts` (1 block) -- `src/flag/flag.ts` (3 blocks) -- `src/global/index.ts` (1 block) -- `src/index.ts` (7 blocks) -- `src/installation/index.ts` (2 blocks) -- `src/telemetry/index.ts` (1 block) - ---- - -## Intentional Differences - -| Item | origin/main | restructure/main | Reason | -|------|-------------|-------------------|--------| -| Package directory | `packages/altimate-code/` | `packages/opencode/` | Upstream naming restored for mergability | -| `bin/altimate-code` | Separate file | Points to `bin/altimate` | Consolidated; both names work via package.json | -| `PROGRESS.md` | Present | Absent | Progress tracking doc, not a feature | -| `opencode` binary | Absent | Present | Upstream compatibility retained | -| `script/upstream/` merge tooling | Absent | Present | NEW: automated merge infrastructure | -| `src/altimate/index.ts` barrel | Absent | Present | NEW: clean import entry point | -| `src/altimate/command/discover.txt` | Absent | Present | NEW: command file | - ---- - -## New on restructure/main (Intentional Additions) - -These are infrastructure improvements added during the restructure: - -1. **`script/upstream/`** — Automated merge tooling (analyze.ts, merge.ts, transforms/, utils/) -2. **`src/altimate/index.ts`** — Barrel export for all custom modules -3. **`src/altimate/command/discover.txt`** — Custom command file -4. **`bin/altimate`** — Unified binary (replaces separate `altimate` + `altimate-code` files) - ---- - -## Conclusion - -**Every custom feature from origin/main exists on restructure/main.** The differences are: -- **Path changes** (tools/bridge/prompts moved into `src/altimate/`) -- **Package naming** (`altimate-code` → `opencode` for upstream compatibility) -- **Binary consolidation** (two files → one file, both names still work) -- **One file intentionally dropped** (`PROGRESS.md` — not a feature) -- **New infrastructure added** (merge tooling, barrel export) diff --git a/FEATURES_MAIN.md b/FEATURES_MAIN.md deleted file mode 100644 index afeb95b824..0000000000 --- a/FEATURES_MAIN.md +++ /dev/null @@ -1,513 +0,0 @@ -# Feature Inventory: origin/main - -## Summary - -- **Total custom features**: 48 -- **Total custom files**: ~340 (across tools, bridge, engine, agent, skills, CI/CD, docs, tests) -- **Forked package name**: `@altimateai/altimate-code` (binaries: `altimate`, `altimate-code`) -- **Core addition**: A full Python sidecar engine (`altimate-engine`) connected to the CLI via JSON-RPC over stdio (the "bridge"), enabling data engineering capabilities (SQL execution, schema introspection, lineage, finops, dbt, PII detection). - ---- - -## Features by Category - ---- - -### Tools (38 features) - -All custom tools live in `packages/altimate-code/src/tool/`. They are grouped below by functional area. - ---- - -#### 1. SQL Core Tools (8 tools) - -- **Files**: - - `packages/altimate-code/src/tool/sql-analyze.ts` - - `packages/altimate-code/src/tool/sql-autocomplete.ts` - - `packages/altimate-code/src/tool/sql-diff.ts` - - `packages/altimate-code/src/tool/sql-execute.ts` - - `packages/altimate-code/src/tool/sql-explain.ts` - - `packages/altimate-code/src/tool/sql-fix.ts` - - `packages/altimate-code/src/tool/sql-format.ts` - - `packages/altimate-code/src/tool/sql-optimize.ts` - - `packages/altimate-code/src/tool/sql-rewrite.ts` - - `packages/altimate-code/src/tool/sql-translate.ts` -- **Description**: Core SQL tools that execute against connected data warehouses via the Python bridge. `sql_execute` runs queries and returns tabular results. `sql_analyze` performs static anti-pattern detection (SELECT *, cartesian joins, missing LIMIT, etc.). `sql_autocomplete` provides schema-aware completion from the indexed cache. `sql_diff` compares two SQL queries with unified diff output. `sql_explain` runs EXPLAIN/EXPLAIN ANALYZE against a warehouse. `sql_fix` diagnoses SQL errors and suggests corrections. `sql_format` formats SQL with dialect-aware indentation. `sql_optimize` applies sqlglot optimizer passes and detects anti-patterns. `sql_rewrite` performs deterministic AST transforms (SELECT * expansion, sargable predicates, large IN lists to CTE). `sql_translate` converts SQL between dialects (Snowflake, BigQuery, Postgres, MySQL, TSQL, Redshift, DuckDB, etc.). -- **Category**: Tool - ---- - -#### 2. Schema Tools (6 tools) - -- **Files**: - - `packages/altimate-code/src/tool/schema-cache-status.ts` - - `packages/altimate-code/src/tool/schema-detect-pii.ts` - - `packages/altimate-code/src/tool/schema-diff.ts` - - `packages/altimate-code/src/tool/schema-index.ts` - - `packages/altimate-code/src/tool/schema-inspect.ts` - - `packages/altimate-code/src/tool/schema-search.ts` - - `packages/altimate-code/src/tool/schema-tags.ts` -- **Description**: Schema metadata tools backed by the Python bridge. `schema_index` crawls a connected warehouse and writes a local cache. `schema_inspect` describes a specific table (columns, types, constraints, row count). `schema_search` performs natural-language search over indexed metadata. `schema_cache_status` shows cache state (tables, columns, last refresh). `schema_detect_pii` scans column names for PII patterns (SSN, email, phone, credit card, etc.). `schema_diff` compares two SQL model versions for breaking column-level changes using sqlglot (no warehouse needed). `schema_tags` lists Snowflake object tags (tag name, value, object type). -- **Category**: Tool - ---- - -#### 3. Warehouse Connection Tools (5 tools) - -- **Files**: - - `packages/altimate-code/src/tool/warehouse-add.ts` - - `packages/altimate-code/src/tool/warehouse-discover.ts` - - `packages/altimate-code/src/tool/warehouse-list.ts` - - `packages/altimate-code/src/tool/warehouse-remove.ts` - - `packages/altimate-code/src/tool/warehouse-test.ts` -- **Description**: Warehouse connection lifecycle management. `warehouse_add` stores connection credentials (in OS keyring when available, metadata in connections.json). `warehouse_list` shows all configured connections. `warehouse_test` verifies connectivity. `warehouse_remove` deletes a connection. `warehouse_discover` auto-detects database containers running in Docker (PostgreSQL, MySQL, SQL Server) by inspecting port mappings and environment variables. -- **Category**: Tool - ---- - -#### 4. dbt Integration Tools (4 tools) - -- **Files**: - - `packages/altimate-code/src/tool/dbt-lineage.ts` - - `packages/altimate-code/src/tool/dbt-manifest.ts` - - `packages/altimate-code/src/tool/dbt-profiles.ts` - - `packages/altimate-code/src/tool/dbt-run.ts` -- **Description**: dbt project integration via the Python bridge. `dbt_run` executes dbt CLI commands (run, test, build, compile, seed, snapshot) and captures stdout/stderr. `dbt_manifest` parses manifest.json to extract models, sources, tests, seeds, and dependency graph. `dbt_lineage` computes column-level lineage for a dbt model using the Rust-based altimate-core engine (reads compiled SQL from manifest). `dbt_profiles` discovers and parses ~/.dbt/profiles.yml to extract warehouse connection configs (Snowflake, BigQuery, Databricks, Postgres, Redshift, MySQL, DuckDB). -- **Category**: Tool - ---- - -#### 5. Lineage Tool (1 tool) - -- **Files**: - - `packages/altimate-code/src/tool/lineage-check.ts` -- **Description**: `lineage_check` traces column-level data flow through a SQL query using the Rust-based altimate-core engine. Accepts schema context (table-to-column mappings) for accurate source-to-output column tracking. Used by analyst, validator, and migrator agents. -- **Category**: Tool - ---- - -#### 6. FinOps Tools (7 tools) - -- **Files**: - - `packages/altimate-code/src/tool/finops-analyze-credits.ts` - - `packages/altimate-code/src/tool/finops-expensive-queries.ts` - - `packages/altimate-code/src/tool/finops-formatting.ts` - - `packages/altimate-code/src/tool/finops-query-history.ts` - - `packages/altimate-code/src/tool/finops-role-access.ts` - - `packages/altimate-code/src/tool/finops-unused-resources.ts` - - `packages/altimate-code/src/tool/finops-warehouse-advice.ts` -- **Description**: Snowflake cost and governance analytics tools. `finops_analyze_credits` queries SNOWFLAKE.ACCOUNT_USAGE to break down credit consumption by warehouse with daily trends. `finops_expensive_queries` identifies the most resource-intensive queries. `finops_query_history` retrieves query execution history with filters. `finops_warehouse_advice` analyzes warehouse sizing, load, and queue times to suggest right-sizing. `finops_unused_resources` finds unused tables, warehouses, and roles. `finops_role_access` inspects role grants and privilege hierarchy (role_grants, role_hierarchy, user_roles sub-operations). `finops-formatting.ts` is a shared utility for byte/query formatting. Supports the executive agent's business-language cost reporting. -- **Category**: Tool - ---- - -#### 7. altimate-core Tools — Phase 1: Basic SQL Analysis (6 tools) - -These tools call the Rust-based `altimate-core` library (from Altimate's own package) via the Python bridge. - -- **Files**: - - `packages/altimate-code/src/tool/altimate-core-validate.ts` - - `packages/altimate-code/src/tool/altimate-core-lint.ts` - - `packages/altimate-code/src/tool/altimate-core-safety.ts` - - `packages/altimate-code/src/tool/altimate-core-transpile.ts` - - `packages/altimate-code/src/tool/altimate-core-check.ts` - - `packages/altimate-code/src/tool/altimate-core-is-safe.ts` -- **Description**: `altimate_core_validate` validates SQL syntax and schema references. `altimate_core_lint` detects anti-patterns (NULL comparisons, implicit casts, unused CTEs). `altimate_core_safety` scans for injection patterns and destructive statements (DROP, TRUNCATE). `altimate_core_transpile` converts SQL between dialects using the Rust engine. `altimate_core_check` runs the full pipeline (validate + lint + safety + PII) in a single call. `altimate_core_is_safe` returns a quick boolean safety verdict. -- **Category**: Tool - ---- - -#### 8. altimate-core Tools — Phase 2: Advanced Analysis (6 tools) - -- **Files**: - - `packages/altimate-code/src/tool/altimate-core-fix.ts` - - `packages/altimate-code/src/tool/altimate-core-policy.ts` - - `packages/altimate-code/src/tool/altimate-core-semantics.ts` - - `packages/altimate-code/src/tool/altimate-core-testgen.ts` - - `packages/altimate-code/src/tool/altimate-core-equivalence.ts` - - `packages/altimate-code/src/tool/altimate-core-migration.ts` -- **Description**: `altimate_core_fix` auto-fixes SQL errors using fuzzy matching and iterative re-validation. `altimate_core_policy` checks SQL against YAML-based governance guardrails (allowed tables, forbidden ops). `altimate_core_semantics` detects logical issues (cartesian products, wrong JOINs, NULL misuse, type mismatches). `altimate_core_testgen` generates automated SQL test cases (boundary values, NULL handling, edge cases). `altimate_core_equivalence` checks semantic equivalence of two queries. `altimate_core_migration` analyzes DDL migration safety (data loss, type narrowing, missing defaults). -- **Category**: Tool - ---- - -#### 9. altimate-core Tools — Phase 3: Schema & Lineage Intelligence (13 tools) - -- **Files**: - - `packages/altimate-code/src/tool/altimate-core-classify-pii.ts` - - `packages/altimate-code/src/tool/altimate-core-column-lineage.ts` - - `packages/altimate-code/src/tool/altimate-core-compare.ts` - - `packages/altimate-code/src/tool/altimate-core-complete.ts` - - `packages/altimate-code/src/tool/altimate-core-correct.ts` - - `packages/altimate-code/src/tool/altimate-core-export-ddl.ts` - - `packages/altimate-code/src/tool/altimate-core-extract-metadata.ts` - - `packages/altimate-code/src/tool/altimate-core-fingerprint.ts` - - `packages/altimate-code/src/tool/altimate-core-format.ts` - - `packages/altimate-code/src/tool/altimate-core-grade.ts` - - `packages/altimate-code/src/tool/altimate-core-import-ddl.ts` - - `packages/altimate-code/src/tool/altimate-core-introspection-sql.ts` - - `packages/altimate-code/src/tool/altimate-core-optimize-context.ts` - - `packages/altimate-code/src/tool/altimate-core-optimize-for-query.ts` - - `packages/altimate-code/src/tool/altimate-core-parse-dbt.ts` - - `packages/altimate-code/src/tool/altimate-core-prune-schema.ts` - - `packages/altimate-code/src/tool/altimate-core-query-pii.ts` - - `packages/altimate-code/src/tool/altimate-core-resolve-term.ts` - - `packages/altimate-code/src/tool/altimate-core-rewrite.ts` - - `packages/altimate-code/src/tool/altimate-core-schema-diff.ts` - - `packages/altimate-code/src/tool/altimate-core-track-lineage.ts` -- **Description**: Advanced schema and lineage intelligence tools. - - `altimate_core_classify_pii`: Classifies PII columns in a schema by name patterns and data types. - - `altimate_core_column_lineage`: Traces schema-aware column lineage through a query (requires API key init). - - `altimate_core_compare`: Structurally compares two SQL queries (tables, joins, filters, projections). - - `altimate_core_complete`: Cursor-aware SQL completion (table names, column names, functions, keywords). - - `altimate_core_correct`: Iterative propose-verify-refine correction loop for SQL. - - `altimate_core_export_ddl`: Exports YAML/JSON schema as CREATE TABLE DDL statements. - - `altimate_core_extract_metadata`: Extracts tables, columns, functions, CTEs from a SQL query. - - `altimate_core_fingerprint`: Computes SHA-256 fingerprint of a schema for cache invalidation. - - `altimate_core_format`: Fast Rust-based SQL formatter with dialect-aware keyword casing. - - `altimate_core_grade`: Grades SQL quality A-F (readability, performance, correctness, best practices). - - `altimate_core_import_ddl`: Converts CREATE TABLE DDL into YAML schema definition. - - `altimate_core_introspection_sql`: Generates INFORMATION_SCHEMA queries for a target database type. - - `altimate_core_optimize_context`: Applies progressive schema disclosure (5 levels) to reduce LLM context size. - - `altimate_core_optimize_for_query`: Prunes schema to only tables/columns referenced by a specific query. - - `altimate_core_parse_dbt`: Parses a dbt project directory (models, sources, tests, seeds) via Rust engine. - - `altimate_core_prune_schema`: Filters schema to SQL-referenced elements only. - - `altimate_core_query_pii`: Checks if a SQL query accesses PII-classified columns and reports exposure risk. - - `altimate_core_resolve_term`: Fuzzy-maps business glossary terms to actual table/column names. - - `altimate_core_rewrite`: Suggests concrete query optimization rewrites. - - `altimate_core_schema_diff`: Diffs two schemas (altimate-core variant, breaking change detection). - - `altimate_core_track_lineage`: Builds a combined lineage graph across multiple SQL queries. -- **Category**: Tool - ---- - -### Bridge (1 feature) - -#### 10. Python Bridge (JSON-RPC over stdio) - -- **Files**: - - `packages/altimate-code/src/bridge/client.ts` - - `packages/altimate-code/src/bridge/engine.ts` - - `packages/altimate-code/src/bridge/protocol.ts` -- **Description**: A typed JSON-RPC 2.0 over stdio bridge between the TypeScript CLI and the Python altimate-engine sidecar. `protocol.ts` defines the full type-safe contract for all ~60 RPC methods (params and result types). `client.ts` manages the child process lifecycle (spawn, restart up to 2 times on failure, 30-second call timeout), serializes requests, deserializes responses, buffers multi-line JSON, and records telemetry for every call. `engine.ts` handles bootstrapping: downloads the `uv` Python package manager, creates an isolated venv, installs `altimate-engine` from PyPI, and maintains a `manifest.json` with version metadata. A mutex prevents concurrent installs from corrupting state. All bridge calls are tracked in telemetry with method name, status, and duration. -- **Category**: Bridge - ---- - -### Prompt/Agent (1 feature) - -#### 11. Custom Agent Modes (5 agents) - -- **Files**: - - `packages/altimate-code/src/agent/agent.ts` - - `packages/altimate-code/src/agent/prompt/analyst.txt` - - `packages/altimate-code/src/agent/prompt/builder.txt` - - `packages/altimate-code/src/agent/prompt/executive.txt` - - `packages/altimate-code/src/agent/prompt/migrator.txt` - - `packages/altimate-code/src/agent/prompt/validator.txt` -- **Description**: Five domain-specific agent modes tailored for data engineering workflows, all registered as `native: true` primary agents: - - **builder**: Full read/write access. Specializes in creating/modifying dbt models, SQL files, YAML configs. Encourages use of `sql_analyze`, `schema_inspect`, `lineage_check`. - - **analyst**: Read-only data exploration. Restricted permission set — can run SELECT queries, validate SQL, inspect schemas, check lineage, browse query history. Cannot modify files or run destructive SQL. - - **executive**: Same analytical capabilities as analyst but communicates exclusively in business terms. Never shows SQL, column names in backticks, or engineering jargon. Translates all findings into business impact (revenue at risk, cost, compliance exposure). Designed for slide decks and executive emails. - - **validator**: Read-only quality and integrity verification. Focuses on `sql_analyze` (18 anti-pattern checks), `altimate_core_validate`, `lineage_check`, and the `/lineage-diff` skill. Reports issues with severity levels. - - **migrator**: Cross-warehouse SQL migration. Read/write access. Specializes in dialect conversion, source/target schema comparison, lineage integrity verification post-migration. -- **Category**: Prompt/Agent - ---- - -### Telemetry (1 feature) - -#### 12. Azure Application Insights Telemetry - -- **Files**: - - `packages/altimate-code/src/telemetry/index.ts` - - `packages/altimate-code/test/telemetry/telemetry.test.ts` - - `docs/docs/configure/telemetry.md` -- **Description**: A buffered, batched telemetry system sending structured events to Azure Application Insights via the `/v2/track` endpoint. Events are typed and include: `session_start`, `session_end`, `generation`, `tool_call`, `bridge_call`, `error`, `command`, `context_overflow_recovered`, `compaction_triggered`, `tool_outputs_pruned`, `auth_login`, `auth_logout`, `mcp_server_status`, `provider_error`, `engine_started`, `engine_error`, `upgrade_attempted`, `session_forked`, `permission_denied`, `doom_loop_detected`, `environment_census`, `context_utilization`, `agent_outcome`, `error_recovered`, `mcp_server_census`. Buffered in-memory (max 200 events), flushed every 5 seconds and on shutdown. User email is SHA-256 hashed before transmission. Can be disabled via `ALTIMATE_TELEMETRY_DISABLED=true` env var or `telemetry.disabled` config key. The instrumentation key is intentionally hardcoded (public telemetry pattern). Overridable via `APPLICATIONINSIGHTS_CONNECTION_STRING` for dev/testing. -- **Category**: Telemetry - ---- - -### CLI (3 features) - -#### 13. Engine CLI Command - -- **Files**: - - `packages/altimate-code/src/cli/cmd/engine.ts` -- **Description**: A `altimate-code engine` top-level command with three subcommands: `status` (shows uv installation state, Python version, engine version, CLI version, install timestamp), `reset` (removes the engine directory and reinstalls from scratch), and `path` (prints the engine directory path). Used for debugging the Python sidecar setup. -- **Category**: CLI - ---- - -#### 14. Custom Binary Names and Launcher Scripts - -- **Files**: - - `packages/altimate-code/bin/altimate` - - `packages/altimate-code/bin/altimate-code` - - `packages/altimate-code/package.json` (bin entries: `altimate` and `altimate-code`) -- **Description**: The package exposes two binary names (`altimate` and `altimate-code`) under `@altimateai` npm scope. The launcher scripts perform platform/architecture detection (darwin/linux/win32, x64/arm64, musl vs glibc on Linux, AVX2 support for x64 baseline fallback) and locate the correct pre-built native binary from the appropriate scoped package (e.g., `@altimateai/altimate-code-darwin-arm64`). Supports `ALTIMATE_CODE_BIN_PATH` env var override and a cached `.altimate-code` binary in the bin directory. -- **Category**: CLI - ---- - -#### 15. Feature Flags (ALTIMATE_CLI_* env vars) - -- **Files**: - - `packages/altimate-code/src/flag/flag.ts` -- **Description**: A comprehensive set of environment variable feature flags all prefixed `ALTIMATE_CLI_*`. Custom flags added on top of upstream include: `ALTIMATE_CLI_DISABLE_CLAUDE_CODE` (disables Claude Code integration), `ALTIMATE_CLI_DISABLE_CLAUDE_CODE_PROMPT`, `ALTIMATE_CLI_DISABLE_CLAUDE_CODE_SKILLS`, `ALTIMATE_CLI_DISABLE_EXTERNAL_SKILLS`, `ALTIMATE_CLI_ENABLE_QUESTION_TOOL`, plus all existing upstream flags renamed to the Altimate prefix convention. Dynamic getters via `Object.defineProperty` ensure `ALTIMATE_CLI_DISABLE_PROJECT_CONFIG`, `ALTIMATE_CLI_CONFIG_DIR`, and `ALTIMATE_CLI_CLIENT` are evaluated at access time rather than module load time. -- **Category**: CLI - ---- - -### Python Engine (1 feature) - -#### 16. altimate-engine Python Package - -- **Files** (complete package at `packages/altimate-engine/`): - - `src/altimate_engine/server.py` — JSON-RPC stdio server (entry point) - - `src/altimate_engine/connections.py` — Connection registry - - `src/altimate_engine/credential_store.py` — OS keyring integration - - `src/altimate_engine/models.py` — Pydantic request/response models (~60 RPC types) - - `src/altimate_engine/connectors/` — Database connectors: - - `base.py`, `bigquery.py`, `databricks.py`, `duckdb.py`, `mysql.py`, `postgres.py`, `redshift.py`, `snowflake.py`, `sqlserver.py` - - `src/altimate_engine/dbt/` — dbt integration: - - `lineage.py`, `manifest.py`, `profiles.py`, `runner.py` - - `src/altimate_engine/docker_discovery.py` — Docker container detection - - `src/altimate_engine/finops/` — Cost analytics: - - `credit_analyzer.py`, `query_history.py`, `role_access.py`, `unused_resources.py`, `warehouse_advisor.py` - - `src/altimate_engine/local/` — Local testing: - - `schema_sync.py`, `test_local.py` - - `src/altimate_engine/schema/` — Schema intelligence: - - `cache.py`, `inspector.py`, `pii_detector.py`, `tags.py` - - `src/altimate_engine/sql/` — SQL processing: - - `autocomplete.py`, `diff.py`, `executor.py`, `explainer.py`, `guard.py` - - `src/altimate_engine/ssh_tunnel.py` — SSH tunnel support - - `tests/` — 20+ test files covering all modules - - `pyproject.toml` — Package definition (name: `altimate-engine`, version: `0.2.0`) -- **Description**: A Python sidecar process exposing ~60 JSON-RPC methods over stdio. Implements: SQL execution across 8 warehouse connectors (Snowflake, BigQuery, Databricks, DuckDB, MySQL, PostgreSQL, Redshift, SQL Server), schema inspection and caching, SQL analysis/formatting/optimization/translation via sqlglot, PII column detection, finops analytics (Snowflake ACCOUNT_USAGE queries), dbt project parsing and command execution, Docker-based database discovery, SSH tunnel support for remote databases, OS keyring credential storage, and integration with the Rust-based `altimate-core` PyPI package for advanced lineage/validation. Installed in an isolated managed venv via the `uv` package manager (see Bridge feature). -- **Category**: Python Engine - ---- - -### Skills (1 feature) - -#### 17. Data Engineering Skills (11 skills) - -- **Files** (all in `.altimate-code/skills/`): - - `.altimate-code/skills/cost-report/SKILL.md` - - `.altimate-code/skills/dbt-docs/SKILL.md` - - `.altimate-code/skills/generate-tests/SKILL.md` - - `.altimate-code/skills/impact-analysis/SKILL.md` - - `.altimate-code/skills/incremental-logic/SKILL.md` - - `.altimate-code/skills/lineage-diff/SKILL.md` - - `.altimate-code/skills/medallion-patterns/SKILL.md` - - `.altimate-code/skills/model-scaffold/SKILL.md` - - `.altimate-code/skills/query-optimize/SKILL.md` - - `.altimate-code/skills/sql-translate/SKILL.md` - - `.altimate-code/skills/yaml-config/SKILL.md` -- **Description**: Eleven bundled skills (slash commands) for data engineering workflows: - - **cost-report**: Analyze Snowflake query costs and identify optimization opportunities. - - **dbt-docs**: Generate or improve dbt model documentation (column descriptions, model descriptions, doc blocks). - - **generate-tests**: Generate dbt tests for a model by inspecting schema and SQL, producing schema.yml test definitions. - - **impact-analysis**: Analyze downstream impact of changes to a dbt model by combining column-level lineage with the dbt dependency graph. - - **incremental-logic**: Add or fix incremental materialization logic in dbt models (is_incremental(), unique keys, merge strategies). - - **lineage-diff**: Compare column-level lineage between two SQL query versions to show added, removed, and changed data flow edges. - - **medallion-patterns**: Apply medallion architecture (bronze/silver/gold) patterns to organize dbt models into clean data layers. - - **model-scaffold**: Scaffold a new dbt model following staging/intermediate/mart patterns with proper naming, materialization, and structure. - - **query-optimize**: Analyze and optimize SQL queries for better performance. - - **sql-translate**: Translate SQL queries between database dialects. - - **yaml-config**: Generate dbt YAML configuration files (sources.yml, schema.yml, properties.yml) from warehouse schema or existing models. -- **Category**: Skill - ---- - -### CI/CD (3 features) - -#### 18. CI Workflow (TypeScript + Python tests) - -- **Files**: - - `.github/workflows/ci.yml` -- **Description**: Runs on push/PR to main. Three parallel jobs: (1) TypeScript — installs Bun 1.3.9 with cache, configures git for tests, runs `bun test` in `packages/altimate-code`; (2) Lint — installs ruff 0.9.10, runs `ruff check src` on `packages/altimate-engine`; (3) Python matrix — tests the Python engine across Python versions. -- **Category**: CI/CD - ---- - -#### 19. Release Workflow (Multi-platform binary builds) - -- **Files**: - - `.github/workflows/release.yml` -- **Description**: Triggered on `v*` tags. Builds native binaries for linux, darwin, and win32 using Bun's cross-compilation. Uploads to GitHub Releases. Then publishes to npm under `@altimateai` scope (not the upstream `opencode` scope). Injects `ALTIMATE_CLI_VERSION`, `ALTIMATE_CLI_CHANNEL`, `ALTIMATE_CLI_RELEASE`, and `GH_REPO=AltimateAI/altimate-code` at build time. -- **Category**: CI/CD - ---- - -#### 20. Publish Engine Workflow (PyPI) - -- **Files**: - - `.github/workflows/publish-engine.yml` -- **Description**: Triggered on `engine-v*` tags. Builds the `altimate-engine` Python package with `python -m build` (hatchling backend) and publishes to PyPI using `pypa/gh-action-pypi-publish` with OIDC trusted publishing. Allows skipping existing versions. This is what the bridge's `ensureEngine()` function installs in the managed venv. -- **Category**: CI/CD - ---- - -### Docs (1 feature) - -#### 21. Data Engineering Documentation Site - -- **Files** (under `docs/`): - - `docs/mkdocs.yml` — MkDocs Material configuration - - `docs/docs/data-engineering/agent-modes.md` - - `docs/docs/data-engineering/guides/cost-optimization.md` - - `docs/docs/data-engineering/guides/migration.md` - - `docs/docs/data-engineering/guides/using-with-claude-code.md` - - `docs/docs/data-engineering/guides/using-with-codex.md` - - `docs/docs/data-engineering/tools/dbt-tools.md` - - `docs/docs/data-engineering/tools/finops-tools.md` - - `docs/docs/data-engineering/tools/lineage-tools.md` - - `docs/docs/data-engineering/tools/schema-tools.md` - - `docs/docs/data-engineering/tools/sql-tools.md` - - `docs/docs/data-engineering/tools/warehouse-tools.md` - - `docs/docs/configure/telemetry.md` - - Plus top-level project docs: `docs/docs/index.md`, `docs/docs/getting-started.md`, `docs/docs/security-faq.md`, `docs/docs/network.md`, `docs/docs/troubleshooting.md`, `docs/docs/windows-wsl.md` - - `docs/docs/assets/images/altimate-code-banner.png`, `favicon.png`, `logo.png` - - `.github/workflows/docs.yml` -- **Description**: A full MkDocs Material documentation site for the altimate-code product. Includes a dedicated "Data Engineering" section covering: agent modes (builder, analyst, executive, validator, migrator), workflow guides (cost optimization, migration, integration with Claude Code and Codex), and reference pages for all custom tool categories (SQL, schema, warehouse, dbt, lineage, finops). Also covers telemetry configuration. -- **Category**: Docs - ---- - -### Tests (1 feature) - -#### 22. Custom Feature Tests - -- **Files**: - - `packages/altimate-code/test/bridge/client.test.ts` - - `packages/altimate-code/test/bridge/engine.test.ts` - - `packages/altimate-code/test/acp/agent-interface.test.ts` - - `packages/altimate-code/test/acp/event-subscription.test.ts` - - `packages/altimate-code/test/telemetry/telemetry.test.ts` - - `packages/altimate-engine/tests/test_autocomplete.py` - - `packages/altimate-engine/tests/test_connections.py` - - `packages/altimate-engine/tests/test_connectors.py` - - `packages/altimate-engine/tests/test_credential_store.py` - - `packages/altimate-engine/tests/test_dbt_profiles.py` - - `packages/altimate-engine/tests/test_diff.py` - - `packages/altimate-engine/tests/test_docker_discovery.py` - - `packages/altimate-engine/tests/test_enterprise_connectors.py` - - `packages/altimate-engine/tests/test_env_detect.py` - - `packages/altimate-engine/tests/test_executor.py` - - `packages/altimate-engine/tests/test_explainer.py` - - `packages/altimate-engine/tests/test_finops.py` - - `packages/altimate-engine/tests/test_guard.py` - - `packages/altimate-engine/tests/test_guard_new.py` - - `packages/altimate-engine/tests/test_local.py` - - `packages/altimate-engine/tests/test_manifest.py` - - `packages/altimate-engine/tests/test_pii_detector.py` - - `packages/altimate-engine/tests/test_schema_cache.py` - - `packages/altimate-engine/tests/test_server.py` - - `packages/altimate-engine/tests/test_server_guard.py` - - `packages/altimate-engine/tests/test_server_guard_new.py` - - `packages/altimate-engine/tests/test_ssh_tunnel.py` - - `packages/altimate-engine/tests/test_tags.py` -- **Description**: Tests covering the custom bridge client (Python process management, restart behavior, timeout handling), engine bootstrapping, ACP protocol compliance, telemetry buffering/flushing, and the entire Python engine test suite (SQL execution, autocomplete, schema caching, PII detection, finops queries, Docker discovery, SSH tunnels, dbt manifest parsing, SQL guard/safety). -- **Category**: Test - ---- - -### Other / Platform (3 features) - -#### 23. ACP (Agent Client Protocol) Server - -- **Files**: - - `packages/altimate-code/src/acp/agent.ts` - - `packages/altimate-code/src/acp/session.ts` - - `packages/altimate-code/src/acp/types.ts` - - `packages/altimate-code/src/acp/README.md` - - `packages/altimate-code/src/cli/cmd/acp.ts` - - `packages/altimate-code/test/acp/agent-interface.test.ts` - - `packages/altimate-code/test/acp/event-subscription.test.ts` -- **Description**: A protocol-compliant implementation of the Agent Client Protocol (ACP) using `@agentclientprotocol/sdk`. Exposes altimate-code as an ACP-compatible agent server, enabling integration with clients such as Zed. Implements: `initialize` with capability negotiation, `session/new` (creates internal sessions), `session/load` (basic resume), `session/prompt` (processes messages, returns responses). Session state management maps ACP sessions to internal altimate-code sessions with working directory context. The ACP server starts via `altimate-code acp` CLI command. `ALTIMATE_CLI_ENABLE_QUESTION_TOOL` env var enables the question tool for ACP clients that support interactive prompts. -- **Category**: Other - ---- - -#### 24. Rebranded Package Identity - -- **Files**: - - `packages/altimate-code/package.json` (name: `@altimateai/altimate-code`) - - `packages/altimate-code/bin/altimate` - - `packages/altimate-code/bin/altimate-code` - - `README.md` (root) - - `CHANGELOG.md` - - `CODE_OF_CONDUCT.md` - - `CONTRIBUTING.md` - - `RELEASING.md` - - `SECURITY.md` - - `PROGRESS.md` - - `packages/altimate-code/AGENTS.md` - - `packages/altimate-code/Dockerfile` - - `packages/altimate-code/src/cli/cmd/tui/context/theme/altimate-code.json` -- **Description**: Complete rebranding of the upstream `opencode` project to `altimate-code` under the `@altimateai` npm scope. Includes a custom Altimate Code TUI theme (`altimate-code.json`), a Dockerfile for containerized deployment, updated project governance documents (CODE_OF_CONDUCT, CONTRIBUTING, RELEASING, SECURITY), and a `PROGRESS.md` tracking the upstream merge state. -- **Category**: Other - ---- - -#### 25. Local Schema Sync and SQL Testing - -- **Files**: - - `packages/altimate-engine/src/altimate_engine/local/schema_sync.py` - - `packages/altimate-engine/src/altimate_engine/local/test_local.py` - - `packages/altimate-engine/tests/test_local.py` -- **Description**: A local-first SQL testing workflow accessible via the bridge (`local.schema_sync` and `local.test`). `schema_sync` pulls schema metadata from a live warehouse into a local YAML file (tables, columns, sample rows). `local.test` executes a SQL query against the locally synced schema using DuckDB as a local executor — with optional dialect transpilation via sqlglot. This enables SQL testing without a live warehouse connection, useful for CI environments. -- **Category**: Other - ---- - -## Consolidated Tool Count by Type - -| Category | Tool Names | -|---|---| -| SQL Core | sql_execute, sql_analyze, sql_autocomplete, sql_diff, sql_explain, sql_fix, sql_format, sql_optimize, sql_rewrite, sql_translate | -| Schema | schema_cache_status, schema_detect_pii, schema_diff, schema_index, schema_inspect, schema_search, schema_tags | -| Warehouse | warehouse_add, warehouse_discover, warehouse_list, warehouse_remove, warehouse_test | -| dbt | dbt_lineage, dbt_manifest, dbt_profiles, dbt_run | -| Lineage | lineage_check | -| FinOps | finops_analyze_credits, finops_expensive_queries, finops_query_history, finops_role_grants, finops_role_hierarchy, finops_unused_resources, finops_user_roles, finops_warehouse_advice | -| altimate-core Phase 1 | altimate_core_validate, altimate_core_lint, altimate_core_safety, altimate_core_transpile, altimate_core_check, altimate_core_is_safe | -| altimate-core Phase 2 | altimate_core_fix, altimate_core_policy, altimate_core_semantics, altimate_core_testgen, altimate_core_equivalence, altimate_core_migration | -| altimate-core Phase 3 | altimate_core_classify_pii, altimate_core_column_lineage, altimate_core_compare, altimate_core_complete, altimate_core_correct, altimate_core_export_ddl, altimate_core_extract_metadata, altimate_core_fingerprint, altimate_core_format, altimate_core_grade, altimate_core_import_ddl, altimate_core_introspection_sql, altimate_core_optimize_context, altimate_core_optimize_for_query, altimate_core_parse_dbt, altimate_core_prune_schema, altimate_core_query_pii, altimate_core_resolve_term, altimate_core_rewrite, altimate_core_schema_diff, altimate_core_track_lineage | - -**Total custom tools: ~56 tool functions across 38 TypeScript tool files** - ---- - -## Bridge RPC Method Registry (complete) - -The bridge (`packages/altimate-code/src/bridge/protocol.ts`) defines 61 typed RPC methods: - -``` -sql.execute, sql.analyze, sql.optimize, sql.translate, sql.explain, -sql.format, sql.fix, sql.autocomplete, sql.diff, sql.rewrite, sql.schema_diff -schema.inspect, schema.index, schema.search, schema.cache_status, -schema.detect_pii, schema.tags, schema.tags_list -lineage.check -dbt.run, dbt.manifest, dbt.lineage, dbt.profiles -warehouse.list, warehouse.test, warehouse.add, warehouse.remove, warehouse.discover -finops.query_history, finops.analyze_credits, finops.expensive_queries, -finops.warehouse_advice, finops.unused_resources, finops.role_grants, -finops.role_hierarchy, finops.user_roles -local.schema_sync, local.test -altimate_core.validate, altimate_core.lint, altimate_core.safety, -altimate_core.transpile, altimate_core.explain, altimate_core.check, -altimate_core.fix, altimate_core.policy, altimate_core.semantics, altimate_core.testgen, -altimate_core.equivalence, altimate_core.migration, altimate_core.schema_diff, -altimate_core.rewrite, altimate_core.correct, altimate_core.grade, -altimate_core.classify_pii, altimate_core.query_pii, altimate_core.resolve_term, -altimate_core.column_lineage, altimate_core.track_lineage, -altimate_core.format, altimate_core.metadata, altimate_core.compare, -altimate_core.complete, altimate_core.optimize_context, altimate_core.optimize_for_query, -altimate_core.prune_schema, altimate_core.import_ddl, altimate_core.export_ddl, -altimate_core.fingerprint, altimate_core.introspection_sql, altimate_core.parse_dbt, -altimate_core.is_safe -ping -``` diff --git a/FEATURES_RESTRUCTURE.md b/FEATURES_RESTRUCTURE.md deleted file mode 100644 index bcc79ced81..0000000000 --- a/FEATURES_RESTRUCTURE.md +++ /dev/null @@ -1,550 +0,0 @@ -# Feature Inventory: restructure/main - -## Summary - -- **Total custom features**: 60 -- **Total custom files**: ~244 (altimate-specific) across the entire branch -- **Product name**: altimate-code (fork of opencode, re-branded as a data engineering platform) -- **Binary names added**: `altimate`, `altimate-code` (alongside upstream `opencode`) -- **Core architecture**: TypeScript CLI (opencode fork) + Python engine sidecar (`altimate-engine`) communicating over JSON-RPC via stdio - ---- - -## Features by Category - ---- - -### Bridge (3 features) - -The bridge subsystem connects the TypeScript CLI to the Python `altimate-engine` sidecar via JSON-RPC over stdio. - -#### 1. Bridge Client (JSON-RPC over stdio) -- **Files**: `packages/opencode/src/altimate/bridge/client.ts` -- **Description**: Spawns the Python `altimate-engine` sidecar as a child process and communicates with it via newline-delimited JSON-RPC. Handles timeouts (30s), automatic restart on crash (up to 2 restarts), pending request tracking, and per-call telemetry instrumentation. -- **Category**: Bridge - -#### 2. Bridge Engine Bootstrap -- **Files**: `packages/opencode/src/altimate/bridge/engine.ts` -- **Description**: Downloads and manages the `uv` Python package manager, creates an isolated Python venv, installs the `altimate-engine` PyPI package at the pinned version embedded at build time, and stores a manifest file with version metadata. Supports cross-platform (macOS arm64/x64, Linux arm64/x64, Windows x64). Provides `ensureEngine()`, `engineStatus()`, `resetEngine()`, and `enginePythonPath()` functions. Uses a mutex to prevent concurrent install races. -- **Category**: Bridge - -#### 3. Bridge Protocol (RPC Contract) -- **Files**: `packages/opencode/src/altimate/bridge/protocol.ts` -- **Description**: Complete TypeScript type definitions for all ~70 JSON-RPC methods between the CLI and Python engine. Defines request/response types for SQL, schema, lineage, dbt, warehouse, FinOps, altimate-core, and local testing namespaces. Acts as the single source of truth for the CLI↔engine interface. -- **Category**: Bridge - ---- - -### Tools (5 groups, 83 tools total) - -All custom tools live in `packages/opencode/src/altimate/tools/` and are registered in `packages/opencode/src/tool/registry.ts` via `altimate_change` markers. - -#### 4. SQL Tools (10 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/sql-analyze.ts` - - `packages/opencode/src/altimate/tools/sql-autocomplete.ts` - - `packages/opencode/src/altimate/tools/sql-diff.ts` - - `packages/opencode/src/altimate/tools/sql-execute.ts` - - `packages/opencode/src/altimate/tools/sql-explain.ts` - - `packages/opencode/src/altimate/tools/sql-fix.ts` - - `packages/opencode/src/altimate/tools/sql-format.ts` - - `packages/opencode/src/altimate/tools/sql-optimize.ts` - - `packages/opencode/src/altimate/tools/sql-rewrite.ts` - - `packages/opencode/src/altimate/tools/sql-translate.ts` -- **Description**: Comprehensive SQL developer experience tools. `sql_execute` runs queries against connected warehouses. `sql_analyze` detects anti-patterns (19 rules: SELECT *, cartesian products, correlated subqueries, missing LIMIT, etc.). `sql_optimize` suggests query optimizations. `sql_translate` converts between dialects. `sql_explain` fetches query execution plans. `sql_format` formats SQL. `sql_fix` repairs broken SQL given an error message. `sql_autocomplete` provides schema-aware completion suggestions. `sql_diff` diffs two SQL strings. `sql_rewrite` applies automated rewrites (SELECT_STAR, NON_SARGABLE, LARGE_IN_LIST rules). -- **Category**: Tool - -#### 5. Schema Tools (8 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/schema-cache-status.ts` - - `packages/opencode/src/altimate/tools/schema-detect-pii.ts` - - `packages/opencode/src/altimate/tools/schema-diff.ts` - - `packages/opencode/src/altimate/tools/schema-index.ts` - - `packages/opencode/src/altimate/tools/schema-inspect.ts` - - `packages/opencode/src/altimate/tools/schema-search.ts` - - `packages/opencode/src/altimate/tools/schema-tags.ts` (2 tools: `schema_tags`, `schema_tags_list`) -- **Description**: Schema management and discovery tools. `schema_inspect` describes a table's columns/types. `schema_index` crawls a warehouse and builds a local SQLite search index. `schema_search` performs full-text search across indexed tables and columns. `schema_cache_status` reports index freshness per warehouse. `schema_detect_pii` scans column names for PII patterns (30+ categories). `schema_diff` detects breaking schema changes between two DDL versions. `schema_tags` and `schema_tags_list` query Snowflake metadata tags. -- **Category**: Tool - -#### 6. Warehouse Tools (5 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/warehouse-add.ts` - - `packages/opencode/src/altimate/tools/warehouse-discover.ts` - - `packages/opencode/src/altimate/tools/warehouse-list.ts` - - `packages/opencode/src/altimate/tools/warehouse-remove.ts` - - `packages/opencode/src/altimate/tools/warehouse-test.ts` -- **Description**: Warehouse connection lifecycle management. `warehouse_list` enumerates configured connections. `warehouse_test` verifies connectivity. `warehouse_add` saves a new connection configuration. `warehouse_remove` deletes a connection. `warehouse_discover` scans running Docker containers and extracts database connection details (PostgreSQL, MySQL/MariaDB, SQL Server) from port mappings and environment variables. -- **Category**: Tool - -#### 7. dbt Tools (4 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/dbt-lineage.ts` - - `packages/opencode/src/altimate/tools/dbt-manifest.ts` - - `packages/opencode/src/altimate/tools/dbt-profiles.ts` - - `packages/opencode/src/altimate/tools/dbt-run.ts` -- **Description**: dbt integration tools. `dbt_run` executes dbt commands (run, test, compile, etc.) with selector support. `dbt_manifest` parses a `manifest.json` and returns models, sources, tests, snapshots, and seeds with counts. `dbt_lineage` extracts compiled SQL and column-level lineage from a specific dbt model via the manifest. `dbt_profiles` discovers database connections from `~/.dbt/profiles.yml`. -- **Category**: Tool - -#### 8. FinOps Tools (7 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/finops-analyze-credits.ts` - - `packages/opencode/src/altimate/tools/finops-expensive-queries.ts` - - `packages/opencode/src/altimate/tools/finops-formatting.ts` - - `packages/opencode/src/altimate/tools/finops-query-history.ts` - - `packages/opencode/src/altimate/tools/finops-role-access.ts` (3 tools: `finops_role_grants`, `finops_role_hierarchy`, `finops_user_roles`) - - `packages/opencode/src/altimate/tools/finops-unused-resources.ts` - - `packages/opencode/src/altimate/tools/finops-warehouse-advice.ts` -- **Description**: Cloud cost optimization and governance tools (primarily Snowflake-focused). `finops_query_history` retrieves query execution history with cost metadata. `finops_analyze_credits` analyzes daily/warehouse credit consumption trends. `finops_expensive_queries` identifies the costliest queries over a time window. `finops_warehouse_advice` provides warehouse sizing recommendations. `finops_unused_resources` identifies stale tables and idle warehouses. `finops_role_grants`, `finops_role_hierarchy`, and `finops_user_roles` analyze Snowflake RBAC structure. -- **Category**: Tool - -#### 9. altimate-core Tools (29 tools) -- **Files** (all in `packages/opencode/src/altimate/tools/`): - - `altimate-core-check.ts` — `altimate_core_check` - - `altimate-core-classify-pii.ts` — `altimate_core_classify_pii` - - `altimate-core-column-lineage.ts` — `altimate_core_column_lineage` - - `altimate-core-compare.ts` — `altimate_core_compare` - - `altimate-core-complete.ts` — `altimate_core_complete` - - `altimate-core-correct.ts` — `altimate_core_correct` - - `altimate-core-equivalence.ts` — `altimate_core_equivalence` - - `altimate-core-export-ddl.ts` — `altimate_core_export_ddl` - - `altimate-core-extract-metadata.ts` — `altimate_core_metadata` - - `altimate-core-fingerprint.ts` — `altimate_core_fingerprint` - - `altimate-core-fix.ts` — `altimate_core_fix` - - `altimate-core-format.ts` — `altimate_core_format` - - `altimate-core-grade.ts` — `altimate_core_grade` - - `altimate-core-import-ddl.ts` — `altimate_core_import_ddl` - - `altimate-core-introspection-sql.ts` — `altimate_core_introspection_sql` - - `altimate-core-is-safe.ts` — `altimate_core_is_safe` - - `altimate-core-lint.ts` — `altimate_core_lint` - - `altimate-core-migration.ts` — `altimate_core_migration` - - `altimate-core-optimize-context.ts` — `altimate_core_optimize_context` - - `altimate-core-optimize-for-query.ts` — `altimate_core_optimize_for_query` - - `altimate-core-parse-dbt.ts` — `altimate_core_parse_dbt` - - `altimate-core-policy.ts` — `altimate_core_policy` - - `altimate-core-prune-schema.ts` — `altimate_core_prune_schema` - - `altimate-core-query-pii.ts` — `altimate_core_query_pii` - - `altimate-core-resolve-term.ts` — `altimate_core_resolve_term` - - `altimate-core-rewrite.ts` — `altimate_core_rewrite` - - `altimate-core-safety.ts` — `altimate_core_safety` - - `altimate-core-schema-diff.ts` — `altimate_core_schema_diff` - - `altimate-core-semantics.ts` — `altimate_core_semantics` - - `altimate-core-testgen.ts` — `altimate_core_testgen` - - `altimate-core-track-lineage.ts` — `altimate_core_track_lineage` - - `altimate-core-transpile.ts` — `altimate_core_transpile` - - `altimate-core-validate.ts` — `altimate_core_validate` -- **Description**: Wrappers around the Rust-based `altimate-core` library (accessed via the Python bridge). Organized in three phases: P0 (validate, lint, safety, transpile, check, fix, policy, semantics, testgen), P1 (equivalence, migration, schema_diff, rewrite, correct, grade), and P2 (classify_pii, query_pii, resolve_term, column_lineage, track_lineage, format, metadata, compare, complete, optimize_context, optimize_for_query, prune_schema, import_ddl, export_ddl, fingerprint, introspection_sql, parse_dbt, is_safe). These provide deep SQL static analysis, semantic understanding, lineage tracking, PII classification, query equivalence checking, DDL migration, and policy enforcement. -- **Category**: Tool - -#### 10. Miscellaneous Tools (2 tools) -- **Files**: - - `packages/opencode/src/altimate/tools/project-scan.ts` - - `packages/opencode/src/altimate/tools/lineage-check.ts` -- **Description**: `project_scan` detects the full data engineering environment: git repository details, dbt project structure (models/sources/tests counts), configured and newly discovered warehouse connections (from dbt profiles, Docker containers, and env vars), schema cache status, and installed data tools (dbt, sqlfluff, etc.). `lineage_check` computes column-level data lineage for a SQL query, tracing source-to-target column flows through joins, transforms, and CTEs. -- **Category**: Tool - ---- - -### Prompt/Agent (5 features) - -Custom agent modes added to `packages/opencode/src/agent/agent.ts` via `altimate_change` markers. Each mode has a custom system prompt and a permission ruleset controlling which tools are available. - -#### 11. Builder Agent Mode -- **Files**: `packages/opencode/src/altimate/prompts/builder.txt`, `packages/opencode/src/agent/agent.ts` -- **Description**: Full read/write data engineering agent for creating and modifying dbt models, SQL files, and YAML configs. Enforces a mandatory pre-execution protocol (analyze → validate → execute) before any SQL execution. Includes a dbt verification workflow and self-review requirement before declaring tasks complete. Replaces the upstream default `build` agent. -- **Category**: Prompt/Agent - -#### 12. Analyst Agent Mode -- **Files**: `packages/opencode/src/altimate/prompts/analyst.txt`, `packages/opencode/src/agent/agent.ts` -- **Description**: Read-only data exploration agent with a restricted permission set (denies all write tools, allows SQL/schema/lineage/FinOps read tools). Enforces cost-conscious exploration protocols (LIMIT clauses, iterative optimization, session cost tracking). Surfaces available read-only skills. -- **Category**: Prompt/Agent - -#### 13. Executive Agent Mode -- **Files**: `packages/opencode/src/altimate/prompts/executive.txt`, `packages/opencode/src/agent/agent.ts` -- **Description**: Read-only agent calibrated for non-technical business stakeholders. Strictly prohibits SQL, jargon, and technical notation in output. Translates all technical findings into business impact (revenue, cost, compliance risk). Formats output for slide decks and executive emails. -- **Category**: Prompt/Agent - -#### 14. Migrator Agent Mode -- **Files**: `packages/opencode/src/altimate/prompts/migrator.txt`, `packages/opencode/src/agent/agent.ts` -- **Description**: Cross-warehouse SQL migration agent with read/write access scoped to migration tasks. Validates source SQL, converts between dialects, verifies lineage preservation, compares schemas between source and target, and documents incompatibilities. Has access to sql-translate, lineage-diff, and all dbt skills. -- **Category**: Prompt/Agent - -#### 15. Validator Agent Mode -- **Files**: `packages/opencode/src/altimate/prompts/validator.txt`, `packages/opencode/src/agent/agent.ts` -- **Description**: Read-only data quality and integrity verification agent. Uses a structured findings format (Critical/Warning/Info severity tiers) and a dbt model verification checklist covering correctness, testing, performance, and documentation. Cannot modify files. Enforces a comprehensive validation protocol across SQL analysis, lineage, and dbt test coverage. -- **Category**: Prompt/Agent - ---- - -### Plugin (1 feature) - -#### 16. Anthropic OAuth Plugin -- **Files**: `packages/opencode/src/altimate/plugin/anthropic.ts` -- **Description**: Custom plugin implementing Anthropic OAuth 2.0 authentication via PKCE flow. Supports two login modes: Claude Pro/Max subscription (claude.ai) and API key creation via console (console.anthropic.com). Handles token refresh, injects required OAuth beta headers, prefixes all tool names with `mcp_` as required by Anthropic's OAuth endpoint, strips the prefix in streaming responses, and sanitizes system prompts (replaces "OpenCode" with "Claude Code"). Also zeroes out model costs for Pro/Max subscribers. -- **Category**: Plugin - ---- - -### CLI (1 feature) - -#### 17. Engine CLI Command -- **Files**: `packages/opencode/src/altimate/cli/engine.ts` -- **Description**: Adds an `engine` subcommand group to the CLI with three sub-commands: `status` (shows uv, Python, and engine versions + install path), `reset` (removes and reinstalls the engine), and `path` (prints the engine directory). Registered at the CLI root alongside standard opencode commands. -- **Category**: CLI - ---- - -### Config (4 features) - -These are modifications to upstream opencode files, marked with `// altimate_change` comments. - -#### 18. Dual Config Directory Support -- **Files**: `packages/opencode/src/config/config.ts` -- **Description**: Extends config file discovery to look in both `.altimate-code/` and `.opencode/` directories, enabling users migrating from opencode to pick up their existing config without renaming the directory. -- **Category**: Config - -#### 19. CLI Script Name and Binary Aliases -- **Files**: `packages/opencode/src/index.ts`, `packages/opencode/bin/altimate`, `packages/opencode/package.json` -- **Description**: Sets the yargs script name to `altimate-code`, adds `altimate` and `altimate-code` bin entries in package.json pointing to `./bin/altimate`, and sets `process.env.DATAPILOT = "1"` as a runtime identifier. The original `opencode` binary is retained for backward compatibility. -- **Category**: Config - -#### 20. ALTIMATE_CLI_CLIENT Flag -- **Files**: `packages/opencode/src/flag/flag.ts` -- **Description**: Adds `ALTIMATE_CLI_CLIENT` as a dual-env-var flag (primary: `ALTIMATE_CLI_CLIENT`, fallback: `OPENCODE_CLIENT`) with helper functions `altTruthy()` and `altEnv()` for reading flags that support both naming conventions. -- **Category**: Config - -#### 21. App Name and Data Directory Branding -- **Files**: `packages/opencode/src/global/index.ts`, `packages/opencode/src/installation/index.ts` -- **Description**: Changes the application data directory name from `opencode` to `altimate-code` (XDG data/cache/config/state paths), updates the database marker file name, and sets the user-agent string to `altimate-code/{CHANNEL}/{VERSION}/{CLIENT}`. The database marker (`altimate-code.db`) prevents re-running one-time migrations. -- **Category**: Config - ---- - -### Telemetry (1 feature) - -#### 22. Altimate Telemetry System -- **Files**: `packages/opencode/src/altimate/telemetry/index.ts`, `packages/opencode/src/telemetry/index.ts` -- **Description**: Full telemetry pipeline sending events to Azure Application Insights (hardcoded instrumentation key, overridable via `APPLICATIONINSIGHTS_CONNECTION_STRING`). Tracks 30+ event types including: `session_start/end`, `generation`, `tool_call` (with category classification for SQL/schema/dbt/finops/warehouse/lineage/file tools), `bridge_call` (Python RPC timing), `engine_started/error`, `auth_login/logout`, `mcp_server_status/census`, `context_overflow_recovered`, `compaction_triggered`, `doom_loop_detected`, `environment_census` (warehouse types, dbt detection, MCP count, OS), `context_utilization`, `agent_outcome`, `error_recovered`, `upgrade_attempted`, `permission_denied`, and `session_forked`. User email is SHA-256 hashed before sending. Supports opt-out via `ALTIMATE_TELEMETRY_DISABLED=true` or config. Uses a 5-second flush interval with a 200-event buffer and retry logic. -- **Category**: Telemetry - ---- - -### Python Engine (14 features) - -The `packages/altimate-engine/` package is a complete Python application published to PyPI as `altimate-engine`. It runs as a JSON-RPC sidecar process. - -#### 23. JSON-RPC Server -- **Files**: `packages/altimate-engine/src/altimate_engine/server.py` -- **Description**: Reads newline-delimited JSON-RPC 2.0 requests from stdin, dispatches to handlers across all engine modules, and writes responses to stdout. Entry point for the sidecar process (`python -m altimate_engine.server`). -- **Category**: Python Engine - -#### 24. Database Connector Framework -- **Files**: - - `packages/altimate-engine/src/altimate_engine/connectors/base.py` - - `packages/altimate-engine/src/altimate_engine/connectors/bigquery.py` - - `packages/altimate-engine/src/altimate_engine/connectors/databricks.py` - - `packages/altimate-engine/src/altimate_engine/connectors/duckdb.py` - - `packages/altimate-engine/src/altimate_engine/connectors/mysql.py` - - `packages/altimate-engine/src/altimate_engine/connectors/postgres.py` - - `packages/altimate-engine/src/altimate_engine/connectors/redshift.py` - - `packages/altimate-engine/src/altimate_engine/connectors/snowflake.py` - - `packages/altimate-engine/src/altimate_engine/connectors/sqlserver.py` -- **Description**: Abstract `Connector` base class with concrete implementations for 8 warehouse types (Snowflake, BigQuery, Databricks, PostgreSQL, MySQL, Redshift, DuckDB, SQL Server). Each connector implements `connect()`, `execute()`, `list_schemas()`, `list_tables()`, `describe_table()`, and `close()`. Optional `set_statement_timeout()` for query time limits. -- **Category**: Python Engine - -#### 25. Connection Registry -- **Files**: `packages/altimate-engine/src/altimate_engine/connections.py` -- **Description**: Manages named warehouse connections loaded from `~/.altimate-code/connections.json`, `.altimate-code/connections.json` (project-level), and `ALTIMATE_CODE_CONN_*` environment variables. Resolves credentials from the keyring store and transparently starts SSH tunnels when tunnel configuration is present. -- **Category**: Python Engine - -#### 26. Credential Store -- **Files**: `packages/altimate-engine/src/altimate_engine/credential_store.py` -- **Description**: Stores and retrieves sensitive connection fields (password, private_key_passphrase, access_token, ssh_password, connection_string) in the OS keyring (`keyring` library) under the `altimate-code` service name. Falls back gracefully when keyring is unavailable. Integrates with `ConnectionRegistry` to resolve stored credentials at connection time. -- **Category**: Python Engine - -#### 27. SQL Executor -- **Files**: `packages/altimate-engine/src/altimate_engine/sql/executor.py` -- **Description**: Executes SQL against any registered warehouse connector, returns rows as lists of lists with column names, and truncates results at a configurable limit. Used by the `sql_execute` tool. -- **Category**: Python Engine - -#### 28. SQL Static Analyzer -- **Files**: `packages/altimate-engine/src/altimate_engine/sql/` (analyzer component) -- **Description**: Rule-based SQL anti-pattern detector with 19 rules (SELECT_STAR, SELECT_STAR_IN_SUBQUERY, CARTESIAN_PRODUCT, IMPLICIT_CARTESIAN, MISSING_LIMIT, ORDER_BY_WITHOUT_LIMIT, ORDER_BY_IN_SUBQUERY, CORRELATED_SUBQUERY, NOT_IN_WITH_SUBQUERY, LARGE_IN_LIST, LIKE_LEADING_WILDCARD, NON_EQUI_JOIN, OR_IN_JOIN, UNION_INSTEAD_OF_UNION_ALL, UNUSED_CTE, FUNCTION_IN_FILTER, FUNCTION_IN_JOIN, WINDOW_WITHOUT_PARTITION, GROUP_BY_PRIMARY_KEY). Each issue includes severity, recommendation, location, and per-issue confidence scoring via `ConfidenceTracker`. Benchmarked at F1=1.00 on 1,077 test queries at 0.48ms/query average latency. -- **Category**: Python Engine - -#### 29. SQL Diff Engine -- **Files**: `packages/altimate-engine/src/altimate_engine/sql/diff.py` -- **Description**: Computes a unified diff between two SQL strings, returning additions, deletions, similarity ratio, and a structured list of changes. Used by the `sql_diff` tool. -- **Category**: Python Engine - -#### 30. SQL Explainer -- **Files**: `packages/altimate-engine/src/altimate_engine/sql/explainer.py` -- **Description**: Fetches query execution plans from connected warehouses (EXPLAIN / EXPLAIN ANALYZE), returning the plan as text and structured rows. Warehouse-aware: handles Snowflake, PostgreSQL, MySQL, and BigQuery plan output formats. -- **Category**: Python Engine - -#### 31. SQL Guard (Safety) -- **Files**: `packages/altimate-engine/src/altimate_engine/sql/guard.py` -- **Description**: Thin wrapper around the `altimate-core` Rust bindings providing graceful fallback when the Rust library is not installed. Bridges Python server handlers to the `altimate_core` module functions for validate, lint, safety, transpile, and all Phase 1-3 operations. Handles schema resolution from file paths or inline JSON dictionaries. -- **Category**: Python Engine - -#### 32. Schema Cache (SQLite Index) -- **Files**: `packages/altimate-engine/src/altimate_engine/schema/cache.py` -- **Description**: Builds and queries a local SQLite database indexing all warehouse metadata (databases, schemas, tables, columns). Enables fast full-text search without live warehouse queries. Tracks indexing timestamps and row counts per warehouse. Described internally as "altimate-code's answer to Snowflake's Horizon Catalog integration." -- **Category**: Python Engine - -#### 33. Schema Inspector -- **Files**: `packages/altimate-engine/src/altimate_engine/schema/inspector.py` -- **Description**: Inspects a specific table's column definitions using the `ConnectionRegistry`. Falls back to direct Postgres connection string for backward compatibility. Returns column names, data types, nullability, and primary key flags. -- **Category**: Python Engine - -#### 34. PII Detector -- **Files**: `packages/altimate-engine/src/altimate_engine/schema/pii_detector.py` -- **Description**: Detects columns likely to contain PII using regex patterns against column names. Covers 30+ PII categories (SSN, passport, drivers license, email, phone, address, names, credit cards, bank accounts, salary, dates of birth, passwords/tokens, IP addresses, health data, biometric data, demographics, geolocation). Each match includes a confidence level (high/medium/low). Filters out metadata columns (e.g., `email_sent_count`) that reference PII without containing it. -- **Category**: Python Engine - -#### 35. Metadata Tags -- **Files**: `packages/altimate-engine/src/altimate_engine/schema/tags.py` -- **Description**: Queries Snowflake metadata tag assignments and tag definitions using `SNOWFLAKE.ACCOUNT_USAGE` views. Returns tag-to-object mappings and a summary of tag usage counts. -- **Category**: Python Engine - -#### 36. FinOps Modules -- **Files**: - - `packages/altimate-engine/src/altimate_engine/finops/credit_analyzer.py` - - `packages/altimate-engine/src/altimate_engine/finops/query_history.py` - - `packages/altimate-engine/src/altimate_engine/finops/role_access.py` - - `packages/altimate-engine/src/altimate_engine/finops/unused_resources.py` - - `packages/altimate-engine/src/altimate_engine/finops/warehouse_advisor.py` -- **Description**: Python implementations for all FinOps analytics. `credit_analyzer` queries `SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY` for daily/per-warehouse credit consumption with configurable time windows. `query_history` retrieves `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`. `role_access` queries role grants, role hierarchy (`SHOW ROLES`), and user-role assignments. `unused_resources` identifies tables not accessed in N days and idle warehouses. `warehouse_advisor` queries warehouse load and performance metrics. -- **Category**: Python Engine - ---- - -### Skills (11 features) - -Skills are `.opencode/skills//SKILL.md` prompt files invoked via `/skill-name` commands in chat. - -#### 37. cost-report Skill -- **Files**: `.opencode/skills/cost-report/SKILL.md` -- **Description**: Analyzes Snowflake query costs by querying `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`, groups results by user/warehouse/query type, runs `sql_analyze` on the top 10 most expensive queries, classifies queries into cost tiers, and produces prioritized optimization recommendations. -- **Category**: Skill - -#### 38. dbt-docs Skill -- **Files**: `.opencode/skills/dbt-docs/SKILL.md` -- **Description**: Generates or improves dbt model and column descriptions in `schema.yml` files. Inspects the source table schema, reviews existing model SQL, and produces business-friendly documentation. -- **Category**: Skill - -#### 39. generate-tests Skill -- **Files**: `.opencode/skills/generate-tests/SKILL.md` -- **Description**: Auto-generates dbt test definitions (not_null, unique, relationships, accepted_values) for specified models or all models in the project. Reads the dbt manifest to understand model grain and key columns. -- **Category**: Skill - -#### 40. impact-analysis Skill -- **Files**: `.opencode/skills/impact-analysis/SKILL.md` -- **Description**: Performs downstream impact analysis for a given model or column using lineage data and the dbt manifest. Identifies which models, dashboards, and consumers are affected by a proposed change. -- **Category**: Skill - -#### 41. incremental-logic Skill -- **Files**: `.opencode/skills/incremental-logic/SKILL.md` -- **Description**: Guides implementation of incremental materialization strategies in dbt, including merge keys, delete+insert patterns, and partition-aware incrementals. -- **Category**: Skill - -#### 42. lineage-diff Skill -- **Files**: `.opencode/skills/lineage-diff/SKILL.md` -- **Description**: Compares column-level lineage between two versions of a SQL query or dbt model, highlighting columns whose data flow has changed or been broken. -- **Category**: Skill - -#### 43. medallion-patterns Skill -- **Files**: `.opencode/skills/medallion-patterns/SKILL.md` -- **Description**: Guides implementing bronze/silver/gold (medallion) data architecture patterns in dbt, including layer separation, materialization choices, and data quality contracts. -- **Category**: Skill - -#### 44. model-scaffold Skill -- **Files**: `.opencode/skills/model-scaffold/SKILL.md` -- **Description**: Scaffolds new dbt staging, intermediate, or mart models following project naming conventions. Generates the SQL skeleton, schema.yml entry, and recommended tests. -- **Category**: Skill - -#### 45. query-optimize Skill -- **Files**: `.opencode/skills/query-optimize/SKILL.md` -- **Description**: Optimizes a SQL query by running `sql_analyze` for anti-pattern detection and `sql_optimize` for rewrite suggestions, then produces a before/after comparison with cost impact estimates. -- **Category**: Skill - -#### 46. sql-translate Skill -- **Files**: `.opencode/skills/sql-translate/SKILL.md` -- **Description**: Translates SQL from one dialect to another (e.g., Snowflake to BigQuery), validates the result with `sql_validate`, and documents any functions or features that require manual adjustment. -- **Category**: Skill - -#### 47. yaml-config Skill -- **Files**: `.opencode/skills/yaml-config/SKILL.md` -- **Description**: Generates `sources.yml` and `schema.yml` configuration files from a live warehouse schema inspection, pre-populating table and column definitions. -- **Category**: Skill - ---- - -### Python Engine (Additional features) - -#### 48. dbt Integration Modules -- **Files**: - - `packages/altimate-engine/src/altimate_engine/dbt/lineage.py` - - `packages/altimate-engine/src/altimate_engine/dbt/manifest.py` - - `packages/altimate-engine/src/altimate_engine/dbt/profiles.py` - - `packages/altimate-engine/src/altimate_engine/dbt/runner.py` -- **Description**: Python modules backing the dbt tools. `manifest.py` parses `manifest.json` to extract models, sources, tests, snapshots, and seeds. `lineage.py` extracts compiled SQL and column-level lineage for a specific model from the manifest. `profiles.py` reads `~/.dbt/profiles.yml` and converts profiles into connection configs. `runner.py` executes dbt CLI commands as subprocesses, capturing stdout/stderr and exit codes. -- **Category**: Python Engine - -#### 49. Docker Discovery -- **Files**: `packages/altimate-engine/src/altimate_engine/docker_discovery.py` -- **Description**: Scans running Docker containers using the Docker SDK, matches known database images (postgres, mysql, mariadb, SQL Server), extracts host port mappings, and infers credentials from container environment variables. Returns a list of discovered database containers ready for `warehouse_add`. -- **Category**: Python Engine - -#### 50. SSH Tunnel Manager -- **Files**: `packages/altimate-engine/src/altimate_engine/ssh_tunnel.py` -- **Description**: Starts and manages SSH tunnels using `sshtunnel` and `paramiko`. Supports key-based and password-based authentication. Tunnels are registered by connection name and automatically stopped on process exit via `atexit`. Integrates transparently with `ConnectionRegistry` when tunnel config fields are present. -- **Category**: Python Engine - -#### 51. Local Schema Sync & Testing -- **Files**: - - `packages/altimate-engine/src/altimate_engine/local/schema_sync.py` - - `packages/altimate-engine/src/altimate_engine/local/test_local.py` -- **Description**: `schema_sync` syncs a remote warehouse schema into a local DuckDB database (empty stub tables, optionally with sample rows), enabling offline SQL development and testing. `test_local` executes SQL against a local DuckDB snapshot with optional dialect transpilation from the warehouse's native dialect. -- **Category**: Python Engine - ---- - -### Tests (3 features) - -Custom tests for the altimate-specific components. - -#### 52. Bridge Client Tests -- **Files**: `packages/opencode/test/bridge/client.test.ts` -- **Description**: Tests the `resolvePython()` function in `bridge/client.ts`, verifying Python binary resolution priority: `OPENCODE_PYTHON` env var, local dev venv, cwd venv, managed engine venv, and `python3` fallback. Uses `mock.module()` to stub `bridge/engine`. -- **Category**: Test - -#### 53. Bridge Engine Tests -- **Files**: `packages/opencode/test/bridge/engine.test.ts` -- **Description**: E2E tests verifying that `execFileSync` with `{ stdio: "pipe" }` prevents subprocess output from leaking to the parent process stdout/stderr. Validates the noise-suppression approach used in `engine.ts` for all subprocess calls (uv, python, tar). -- **Category**: Test - -#### 54. Python Engine Tests -- **Files**: `packages/altimate-engine/tests/` (27 test files) -- **Description**: Comprehensive pytest test suite for `altimate-engine`. Tests cover: autocomplete, connections, connectors (per-warehouse), credential store, dbt profiles, SQL diff, Docker discovery, enterprise connectors, environment detection, SQL execution, SQL explain, FinOps modules, SQL guard, local schema sync/testing, dbt manifest parsing, PII detection, schema cache, JSON-RPC server, server-level guard, SSH tunnels, and metadata tags. -- **Category**: Test - ---- - -### Experiments (2 features) - -#### 55. SQL Analyze Validation Benchmark -- **Files**: - - `experiments/sql_analyze_validation/__init__.py` - - `experiments/sql_analyze_validation/generate_queries.py` - - `experiments/sql_analyze_validation/queries.json` - - `experiments/sql_analyze_validation/report.py` - - `experiments/sql_analyze_validation/run_benchmark.py` - - `experiments/sql_analyze_validation/results/` (benchmark result files) -- **Description**: Accuracy benchmark framework for the `sql.analyze` engine. Generates 1,077 test queries across 18 categories with ground-truth expected positive/negative rules, runs the `StaticQueryAnalyzer`, computes per-rule precision/recall/F1, and generates a benchmark report. Published results show F1=1.00 on all 19 rules at 0.48ms/query average latency. -- **Category**: Experiment - -#### 56. Lineage Validation Benchmark -- **Files**: - - `experiments/lineage_validation/__init__.py` - - `experiments/lineage_validation/generate_lineage_queries.py` - - `experiments/lineage_validation/lineage_queries.json` - - `experiments/lineage_validation/report_lineage.py` - - `experiments/lineage_validation/run_lineage_benchmark.py` - - `experiments/lineage_validation/results/` (benchmark result files) - - `experiments/BENCHMARKS.md` -- **Description**: Accuracy benchmark for the `lineage.check` engine. Generates queries with expected column-level lineage edges, runs `check_lineage`, computes edge-level precision/recall/F1 using multiset matching (handles duplicate edges), and generates timestamped result files. -- **Category**: Experiment - ---- - -### CI/CD (2 features) - -#### 57. Publish Engine CI Workflow -- **Files**: `.github/workflows/publish-engine.yml` -- **Description**: GitHub Actions workflow that publishes the `altimate-engine` Python package to PyPI on `engine-v*` git tags. Uses OIDC trusted publishing (`id-token: write`) via the official PyPA action. Builds with `python -m build` and skips existing versions. -- **Category**: CI/CD - -#### 58. Upstream Merge Tooling -- **Files**: - - `script/upstream/merge.ts` - - `script/upstream/merge-config.json` - - `script/upstream/analyze.ts` - - `script/upstream/package.json` - - `script/upstream/transforms/keep-ours.ts` - - `script/upstream/transforms/lock-files.ts` - - `script/upstream/transforms/skip-files.ts` - - `script/upstream/utils/config.ts` - - `script/upstream/utils/git.ts` -- **Description**: Automated tooling to merge upstream opencode releases into the fork. Given a version tag, the script: validates prerequisites, creates a merge branch, runs `git merge`, automatically resolves conflicts using three strategies (keep-ours for custom code paths like `src/altimate/**`, skip-files for unused upstream packages like `packages/app/**`, and lock-files), reports remaining conflicts for manual resolution, and regenerates the lockfile. The `analyze.ts` script does a dry-run conflict analysis. Configuration in `merge-config.json` specifies `keepOurs` paths, `skipFiles` patterns, package name mappings, and the `altimate_change` marker name. -- **Category**: CI/CD - ---- - -### Docs (1 feature) - -#### 59. Data Engineering Documentation Site -- **Files** (all under `docs/docs/data-engineering/`): - - `agent-modes.md` — Builder, Analyst, Validator, Migrator, Executive mode docs - - `guides/cost-optimization.md` - - `guides/index.md` - - `guides/migration.md` - - `guides/using-with-claude-code.md` - - `guides/using-with-codex.md` - - `tools/dbt-tools.md` - - `tools/finops-tools.md` - - `tools/index.md` - - `tools/lineage-tools.md` - - `tools/schema-tools.md` - - `tools/sql-tools.md` - - `tools/warehouse-tools.md` - - `docs/docs/index.md` (custom homepage: "The data engineering agent for dbt, SQL, and cloud warehouses") - - `docs/docs/assets/` (custom logo, banner, favicon, CSS) - - `CHANGELOG.md`, `RELEASING.md`, `CODE_OF_CONDUCT.md` -- **Description**: Complete documentation site for the altimate-code product. Covers all five agent modes with examples, all 55+ custom tools organized by category (SQL, schema, dbt, FinOps, warehouse, lineage), integration guides for Claude Code and Codex, and the release/publishing process. Custom branding assets (altimate-code logo, banner, favicon) and theme-aware CSS. The RELEASING.md documents the dual-package (npm + PyPI) release process with version bumping scripts. -- **Category**: Docs - ---- - -### Other (1 feature) - -#### 60. Paid Context Management Feature Planning -- **Files**: `packages/opencode/src/altimate/session/PAID_CONTEXT_FEATURES.md` -- **Description**: Design document for six planned paid-tier context management features to be implemented in `altimate-core` (Rust) behind license key verification: (1) Precise token counting via tiktoken-rs, (2) Smart context scoring via local embedding-based relevance, (3) Schema compression using ILP optimization (~2x token reduction), (4) Lineage-aware context selection from the dbt DAG, (5) Semantic schema catalog generation (YAML-based business descriptions), and (6) Context budget allocator with per-category token allocation. -- **Category**: Other - ---- - -## Additional Modified Upstream Files (via `altimate_change` markers) - -These upstream opencode files were modified to wire in the custom code: - -| File | Change | -|------|--------| -| `packages/opencode/src/tool/registry.ts` | Imports and registers all 83 custom tools | -| `packages/opencode/src/agent/agent.ts` | Imports 5 custom agent mode prompts and adds builder/analyst/executive/migrator/validator agents | -| `packages/opencode/src/index.ts` | Sets script name, `DATAPILOT` env var, telemetry init, DB marker name | -| `packages/opencode/src/flag/flag.ts` | Adds `ALTIMATE_CLI_CLIENT` dual env var flag | -| `packages/opencode/src/global/index.ts` | Changes app name for XDG data directories | -| `packages/opencode/src/installation/index.ts` | Updates user-agent string, imports telemetry | -| `packages/opencode/src/config/config.ts` | Adds `.altimate-code` config dir support | -| `packages/opencode/src/telemetry/index.ts` | Re-exports from altimate telemetry module | -| `packages/opencode/src/altimate/cli/theme/altimate-code.json` | Custom dark/light color theme | - ---- - -## File Count Summary - -| Area | Files | -|------|-------| -| `packages/opencode/src/altimate/` | 86 files | -| `packages/altimate-engine/src/` | 38 files | -| `packages/altimate-engine/tests/` | 27 files | -| `.opencode/skills/` | 11 files | -| `experiments/` | 20 files | -| `docs/docs/data-engineering/` | 13 files | -| `docs/docs/` (other custom) | ~10 files | -| `.github/workflows/publish-engine.yml` | 1 file | -| `script/upstream/` | 9 files | -| `CHANGELOG.md`, `RELEASING.md`, `CODE_OF_CONDUCT.md` | 3 files | -| Modified upstream files | ~9 files | -| **Total custom/modified** | **~227 files** | diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md deleted file mode 100644 index 803cf2d37c..0000000000 --- a/IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,172 +0,0 @@ -# Implementation Plan: Verify Restructured Branch Completeness - -## Summary - -The draft plan is fundamentally sound. I've verified: - -- ✅ All 68 custom tools moved to `src/altimate/tools/` -- ✅ All 3 bridge files moved to `src/altimate/bridge/` -- ✅ All 5 prompt files moved to `src/altimate/prompts/` -- ✅ Python engine (71 files) preserved -- ✅ Docs, CI/CD, skills present -- ✅ Merge tooling added in `script/upstream/` - -**Key findings requiring adjustment:** - -1. Additional Altimate files exist on restructure/main not in draft: `cli/`, `plugin/`, `command/`, `session/` -2. Need to verify `altimate_change` marker blocks capture original modifications -3. Session feature was added during restructure (not in prep/revert-at-main) - ---- - -## Files to Create - -### 1. `script/verify-restructure/verify.ts` - -Main verification script. - ---- - -## Step-by-step Approach - -### Step 1: Create Verification Script - -Create `script/verify-restructure/verify.ts` that: - -```typescript -#!/usr/bin/env bun - -const OLD_BRANCH = "prep/revert-at-main" -const NEW_BRANCH = "restructure/main" -const BASE_BRANCH = "v1.2.18" - -const pathMap = [ - // Tools - { - from: /^packages\/opencode\/src\/tool\/altimate-core-(.+)\.ts$/, - to: "packages/opencode/src/altimate/tools/altimate-core-$1.ts", - }, - { - from: /^packages\/opencode\/src\/tool\/(sql|warehouse|schema|finops|dbt|lineage|project)-(.+)\.ts$/, - to: "packages/opencode/src/altimate/tools/$1-$2.ts", - }, - // Bridge - { from: /^packages\/opencode\/src\/bridge\/(.+)$/, to: "packages/opencode/src/altimate/bridge/$1" }, - // Prompts - { - from: /^packages\/opencode\/src\/agent\/prompt\/(analyst|builder|executive|migrator|validator)\.txt$/, - to: "packages/opencode/src/altimate/prompts/$1.txt", - }, - // Other altimate files - { from: /^packages\/opencode\/src\/altimate\/cli\/(.+)$/, to: "packages/opencode/src/altimate/cli/$1" }, - { from: /^packages\/opencode\/src\/altimate\/plugin\/(.+)$/, to: "packages/opencode/src/altimate/plugin/$1" }, - { from: /^packages\/opencode\/src\/altimate\/index\.ts$/, to: "packages/opencode/src/altimate/index.ts" }, - // Python engine (unchanged) - { from: /^packages\/altimate-engine\/(.+)$/, to: "packages/altimate-engine/$1" }, - // Everything else (docs, CI/CD, skills, etc.) - { from: /^(.+)$/, to: "$1" }, -] -``` - -### Step 2: Run Tool Category Verification - -Execute file-by-file comparison: - -- Extract custom files from `prep/revert-at-main` (not in v1.2.18) -- Map each to restructure/main path -- Compare and categorize: MATCH / MOVED / MODIFIED / MISSING - -### Step 3: Run Python Engine Verification - -```bash -diff -rq <(git show prep/revert-at-main:packages/altimate-engine/) \ - <(git show restructure/main:packages/altimate-engine/) -``` - -### Step 4: Verify altimate_change Blocks - -For each file modified with markers on restructure/main: - -- Extract content between `// altimate_change start` and `// altimate_change end` -- Compare against equivalent modification on prep/revert-at-main - -### Step 5: Build & Test - -```bash -cd packages/opencode && bun run build -cd packages/opencode && bun test -``` - ---- - -## Key Decisions - -### 1. Why path mapping instead of content hashing? - -Files may have internal reference changes. Content comparison ensures functional equivalence, not just filename matching. - -### 2. Why not use git rename detection? - -As noted in draft, rename detection is unreliable with 4000+ files. Explicit path mapping is deterministic. - -### 3. Why verify `altimate_change` blocks separately? - -These are critical modifications to upstream code. Need to ensure markers capture the exact original changes. - ---- - -## Edge Cases - -### 1. New files on restructure/main (not in prep/revert-at-main) - -**Scenario:** Files added during restructure that weren't in old main (e.g., merge tooling) -**Handling:** These are expected additions, mark as "NEW" and verify they're intentional - -### 2. Session feature discrepancy - -**Scenario:** `src/altimate/session/PAID_CONTEXT_FEATURES.md` exists on restructure/main but not prep/revert-at-main -**Handling:** Verify it's new functionality added during restructure, not a loss - -### 3. Binary files in docs - -**Scenario:** PNG files in docs/ -**Handling:** Use binary diff, report if content differs - -### 4. Large diffs in modified files - -**Handling:** Show first 50 lines of diff, offer to show full with flag - ---- - -## Verification Checklist - -| Category | Count (old) | Count (new) | Status | -| ---------------------- | ----------- | ----------- | ------ | -| altimate-core-\* tools | 33 | 33 | ⏳ | -| sql-\* tools | 10 | 10 | ⏳ | -| warehouse-\* tools | 5 | 5 | ⏳ | -| schema-\* tools | 6 | 6 | ⏳ | -| finops-\* tools | 6 | 6 | ⏳ | -| dbt-\* tools | 4 | 4 | ⏳ | -| lineage-\* tools | 1 | 1 | ⏳ | -| project-scan | 1 | 1 | ⏳ | -| Bridge | 3 | 3 | ⏳ | -| Prompts | 5 | 5 | ⏳ | -| Telemetry | 1 | 1 | ⏳ | -| CLI/Plugin/Other | ? | 7 | ⏳ | -| Python engine | 71 | 71 | ⏳ | -| Docs | ? | ? | ⏳ | -| Skills | 11 | 11 | ⏳ | -| CI/CD workflows | ? | ? | ⏳ | - ---- - -## Command to Run - -```bash -# Create verification script -mkdir -p script/verify-restructure - -# Run verification (after script created) -bun run script/verify-restructure/verify.ts -``` diff --git a/LICENSE b/LICENSE index 6439474bee..ba8c6f3e8c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ MIT License +Copyright (c) 2026 Altimate AI Copyright (c) 2025 opencode Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/PLATFORM_ENGINEER_SIMULATION.md b/PLATFORM_ENGINEER_SIMULATION.md deleted file mode 100644 index 985be55742..0000000000 --- a/PLATFORM_ENGINEER_SIMULATION.md +++ /dev/null @@ -1,604 +0,0 @@ -# Platform Engineer Simulation: Databricks + Unity Catalog + PySpark - -**Persona:** Data Platform Engineer at fintech (SOC2 + PCI-DSS compliance) -**Stack:** Databricks + Unity Catalog + Delta Lake + PySpark + dbt-databricks -**Team:** 8 engineers -**Date:** 2026-03-15 - ---- - -## Executive Summary - -Training system coverage: **~25-30%** of daily PySpark work. Compliance gap: **Critical**. Production readiness: **Not suitable** without major architectural changes. - -| Aspect | Training System | CLAUDE.md + Git | Winner | -|--------|-----------------|-----------------|--------| -| PySpark patterns | Limited (SQL-only scan) | N/A | Tie (missing) | -| Compliance audit trail | None | Full history + PR reviews | Git (clear win) | -| Approval workflow | Missing | PRs + code review | Git (clear win) | -| Environment-specific rules | None | Section-based | Git (clear win) | -| Version history | Flat (updated timestamp) | Full git blame | Git (clear win) | -| Multi-team governance | Single scope (global/project) | CODEOWNERS + teams | Git (clear win) | - ---- - -## Part 1: PySpark Problem - -### What the Training System Actually Finds - -**Training Scan Targets** (training-scan.ts line 15-21): -```typescript -const TARGET_GLOBS: Record = { - models: ["**/models/**/*.sql", "**/staging/**/*.sql", ...], // SQL ONLY - sql: ["**/*.sql"], // SQL ONLY - config: ["**/dbt_project.yml", "**/packages.yml", ...], - tests: ["**/tests/**/*.sql", "**/tests/**/*.yml", ...], - docs: ["**/*.md", ...], -} -``` - -**Result:** No Python scanning. Your team's PySpark code is invisible: -- `spark.read.table()` patterns → not found -- `df.filter()` chains → not found -- `df.write.mode("overwrite")` → not found -- Unity Catalog namespacing (`catalog.schema.table`) → not found -- Databricks-specific patterns (MERGE INTO, Z-order, OPTIMIZE) → not found - -**Coverage:** ~0% of PySpark work. Your team writes 70% PySpark, 30% SQL + dbt. - ---- - -### Gap 1: No Python File Scanning - -You need to add to training-scan.ts: -```typescript -python: ["**/*.py", "**/dbt_packages/**/*.py"], -``` - -But even then, keyword extraction (line 274-294) won't understand: -- DataFrame transformations (`.select()`, `.filter()`, `.groupBy()`) -- PySpark patterns (broadcast variables, window functions) -- Databricks APIs (`spark.sql()`, `sql()` magic commands) -- dbt-databricks macros (`dbt_utils.get_column_list()`) - ---- - -### Gap 2: 2500-Character Pattern Limit - -Your PySpark pattern: -```python -from pyspark.sql.functions import col, sum as spark_sum - -df = spark.read.table("bronze.raw_customers") -df_clean = df.filter(col("is_valid") == True).select( - "customer_id", "name", "email" -).repartition(10, "customer_id") - -df_clean.write.format("delta") \ - .mode("overwrite") \ - .option("mergeSchema", "true") \ - .partitionBy("customer_id") \ - .bucketBy(10, "customer_id") \ - .saveAsTable("silver.customers") - -spark.sql("OPTIMIZE silver.customers ZORDER BY customer_id") -``` - -After imports + formatting: ~650 characters. Fits within MemoryBlock content limit (2048 chars). - -But try this Unity Catalog + dynamic partition pattern: -```python -# Read from Bronze (catalog.schema.table) -df = spark.read.table(f"{bronze_catalog}.raw.events") - -# Complex transformation chain with window functions -from pyspark.sql.window import Window -from pyspark.sql.functions import row_number, dense_rank, lag - -w = Window.partitionBy("customer_id").orderBy(desc("event_timestamp")) -df_ranked = df.select("*", - row_number().over(w).alias("rn"), - lag("event_type").over(w).alias("prev_event") -) - -# Write to Silver with MERGE (idempotent upsert) -df_silver = df_ranked.filter(col("rn") == 1) - -# Can't express this pattern! No good way to show: -# - MERGE INTO ... MATCHED/NOT MATCHED clauses -# - Dynamic SQL construction -# - Partition pruning optimization -# - Z-order clustering strategy -``` - -**Result:** Complex PySpark patterns (MERGE, dynamic SQL, partition strategies) exceed 2500 chars or can't be captured as simple text. - ---- - -### Gap 3: No DataFrames = No Databricks Patterns - -Your team's most critical patterns: - -1. **MERGE Pattern** (Databricks Delta Lake idempotent upsert) - ```python - # No way to express this in training system - # SQL: MERGE INTO silver.customers USING df ... - # But we need to show: how to structure the logic, handle type mismatches, etc. - ``` - -2. **Z-order + OPTIMIZE** (critical for cost optimization) - ```python - spark.sql(f"OPTIMIZE {table_name} ZORDER BY ({zorder_cols})") - ``` - This single line represents: - - When to OPTIMIZE (file sizes > threshold) - - Which columns to Z-order (query predicates) - - Cost implications (can't show without context) - -3. **Unity Catalog Namespacing** - ```python - # Pattern: Always use three-part names for multi-workspace support - df = spark.read.table("fintech_prod.bronze.transactions") - - # Anti-pattern: Single/two-part names (breaks in other workspaces) - df = spark.read.table("bronze.transactions") # ❌ - ``` - -Training validation can't catch this — it just looks for strings like "transactions". - ---- - -## Part 2: Compliance Problem - -### Metadata Gaps - -**Current metadata** (types.ts line 13-19): -```typescript -export const TrainingBlockMeta = z.object({ - kind: TrainingKind, - source: z.string().optional(), - applied: z.number().int().min(0).default(0), - accepted: z.number().int().min(0).default(0), - rejected: z.number().int().min(0).default(0), -}) -``` - -**Missing fields for compliance:** -- ❌ `created_by: string` — Who added this rule? -- ❌ `approved_by: string` — Who approved it? -- ❌ `approval_date: ISO8601` — When was it approved? -- ❌ `reason: string` — Why does this rule exist? -- ❌ `impact: string` — What breaks if we ignore it? -- ❌ `reviewer_notes: string` — What did the reviewer check? - -**Audit trail comparison:** - -| Requirement | Training System | Git + CLAUDE.md | -|-------------|-----------------|-----------------| -| Who created rule | ❌ No | ✅ git log (author) | -| When created | ✅ created timestamp | ✅ git log (date) | -| Who approved | ❌ No | ✅ PR reviewers | -| Approval date | ❌ No | ✅ Merge commit | -| Change history | ❌ Flat (updated overwrites) | ✅ Full diff history | -| Compliance proof | ❌ No | ✅ PR description + approval | -| Review notes | ❌ No | ✅ PR comments + thread | -| Enforcement evidence | ❌ No | ✅ Commit messages | - ---- - -### Approval Workflow: Missing - -Store.ts has `accepted`/`rejected` counters (line 16) but: -- No workflow to set them -- No endpoint to approve/reject -- No user interface for approval -- No audit log of who approved what - -**Your compliance requirement:** -> "PII tagging rules must be enforced, not advisory. Audit trail: who added each rule, when, approved by whom." - -**Training system answer:** Rule exists, applied 5 times, 0 approvals recorded. - -**Git answer:** -``` -commit abc123 (PR #1234 by alice, approved by bob) -Author: alice -Date: 2025-11-15 - - feat: [AI-201] enforce PII tagging on sensitive columns - - Rule: Never store SSN, credit_card, or account_number without PII tag - Impact: Prevents accidental data exposure in non-sensitive systems - - Co-Authored-By: bob -``` - -You can prove: alice wrote it, bob reviewed, approved 2025-11-15. - ---- - -## Part 3: Multi-Environment Problem - -### Scenario: OPTIMIZE Rule - -**Rule:** "Always OPTIMIZE after writes > 1GB" - -**Environment variance:** -- **Dev**: Optional (lots of small writes, cost not critical) -- **Staging**: Recommended (some cost, helps catch issues) -- **Prod**: Mandatory (cost critical, SLAs matter) - -**Training system:** No environment concept. -```typescript -export interface TrainingEntry { - scope: "global" | "project", // That's it - ... -} -``` - -Save rule as global → applies everywhere. Applies same way in dev/prod. - -**CLAUDE.md approach:** -```markdown -## Databricks Optimization Rules - -### Dev Environment -- OPTIMIZE is optional -- Focus on correctness over cost - -### Staging Environment -- OPTIMIZE recommended for tables > 1GB -- Use for pre-prod validation - -### Prod Environment -- OPTIMIZE mandatory after writes > 1GB -- Monitor Z-order effectiveness -- Alert if skipped -``` - -**Implementation comparison:** - -| Scenario | Training | CLAUDE.md | -|----------|----------|-----------| -| Dev team pushes expensive OPTIMIZE | Applied everywhere ✅ (but not enforced) | Docs say optional, code can skip ✅ | -| Prod engineer forgets OPTIMIZE | Ignored ❌ (advisory) | Code review catches ✅ (CODEOWNERS) | -| New rule added mid-project | Updated immediately (affects all) ⚠️ | PR discussion, approved first ✅ | -| Rollback old rule | Delete entry, no history | `git revert` with full context ✅ | - ---- - -## Part 4: The Validation Problem - -### What `training_validate` Actually Does - -**Validation logic** (training-validate.ts line 136-151): -```typescript -// Check for violation indicators (negative rules) -const negativeKeywords = extractNegativeKeywords(entry.content) -for (const neg of negativeKeywords) { - if (contentLower.includes(neg.toLowerCase())) { - violationCount++ // Found a violation! - } -} -``` - -**Example: PII Rule** - -Rule: -``` -Never store SSN or credit_card in non-sensitive systems. -Don't use float for financial amounts — use DECIMAL(18,2). -``` - -Extract negative keywords: -- "SSN" -- "credit_card" -- "float" - -Scan 10 random .sql/.py files: -- File 1: `SELECT * FROM temp_ssn_lookup` → **VIOLATION DETECTED** (found "ssn") -- File 2: `-- legacy: using float (deprecated)` → **VIOLATION DETECTED** (found "float") -- File 3: `CAST(amount AS DECIMAL(18,2))` → NO VIOLATION - -**Problem:** Can't distinguish: -- ✅ "SSN in sensitive system (allowed)" -- ❌ "SSN in non-sensitive system (violation)" - -Training validation just looks for keywords. No scope understanding. - ---- - -### Practical Example: Your Team's Compliance Rule - -**Rule you want to enforce:** -``` -PII tagging rule: -- Columns with PII must have @pii tag in schema -- Systems: fintech_sensitive only -- Not enforced in: fintech_dev, fintech_analytics - -Example: - - Column: customer_ssn → MUST have @pii (in fintech_sensitive) - - Column: customer_email → SHOULD have @pii (in fintech_sensitive) - - Column: aggregated_customer_id → No @pii needed (in fintech_analytics) -``` - -**What training_validate finds:** -- "Files with @pii: 15/20 (75%)" ✅ -- "Files with SSN tag: 20/20 (100%)" ✅ -- Verdict: "Followed" - -**What audit needs:** -- "In fintech_sensitive: SSN/email/phone have @pii (100%)" -- "In fintech_dev: No @pii required (0/0)" -- "In fintech_analytics: @pii correctly absent (100%)" -- "Approved by bob@fintech.com on 2025-11-15" -- "Last audit: 2026-02-15 (passed)" - -**Training system:** Can't provide this. - ---- - -## Part 5: Version History & Drift - -### Scenario: Rule Changed Without Team Knowing - -**Original rule** (2025-11-01): -``` -Use DECIMAL(18,2) for all financial amounts. -Reason: Avoid rounding errors. -``` - -**Rule updated** (2025-12-15, by you): -``` -Use DECIMAL(38,10) for financial amounts. -Reason: New reporting requirement needs more precision. -``` - -**Training system:** `updated: "2025-12-15"`. No version history. - -**What happened:** -- ✅ New code follows DECIMAL(38,10) -- ❌ Old code still has DECIMAL(18,2) -- ❌ No one knows rule changed -- ❌ Can't compare old vs new -- ❌ Can't audit who decided why - -**Git history:** -```bash -git log --follow -- CLAUDE.md | grep -A5 "DECIMAL" - -commit abc123 (2025-12-15 by you) - fix: update decimal precision for new reporting - -commit def456 (2025-11-01 by alice) - feat: enforce decimal financial types - -git show abc123 -- CLAUDE.md | grep -B2 -A2 DECIMAL - # Shows exact change -``` - -**Compliance answer:** "Rule changed 2025-12-15. Old version had DECIMAL(18,2). All code updated in PR #1234. Approved by bob." - ---- - -## Part 6: The Reality Check - -### Coverage Percentage: Your Daily Work - -**Daily work breakdown (70% PySpark team):** - -1. **DataFrame transformations** (40% of time) - - `.select()`, `.filter()`, `.groupBy()`, `.join()` - - Window functions - - Custom UDFs - - Training coverage: ❌ **0%** (no Python scanning) - -2. **Databricks-specific patterns** (25% of time) - - MERGE INTO (idempotent upserts) - - OPTIMIZE + Z-order (cost management) - - Unity Catalog namespacing - - Delta Lake features - - Training coverage: ❌ **0%** (no Databricks-specific scanning) - -3. **dbt-databricks integration** (20% of time) - - `dbt-databricks` adapter-specific macros - - Python models in dbt - - Incremental strategy (merge vs insert) - - Training coverage: ⚠️ **5%** (finds dbt_project.yml, misses Python models) - -4. **Compliance checks** (10% of time) - - PII tagging validation - - Data governance (Unity Catalog levels) - - Audit logging - - Training coverage: ❌ **0%** (no approval/audit trail) - -5. **SQL + analytics** (5% of time) - - Raw SQL queries - - Testing/validation - - Training coverage: ✅ **100%** (full SQL scanning) - -**Realistic coverage: ~5-10%** of your team's daily work. - ---- - -## Part 7: Security Team Evaluation - -### Would Security Approve Training for Prod? - -**Compliance Officer Checklist:** - -| Requirement | Status | Risk | -|-------------|--------|------| -| Audit trail (who, when) | ❌ Partial | Medium | -| Approval workflow | ❌ Missing | High | -| Enforcement proof | ❌ No | High | -| Version history | ❌ No | Medium | -| Rollback capability | ❌ Limited | Medium | -| Cross-environment rules | ❌ Not supported | High | -| PII/sensitivity scoping | ❌ No | Critical | -| Integration with SIEM | ❌ No | High | - -**Security verdict:** -> "Training system cannot be approved for production compliance enforcement. It lacks: -> 1. Formal approval workflows -> 2. Audit trail of approvals (who, when, why) -> 3. Scope/environment differentiation -> 4. Version control + rollback -> 5. Integration with compliance monitoring -> -> Recommendation: Use git + CLAUDE.md for compliance-critical rules. Use training for patterns/context only." - ---- - -## Part 8: Specific Changes Needed - -### To Make Training Production-Ready - -#### 1. Add Approval Workflow - -```typescript -export interface TrainingBlockMeta extends z.infer { - created_by: string // User who created - created_date: ISO8601 // Timestamp - approved_by?: string // User who approved - approved_date?: ISO8601 // Approval timestamp - approval_status: "pending" | "approved" | "rejected" - rejection_reason?: string - compliance_required: boolean - environment_scope?: "dev" | "staging" | "prod" | "all" -} -``` - -#### 2. Add Python Scanning - -```typescript -const TARGET_GLOBS = { - python: ["**/*.py", "!**/__pycache__/**"], - pyspark: ["**/spark_*.py", "**/dataframe_*.py"], - dbt_python: ["dbt/models/**/*.py"], -} -``` - -#### 3. Environment-Aware Validation - -```typescript -export async function validateInEnvironment( - entry: TrainingEntry, - environment: "dev" | "staging" | "prod" -): Promise { - // Filter files by environment-specific patterns - // Apply environment-specific rules - // Check approval status for prod -} -``` - -#### 4. Integration with Git - -```typescript -// Store training metadata in git as well -// Enable `git blame` on training rules -// Link training to PRs/issues -export async function exportToGitCLAUDE( - training: TrainingEntry[] -): Promise { - // Generate CLAUDE.md section from training entries -} -``` - ---- - -## Summary & Recommendation - -### Training System: Best Use Cases ✅ - -1. **Pattern discovery** — find structural conventions -2. **Knowledge sharing** — disseminate learned patterns -3. **Context building** — capture "why" decisions -4. **Playbooks** — step-by-step procedures -5. **Glossary** — domain term definitions - -### Training System: Not Suitable ❌ - -1. **Compliance rules** — no approval/audit trail -2. **Environment-specific policies** — no scope differentiation -3. **PII/security enforcement** — no granular scoping -4. **Critical operational rules** — no version history/rollback -5. **Multi-team governance** — no CODEOWNERS integration - -### Recommendation for Your Stack - -**Hybrid approach:** - -| Category | Use | Tool | -|----------|-----|------| -| PySpark patterns | How to use DataFrame API | Training | -| Databricks best practices | Z-order, OPTIMIZE patterns | Training | -| dbt-databricks patterns | Macros, incremental strategy | Training | -| **PII rules** | **What is PII, enforcement** | **Git + CLAUDE.md** | -| **Compliance policies** | **Data retention, governance** | **Git + CLAUDE.md** | -| **Environment rules** | **Dev vs prod behavior** | **Git + CLAUDE.md** | -| **Approvals** | **Who approved what** | **GitHub PRs + reviews** | -| **Version history** | **Track changes over time** | **Git + git log** | - -**Action items:** - -1. ✅ Document PySpark patterns in training (fills 40% gap) -2. ✅ Document dbt-databricks patterns in training (fills 20% gap) -3. ✅ Keep PII/compliance rules in CLAUDE.md (remains 100% auditable) -4. ✅ Link training discoveries back to CLAUDE.md for compliance sync -5. ✅ Use git for version control + approval trail -6. ❌ Don't use training for compliance-critical enforcement - -**Coverage after implementation:** -- PySpark patterns: 35-40% (up from 0%) -- Compliance rules: 100% (via CLAUDE.md) -- Overall production readiness: 60-70% - ---- - -## Appendix: Scan Results If Running on Sample PySpark - -**If you added Python scanning, running `training_scan target:python` would find:** - -```markdown -## Scan Results: python - -Scanned **20** files in `dataframe_transforms/` - -| Type | Count | -|------|-------| -| Python files | 20 | - -### Discovered Patterns - -**Naming Conventions**: `stg_*` (3 files), `fct_*` (2 files), `dim_*` (1 file) - -**Common Patterns**: -- Uses `spark.read.table()`: 15/20 files (75%) -- Uses `df.filter()` chains: 18/20 files (90%) -- Uses `partition` or `bucket`: 8/20 files (40%) -- Uses `OPTIMIZE` or Z-order: 3/20 files (15%) -- Uses `MERGE INTO`: 2/20 files (10%) -- Uses Unity Catalog three-part names: 5/20 files (25%) - -### Key Observations - -- Most code uses `.write.mode("overwrite")` instead of MERGE -- Z-order/OPTIMIZE only used in 15% — opportunity to standardize -- Unity Catalog adoption at 25% — needs team migration plan -- No custom UDFs found — may be in separate utility files - -### Recommendations - -Could teach patterns: -- "Idempotent upsert pattern using MERGE" -- "Z-order clustering for query performance" -- "Three-part table naming for multi-workspace support" -- "Partition strategy for Bronze→Silver→Gold" -``` - -But validation would still be weak: -- Can't distinguish "MERGE in prod" vs "MERGE in dev" -- Can't validate "PII columns tagged" -- Can't prove "rule approved by security team" diff --git a/README.md b/README.md index 81e02562aa..6ffbeeadcf 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ altimate-code -# altimate - **The data engineering agent for dbt, SQL, and cloud warehouses.** An AI-powered CLI with 99+ specialized tools — SQL analysis, schema inspection, diff --git a/SECURITY.md b/SECURITY.md index e7eb27511f..524826c90b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -44,4 +44,4 @@ The team will send a response indicating the next steps in handling your report. ## Escalation -If you do not receive an acknowledgement of your report within 6 business days, you may send an email to security@anoma.ly +If you do not receive an acknowledgement of your report within 6 business days, you may send an email to security@altimate.ai diff --git a/STATS.md b/STATS.md deleted file mode 100644 index 44819a6eb8..0000000000 --- a/STATS.md +++ /dev/null @@ -1,217 +0,0 @@ -# Download Stats - -| Date | GitHub Downloads | npm Downloads | Total | -| ---------- | -------------------- | -------------------- | --------------------- | -| 2025-06-29 | 18,789 (+0) | 39,420 (+0) | 58,209 (+0) | -| 2025-06-30 | 20,127 (+1,338) | 41,059 (+1,639) | 61,186 (+2,977) | -| 2025-07-01 | 22,108 (+1,981) | 43,745 (+2,686) | 65,853 (+4,667) | -| 2025-07-02 | 24,814 (+2,706) | 46,168 (+2,423) | 70,982 (+5,129) | -| 2025-07-03 | 27,834 (+3,020) | 49,955 (+3,787) | 77,789 (+6,807) | -| 2025-07-04 | 30,608 (+2,774) | 54,758 (+4,803) | 85,366 (+7,577) | -| 2025-07-05 | 32,524 (+1,916) | 58,371 (+3,613) | 90,895 (+5,529) | -| 2025-07-06 | 33,766 (+1,242) | 59,694 (+1,323) | 93,460 (+2,565) | -| 2025-07-08 | 38,052 (+4,286) | 64,468 (+4,774) | 102,520 (+9,060) | -| 2025-07-09 | 40,924 (+2,872) | 67,935 (+3,467) | 108,859 (+6,339) | -| 2025-07-10 | 43,796 (+2,872) | 71,402 (+3,467) | 115,198 (+6,339) | -| 2025-07-11 | 46,982 (+3,186) | 77,462 (+6,060) | 124,444 (+9,246) | -| 2025-07-12 | 49,302 (+2,320) | 82,177 (+4,715) | 131,479 (+7,035) | -| 2025-07-13 | 50,803 (+1,501) | 86,394 (+4,217) | 137,197 (+5,718) | -| 2025-07-14 | 53,283 (+2,480) | 87,860 (+1,466) | 141,143 (+3,946) | -| 2025-07-15 | 57,590 (+4,307) | 91,036 (+3,176) | 148,626 (+7,483) | -| 2025-07-16 | 62,313 (+4,723) | 95,258 (+4,222) | 157,571 (+8,945) | -| 2025-07-17 | 66,684 (+4,371) | 100,048 (+4,790) | 166,732 (+9,161) | -| 2025-07-18 | 70,379 (+3,695) | 102,587 (+2,539) | 172,966 (+6,234) | -| 2025-07-19 | 73,497 (+3,117) | 105,904 (+3,317) | 179,401 (+6,434) | -| 2025-07-20 | 76,453 (+2,956) | 109,044 (+3,140) | 185,497 (+6,096) | -| 2025-07-21 | 80,197 (+3,744) | 113,537 (+4,493) | 193,734 (+8,237) | -| 2025-07-22 | 84,251 (+4,054) | 118,073 (+4,536) | 202,324 (+8,590) | -| 2025-07-23 | 88,589 (+4,338) | 121,436 (+3,363) | 210,025 (+7,701) | -| 2025-07-24 | 92,469 (+3,880) | 124,091 (+2,655) | 216,560 (+6,535) | -| 2025-07-25 | 96,417 (+3,948) | 126,985 (+2,894) | 223,402 (+6,842) | -| 2025-07-26 | 100,646 (+4,229) | 131,411 (+4,426) | 232,057 (+8,655) | -| 2025-07-27 | 102,644 (+1,998) | 134,736 (+3,325) | 237,380 (+5,323) | -| 2025-07-28 | 105,446 (+2,802) | 136,016 (+1,280) | 241,462 (+4,082) | -| 2025-07-29 | 108,998 (+3,552) | 137,542 (+1,526) | 246,540 (+5,078) | -| 2025-07-30 | 113,544 (+4,546) | 140,317 (+2,775) | 253,861 (+7,321) | -| 2025-07-31 | 118,339 (+4,795) | 143,344 (+3,027) | 261,683 (+7,822) | -| 2025-08-01 | 123,539 (+5,200) | 146,680 (+3,336) | 270,219 (+8,536) | -| 2025-08-02 | 127,864 (+4,325) | 149,236 (+2,556) | 277,100 (+6,881) | -| 2025-08-03 | 131,397 (+3,533) | 150,451 (+1,215) | 281,848 (+4,748) | -| 2025-08-04 | 136,266 (+4,869) | 153,260 (+2,809) | 289,526 (+7,678) | -| 2025-08-05 | 141,596 (+5,330) | 155,752 (+2,492) | 297,348 (+7,822) | -| 2025-08-06 | 147,067 (+5,471) | 158,309 (+2,557) | 305,376 (+8,028) | -| 2025-08-07 | 152,591 (+5,524) | 160,889 (+2,580) | 313,480 (+8,104) | -| 2025-08-08 | 158,187 (+5,596) | 163,448 (+2,559) | 321,635 (+8,155) | -| 2025-08-09 | 162,770 (+4,583) | 165,721 (+2,273) | 328,491 (+6,856) | -| 2025-08-10 | 165,695 (+2,925) | 167,109 (+1,388) | 332,804 (+4,313) | -| 2025-08-11 | 169,297 (+3,602) | 167,953 (+844) | 337,250 (+4,446) | -| 2025-08-12 | 176,307 (+7,010) | 171,876 (+3,923) | 348,183 (+10,933) | -| 2025-08-13 | 182,997 (+6,690) | 177,182 (+5,306) | 360,179 (+11,996) | -| 2025-08-14 | 189,063 (+6,066) | 179,741 (+2,559) | 368,804 (+8,625) | -| 2025-08-15 | 193,608 (+4,545) | 181,792 (+2,051) | 375,400 (+6,596) | -| 2025-08-16 | 198,118 (+4,510) | 184,558 (+2,766) | 382,676 (+7,276) | -| 2025-08-17 | 201,299 (+3,181) | 186,269 (+1,711) | 387,568 (+4,892) | -| 2025-08-18 | 204,559 (+3,260) | 187,399 (+1,130) | 391,958 (+4,390) | -| 2025-08-19 | 209,814 (+5,255) | 189,668 (+2,269) | 399,482 (+7,524) | -| 2025-08-20 | 214,497 (+4,683) | 191,481 (+1,813) | 405,978 (+6,496) | -| 2025-08-21 | 220,465 (+5,968) | 194,784 (+3,303) | 415,249 (+9,271) | -| 2025-08-22 | 225,899 (+5,434) | 197,204 (+2,420) | 423,103 (+7,854) | -| 2025-08-23 | 229,005 (+3,106) | 199,238 (+2,034) | 428,243 (+5,140) | -| 2025-08-24 | 232,098 (+3,093) | 201,157 (+1,919) | 433,255 (+5,012) | -| 2025-08-25 | 236,607 (+4,509) | 202,650 (+1,493) | 439,257 (+6,002) | -| 2025-08-26 | 242,783 (+6,176) | 205,242 (+2,592) | 448,025 (+8,768) | -| 2025-08-27 | 248,409 (+5,626) | 205,242 (+0) | 453,651 (+5,626) | -| 2025-08-28 | 252,796 (+4,387) | 205,242 (+0) | 458,038 (+4,387) | -| 2025-08-29 | 256,045 (+3,249) | 211,075 (+5,833) | 467,120 (+9,082) | -| 2025-08-30 | 258,863 (+2,818) | 212,397 (+1,322) | 471,260 (+4,140) | -| 2025-08-31 | 262,004 (+3,141) | 213,944 (+1,547) | 475,948 (+4,688) | -| 2025-09-01 | 265,359 (+3,355) | 215,115 (+1,171) | 480,474 (+4,526) | -| 2025-09-02 | 270,483 (+5,124) | 217,075 (+1,960) | 487,558 (+7,084) | -| 2025-09-03 | 274,793 (+4,310) | 219,755 (+2,680) | 494,548 (+6,990) | -| 2025-09-04 | 280,430 (+5,637) | 222,103 (+2,348) | 502,533 (+7,985) | -| 2025-09-05 | 283,769 (+3,339) | 223,793 (+1,690) | 507,562 (+5,029) | -| 2025-09-06 | 286,245 (+2,476) | 225,036 (+1,243) | 511,281 (+3,719) | -| 2025-09-07 | 288,623 (+2,378) | 225,866 (+830) | 514,489 (+3,208) | -| 2025-09-08 | 293,341 (+4,718) | 227,073 (+1,207) | 520,414 (+5,925) | -| 2025-09-09 | 300,036 (+6,695) | 229,788 (+2,715) | 529,824 (+9,410) | -| 2025-09-10 | 307,287 (+7,251) | 233,435 (+3,647) | 540,722 (+10,898) | -| 2025-09-11 | 314,083 (+6,796) | 237,356 (+3,921) | 551,439 (+10,717) | -| 2025-09-12 | 321,046 (+6,963) | 240,728 (+3,372) | 561,774 (+10,335) | -| 2025-09-13 | 324,894 (+3,848) | 245,539 (+4,811) | 570,433 (+8,659) | -| 2025-09-14 | 328,876 (+3,982) | 248,245 (+2,706) | 577,121 (+6,688) | -| 2025-09-15 | 334,201 (+5,325) | 250,983 (+2,738) | 585,184 (+8,063) | -| 2025-09-16 | 342,609 (+8,408) | 255,264 (+4,281) | 597,873 (+12,689) | -| 2025-09-17 | 351,117 (+8,508) | 260,970 (+5,706) | 612,087 (+14,214) | -| 2025-09-18 | 358,717 (+7,600) | 266,922 (+5,952) | 625,639 (+13,552) | -| 2025-09-19 | 365,401 (+6,684) | 271,859 (+4,937) | 637,260 (+11,621) | -| 2025-09-20 | 372,092 (+6,691) | 276,917 (+5,058) | 649,009 (+11,749) | -| 2025-09-21 | 377,079 (+4,987) | 280,261 (+3,344) | 657,340 (+8,331) | -| 2025-09-22 | 382,492 (+5,413) | 284,009 (+3,748) | 666,501 (+9,161) | -| 2025-09-23 | 387,008 (+4,516) | 289,129 (+5,120) | 676,137 (+9,636) | -| 2025-09-24 | 393,325 (+6,317) | 294,927 (+5,798) | 688,252 (+12,115) | -| 2025-09-25 | 398,879 (+5,554) | 301,663 (+6,736) | 700,542 (+12,290) | -| 2025-09-26 | 404,334 (+5,455) | 306,713 (+5,050) | 711,047 (+10,505) | -| 2025-09-27 | 411,618 (+7,284) | 317,763 (+11,050) | 729,381 (+18,334) | -| 2025-09-28 | 414,910 (+3,292) | 322,522 (+4,759) | 737,432 (+8,051) | -| 2025-09-29 | 419,919 (+5,009) | 328,033 (+5,511) | 747,952 (+10,520) | -| 2025-09-30 | 427,991 (+8,072) | 336,472 (+8,439) | 764,463 (+16,511) | -| 2025-10-01 | 433,591 (+5,600) | 341,742 (+5,270) | 775,333 (+10,870) | -| 2025-10-02 | 440,852 (+7,261) | 348,099 (+6,357) | 788,951 (+13,618) | -| 2025-10-03 | 446,829 (+5,977) | 359,937 (+11,838) | 806,766 (+17,815) | -| 2025-10-04 | 452,561 (+5,732) | 370,386 (+10,449) | 822,947 (+16,181) | -| 2025-10-05 | 455,559 (+2,998) | 374,745 (+4,359) | 830,304 (+7,357) | -| 2025-10-06 | 460,927 (+5,368) | 379,489 (+4,744) | 840,416 (+10,112) | -| 2025-10-07 | 467,336 (+6,409) | 385,438 (+5,949) | 852,774 (+12,358) | -| 2025-10-08 | 474,643 (+7,307) | 394,139 (+8,701) | 868,782 (+16,008) | -| 2025-10-09 | 479,203 (+4,560) | 400,526 (+6,387) | 879,729 (+10,947) | -| 2025-10-10 | 484,374 (+5,171) | 406,015 (+5,489) | 890,389 (+10,660) | -| 2025-10-11 | 488,427 (+4,053) | 414,699 (+8,684) | 903,126 (+12,737) | -| 2025-10-12 | 492,125 (+3,698) | 418,745 (+4,046) | 910,870 (+7,744) | -| 2025-10-14 | 505,130 (+13,005) | 429,286 (+10,541) | 934,416 (+23,546) | -| 2025-10-15 | 512,717 (+7,587) | 439,290 (+10,004) | 952,007 (+17,591) | -| 2025-10-16 | 517,719 (+5,002) | 447,137 (+7,847) | 964,856 (+12,849) | -| 2025-10-17 | 526,239 (+8,520) | 457,467 (+10,330) | 983,706 (+18,850) | -| 2025-10-18 | 531,564 (+5,325) | 465,272 (+7,805) | 996,836 (+13,130) | -| 2025-10-19 | 536,209 (+4,645) | 469,078 (+3,806) | 1,005,287 (+8,451) | -| 2025-10-20 | 541,264 (+5,055) | 472,952 (+3,874) | 1,014,216 (+8,929) | -| 2025-10-21 | 548,721 (+7,457) | 479,703 (+6,751) | 1,028,424 (+14,208) | -| 2025-10-22 | 557,949 (+9,228) | 491,395 (+11,692) | 1,049,344 (+20,920) | -| 2025-10-23 | 564,716 (+6,767) | 498,736 (+7,341) | 1,063,452 (+14,108) | -| 2025-10-24 | 572,692 (+7,976) | 506,905 (+8,169) | 1,079,597 (+16,145) | -| 2025-10-25 | 578,927 (+6,235) | 516,129 (+9,224) | 1,095,056 (+15,459) | -| 2025-10-26 | 584,409 (+5,482) | 521,179 (+5,050) | 1,105,588 (+10,532) | -| 2025-10-27 | 589,999 (+5,590) | 526,001 (+4,822) | 1,116,000 (+10,412) | -| 2025-10-28 | 595,776 (+5,777) | 532,438 (+6,437) | 1,128,214 (+12,214) | -| 2025-10-29 | 606,259 (+10,483) | 542,064 (+9,626) | 1,148,323 (+20,109) | -| 2025-10-30 | 613,746 (+7,487) | 542,064 (+0) | 1,155,810 (+7,487) | -| 2025-10-30 | 617,846 (+4,100) | 555,026 (+12,962) | 1,172,872 (+17,062) | -| 2025-10-31 | 626,612 (+8,766) | 564,579 (+9,553) | 1,191,191 (+18,319) | -| 2025-11-01 | 636,100 (+9,488) | 581,806 (+17,227) | 1,217,906 (+26,715) | -| 2025-11-02 | 644,067 (+7,967) | 590,004 (+8,198) | 1,234,071 (+16,165) | -| 2025-11-03 | 653,130 (+9,063) | 597,139 (+7,135) | 1,250,269 (+16,198) | -| 2025-11-04 | 663,912 (+10,782) | 608,056 (+10,917) | 1,271,968 (+21,699) | -| 2025-11-05 | 675,074 (+11,162) | 619,690 (+11,634) | 1,294,764 (+22,796) | -| 2025-11-06 | 686,252 (+11,178) | 630,885 (+11,195) | 1,317,137 (+22,373) | -| 2025-11-07 | 696,646 (+10,394) | 642,146 (+11,261) | 1,338,792 (+21,655) | -| 2025-11-08 | 706,035 (+9,389) | 653,489 (+11,343) | 1,359,524 (+20,732) | -| 2025-11-09 | 713,462 (+7,427) | 660,459 (+6,970) | 1,373,921 (+14,397) | -| 2025-11-10 | 722,288 (+8,826) | 668,225 (+7,766) | 1,390,513 (+16,592) | -| 2025-11-11 | 729,769 (+7,481) | 677,501 (+9,276) | 1,407,270 (+16,757) | -| 2025-11-12 | 740,180 (+10,411) | 686,454 (+8,953) | 1,426,634 (+19,364) | -| 2025-11-13 | 749,905 (+9,725) | 696,157 (+9,703) | 1,446,062 (+19,428) | -| 2025-11-14 | 759,928 (+10,023) | 705,237 (+9,080) | 1,465,165 (+19,103) | -| 2025-11-15 | 765,955 (+6,027) | 712,870 (+7,633) | 1,478,825 (+13,660) | -| 2025-11-16 | 771,069 (+5,114) | 716,596 (+3,726) | 1,487,665 (+8,840) | -| 2025-11-17 | 780,161 (+9,092) | 723,339 (+6,743) | 1,503,500 (+15,835) | -| 2025-11-18 | 791,563 (+11,402) | 732,544 (+9,205) | 1,524,107 (+20,607) | -| 2025-11-19 | 804,409 (+12,846) | 747,624 (+15,080) | 1,552,033 (+27,926) | -| 2025-11-20 | 814,620 (+10,211) | 757,907 (+10,283) | 1,572,527 (+20,494) | -| 2025-11-21 | 826,309 (+11,689) | 769,307 (+11,400) | 1,595,616 (+23,089) | -| 2025-11-22 | 837,269 (+10,960) | 780,996 (+11,689) | 1,618,265 (+22,649) | -| 2025-11-23 | 846,609 (+9,340) | 795,069 (+14,073) | 1,641,678 (+23,413) | -| 2025-11-24 | 856,733 (+10,124) | 804,033 (+8,964) | 1,660,766 (+19,088) | -| 2025-11-25 | 869,423 (+12,690) | 817,339 (+13,306) | 1,686,762 (+25,996) | -| 2025-11-26 | 881,414 (+11,991) | 832,518 (+15,179) | 1,713,932 (+27,170) | -| 2025-11-27 | 893,960 (+12,546) | 846,180 (+13,662) | 1,740,140 (+26,208) | -| 2025-11-28 | 901,741 (+7,781) | 856,482 (+10,302) | 1,758,223 (+18,083) | -| 2025-11-29 | 908,689 (+6,948) | 863,361 (+6,879) | 1,772,050 (+13,827) | -| 2025-11-30 | 916,116 (+7,427) | 870,194 (+6,833) | 1,786,310 (+14,260) | -| 2025-12-01 | 925,898 (+9,782) | 876,500 (+6,306) | 1,802,398 (+16,088) | -| 2025-12-02 | 939,250 (+13,352) | 890,919 (+14,419) | 1,830,169 (+27,771) | -| 2025-12-03 | 952,249 (+12,999) | 903,713 (+12,794) | 1,855,962 (+25,793) | -| 2025-12-04 | 965,611 (+13,362) | 916,471 (+12,758) | 1,882,082 (+26,120) | -| 2025-12-05 | 977,996 (+12,385) | 930,616 (+14,145) | 1,908,612 (+26,530) | -| 2025-12-06 | 987,884 (+9,888) | 943,773 (+13,157) | 1,931,657 (+23,045) | -| 2025-12-07 | 994,046 (+6,162) | 951,425 (+7,652) | 1,945,471 (+13,814) | -| 2025-12-08 | 1,000,898 (+6,852) | 957,149 (+5,724) | 1,958,047 (+12,576) | -| 2025-12-09 | 1,011,488 (+10,590) | 973,922 (+16,773) | 1,985,410 (+27,363) | -| 2025-12-10 | 1,025,891 (+14,403) | 991,708 (+17,786) | 2,017,599 (+32,189) | -| 2025-12-11 | 1,045,110 (+19,219) | 1,010,559 (+18,851) | 2,055,669 (+38,070) | -| 2025-12-12 | 1,061,340 (+16,230) | 1,030,838 (+20,279) | 2,092,178 (+36,509) | -| 2025-12-13 | 1,073,561 (+12,221) | 1,044,608 (+13,770) | 2,118,169 (+25,991) | -| 2025-12-14 | 1,082,042 (+8,481) | 1,052,425 (+7,817) | 2,134,467 (+16,298) | -| 2025-12-15 | 1,093,632 (+11,590) | 1,059,078 (+6,653) | 2,152,710 (+18,243) | -| 2025-12-16 | 1,120,477 (+26,845) | 1,078,022 (+18,944) | 2,198,499 (+45,789) | -| 2025-12-17 | 1,151,067 (+30,590) | 1,097,661 (+19,639) | 2,248,728 (+50,229) | -| 2025-12-18 | 1,178,658 (+27,591) | 1,113,418 (+15,757) | 2,292,076 (+43,348) | -| 2025-12-19 | 1,203,485 (+24,827) | 1,129,698 (+16,280) | 2,333,183 (+41,107) | -| 2025-12-20 | 1,223,000 (+19,515) | 1,146,258 (+16,560) | 2,369,258 (+36,075) | -| 2025-12-21 | 1,242,675 (+19,675) | 1,158,909 (+12,651) | 2,401,584 (+32,326) | -| 2025-12-22 | 1,262,522 (+19,847) | 1,169,121 (+10,212) | 2,431,643 (+30,059) | -| 2025-12-23 | 1,286,548 (+24,026) | 1,186,439 (+17,318) | 2,472,987 (+41,344) | -| 2025-12-24 | 1,309,323 (+22,775) | 1,203,767 (+17,328) | 2,513,090 (+40,103) | -| 2025-12-25 | 1,333,032 (+23,709) | 1,217,283 (+13,516) | 2,550,315 (+37,225) | -| 2025-12-26 | 1,352,411 (+19,379) | 1,227,615 (+10,332) | 2,580,026 (+29,711) | -| 2025-12-27 | 1,371,771 (+19,360) | 1,238,236 (+10,621) | 2,610,007 (+29,981) | -| 2025-12-28 | 1,390,388 (+18,617) | 1,245,690 (+7,454) | 2,636,078 (+26,071) | -| 2025-12-29 | 1,415,560 (+25,172) | 1,257,101 (+11,411) | 2,672,661 (+36,583) | -| 2025-12-30 | 1,445,450 (+29,890) | 1,272,689 (+15,588) | 2,718,139 (+45,478) | -| 2025-12-31 | 1,479,598 (+34,148) | 1,293,235 (+20,546) | 2,772,833 (+54,694) | -| 2026-01-01 | 1,508,883 (+29,285) | 1,309,874 (+16,639) | 2,818,757 (+45,924) | -| 2026-01-02 | 1,563,474 (+54,591) | 1,320,959 (+11,085) | 2,884,433 (+65,676) | -| 2026-01-03 | 1,618,065 (+54,591) | 1,331,914 (+10,955) | 2,949,979 (+65,546) | -| 2026-01-04 | 1,672,656 (+39,702) | 1,339,883 (+7,969) | 3,012,539 (+62,560) | -| 2026-01-05 | 1,738,171 (+65,515) | 1,353,043 (+13,160) | 3,091,214 (+78,675) | -| 2026-01-06 | 1,960,988 (+222,817) | 1,377,377 (+24,334) | 3,338,365 (+247,151) | -| 2026-01-07 | 2,123,239 (+162,251) | 1,398,648 (+21,271) | 3,521,887 (+183,522) | -| 2026-01-08 | 2,272,630 (+149,391) | 1,432,480 (+33,832) | 3,705,110 (+183,223) | -| 2026-01-09 | 2,443,565 (+170,935) | 1,469,451 (+36,971) | 3,913,016 (+207,906) | -| 2026-01-10 | 2,632,023 (+188,458) | 1,503,670 (+34,219) | 4,135,693 (+222,677) | -| 2026-01-11 | 2,836,394 (+204,371) | 1,530,479 (+26,809) | 4,366,873 (+231,180) | -| 2026-01-12 | 3,053,594 (+217,200) | 1,553,671 (+23,192) | 4,607,265 (+240,392) | -| 2026-01-13 | 3,297,078 (+243,484) | 1,595,062 (+41,391) | 4,892,140 (+284,875) | -| 2026-01-14 | 3,568,928 (+271,850) | 1,645,362 (+50,300) | 5,214,290 (+322,150) | -| 2026-01-16 | 4,121,550 (+552,622) | 1,754,418 (+109,056) | 5,875,968 (+661,678) | -| 2026-01-17 | 4,389,558 (+268,008) | 1,805,315 (+50,897) | 6,194,873 (+318,905) | -| 2026-01-18 | 4,627,623 (+238,065) | 1,839,171 (+33,856) | 6,466,794 (+271,921) | -| 2026-01-19 | 4,861,108 (+233,485) | 1,863,112 (+23,941) | 6,724,220 (+257,426) | -| 2026-01-20 | 5,128,999 (+267,891) | 1,903,665 (+40,553) | 7,032,664 (+308,444) | -| 2026-01-21 | 5,444,842 (+315,843) | 1,962,531 (+58,866) | 7,407,373 (+374,709) | -| 2026-01-22 | 5,766,340 (+321,498) | 2,029,487 (+66,956) | 7,795,827 (+388,454) | -| 2026-01-23 | 6,096,236 (+329,896) | 2,096,235 (+66,748) | 8,192,471 (+396,644) | -| 2026-01-24 | 6,371,019 (+274,783) | 2,156,870 (+60,635) | 8,527,889 (+335,418) | -| 2026-01-25 | 6,639,082 (+268,063) | 2,187,853 (+30,983) | 8,826,935 (+299,046) | -| 2026-01-26 | 6,941,620 (+302,538) | 2,232,115 (+44,262) | 9,173,735 (+346,800) | -| 2026-01-27 | 7,208,093 (+266,473) | 2,280,762 (+48,647) | 9,488,855 (+315,120) | -| 2026-01-28 | 7,489,370 (+281,277) | 2,314,849 (+34,087) | 9,804,219 (+315,364) | -| 2026-01-29 | 7,815,471 (+326,101) | 2,374,982 (+60,133) | 10,190,453 (+386,234) | diff --git a/RELEASING.md b/docs/RELEASING.md similarity index 100% rename from RELEASING.md rename to docs/RELEASING.md diff --git a/docs/design/ai-teammate-repositioning.md b/docs/design/ai-teammate-repositioning.md deleted file mode 100644 index 4256318c69..0000000000 --- a/docs/design/ai-teammate-repositioning.md +++ /dev/null @@ -1,566 +0,0 @@ -# altimate: From AI Tool to AI Teammate - -## The Core Repositioning - -**Current**: "The data engineering agent for dbt, SQL, and cloud warehouses" — a sophisticated CLI tool with 55+ features. - -**Proposed**: "Your data engineering teammate" — an AI colleague you onboard, train on your team's standards, and who gets better the more you work together. - -### Why This Matters - -The "AI tool" framing puts altimate in a crowded market of CLI tools and coding agents. Users evaluate it feature-by-feature against Claude Code, Cursor, Copilot, etc. - -The "AI teammate" framing creates a different mental model entirely: -- **Tools are disposable; teammates are invested in.** You don't "configure" a teammate — you onboard them, teach them your ways, and they earn your trust over time. -- **Tools are generic; teammates know your context.** A teammate knows your naming conventions, your warehouse quirks, your team's review standards, your domain vocabulary. -- **Tools wait for instructions; teammates are proactive.** A teammate notices when a PR introduces an anti-pattern, flags cost anomalies, and suggests improvements without being asked. - -### Inspiration: OpenClaw & the "Trainable Agent" Pattern - -**OpenClaw** (247K+ GitHub stars, fastest-growing open-source project ever) proved the "teammate" framing works when backed by real architecture. Key lessons: - -1. **Meet users where they are.** OpenClaw's UX *is* your existing messaging apps (WhatsApp, Telegram, Slack, Signal). Zero learning curve. For altimate, the equivalent: meet data engineers in their terminal, their dbt workflow, their Slack — don't force them into a separate app. - -2. **Self-improving memory.** OpenClaw captures learnings, errors, and corrections in structured files (`LEARNINGS.md`, `ERRORS.md`). When patterns recur 3+ times across 2+ tasks within 30 days, they auto-promote into permanent system prompt files (`CLAUDE.md`, `SOUL.md`). This is the model for altimate's training system — learning should be automatic, not manual. - -3. **Proactive heartbeat.** A scheduler wakes the agent at intervals so it can act without being prompted — checking email, running tasks, flagging issues. For altimate: imagine the teammate running nightly cost checks, freshness monitors, or schema drift detection without being asked. - -4. **Persistent identity.** One agent instance across all channels with shared memory and context. For altimate: the same teammate across TUI, web, CI/CD, and Slack — always knowing your project, your standards, your history. - -**Devin** ($10.2B valuation, $73M ARR) proved the market appetite: they market as "a collaborative AI teammate" and "the first AI software engineer," but candidly advise treating it as "a junior coding partner." The honesty works — users understand the capability boundary. - -**Factory AI** positions autonomous "Droids" that embed into existing workflows (VS Code, JetBrains, Slack, Linear). Their insight: "delegate complete tasks like refactors, incident response, and migrations without changing your tools." - -The **World Economic Forum** outlines the industry evolution: -1. **Copilots** (assisted intelligence) — suggestions, human controls -2. **Agents** (autonomous task execution) — limited decisions, task-oriented -3. **AI Teammates** (collaborative intelligence) — adapt, learn, achieve shared objectives - -altimate should skip straight to level 3 for data engineering. - -> The best AI teammate is one that adapts to YOUR team, not one that forces your team to adapt to IT. - ---- - -## Design: The Teammate Interface - -### 1. Onboarding (replacing "Configuration") - -Instead of a config file with warehouse credentials, the first experience is an **onboarding conversation**: - -``` -Welcome! I'm your new data engineering teammate. - -Let me get set up. I'll learn about your data stack, coding standards, -and how your team works. - -Step 1/4: Discovering your environment... - Found: dbt project (dbt-core 1.7.4) - Found: Snowflake warehouse (from ~/.dbt/profiles.yml) - Found: 3 dbt packages (dbt_utils, dbt_expectations, elementary) - Found: sqlfluff config (.sqlfluff) - -Step 2/4: Learning your coding style... - Analyzing 47 existing models... - Detected patterns: - - CTE-heavy style (avg 3.2 CTEs per model) - - snake_case naming (100% consistent) - - Staging models use "stg_" prefix - - Intermediate models use "int_" prefix - - Mart models use no prefix - - Primary keys named "_id" - - Created/updated timestamps always present - -Step 3/4: Understanding your review standards... - No .altimate/standards found. - Would you like to teach me your standards now, or learn as we go? - -Step 4/4: Learning your domain vocabulary... - Scanned model descriptions and column comments. - Built glossary: 23 domain terms (ARR, MRR, churn_date, cohort, ...) - -Ready! I'll keep learning as we work together. -``` - -**Key shift**: The user doesn't fill out a config file. They have a conversation. The AI discovers, learns, and confirms. - -### 2. Training (replacing "Skills/Prompts") - -Training is how the teammate gets better at YOUR team's specific way of working. Three training mechanisms: - -#### a) Learn-by-Example (`/teach`) - -``` -You: /teach staging-model -Teammate: Show me an example of a well-written staging model in your project. -You: @models/staging/stg_orders.sql -Teammate: Got it. I've learned these patterns from your staging model: - - Source reference via {{ source('stripe', 'orders') }} - - Rename columns to snake_case in CTE - - Cast types explicitly - - Add surrogate key via dbt_utils.generate_surrogate_key - - Final SELECT with column ordering: keys → dimensions → measures → timestamps - I'll follow this pattern for all future staging models. -``` - -The teammate stores this as a **learned pattern** (not just a prompt — structured knowledge it can reference and apply): - -```yaml -# .altimate/learned/staging-model.yml -name: staging-model -type: pattern -learned_from: models/staging/stg_orders.sql -learned_at: 2025-12-15T10:30:00Z -patterns: - - source_reference: "{{ source('', '') }}" - - cte_strategy: rename_and_cast - - surrogate_key: dbt_utils.generate_surrogate_key - - column_order: [keys, dimensions, measures, timestamps] - - naming: snake_case -``` - -#### b) Learn-by-Correction (Implicit + `/feedback`) - -When the teammate does something wrong, instead of just fixing it, you teach it: - -``` -You: Actually, we never use FLOAT — always use NUMERIC(18,2) for financial amounts -Teammate: Noted! I've updated my understanding: - Rule: Use NUMERIC(18,2) instead of FLOAT for financial columns - Applies to: columns matching *_amount, *_price, *_cost, *_revenue, *_total - I'll apply this in all future work. -``` - -Stored as a **learned rule**: - -```yaml -# .altimate/learned/rules/numeric-over-float.yml -name: numeric-over-float -type: rule -learned_at: 2025-12-15T11:00:00Z -rule: "Use NUMERIC(18,2) instead of FLOAT for financial amounts" -applies_to: - column_patterns: ["*_amount", "*_price", "*_cost", "*_revenue", "*_total"] -severity: high -``` - -**Auto-promotion (inspired by OpenClaw's self-improving agent):** - -When a correction pattern recurs 3+ times across 2+ sessions within 30 days, it auto-promotes from a "soft preference" to a "hard rule" in the system prompt. The teammate proactively tells you: - -``` -Teammate: I've noticed you've corrected my use of LEFT JOIN → INNER JOIN - in staging models 4 times now. Should I make this a permanent rule? - [Yes, always use INNER JOIN in staging] [No, keep case-by-case] -``` - -This mirrors OpenClaw's `LEARNINGS.md` → `CLAUDE.md` promotion pipeline, but adapted for structured data engineering rules rather than free-form text. - -#### c) Learn-by-Document (`/train`) - -For more formal training, point the teammate at documents: - -``` -You: /train https://wiki.company.com/data-team/sql-style-guide -Teammate: I've read your SQL style guide. Here's what I've learned: - - 14 SQL formatting rules - - 7 naming conventions - - 5 prohibited patterns (no SELECT *, no implicit joins, etc.) - - 3 performance guidelines - Want me to summarize what I'll enforce? -``` - -Also works with local files: - -``` -You: /train .altimate/standards/review-checklist.md -You: /train @models/README.md -``` - -### 3. Teammate Roles (replacing "Agent Modes") - -Instead of "switching agent modes," you ask your teammate to work in different roles. Same teammate, different hat: - -| Current (Mode) | Proposed (Role) | Framing | -|---|---|---| -| Builder | **Build** | "Help me build this model" | -| Analyst | **Investigate** | "Help me understand this data" | -| Validator | **Review** | "Review this PR / model" | -| Migrator | **Migrate** | "Help me move this to BigQuery" | -| Executive | **Brief** | "Summarize this for my VP" | -| *NEW* | **Deep Research** | "Do a deep dive on X" | - -The role selection becomes more natural: - -``` -You: @stg_orders.sql review this -Teammate: [automatically enters Review role] - -You: I need to understand why revenue numbers are off -Teammate: [automatically enters Investigate role, then Deep Research if complex] -``` - -**Auto-role detection**: The teammate infers the right role from context, rather than requiring explicit mode switching. You CAN still force a role with `/role build` etc. - -### 4. Deep Research Mode (NEW) - -This is a powerful new capability that differentiates altimate. When the user asks a complex question, the teammate enters Deep Research mode — a multi-step, thorough investigation. - -#### What Deep Research Does - -``` -You: Why did our Snowflake costs spike 40% last month? - -Teammate: [Entering Deep Research mode — this may take a few minutes] - -Phase 1: Gathering data - ├─ Pulling credit consumption history (90 days) - ├─ Analyzing query history (top 100 by cost) - ├─ Checking warehouse utilization patterns - └─ Comparing against previous month baseline - -Phase 2: Analyzing patterns - ├─ Identified 3 warehouses with anomalous usage - ├─ Found 12 queries with 10x+ cost increase - ├─ Detected new recurring job (started March 3) - └─ Cross-referencing with git history for pipeline changes - -Phase 3: Tracing root causes - ├─ PR #847 (merged March 2): Changed incremental model to full refresh - ├─ New dashboard query running every 15 min (no result caching) - ├─ ANALYTICS_WH auto-scaled to 2XL due to queue depth - └─ 3 zombie queries from failed Airflow tasks - -Phase 4: Synthesizing report - ... - -━━━ Deep Research Report ━━━ - -## Snowflake Cost Spike Analysis - -### Summary -Credits increased 40% ($12,400 → $17,360) driven by three root causes: - -### Root Cause 1: Full Refresh Regression (est. $2,800) -PR #847 changed `fct_orders` from incremental to full refresh. -This was likely unintentional — the PR description says "fix deduplication" -but the materialization change wasn't mentioned in review. -→ Recommendation: Revert to incremental, fix dedup with window function - -### Root Cause 2: Dashboard Over-Polling (est. $1,200) -The new "Real-Time Revenue" dashboard queries `mart_revenue` every 15 min. -This model takes ~45 seconds per run and doesn't use result caching. -→ Recommendation: Enable result caching, reduce frequency to hourly - -### Root Cause 3: Warehouse Auto-Scaling (est. $960) -ANALYTICS_WH scaled from L to 2XL due to queue depth from the above. -→ Recommendation: Will auto-resolve once causes 1-2 are fixed - -### Action Items -1. [ ] Revert fct_orders to incremental materialization -2. [ ] Enable result caching for mart_revenue -3. [ ] Kill 3 zombie queries (IDs: ...) -4. [ ] Add cost alerting threshold at $14,000/month -``` - -#### Deep Research Architecture - -Deep Research works by orchestrating multiple tools in a planned sequence: - -``` -User Question - │ - ▼ -┌─────────────────┐ -│ Research Planner│ ← Breaks question into investigation steps -└────────┬────────┘ - │ - ┌────┴────┐ - ▼ ▼ -┌────────┐ ┌────────┐ -│ Worker │ │ Worker │ ← Parallel data gathering (sub-agents) -│ (SQL) │ │ (Schema│ -└───┬────┘ └───┬────┘ - │ │ - ▼ ▼ -┌─────────────────┐ -│ Synthesizer │ ← Combines findings, identifies patterns -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Report Generator│ ← Produces structured output with actions -└─────────────────┘ -``` - -Key properties: -- **Multi-step**: Not a single LLM call — a planned investigation -- **Tool-heavy**: Uses warehouse queries, lineage tracing, cost analysis, git history -- **Parallel**: Gathers data concurrently where possible -- **Progressive**: Shows progress as it works (not a black box) -- **Actionable**: Always ends with specific recommendations and action items - -#### Deep Research Use Cases - -| Question | What it investigates | -|---|---| -| "Why did costs spike?" | Credit history, query costs, warehouse scaling, recent PRs | -| "Is this model safe to deprecate?" | Downstream lineage, dashboard dependencies, query history | -| "What's the data quality status of our pipeline?" | Test results, freshness, row counts, schema changes | -| "How should we migrate to BigQuery?" | Dialect differences, feature usage, cost projections | -| "What PII exposure do we have?" | Column scanning, lineage to PII sources, access controls | - -### 5. Memory & Continuity - -A teammate remembers. Three types of memory: - -#### a) Session Memory (Short-term) -What you discussed in this session. Already exists via session management. - -#### b) Project Memory (Medium-term) -Learned patterns, rules, and preferences stored in `.altimate/learned/`. Persists across sessions. Committed to git so the whole team benefits. - -``` -.altimate/ - learned/ - patterns/ # Learned from /teach - staging-model.yml - incremental-pattern.yml - rules/ # Learned from corrections - numeric-over-float.yml - no-select-star.yml - glossary/ # Domain terms - business-terms.yml - standards/ # Learned from /train - sql-style.yml - review-checklist.yml - training-log.yml # History of all training events -``` - -#### c) Team Memory (Long-term) -When learned patterns are committed to git, every team member's altimate instance picks them up. One person teaches, the whole team benefits. - -``` -You: /teach staging-model @models/staging/stg_orders.sql -Teammate: Learned! This pattern is saved to .altimate/learned/patterns/staging-model.yml - Commit this file so your teammates' altimate instances learn it too. -``` - -### 6. Proactive Behaviors (inspired by OpenClaw's Heartbeat) - -A real teammate doesn't wait to be asked. OpenClaw proved this with its heartbeat scheduler — a background process that wakes the agent at intervals to act autonomously. - -For altimate, proactive behaviors are domain-specific and high-value: - -#### Background Monitors (opt-in) - -```yaml -# .altimate/monitors.yml -cost_alert: - schedule: daily - threshold: 120% # Alert if costs >120% of 30-day average - action: slack_notify # or: session_note, email, github_issue - -freshness_check: - schedule: hourly - models: ["fct_orders", "fct_revenue", "dim_customers"] - stale_after: 4h - action: slack_notify - -schema_drift: - schedule: daily - watch: ["raw.*"] # Watch raw layer for upstream changes - action: session_note # Note it in next session - -pii_scan: - schedule: weekly - scope: all_indexed_schemas - action: github_issue -``` - -#### In-Session Proactive Behaviors - -``` -You: Can you build a staging model for the new payments table? - -Teammate: Sure! Before I start — I noticed something while looking at the - payments table: - - Column `card_number` matches PII pattern (credit card) - - Column `email` matches PII pattern (email address) - These should probably be masked or excluded in staging. - Want me to add masking, or exclude them? -``` - -The teammate doesn't wait to be asked about PII — it checks proactively because that's what a good data engineering teammate does. - -### 7. The Training Dashboard (`/training-status`) - -Show users what their teammate has learned: - -``` -You: /training-status - -━━━ Training Status ━━━ - -Patterns learned: 7 (staging, intermediate, mart, incremental, snapshot, source, test) -Rules learned: 12 (naming, types, performance, safety) -Glossary terms: 34 (business domain terms) -Standards loaded: 2 (SQL style guide, review checklist) - -Last training: 2 days ago (learned "no-cartesian-joins" rule) -Confidence: High (92% of suggestions accepted in last 30 days) - -Recent corrections: - - Dec 13: "Use NUMERIC not FLOAT for money" → applied 4 times since - - Dec 10: "staging models should have a _loaded_at timestamp" → applied 2 times - - Dec 8: "Don't use QUALIFY in staging, save for marts" → applied 1 time - -Want to review or modify any learned patterns? Use /teach --list -``` - ---- - -## Implementation Plan - -### Phase 1: Foundation (Training Infrastructure) - -**Goal**: Build the learned knowledge system and `/teach`, `/feedback`, `/train` commands. - -1. **Learned knowledge store** (`.altimate/learned/`) - - YAML-based storage for patterns, rules, glossary, standards - - Schema definitions for each knowledge type - - Loader that injects learned knowledge into system prompts - - File: `packages/opencode/src/altimate/learned/` - -2. **`/teach` skill** - - Accept file references as examples - - Extract patterns using LLM analysis - - Store as structured YAML - - File: `.opencode/skills/teach/SKILL.md` - -3. **`/feedback` implicit learning** - - Detect corrections in conversation ("actually, we prefer X") - - Extract rules and store them - - Apply rules in future sessions - - File: `packages/opencode/src/altimate/learning/` - -4. **`/train` document ingestion** - - Accept URLs and file paths - - Parse and extract actionable standards - - Store as structured knowledge - - File: `.opencode/skills/train/SKILL.md` - -5. **System prompt injection** - - Load all learned knowledge at session start - - Inject as context alongside agent prompts - - Priority: explicit rules > learned patterns > defaults - - File: modify `packages/opencode/src/session/system.ts` - -### Phase 2: Deep Research Mode - -**Goal**: Add a new "research" role that does multi-step investigations. - -1. **Research planner** - - Takes a question, breaks it into investigation steps - - Determines which tools to use for each step - - Plans parallel vs sequential execution - - File: `packages/opencode/src/altimate/research/planner.ts` - -2. **Research agent** - - New agent type with research-specific prompt - - Has access to all read-only tools + warehouse queries - - Progressive output (shows phases as it works) - - File: add to `packages/opencode/src/agent/agent.ts` - -3. **Report generator** - - Synthesizes findings into structured reports - - Always includes: summary, root causes, evidence, action items - - Export as markdown or JSON - - File: `packages/opencode/src/altimate/research/report.ts` - -4. **Auto-detection** - - Detect when a question warrants deep research vs quick answer - - Trigger automatically for complex analytical questions - - User can force with `/research` command - -### Phase 3: Teammate UX Polish - -**Goal**: Rebrand the interface to feel like working with a colleague. - -1. **Rename throughout** - - "Agent mode" → "Role" - - "Select agent" → "Switch role" - - "Skills" → "Abilities" (or keep skills — it works for teammates too) - - "Configuration" → "Training" / "Preferences" - -2. **Onboarding flow** - - Replace first-run config with conversational onboarding - - Auto-discover + confirm with user - - Learn initial patterns from existing codebase - -3. **Training status** - - `/training-status` command showing what's been learned - - Confidence scoring based on acceptance rate - - Suggestions for what to teach next - -4. **Proactive teammate behaviors** - - Suggest training opportunities ("I noticed you corrected my FLOAT usage 3 times — want me to learn this as a rule?") - - Flag when learned rules conflict - - Periodic "how am I doing?" prompts - -### Phase 4: Terminology & Marketing Updates - -1. **README**: "Your data engineering teammate" not "data engineering agent" -2. **CLI welcome**: "Ready to work!" not "Agent initialized" -3. **Tagline options**: - - "The data engineering teammate that learns your standards" - - "An AI teammate for data teams — train it once, benefit forever" - - "Your team's data engineering expert, trained on YOUR codebase" -4. **Key narrative**: "Don't configure another tool. Onboard a teammate." - ---- - -## Competitive Differentiation - -| Product | Framing | Training? | Data-Aware? | Proactive? | -|---|---|---|---|---| -| Claude Code | AI coding assistant | CLAUDE.md only | No | No | -| Cursor | AI-powered IDE | Cursor Rules files | No | No | -| Devin ($10.2B) | AI software engineer | No | No | Yes (async tasks) | -| Factory AI | Autonomous Droids | No | No | Yes (workflow triggers) | -| OpenClaw (247K stars) | Trainable AI agent | Self-improving memory + RL | No | Yes (heartbeat scheduler) | -| **altimate** | **AI data teammate** | **Structured learning (/teach, /train, auto-promote)** | **Yes (55+ tools, warehouse)** | **Yes (cost alerts, schema drift)** | - -### What altimate takes from each: - -| From | What we borrow | How we adapt it | -|---|---|---| -| **OpenClaw** | Self-improving memory, auto-promotion of learnings | Structured YAML rules instead of free-form markdown; domain-specific (SQL patterns, not general tasks) | -| **OpenClaw** | Heartbeat scheduler for proactive behavior | Nightly cost checks, freshness monitors, schema drift detection | -| **OpenClaw** | Meet-users-where-they-are UX | TUI + Web + Slack + CI/CD — same teammate everywhere | -| **Devin** | "Collaborative AI teammate" positioning | Same framing, but specialized: "data engineering teammate" not "software engineer" | -| **Devin** | Honest capability framing ("junior partner") | "Trained on your standards, but you're still the senior engineer" | -| **Factory AI** | Embed into existing workflows, don't replace them | Works inside your dbt workflow, not beside it | - -### The unique combination - -**Trainable + data-domain-specific + warehouse-connected + proactive.** - -No other product lets you: -1. Teach an AI your team's SQL standards (`/teach`) -2. Have it enforce those standards against your actual warehouse metadata and lineage -3. Watch it auto-improve from your corrections over time -4. Wake up to find it already flagged a cost anomaly or schema drift overnight - ---- - -## Summary - -The repositioning from "AI tool" to "AI teammate" is not just marketing — it requires real product changes: - -1. **Training infrastructure** that makes the AI genuinely learn and improve -2. **Deep Research mode** that showcases teammate-level initiative and thoroughness -3. **Memory system** that persists and shares knowledge across team members -4. **UX changes** that frame every interaction as collaboration, not command-and-control - -The result: users don't just use altimate — they invest in it, teach it, and trust it more over time. That's a fundamentally different (and stickier) relationship than "tool I configured once." diff --git a/packages/opencode/src/cli/cmd/tui/thread.ts b/packages/opencode/src/cli/cmd/tui/thread.ts index 6e787c7afd..1fa1540fd8 100644 --- a/packages/opencode/src/cli/cmd/tui/thread.ts +++ b/packages/opencode/src/cli/cmd/tui/thread.ts @@ -189,7 +189,7 @@ export const TuiThreadCommand = cmd({ events: undefined, } : { - url: "http://opencode.internal", + url: "http://altimate-code.internal", fetch: createWorkerFetch(client), events: createEventSource(client), } diff --git a/packages/opencode/src/cli/cmd/tui/worker.ts b/packages/opencode/src/cli/cmd/tui/worker.ts index aef81cc83a..0cd8135f4e 100644 --- a/packages/opencode/src/cli/cmd/tui/worker.ts +++ b/packages/opencode/src/cli/cmd/tui/worker.ts @@ -116,7 +116,7 @@ const startEventStream = (input: { directory: string; workspaceID?: string }) => }) as typeof globalThis.fetch const sdk = createOpencodeClient({ - baseUrl: "http://opencode.internal", + baseUrl: "http://altimate-code.internal", directory: input.directory, experimental_workspaceID: input.workspaceID, fetch: fetchFn, diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 0cde3344ed..f40cdba919 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -1266,6 +1266,7 @@ export namespace Config { .describe( "Automatically enhance prompts with AI before sending (default: false). Uses a small model to rewrite rough prompts into clearer versions.", ), + // altimate_change end // altimate_change start - env fingerprint skill selection toggle env_fingerprint_skill_selection: z .boolean() diff --git a/packages/opencode/src/control-plane/adaptors/worktree.ts b/packages/opencode/src/control-plane/adaptors/worktree.ts index f848909501..92252c5327 100644 --- a/packages/opencode/src/control-plane/adaptors/worktree.ts +++ b/packages/opencode/src/control-plane/adaptors/worktree.ts @@ -36,7 +36,7 @@ export const WorktreeAdaptor: Adaptor = { async fetch(info, input: RequestInfo | URL, init?: RequestInit) { const config = Config.parse(info) const { WorkspaceServer } = await import("../workspace-server/server") - const url = input instanceof Request || input instanceof URL ? input : new URL(input, "http://opencode.internal") + const url = input instanceof Request || input instanceof URL ? input : new URL(input, "http://altimate-code.internal") const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)) headers.set("x-opencode-directory", config.directory) diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index 15fe747123..e0c033bdbf 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -147,5 +147,3 @@ export const SkillTool = Tool.define("skill", async (ctx) => { }, } }) - -// altimate_change end - old partitionByFingerprint + rescueByMessage removed, replaced by selectSkillsWithLLM diff --git a/packages/opencode/test/branding/branding.test.ts b/packages/opencode/test/branding/branding.test.ts index a7758b27c5..6128dd4c29 100644 --- a/packages/opencode/test/branding/branding.test.ts +++ b/packages/opencode/test/branding/branding.test.ts @@ -298,16 +298,6 @@ describe("Workflow branding", () => { expect(content).not.toMatch(/contains\(.*'\/opencode'\)/) }) - test("beta.yml schedule is disabled", () => { - const content = readText(join(repoRoot, ".github", "workflows", "beta.yml")) - // Schedule should be commented out - const lines = content.split("\n") - const cronLines = lines.filter((l) => l.includes("cron:")) - for (const line of cronLines) { - expect(line.trimStart().startsWith("#")).toBe(true) - } - }) - test("opencode.yml model reference does not use opencode/ prefix", () => { const content = readText(join(repoRoot, ".github", "workflows", "opencode.yml")) expect(content).not.toContain("model: opencode/") diff --git a/script/beta.ts b/script/beta.ts deleted file mode 100755 index 61f9cf8620..0000000000 --- a/script/beta.ts +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bun - -import { $ } from "bun" - -interface PR { - number: number - title: string - author: { login: string } - labels: Array<{ name: string }> -} - -interface FailedPR { - number: number - title: string - reason: string -} - -async function commentOnPR(prNumber: number, reason: string) { - const body = `⚠️ **Blocking Beta Release** - -This PR cannot be merged into the beta branch due to: **${reason}** - -Please resolve this issue to include this PR in the next beta release.` - - try { - await $`gh pr comment ${prNumber} --body ${body}` - console.log(` Posted comment on PR #${prNumber}`) - } catch (err) { - console.log(` Failed to post comment on PR #${prNumber}: ${err}`) - } -} - -async function conflicts() { - const out = await $`git diff --name-only --diff-filter=U`.text().catch(() => "") - return out - .split("\n") - .map((x) => x.trim()) - .filter(Boolean) -} - -async function cleanup() { - try { - await $`git merge --abort` - } catch {} - try { - await $`git checkout -- .` - } catch {} - try { - await $`git clean -fd` - } catch {} -} - -async function fix(pr: PR, files: string[]) { - console.log(` Trying to auto-resolve ${files.length} conflict(s) with opencode...`) - const prompt = [ - `Resolve the current git merge conflicts while merging PR #${pr.number} into the beta branch.`, - `Only touch these files: ${files.join(", ")}.`, - "Keep the merge in progress, do not abort the merge, and do not create a commit.", - "When done, leave the working tree with no unmerged files.", - ].join("\n") - - try { - await $`opencode run -m opencode/gpt-5.3-codex ${prompt}` - } catch (err) { - console.log(` opencode failed: ${err}`) - return false - } - - const left = await conflicts() - if (left.length > 0) { - console.log(` Conflicts remain: ${left.join(", ")}`) - return false - } - - console.log(" Conflicts resolved with opencode") - return true -} - -async function main() { - console.log("Fetching open PRs with beta label...") - - const stdout = - await $`gh pr list --state open --draft=false --label beta --json number,title,author,labels --limit 100`.text() - const prs: PR[] = JSON.parse(stdout).sort((a: PR, b: PR) => a.number - b.number) - - console.log(`Found ${prs.length} open PRs with beta label`) - - if (prs.length === 0) { - console.log("No team PRs to merge") - return - } - - console.log("Fetching latest dev branch...") - await $`git fetch origin dev` - - console.log("Checking out beta branch...") - await $`git checkout -B beta origin/dev` - - const applied: number[] = [] - const failed: FailedPR[] = [] - - for (const pr of prs) { - console.log(`\nProcessing PR #${pr.number}: ${pr.title}`) - - console.log(" Fetching PR head...") - try { - await $`git fetch origin pull/${pr.number}/head:pr/${pr.number}` - } catch (err) { - console.log(` Failed to fetch: ${err}`) - failed.push({ number: pr.number, title: pr.title, reason: "Fetch failed" }) - await commentOnPR(pr.number, "Fetch failed") - continue - } - - console.log(" Merging...") - try { - await $`git merge --no-commit --no-ff pr/${pr.number}` - } catch { - const files = await conflicts() - if (files.length > 0) { - console.log(" Failed to merge (conflicts)") - if (!(await fix(pr, files))) { - await cleanup() - failed.push({ number: pr.number, title: pr.title, reason: "Merge conflicts" }) - await commentOnPR(pr.number, "Merge conflicts with dev branch") - continue - } - } else { - console.log(" Failed to merge") - await cleanup() - failed.push({ number: pr.number, title: pr.title, reason: "Merge failed" }) - await commentOnPR(pr.number, "Merge failed") - continue - } - } - - try { - await $`git rev-parse -q --verify MERGE_HEAD`.text() - } catch { - console.log(" No changes, skipping") - continue - } - - try { - await $`git add -A` - } catch { - console.log(" Failed to stage changes") - failed.push({ number: pr.number, title: pr.title, reason: "Staging failed" }) - await commentOnPR(pr.number, "Failed to stage changes") - continue - } - - const commitMsg = `Apply PR #${pr.number}: ${pr.title}` - try { - await $`git commit -m ${commitMsg}` - } catch (err) { - console.log(` Failed to commit: ${err}`) - failed.push({ number: pr.number, title: pr.title, reason: "Commit failed" }) - await commentOnPR(pr.number, "Failed to commit changes") - continue - } - - console.log(" Applied successfully") - applied.push(pr.number) - } - - console.log("\n--- Summary ---") - console.log(`Applied: ${applied.length} PRs`) - applied.forEach((num) => console.log(` - PR #${num}`)) - - if (failed.length > 0) { - console.log(`Failed: ${failed.length} PRs`) - failed.forEach((f) => console.log(` - PR #${f.number}: ${f.reason}`)) - throw new Error(`${failed.length} PR(s) failed to merge`) - } - - console.log("\nChecking if beta branch has changes...") - await $`git fetch origin beta` - - const localTree = await $`git rev-parse beta^{tree}`.text() - const remoteTrees = (await $`git log origin/dev..origin/beta --format=%T`.text()).split("\n") - - const matchIdx = remoteTrees.indexOf(localTree.trim()) - if (matchIdx !== -1) { - if (matchIdx !== 0) { - console.log(`Beta branch contains this sync, but additional commits exist after it. Leaving beta branch as is.`) - } else { - console.log("Beta branch has identical contents, no push needed") - } - return - } - - console.log("Force pushing beta branch...") - await $`git push origin beta --force --no-verify` - - console.log("Successfully synced beta branch") -} - -main().catch((err) => { - console.error("Error:", err) - process.exit(1) -}) diff --git a/script/duplicate-pr.ts b/script/duplicate-pr.ts deleted file mode 100755 index b77737c1d4..0000000000 --- a/script/duplicate-pr.ts +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bun - -import path from "path" -import { pathToFileURL } from "bun" -import { createOpencode } from "@opencode-ai/sdk" -import { parseArgs } from "util" - -async function main() { - const { values, positionals } = parseArgs({ - args: Bun.argv.slice(2), - options: { - file: { type: "string", short: "f" }, - help: { type: "boolean", short: "h", default: false }, - }, - allowPositionals: true, - }) - - if (values.help) { - console.log(` -Usage: bun script/duplicate-pr.ts [options] - -Options: - -f, --file File to attach to the prompt - -h, --help Show this help message - -Examples: - bun script/duplicate-pr.ts -f pr_info.txt "Check the attached file for PR details" -`) - process.exit(0) - } - - const message = positionals.join(" ") - if (!message) { - console.error("Error: message is required") - process.exit(1) - } - - const opencode = await createOpencode({ port: 0 }) - - try { - const parts: Array<{ type: "text"; text: string } | { type: "file"; url: string; filename: string; mime: string }> = - [] - - if (values.file) { - const resolved = path.resolve(process.cwd(), values.file) - const file = Bun.file(resolved) - if (!(await file.exists())) { - console.error(`Error: file not found: ${values.file}`) - process.exit(1) - } - parts.push({ - type: "file", - url: pathToFileURL(resolved).href, - filename: path.basename(resolved), - mime: "text/plain", - }) - } - - parts.push({ type: "text", text: message }) - - const session = await opencode.client.session.create() - const result = await opencode.client.session - .prompt({ - path: { id: session.data!.id }, - body: { - agent: "duplicate-pr", - parts, - }, - signal: AbortSignal.timeout(120_000), - }) - .then((x) => x.data?.parts?.find((y) => y.type === "text")?.text ?? "") - - console.log(result.trim()) - } finally { - opencode.server.close() - } -} - -main() diff --git a/script/stats.ts b/script/stats.ts deleted file mode 100755 index 6d4e5892d2..0000000000 --- a/script/stats.ts +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bun - -async function sendToPostHog(event: string, properties: Record) { - const key = process.env["POSTHOG_KEY"] - - if (!key) { - console.warn("POSTHOG_API_KEY not set, skipping PostHog event") - return - } - - const response = await fetch("https://us.i.posthog.com/i/v0/e/", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - distinct_id: "download", - api_key: key, - event, - properties: { - ...properties, - }, - }), - }).catch(() => null) - - if (response && !response.ok) { - console.warn(`PostHog API error: ${response.status}`) - } -} - -interface Asset { - name: string - download_count: number -} - -interface Release { - tag_name: string - name: string - assets: Asset[] -} - -interface NpmDownloadsRange { - start: string - end: string - package: string - downloads: Array<{ - downloads: number - day: string - }> -} - -async function fetchNpmDownloads(packageName: string): Promise { - try { - // Use a range from 2020 to current year + 5 years to ensure it works forever - const currentYear = new Date().getFullYear() - const endYear = currentYear + 5 - const response = await fetch(`https://api.npmjs.org/downloads/range/2020-01-01:${endYear}-12-31/${packageName}`) - if (!response.ok) { - console.warn(`Failed to fetch npm downloads for ${packageName}: ${response.status}`) - return 0 - } - const data: NpmDownloadsRange = await response.json() - return data.downloads.reduce((total, day) => total + day.downloads, 0) - } catch (error) { - console.warn(`Error fetching npm downloads for ${packageName}:`, error) - return 0 - } -} - -async function fetchReleases(): Promise { - const releases: Release[] = [] - let page = 1 - const per = 100 - - while (true) { - const url = `https://api.github.com/repos/AltimateAI/altimate-code/releases?page=${page}&per_page=${per}` - - const response = await fetch(url) - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status} ${response.statusText}`) - } - - const batch: Release[] = await response.json() - if (batch.length === 0) break - - releases.push(...batch) - console.log(`Fetched page ${page} with ${batch.length} releases`) - - if (batch.length < per) break - page++ - await new Promise((resolve) => setTimeout(resolve, 1000)) - } - - return releases -} - -function calculate(releases: Release[]) { - let total = 0 - const stats = [] - - for (const release of releases) { - let downloads = 0 - const assets = [] - - for (const asset of release.assets) { - downloads += asset.download_count - assets.push({ - name: asset.name, - downloads: asset.download_count, - }) - } - - total += downloads - stats.push({ - tag: release.tag_name, - name: release.name, - downloads, - assets, - }) - } - - return { total, stats } -} - -async function save(githubTotal: number, npmDownloads: number) { - const file = "STATS.md" - const date = new Date().toISOString().split("T")[0] - const total = githubTotal + npmDownloads - - let previousGithub = 0 - let previousNpm = 0 - let previousTotal = 0 - let content = "" - - try { - content = await Bun.file(file).text() - const lines = content.trim().split("\n") - - for (let i = lines.length - 1; i >= 0; i--) { - const line = lines[i].trim() - if (line.startsWith("|") && !line.includes("Date") && !line.includes("---")) { - const match = line.match( - /\|\s*[\d-]+\s*\|\s*([\d,]+)\s*(?:\([^)]*\))?\s*\|\s*([\d,]+)\s*(?:\([^)]*\))?\s*\|\s*([\d,]+)\s*(?:\([^)]*\))?\s*\|/, - ) - if (match) { - previousGithub = parseInt(match[1].replace(/,/g, "")) - previousNpm = parseInt(match[2].replace(/,/g, "")) - previousTotal = parseInt(match[3].replace(/,/g, "")) - break - } - } - } - } catch { - content = - "# Download Stats\n\n| Date | GitHub Downloads | npm Downloads | Total |\n|------|------------------|---------------|-------|\n" - } - - const githubChange = githubTotal - previousGithub - const npmChange = npmDownloads - previousNpm - const totalChange = total - previousTotal - - const githubChangeStr = - githubChange > 0 - ? ` (+${githubChange.toLocaleString()})` - : githubChange < 0 - ? ` (${githubChange.toLocaleString()})` - : " (+0)" - const npmChangeStr = - npmChange > 0 ? ` (+${npmChange.toLocaleString()})` : npmChange < 0 ? ` (${npmChange.toLocaleString()})` : " (+0)" - const totalChangeStr = - totalChange > 0 - ? ` (+${totalChange.toLocaleString()})` - : totalChange < 0 - ? ` (${totalChange.toLocaleString()})` - : " (+0)" - const line = `| ${date} | ${githubTotal.toLocaleString()}${githubChangeStr} | ${npmDownloads.toLocaleString()}${npmChangeStr} | ${total.toLocaleString()}${totalChangeStr} |\n` - - if (!content.includes("# Download Stats")) { - content = - "# Download Stats\n\n| Date | GitHub Downloads | npm Downloads | Total |\n|------|------------------|---------------|-------|\n" - } - - await Bun.write(file, content + line) - await Bun.spawn(["bunx", "prettier", "--write", file]).exited - - console.log( - `\nAppended stats to ${file}: GitHub ${githubTotal.toLocaleString()}${githubChangeStr}, npm ${npmDownloads.toLocaleString()}${npmChangeStr}, Total ${total.toLocaleString()}${totalChangeStr}`, - ) -} - -console.log("Fetching GitHub releases for AltimateAI/altimate-code...\n") - -const releases = await fetchReleases() -console.log(`\nFetched ${releases.length} releases total\n`) - -const { total: githubTotal, stats } = calculate(releases) - -console.log("Fetching npm all-time downloads for @altimateai/altimate-code...\n") -const npmDownloads = await fetchNpmDownloads("@altimateai/altimate-code") -console.log(`Fetched npm all-time downloads: ${npmDownloads.toLocaleString()}\n`) - -await save(githubTotal, npmDownloads) - -await sendToPostHog("download", { - count: githubTotal, - source: "github", -}) - -await sendToPostHog("download", { - count: npmDownloads, - source: "npm", -}) - -const totalDownloads = githubTotal + npmDownloads - -console.log("=".repeat(60)) -console.log(`TOTAL DOWNLOADS: ${totalDownloads.toLocaleString()}`) -console.log(` GitHub: ${githubTotal.toLocaleString()}`) -console.log(` npm: ${npmDownloads.toLocaleString()}`) -console.log("=".repeat(60)) - -console.log("-".repeat(60)) -console.log(`GitHub Total: ${githubTotal.toLocaleString()} downloads across ${releases.length} releases`) -console.log(`npm Total: ${npmDownloads.toLocaleString()} downloads`) -console.log(`Combined Total: ${totalDownloads.toLocaleString()} downloads`)