From 94ac2abaa0b078903acb49c7eb7026441fa99362 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 18 Jul 2026 12:54:29 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(heroku-to-aws):=20live=20discovery=20v?= =?UTF-8?q?ia=20the=20authenticated=20Heroku=20CLI=20Discovery=20previousl?= =?UTF-8?q?y=20required=20Terraform=20with=20heroku=5F*=20resources,=20whi?= =?UTF-8?q?ch=20most=20startups=20on=20Heroku=20don't=20have.=20Adds=20a?= =?UTF-8?q?=20consent-gated,=20read-only=20live=20discovery=20path:=20an?= =?UTF-8?q?=20interactive=20capture=20step=20(exact-command=20whitelist,?= =?UTF-8?q?=20config-var=20key=20names=20only,=20auth:token=20banned)=20wr?= =?UTF-8?q?ites=20raw=20CLI=20output=20to=20live-capture/,=20and=20a=20new?= =?UTF-8?q?=20'live'=20fragment=20parses=20it=20into=20the=20inventory.=20?= =?UTF-8?q?When=20Terraform=20is=20also=20present,=20live=20wins=20for=20c?= =?UTF-8?q?urrent=20state=20and=20disagreements=20surface=20as=20drift=20(?= =?UTF-8?q?unmanaged=20resources,=20plan=20changes,=20config=20conflicts)?= =?UTF-8?q?=20=E2=80=94=20never=20silently=20resolved.=20Includes=20replay?= =?UTF-8?q?=20fixtures=20with=20a=20drift-assertion=20set,=20validated=20b?= =?UTF-8?q?y=20end-to-end=20fresh-agent=20runs=20of=20the=20Discover=20pha?= =?UTF-8?q?se.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- migrate/README.md | 9 +- migrate/plugins/migration-to-aws/README.md | 20 +- .../fixtures/heroku-live-capture/README.md | 54 ++++++ .../heroku-live-capture/expected-drift.json | 82 ++++++++ .../live-capture/addons.json | 50 +++++ .../live-capture/app-acme-staging.json | 23 +++ .../live-capture/app-acme-web.json | 44 +++++ .../live-capture/apps.json | 35 ++++ .../config-keys-acme-staging.json | 6 + .../live-capture/config-keys-acme-web.json | 11 ++ .../live-capture/domains-acme-staging.json | 9 + .../live-capture/domains-acme-web.json | 16 ++ .../live-capture/manifest.json | 68 +++++++ .../live-capture/pg-acme-staging.out | 13 ++ .../live-capture/pg-acme-web.out | 16 ++ .../live-capture/pipeline-acme.json | 7 + .../live-capture/pipelines.json | 8 + .../live-capture/ps-acme-staging.json | 11 ++ .../live-capture/ps-acme-web.json | 29 +++ .../live-capture/redis-acme-web.out | 13 ++ .../live-capture/spaces.json | 1 + .../workspace-terraform/heroku.tf | 47 +++++ .../skills/heroku-to-aws/SKILL.md | 10 +- .../phases/discover/discover-assemble.md | 45 ++++- .../phases/discover/discover-live-capture.md | 162 ++++++++++++++++ .../phases/discover/discover-live.md | 176 ++++++++++++++++++ .../references/phases/discover/discover.md | 65 +++++-- .../shared/schema-discover-heroku.md | 63 +++++-- 28 files changed, 1051 insertions(+), 42 deletions(-) create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/expected-drift.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/addons.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-staging.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-web.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/apps.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-staging.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-web.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-staging.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-web.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/manifest.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-staging.out create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-web.out create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipeline-acme.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipelines.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-staging.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-web.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/redis-acme-web.out create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/spaces.json create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/workspace-terraform/heroku.tf create mode 100644 migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live-capture.md create mode 100644 migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live.md diff --git a/migrate/README.md b/migrate/README.md index 1b5ea629..4fa73558 100644 --- a/migrate/README.md +++ b/migrate/README.md @@ -4,7 +4,7 @@ AI agent skills for migrating workloads to AWS, built for [Claude Code](https:// ## What This Does -Point this plugin at your Terraform files, application code, or billing data. It runs a structured 6-phase assessment — discovering what you have, asking the right questions, designing the AWS architecture, estimating costs with real pricing data, and generating runnable migration artifacts. +Point this plugin at your Heroku account (via your authenticated Heroku CLI, read-only and consent-gated), your Terraform files, application code, or billing data. It runs a structured 6-phase assessment — discovering what you have, asking the right questions, designing the AWS architecture, estimating costs with real pricing data, and generating runnable migration artifacts. **Supported migration sources:** @@ -82,6 +82,7 @@ After installation, just describe what you want to migrate: **Heroku migrations:** - "Migrate my Heroku app to AWS" +- "Discover my Heroku apps and estimate AWS costs" - "Move my Heroku Postgres to RDS" - "Migrate from Heroku to Fargate" - "Migrate from Heroku to Elastic Beanstalk" @@ -90,6 +91,8 @@ After installation, just describe what you want to migrate: The skill creates a `.migration//` directory in the current working directory with all artifacts. +**Live Heroku discovery — how it works:** No Terraform or exports needed. If `heroku login` works in your terminal, just ask — the agent requests your consent, then inventories your account using read-only list/info CLI commands. It captures app names, dyno types, add-on plans and prices, domains, pipelines, and config var **key names only**. It never reads config var values, credentials, or your API token, and never runs a command that creates, changes, or deletes anything. If you also have `heroku_*` Terraform, the agent cross-checks it against your live account and reports drift. + ## What It Detects ### GCP → AWS @@ -164,9 +167,9 @@ The skill creates a `.migration//` directory in the current working dir - Claude Code >=2.1.29, Codex (latest), or [Cursor >= 2.5](https://cursor.com/changelog/2-5) - AWS CLI configured with appropriate credentials -- At least one input source: Terraform files, application code, or billing data +- At least one input source: an authenticated Heroku CLI (Heroku migrations), Terraform files, application code, or billing data - **For GCP AI/agentic migration:** Application source code is required (billing/IaC alone cannot detect agent architecture) -- **For Heroku migration:** Terraform files with `heroku_*` resources are required (Procfile/app.json supplements but cannot stand alone) +- **For Heroku migration:** an authenticated Heroku CLI (recommended — live, read-only discovery with your consent) or Terraform files with `heroku_*` resources (Procfile/app.json supplements but cannot stand alone). When both are available, live data is authoritative for current state and Terraform drift is surfaced. ## Structure diff --git a/migrate/plugins/migration-to-aws/README.md b/migrate/plugins/migration-to-aws/README.md index 98945f52..6263ab06 100644 --- a/migrate/plugins/migration-to-aws/README.md +++ b/migrate/plugins/migration-to-aws/README.md @@ -4,7 +4,7 @@ AI agent skills for migrating workloads to AWS, built for [Claude Code](https:// ## What This Does -Point this plugin at your Terraform files, application code, or billing data. It runs a structured 6-phase assessment — discovering what you have, asking the right questions, designing the AWS architecture, estimating costs with real pricing data, and generating runnable migration artifacts. +Point this plugin at your Heroku account (via your authenticated Heroku CLI, read-only and consent-gated), your Terraform files, application code, or billing data. It runs a structured 6-phase assessment — discovering what you have, asking the right questions, designing the AWS architecture, estimating costs with real pricing data, and generating runnable migration artifacts. **Supported migration sources:** @@ -103,7 +103,7 @@ ln -s "$(pwd)" ~/.cursor/plugins/local/migration-to-aws ### Workflow -1. **Discover** — Scan Terraform files, application code, and/or billing data. Detects infrastructure resources, AI models, agentic frameworks, tools, and orchestration patterns. +1. **Discover** — Scan Terraform files, application code, and/or billing data — or, for Heroku, inventory your account live via the authenticated Heroku CLI (read-only, consent-gated). Detects infrastructure resources, AI models, agentic frameworks, tools, and orchestration patterns. 2. **Clarify** — Ask targeted questions about migration preferences, AI priorities, agentic migration approach, database sizing, and timeline. 3. **Design** — Map source services to AWS equivalents. For AI workloads: select Bedrock models with honest pricing comparison. For agentic workloads: design AgentCore Harness config or Strands architecture. 4. **Estimate** — Calculate monthly AWS costs using real-time pricing data. Compare to current spend. @@ -198,9 +198,21 @@ The `--json` verdict lists each violation with `file`, `line`, `rule`, and `fix_ - Claude Code >=2.1.29, Codex (latest), or [Cursor >= 2.5](https://cursor.com/changelog/2-5) - AWS CLI configured with appropriate credentials -- At least one input source: Terraform files, application code, or billing data +- At least one input source: an authenticated Heroku CLI (Heroku migrations), Terraform files, application code, or billing data - **For GCP AI/agentic migration:** Application source code is required (billing/IaC alone cannot detect agent architecture) -- **For Heroku migration:** Terraform files with `heroku_*` resources are required (Procfile/app.json supplements but cannot stand alone) +- **For Heroku migration:** an authenticated Heroku CLI (recommended) or Terraform files with `heroku_*` resources (Procfile/app.json supplements but cannot stand alone) + +### Live Heroku discovery — how it works + +No Terraform or exports needed. If `heroku login` works in your terminal, just ask +your agent to migrate ("Migrate my Heroku app to AWS" or "Discover my Heroku apps +and estimate AWS costs"). The agent asks for your consent, then inventories your +account using read-only list/info CLI commands — it captures app names, dyno types, +add-on plans and prices, domains, pipelines, and config var **key names only**. It +never reads config var values, credentials, or your API token, and never runs a +command that creates, changes, or deletes anything. If you also have `heroku_*` +Terraform, the agent cross-checks it against your live account and reports drift. + - **For AI execution (llm-to-bedrock skill):** Python 3.10+, `uv`, and Bedrock model access enabled - **`uvx` required for cost estimation:** The `awspricing` MCP server runs via [`uvx`](https://docs.astral.sh/uv/guides/tools/) (part of the `uv` Python package manager). Install with `pip install uv` or `brew install uv`. Without it, the Estimate phase falls back to cached pricing — migration still works but live pricing lookups are unavailable. diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md new file mode 100644 index 00000000..18b2cf22 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md @@ -0,0 +1,54 @@ +# Heroku live-discovery fixtures (replay mode) + +Canned Heroku CLI outputs for testing the `heroku-to-aws` live-discovery path +(`discover-live-capture.md` → `discover-live.md` → `discover-assemble.md`) without +a Heroku account. The fake account is deliberately small but exercises every +designed-for behavior: + +- **acme-web** — production app: 2× Standard-2X web + 1× Standard-1X worker, + Postgres `standard-2` (42.3 GB — large enough to matter for migration-tool + selection), Redis `premium-0` (HA + TLS), Papertrail, one custom domain, and a + config var list whose keys imply an AI workload (`OPENAI_API_KEY`). +- **acme-staging** — small staging app on `essential-0`. +- **acme-data-team** — a team app the captured account cannot read: every per-app + capture is `failed` with a 403 in `manifest.json`. Expected result: + `discovery_status: "discovery_failed"`, confidence `reduced`, run continues. +- **kafka** capture is `skipped` (plugin not installed) — expected to be a + warning, never a halt. +- `spaces.json` is empty — the common startup case. + +## How to replay + +**Scenario A — live-only (no Terraform):** + +1. Create a scratch directory containing NO `.tf` files. +2. Create `.migration/0715-1820/` and copy `live-capture/` into it. +3. Invoke the heroku-to-aws skill ("migrate my Heroku app to AWS"). +4. The Discover phase's source precondition passes via the manifest; the `live` + fragment parses the captures. Expect an inventory with `discovery_sources: + ["live"]`, 2 successful apps + 1 failed, and no `drift` key. + +**Scenario B — live + Terraform (drift exercise):** + +1. As above, but also copy `workspace-terraform/heroku.tf` into the scratch + directory root. +2. `heroku.tf` is deliberately stale — each divergence is commented with the + merge rule it exercises (config conflicts, plan change, terraform-only, + live-only, scaled-to-zero gap-fill). +3. Check the assembled `heroku-resource-inventory.json` against + `expected-drift.json` — it lists the assertions (merged sources, conflict + fields, drift counts, and `must_not_exist` entries). + +**What a run must never produce** (from either scenario): + +- Config var values anywhere (fixture keys like `STRIPE_SECRET_KEY` are key + NAMES; if a value shows up, the keys-only rule broke) +- Clustering fields (`cluster_id`, `edges`, `dependencies`, ...) +- A halt due to the 403 app or the skipped kafka capture + +## Regenerating / extending + +Captures follow the exact command whitelist in `discover-live-capture.md` Step 3 +(same filenames, `.out` extension for text captures). If you add a capture type, +add its row to the whitelist first, then the fixture, then extend +`expected-drift.json`. All IDs, names, and emails are synthetic. diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/expected-drift.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/expected-drift.json new file mode 100644 index 00000000..4d9cdc8e --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/expected-drift.json @@ -0,0 +1,82 @@ +{ + "_comment": "Expected merge/drift outcomes when BOTH workspace-terraform/ and live-capture/ are used (scenario B in README.md). Not a full inventory — these are the assertions an eval run checks against heroku-resource-inventory.json.", + "metadata": { + "total_apps_discovered": 3, + "discovery_sources_must_include": ["terraform", "live"], + "confidence": "reduced", + "confidence_reason": "acme-data-team captures failed (403)" + }, + "apps": { + "acme-web": { + "discovery_status": "success", + "heroku_generation": "cedar", + "app_id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab" + }, + "acme-staging": { "discovery_status": "success", "heroku_generation": "cedar", "unmanaged_by_terraform": true }, + "acme-data-team": { "discovery_status": "discovery_failed", "failure_reason_contains": "403" } + }, + "merged_resources": { + "formation:acme-web:web": { + "source": "live+terraform", + "config": { "quantity": 2, "dyno_type": "standard-2x", "command": "npm start" }, + "expected_config_conflicts": ["quantity", "dyno_type"] + }, + "formation:acme-web:release": { + "source": "terraform", + "config": { "quantity": 0 }, + "note": "gap-fill rule 4 — scaled to zero, invisible live, NOT drift" + }, + "addon:acme-web:heroku-postgresql:standard-2": { + "source": "live+terraform", + "expected_config_conflicts": ["plan"], + "note": "rule 1a treats the plan change (tf standard-0 vs live standard-2) as ONE merged resource with a plan conflict, so source is live+terraform per rule 1", + "config": { "data_size_gb": 42.3, "pg_version": "16.4", "connection_pooling": true, "monthly_price_usd": 200.0 } + }, + "addon:acme-web:scheduler:standard": { "source": "terraform", "not_found_live": true }, + "addon:acme-web:heroku-redis:premium-0": { + "source": "live", + "unmanaged_by_terraform": true, + "config": { + "ha_enabled": true, + "encryption_in_transit": true, + "redis_version": "7.2.5", + "maxmemory_policy": "noeviction", + "monthly_price_usd": 15.0 + } + }, + "addon:acme-web:papertrail:choklad": { "source": "live", "unmanaged_by_terraform": true }, + "addon:acme-staging:heroku-postgresql:essential-0": { + "source": "live", + "unmanaged_by_terraform": true, + "config": { "data_size_gb": 0.4 } + }, + "formation:acme-staging:web": { + "source": "live", + "unmanaged_by_terraform": true, + "config": { "dyno_type": "basic", "quantity": 1 } + }, + "config:acme-web": { "config": { "config_var_keys_count": 9 }, "must_not_contain_values": true }, + "domain:acme-web:www.acme-demo.com": { "source": "live" }, + "pipeline:acme": { + "source": "live", + "config": { + "stages": [{ "stage": "staging", "app": "acme-staging" }, { "stage": "production", "app": "acme-web" }] + } + } + }, + "must_not_exist": [ + "domain:acme-web:acme-web-1a2b3c4d5e6f.herokuapp.com (default herokuapp.com domains are counted, not resources)", + "any resource with cluster_id / edges / dependencies fields", + "any config var VALUE anywhere in the document" + ], + "live_metadata": { + "apps_captured": 2, + "apps_failed": 1, + "limitations_must_include": "formations scaled to zero are not visible to live discovery", + "drift": { + "resources_live_only_min": 5, + "resources_terraform_only": 1, + "config_conflicts_expected_fields": ["quantity", "dyno_type", "plan"] + } + } +} diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/addons.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/addons.json new file mode 100644 index 00000000..7bb39b6a --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/addons.json @@ -0,0 +1,50 @@ +[ + { + "id": "ad-0001", + "name": "postgresql-cubed-12345", + "app": { "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", "name": "acme-web" }, + "addon_service": { "id": "as-0001", "name": "heroku-postgresql", "human_name": "Heroku Postgres" }, + "plan": { + "id": "p-0001", + "name": "heroku-postgresql:standard-2", + "price": { "cents": 20000, "unit": "month", "contract": false } + }, + "state": "provisioned" + }, + { + "id": "ad-0002", + "name": "redis-fluffy-67890", + "app": { "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", "name": "acme-web" }, + "addon_service": { "id": "as-0002", "name": "heroku-redis", "human_name": "Heroku Data for Redis" }, + "plan": { + "id": "p-0002", + "name": "heroku-redis:premium-0", + "price": { "cents": 1500, "unit": "month", "contract": false } + }, + "state": "provisioned" + }, + { + "id": "ad-0003", + "name": "papertrail-solid-11111", + "app": { "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", "name": "acme-web" }, + "addon_service": { "id": "as-0003", "name": "papertrail", "human_name": "Papertrail" }, + "plan": { + "id": "p-0003", + "name": "papertrail:choklad", + "price": { "cents": 0, "unit": "month", "contract": false } + }, + "state": "provisioned" + }, + { + "id": "ad-0004", + "name": "postgresql-round-22222", + "app": { "id": "a1b2c3d4-0002-4a5b-8c6d-0123456789ab", "name": "acme-staging" }, + "addon_service": { "id": "as-0001", "name": "heroku-postgresql", "human_name": "Heroku Postgres" }, + "plan": { + "id": "p-0004", + "name": "heroku-postgresql:essential-0", + "price": { "cents": 500, "unit": "month", "contract": false } + }, + "state": "provisioned" + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-staging.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-staging.json new file mode 100644 index 00000000..12f99964 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-staging.json @@ -0,0 +1,23 @@ +{ + "app": { + "id": "a1b2c3d4-0002-4a5b-8c6d-0123456789ab", + "name": "acme-staging", + "region": { "name": "us" }, + "stack": { "name": "heroku-22" }, + "space": null, + "buildpack_provided_description": "Node.js", + "web_url": "https://acme-staging-9f8e7d6c5b4a.herokuapp.com/" + }, + "addons": [ + { + "id": "ad-0004", + "name": "postgresql-round-22222", + "addon_service": { "name": "heroku-postgresql" }, + "plan": { "name": "heroku-postgresql:essential-0" } + } + ], + "collaborators": [{ "email": "founder@acme-demo.com" }], + "dynos": [ + { "id": "d-0004", "name": "web.1", "type": "web", "size": "Basic", "state": "up", "command": "npm start" } + ] +} diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-web.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-web.json new file mode 100644 index 00000000..86904c5e --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/app-acme-web.json @@ -0,0 +1,44 @@ +{ + "app": { + "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", + "name": "acme-web", + "region": { "name": "us" }, + "stack": { "name": "heroku-22" }, + "space": null, + "buildpack_provided_description": "Node.js", + "web_url": "https://acme-web-1a2b3c4d5e6f.herokuapp.com/" + }, + "addons": [ + { + "id": "ad-0001", + "name": "postgresql-cubed-12345", + "addon_service": { "name": "heroku-postgresql" }, + "plan": { "name": "heroku-postgresql:standard-2" } + }, + { + "id": "ad-0002", + "name": "redis-fluffy-67890", + "addon_service": { "name": "heroku-redis" }, + "plan": { "name": "heroku-redis:premium-0" } + }, + { + "id": "ad-0003", + "name": "papertrail-solid-11111", + "addon_service": { "name": "papertrail" }, + "plan": { "name": "papertrail:choklad" } + } + ], + "collaborators": [{ "email": "founder@acme-demo.com" }], + "dynos": [ + { "id": "d-0001", "name": "web.1", "type": "web", "size": "Standard-2X", "state": "up", "command": "npm start" }, + { "id": "d-0002", "name": "web.2", "type": "web", "size": "Standard-2X", "state": "up", "command": "npm start" }, + { + "id": "d-0003", + "name": "worker.1", + "type": "worker", + "size": "Standard-1X", + "state": "up", + "command": "node worker.js" + } + ] +} diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/apps.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/apps.json new file mode 100644 index 00000000..e6020669 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/apps.json @@ -0,0 +1,35 @@ +[ + { + "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", + "name": "acme-web", + "region": { "id": "r-0001", "name": "us" }, + "stack": { "id": "s-0001", "name": "heroku-22" }, + "space": null, + "web_url": "https://acme-web-1a2b3c4d5e6f.herokuapp.com/", + "maintenance": false, + "created_at": "2024-05-01T12:00:00Z", + "updated_at": "2026-07-01T08:15:00Z" + }, + { + "id": "a1b2c3d4-0002-4a5b-8c6d-0123456789ab", + "name": "acme-staging", + "region": { "id": "r-0001", "name": "us" }, + "stack": { "id": "s-0001", "name": "heroku-22" }, + "space": null, + "web_url": "https://acme-staging-9f8e7d6c5b4a.herokuapp.com/", + "maintenance": false, + "created_at": "2024-05-01T12:05:00Z", + "updated_at": "2026-06-20T14:00:00Z" + }, + { + "id": "a1b2c3d4-0003-4a5b-8c6d-0123456789ab", + "name": "acme-data-team", + "region": { "id": "r-0001", "name": "us" }, + "stack": { "id": "s-0002", "name": "heroku-24" }, + "space": null, + "web_url": "https://acme-data-team-3c4d5e6f7a8b.herokuapp.com/", + "maintenance": false, + "created_at": "2025-01-10T09:00:00Z", + "updated_at": "2026-07-10T11:30:00Z" + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-staging.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-staging.json new file mode 100644 index 00000000..f0688277 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-staging.json @@ -0,0 +1,6 @@ +[ + "DATABASE_URL", + "LANG", + "NODE_ENV", + "SESSION_SECRET" +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-web.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-web.json new file mode 100644 index 00000000..cd98e316 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/config-keys-acme-web.json @@ -0,0 +1,11 @@ +[ + "DATABASE_URL", + "LANG", + "NODE_ENV", + "OPENAI_API_KEY", + "REDIS_TLS_URL", + "REDIS_URL", + "SENTRY_DSN", + "SESSION_SECRET", + "STRIPE_SECRET_KEY" +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-staging.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-staging.json new file mode 100644 index 00000000..f8736f5e --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-staging.json @@ -0,0 +1,9 @@ +[ + { + "id": "do-0003", + "hostname": "acme-staging-9f8e7d6c5b4a.herokuapp.com", + "kind": "heroku", + "status": "none", + "sni_endpoint": null + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-web.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-web.json new file mode 100644 index 00000000..bcea1528 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/domains-acme-web.json @@ -0,0 +1,16 @@ +[ + { + "id": "do-0001", + "hostname": "acme-web-1a2b3c4d5e6f.herokuapp.com", + "kind": "heroku", + "status": "none", + "sni_endpoint": null + }, + { + "id": "do-0002", + "hostname": "www.acme-demo.com", + "kind": "custom", + "status": "succeeded", + "sni_endpoint": { "id": "sni-0001", "name": "tokyo-1234" } + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/manifest.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/manifest.json new file mode 100644 index 00000000..a65dfe05 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/manifest.json @@ -0,0 +1,68 @@ +{ + "captured_at": "2026-07-15T18:20:00Z", + "cli_version": "heroku/10.12.0 darwin-arm64 node-v20.19.0", + "account": "founder@acme-demo.com", + "apps_selected": ["acme-web", "acme-staging", "acme-data-team"], + "captures": [ + { "command": "heroku apps --all --json", "file": "apps.json", "status": "ok", "note": null }, + { "command": "heroku apps:info -a acme-web --json", "file": "app-acme-web.json", "status": "ok", "note": null }, + { + "command": "heroku apps:info -a acme-staging --json", + "file": "app-acme-staging.json", + "status": "ok", + "note": null + }, + { + "command": "heroku apps:info -a acme-data-team --json", + "file": "app-acme-data-team.json", + "status": "failed", + "note": "403 Forbidden — no access to team app" + }, + { "command": "heroku ps -a acme-web --json", "file": "ps-acme-web.json", "status": "ok", "note": null }, + { "command": "heroku ps -a acme-staging --json", "file": "ps-acme-staging.json", "status": "ok", "note": null }, + { + "command": "heroku ps -a acme-data-team --json", + "file": "ps-acme-data-team.json", + "status": "failed", + "note": "403 Forbidden — no access to team app" + }, + { "command": "heroku addons --all --json", "file": "addons.json", "status": "ok", "note": null }, + { + "command": "heroku config -a acme-web --json | jq 'keys'", + "file": "config-keys-acme-web.json", + "status": "ok", + "note": null + }, + { + "command": "heroku config -a acme-staging --json | jq 'keys'", + "file": "config-keys-acme-staging.json", + "status": "ok", + "note": null + }, + { + "command": "heroku config -a acme-data-team --json | jq 'keys'", + "file": "config-keys-acme-data-team.json", + "status": "failed", + "note": "403 Forbidden — no access to team app" + }, + { "command": "heroku domains -a acme-web --json", "file": "domains-acme-web.json", "status": "ok", "note": null }, + { + "command": "heroku domains -a acme-staging --json", + "file": "domains-acme-staging.json", + "status": "ok", + "note": null + }, + { "command": "heroku pg:info -a acme-web", "file": "pg-acme-web.out", "status": "ok", "note": null }, + { "command": "heroku pg:info -a acme-staging", "file": "pg-acme-staging.out", "status": "ok", "note": null }, + { "command": "heroku redis:info -a acme-web", "file": "redis-acme-web.out", "status": "ok", "note": null }, + { + "command": "heroku kafka:info -a acme-web", + "file": "kafka-acme-web.out", + "status": "skipped", + "note": "kafka CLI plugin not installed; no kafka add-on detected" + }, + { "command": "heroku pipelines --json", "file": "pipelines.json", "status": "ok", "note": null }, + { "command": "heroku pipelines:info acme --json", "file": "pipeline-acme.json", "status": "ok", "note": null }, + { "command": "heroku spaces --json", "file": "spaces.json", "status": "ok", "note": null } + ] +} diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-staging.out b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-staging.out new file mode 100644 index 00000000..29ca0e0f --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-staging.out @@ -0,0 +1,13 @@ +=== DATABASE_URL +Plan: Essential 0 +Status: Available +Data Size: 0.4 GB / 1 GB (40.00%) +Tables: 87 +PG Version: 16.4 +Connections: 3/20 +Credentials: 1 +Rollback: Unsupported +Created: 2024-05-01 12:15 UTC +Region: us +Data Encryption: In Use +Maintenance: not required diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-web.out b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-web.out new file mode 100644 index 00000000..a1e3cec7 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pg-acme-web.out @@ -0,0 +1,16 @@ +=== DATABASE_URL +Plan: Standard 2 +Status: Available +Data Size: 42.3 GB / 256 GB (16.52%) +Tables: 87 +PG Version: 16.4 +Connections: 38/400 +Connection Pooling: Available +Credentials: 1 +Fork/Follow: Available +Rollback: earliest from 2026-07-11 18:00 UTC +Created: 2024-05-01 12:10 UTC +Region: us +Data Encryption: In Use +Continuous Protection: On +Maintenance: not required diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipeline-acme.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipeline-acme.json new file mode 100644 index 00000000..ed5b5778 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipeline-acme.json @@ -0,0 +1,7 @@ +{ + "pipeline": { "id": "pl-0001", "name": "acme" }, + "apps": [ + { "id": "a1b2c3d4-0002-4a5b-8c6d-0123456789ab", "name": "acme-staging", "coupling": { "stage": "staging" } }, + { "id": "a1b2c3d4-0001-4a5b-8c6d-0123456789ab", "name": "acme-web", "coupling": { "stage": "production" } } + ] +} diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipelines.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipelines.json new file mode 100644 index 00000000..d4eb30c2 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/pipelines.json @@ -0,0 +1,8 @@ +[ + { + "id": "pl-0001", + "name": "acme", + "created_at": "2024-05-02T09:00:00Z", + "updated_at": "2026-06-01T10:00:00Z" + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-staging.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-staging.json new file mode 100644 index 00000000..58a1d4fb --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-staging.json @@ -0,0 +1,11 @@ +[ + { + "id": "d-0004", + "name": "web.1", + "type": "web", + "size": "Basic", + "state": "up", + "command": "npm start", + "updated_at": "2026-07-14T22:10:00Z" + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-web.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-web.json new file mode 100644 index 00000000..4a42a830 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/ps-acme-web.json @@ -0,0 +1,29 @@ +[ + { + "id": "d-0001", + "name": "web.1", + "type": "web", + "size": "Standard-2X", + "state": "up", + "command": "npm start", + "updated_at": "2026-07-15T06:00:00Z" + }, + { + "id": "d-0002", + "name": "web.2", + "type": "web", + "size": "Standard-2X", + "state": "up", + "command": "npm start", + "updated_at": "2026-07-15T06:00:00Z" + }, + { + "id": "d-0003", + "name": "worker.1", + "type": "worker", + "size": "Standard-1X", + "state": "up", + "command": "node worker.js", + "updated_at": "2026-07-15T06:00:00Z" + } +] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/redis-acme-web.out b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/redis-acme-web.out new file mode 100644 index 00000000..3d258057 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/redis-acme-web.out @@ -0,0 +1,13 @@ +=== redis-fluffy-67890 (REDIS_URL) +Plan: Premium 0 +Status: available +Created: 2024-06-15 10:00 UTC +Version: 7.2.5 +Timeout: 300 +Maxmemory: noeviction +Maintenance: not required +Maintenance window: Mondays 22:00 to Tuesdays 02:00 UTC +Persistence: AOF +HA Status: Available +Requires TLS: Yes +Keyspace Notifications: Disabled diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/spaces.json b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/spaces.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/live-capture/spaces.json @@ -0,0 +1 @@ +[] diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/workspace-terraform/heroku.tf b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/workspace-terraform/heroku.tf new file mode 100644 index 00000000..fb9fea00 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/workspace-terraform/heroku.tf @@ -0,0 +1,47 @@ +# Fixture Terraform — deliberately STALE relative to the live captures in +# ../live-capture/. Each divergence below exercises a specific rule in +# discover-assemble.md § Merge & Drift Rules. Do not "fix" the drift. + +resource "heroku_app" "web" { + name = "acme-web" + region = "us" + stack = "heroku-22" +} + +# Drift: live runs 2x Standard-2X; Terraform says 1x standard-1x. +# → merge rule 1 (config conflicts on quantity + dyno_type, live wins) +resource "heroku_formation" "web" { + app_id = heroku_app.web.id + type = "web" + quantity = 1 + size = "standard-1x" +} + +# Scaled-to-zero release process: invisible to `heroku ps`, declared here. +# → merge rule 4 (gap-fill from Terraform, expected complement not conflict) +resource "heroku_formation" "release" { + app_id = heroku_app.web.id + type = "release" + quantity = 0 + size = "standard-1x" +} + +# Drift: live plan is standard-2; Terraform still says standard-0. +# → merge rule 1a (plan change = config conflict, NOT an add/remove pair) +resource "heroku_addon" "postgres" { + app_id = heroku_app.web.id + plan = "heroku-postgresql:standard-0" +} + +# Declared but never deployed (absent from live captures). +# → merge rule 3 (terraform-only, not_found_live: true) +resource "heroku_addon" "scheduler" { + app_id = heroku_app.web.id + plan = "scheduler:standard" +} + +# NOTE deliberate absences from this file (present in live captures): +# - app acme-staging and everything on it → merge rule 2 (unmanaged_by_terraform) +# - heroku-redis:premium-0 on acme-web → merge rule 2 (unmanaged_by_terraform) +# - papertrail:choklad on acme-web → merge rule 2 (unmanaged_by_terraform) +# - pipeline "acme" and custom domain → merge rule 2 (unmanaged_by_terraform) diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/SKILL.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/SKILL.md index 100fd326..e3c23061 100644 --- a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/SKILL.md +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/SKILL.md @@ -1,6 +1,6 @@ --- name: heroku-to-aws -description: "Migrate workloads from Heroku to AWS. Triggers on: migrate from Heroku, Heroku to AWS, move off Heroku, migrate Heroku app, migrate Heroku Postgres to RDS, migrate Heroku Redis to ElastiCache, migrate Heroku Kafka to MSK, migrate dynos to Elastic Beanstalk, migrate dynos to Fargate, Heroku migration, move from Heroku to AWS, migrate Heroku Private Space, Heroku to Elastic Beanstalk, Heroku to ECS, Heroku to Fargate, leave Heroku, migrate off Heroku platform. Runs a 6-phase process: discover Heroku resources from Terraform files, Procfile/app.json, and optional billing exports, clarify migration requirements, design AWS architecture, estimate costs, generate migration artifacts, and collect optional feedback. Clarify must finish before Design, Estimate, or Generate. Uses a flat resource model (no clustering or dependency graphs) with deterministic mapping tables for core services (Dynos → Elastic Beanstalk by default, Postgres → RDS/Aurora, Redis → ElastiCache, Kafka → MSK) and a fast-path table for 13+ common add-ons. Cedar/Fir generation detection is detect-only in v1. Pipeline/Review Apps are detect-only. Do not use for: GCP or Azure migrations to AWS, AWS-to-Heroku reverse migration, general AWS architecture advice without migration intent, Heroku-to-Heroku refactoring, or multi-cloud deployments that do not involve migrating off Heroku." +description: "Migrate workloads from Heroku to AWS. Triggers on: migrate from Heroku, Heroku to AWS, move off Heroku, migrate Heroku app, migrate Heroku Postgres to RDS, migrate Heroku Redis to ElastiCache, migrate Heroku Kafka to MSK, migrate dynos to Elastic Beanstalk, migrate dynos to Fargate, Heroku migration, move from Heroku to AWS, migrate Heroku Private Space, Heroku to Elastic Beanstalk, Heroku to ECS, Heroku to Fargate, leave Heroku, migrate off Heroku platform. Runs a 6-phase process: discover Heroku resources live via the authenticated Heroku CLI (read-only, consent-gated) and/or from Terraform files, Procfile/app.json, and optional billing exports, clarify migration requirements, design AWS architecture, estimate costs, generate migration artifacts, and collect optional feedback. Clarify must finish before Design, Estimate, or Generate. Uses a flat resource model (no clustering or dependency graphs) with deterministic mapping tables for core services (Dynos → Elastic Beanstalk by default, Postgres → RDS/Aurora, Redis → ElastiCache, Kafka → MSK) and a fast-path table for 13+ common add-ons. Cedar/Fir generation detection is detect-only in v1. Pipeline/Review Apps are detect-only. Do not use for: GCP or Azure migrations to AWS, AWS-to-Heroku reverse migration, general AWS architecture advice without migration intent, Heroku-to-Heroku refactoring, or multi-cloud deployments that do not involve migrating off Heroku." --- # Heroku-to-AWS Migration Skill @@ -13,7 +13,7 @@ description: "Migrate workloads from Heroku to AWS. Triggers on: migrate from He - **Re-platform by default**: Select AWS services that match Heroku workload types (e.g., Dynos → Elastic Beanstalk, Heroku Postgres → RDS/Aurora, Heroku Redis → ElastiCache, Kafka → MSK). - **Dev sizing unless specified**: Default to development-tier capacity (e.g., db.t4g.micro, single AZ). Upgrade only on user direction. - **No human one-time migration costs**: Do not present human labor, professional services, or people-time work as dollar estimates or "one-time migration cost" budget categories. Vendor charges grounded in data (for example Heroku invoice line items in the infra estimate when billing exists) are allowed. -- **Terraform + repo as primary discovery**: Terraform files (`.tf` with `heroku_*` resources) and repo artifacts (Procfile, app.json) are the primary data sources for resource discovery. No Platform API calls in v1. +- **Live-first discovery, read-only and consent-gated**: The user's authenticated Heroku CLI is a first-class discovery source — most startups have no `heroku_*` Terraform, and the account is authoritative for what actually runs. Live capture is strictly read-only (an exact-command whitelist of list/info commands), requires explicit consent, never captures config var values (key names only), and never extracts the API token. Terraform files (`.tf` with `heroku_*` resources) and repo artifacts (Procfile, app.json) remain fully supported; when both live and Terraform data exist, live wins for current state, Terraform supplements structure and provenance, and disagreements are surfaced as drift — never silently resolved. - **Flat resource model**: Heroku resources are organized per-app without dependency graphs or clustering. No topological sorting, typed edges, or cluster formation logic. Resources are processed as a flat list in input order. - **Deterministic mappings**: Core services use fixed lookup tables (Dyno Type Table, Postgres Plan Table, Redis Plan Table, Kafka Plan Table). Common add-ons use the Fast-Path Table. Unknown add-ons hit the specialist gate. - **DMS has Heroku constraints**: AWS DMS cannot perform continuous replication (CDC) with Heroku Postgres because Heroku does not grant the REPLICATION role. DMS is for one-time bulk migration with a cutover window only. The skill must surface this constraint when DMS is selected. @@ -114,7 +114,9 @@ heroku-to-aws/ │ ├── phases/ │ │ ├── discover/ │ │ │ ├── discover.md # Phase 1: Discover orchestrator -│ │ │ ├── discover-terraform.md # Terraform discovery (primary) +│ │ │ ├── discover-terraform.md # Terraform discovery +│ │ │ ├── discover-live-capture.md # Live CLI capture (main-window pre-work, consent-gated) +│ │ │ ├── discover-live.md # Live discovery fragment (parses live-capture/) │ │ │ └── discover-billing.md # Billing data parsing │ │ ├── clarify/ │ │ │ └── clarify.md # Phase 2: Adaptive questions (12–15, batched ≤5) @@ -158,7 +160,7 @@ heroku-to-aws/ - **IaC output**: Terraform configurations, migration scripts, and documentation - **Region**: `us-east-1` (unless user specifies otherwise) - **Sizing**: Development tier (e.g., `db.t4g.micro` for databases, 0.5 CPU for Fargate) -- **Migration mode**: Adapts based on available inputs (Terraform primary, Procfile/app.json supplementary, billing optional) +- **Migration mode**: Adapts based on available inputs (live CLI discovery recommended, Terraform supported, Procfile/app.json supplementary, billing optional) - **Cost currency**: USD - **Timeline assumption**: 2-16 weeks depending on migration complexity — small (2-6 weeks), medium (6-12 weeks), large (12-18 weeks). Complexity tiers are classified per `references/vendored/estimate/complexity-tiers.json`. diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-assemble.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-assemble.md index e73bb160..5ecae8a8 100644 --- a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-assemble.md +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-assemble.md @@ -3,6 +3,7 @@ _assemble: assemble-inventory _of_phase: discover _reads: - terraform (fragment contribution) + - live (fragment contribution) - billing (fragment contribution) _produces: - heroku-resource-inventory.json @@ -11,7 +12,7 @@ _produces: # Discover — Assemble Inventory > **Assembler unit.** Runs after the discover fragments (`discover-terraform.md`, -> `discover-billing.md`) have produced their contributions. It combines them into +> `discover-live.md`, `discover-billing.md`) have produced their contributions. It combines them into > the single `heroku-resource-inventory.json` artifact and owns that artifact's > final contract. See `discover.md` for how this unit is composed into the phase. @@ -31,7 +32,47 @@ After all sub-discoveries complete, assemble `heroku-resource-inventory.json` in - `space` (Private Space name or null) - `procfile_parse_warning`, `app_json_parse_warning` (per-app parse warnings or null) 6. Include `billing_profile` section (if billing data available, with `available`, `total_monthly_cost`, `currency`, `billing_period`, `line_items`). -7. Include `terraform_metadata` section (if Terraform discovery ran, with `found`, `tf_files_scanned`, `resource_types_extracted`, `parse_warnings`). +7. Include `terraform_metadata` section only when `.tf` files with `heroku_*` resources were actually FOUND (the terraform fragment always runs but may exit empty — an empty run contributes no section and no `"terraform"` discovery source). +8. Include `live_metadata` section (if the live fragment ran, with `found`, `captured_at`, `apps_captured`, `apps_failed`, `capture_warnings`, `limitations` — and `drift` per the Merge & Drift Rules below). + +## Merge & Drift Rules (when BOTH terraform and live fragments contributed) + +Resource identity is `resource_id` (both fragments use the same deterministic ID +formats). Merge into ONE entry per `resource_id`. Never resolve a disagreement +silently — every conflict is recorded as drift. + +1. **Same `resource_id` from both:** keep one entry. Live values win field-by-field + in `config` (live reflects current account state; Terraform may be stale). Keep + Terraform provenance fields (`tf_file`, `tf_resource_name`). Set + `source: "live+terraform"`. If any config field disagreed, record it in + `live_metadata.drift.config_conflicts[]` as + `{ "resource_id", "field", "terraform_value", "live_value" }`. + 1a. **Add-on plan changes are conflicts, not add/remove pairs.** Add-on + `resource_id`s embed the plan, so before applying rules 2–3, pair any live-only + and terraform-only addon entries that share the same `heroku_app` + + `addon_service`. Treat such a pair as ONE resource with a `plan` config conflict + (rule 1): keep the live entry, record + `{ "resource_id": , "field": "plan", "terraform_value", "live_value" }` + in `config_conflicts[]`, and do not count the pair in `resources_live_only` / + `resources_terraform_only`. +2. **Live only:** keep the entry (`source: "live"`) and set + `unmanaged_by_terraform: true` — this is click-ops drift the migration plan must + include. +3. **Terraform only:** keep the entry (`source: "terraform"`) and set + `not_found_live: true` — defined but not deployed (or not in the selected app + set). Detect-only: downstream phases decide how to treat it. +4. **Formation gap-fill:** process types that live discovery cannot see (scaled to + zero, i.e. Terraform `quantity: 0`) but Terraform/Procfile declares are kept from + the Terraform contribution — this is the expected complement, not a conflict. Do + NOT set `not_found_live` on them and do NOT count them in + `resources_terraform_only`. +5. **Apps section:** live `apps[]` entries win (they carry the real `app_id` UUID); + merge in Terraform-only apps with `not_found_live: true`. +6. **Drift summary:** set `live_metadata.drift` to + `{ "resources_live_only": N, "resources_terraform_only": M, "config_conflicts": [...] }`. + +When only ONE of the two fragments contributed, no merge occurs and no `drift` key +is written. **If assembly fails** (no valid resources from any source after sub-discoveries ran): this is an unrecoverable error (`INTERPRETER.md` § `_on_error` — `_unrecoverable`). diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live-capture.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live-capture.md new file mode 100644 index 00000000..e078f621 --- /dev/null +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live-capture.md @@ -0,0 +1,162 @@ +# Discover Phase: Live Capture (Main Window — Interactive Pre-Work) + +> Interactive CLI capture step for live discovery. **This file is NOT a fragment.** +> It runs in the MAIN window because it converses with the user and uses the shell — +> two things the dispatched `rw` worker cannot do. `discover.md` § Orientation says +> when to load it: after `_init`, before the phase's work is dispatched. +> +> Its only output is raw CLI captures under `$MIGRATION_DIR/live-capture/` plus a +> `manifest.json` index. The `live` FRAGMENT (`discover-live.md`) parses those files +> into inventory entries inside the worker. This file never writes inventory entries. + +**Execute ALL steps in order. Do not skip or optimize.** + +--- + +## Security Contract (applies to every step) + +1. **Exact-command whitelist.** Run ONLY commands that appear as rows in the Capture + Command Table (Step 3). Read and write commands share namespaces in the Heroku CLI + (`addons` lists; `addons:upgrade` mutates; `addons:upgrade` and `addons:downgrade` + are the same command). No namespace-prefix reasoning: if a command is not a table + row, do not run it. +2. **Never capture secrets.** + - NEVER run `heroku auth:token` — it prints the API token to stdout. + - Config vars: capture KEY NAMES ONLY, using the key filter in Step 3 row 5. NEVER + write raw `heroku config` output (values are secrets) to disk, to chat, or into + any artifact. If neither `jq` nor `python3` is available for filtering, SKIP + config capture entirely and record a `skipped` entry in the manifest. +3. **Always explicit flags.** Some commands change scope based on the working + directory (`heroku addons` implies `--app` inside an app's git repo). Always pass + `--all` or `-a ` explicitly. Never use the global `--prompt` flag. +4. **No mutations, no logins.** Never run `heroku login` (browser-interactive — hand + off to the user per Step 2). Never run any create / set / add / attach / scale / + upgrade / destroy / rename / remove command. +5. **Capture to files, not chat.** Redirect stdout to files under + `$MIGRATION_DIR/live-capture/`. Do not paste large outputs into the conversation. + (`.migration/` is gitignored by `_init`, so captures cannot be committed.) + +--- + +## Step 1: Consent Gate + +Output exactly, then wait for the user's choice: + +``` +─── Live Heroku Discovery (read-only) ─── + +I can inventory your Heroku account directly using your authenticated +Heroku CLI. This runs LIST/INFO commands only: + + ✓ Captured: app names, regions, stacks, dyno types and counts, + add-on plans and prices, domain names, pipeline stages, Private + Space peering info, and config var KEY NAMES. + ✗ Never captured: config var values, credentials, API tokens, + source code, or database contents. No command that creates, + changes, or deletes anything will run. + +Output is written to .migration//live-capture/ (gitignored). + +[A] Proceed with live discovery +[B] Skip — use workspace files only +``` + +- **[A]** → continue to Step 2. +- **[B]** → do not run any Heroku command. Return to `discover.md` and record that + live capture was declined. If no `heroku_*` Terraform exists either, the phase's + `_preconditions` will fail normally. + +## Step 2: Preflight + +1. **CLI installed:** run `heroku --version`. + - Missing → tell the user: "The Heroku CLI isn't installed. Install it + (https://devcenter.heroku.com/articles/heroku-cli) and tell me to continue, or + choose to skip live discovery." Wait. If skipped → exit as in Step 1 [B]. +2. **Authenticated:** run `heroku auth:whoami`. + - Success → record the account email for the manifest. + - Failure (not logged in, or token expired) → tell the user: "Your Heroku CLI + isn't authenticated (or the session expired). Run `heroku login` in your + terminal — it needs a browser, so I can't run it for you — then tell me to + continue." Wait. If the user declines → exit as in Step 1 [B]. + +## Step 3: Capture + +Create `$MIGRATION_DIR/live-capture/`. Then: + +**3a. App list and selection guard.** Run row 1 first. If it returns more than 25 +apps, list the app names and ask the user which apps to include (`all` is a valid +answer). Record the selected set as `apps_selected`. All per-app rows below run only +for selected apps. + +**3b. Capture Command Table.** Run each applicable row, redirecting stdout to the +named file. ``, ``, `` iterate over the selected apps and the +account's pipelines/spaces from rows 9 and 11. + +| # | Command | Output file | Scope | +| -- | -------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `heroku apps --all --json` | `apps.json` | account | +| 2 | `heroku apps:info -a --json` | `app-.json` | per app | +| 3 | `heroku ps -a --json` | `ps-.json` | per app | +| 4 | `heroku addons --all --json` | `addons.json` | account | +| 5 | `heroku config -a --json \| jq 'keys'` | `config-keys-.json` | per app — KEYS ONLY (see fallback below) | +| 6 | `heroku domains -a --json` | `domains-.json` | per app | +| 7 | `heroku pg:info -a ` | `pg-.out` | apps with a heroku-postgresql add-on (from row 4) | +| 8 | `heroku redis:info -a ` | `redis-.out` | apps with a heroku-redis add-on (from row 4) | +| 8b | `heroku kafka:info -a ` | `kafka-.out` | apps with a heroku-kafka add-on; needs the kafka CLI plugin — if the command is unavailable, record `skipped` (plan data from row 4 is sufficient for sizing) | +| 9 | `heroku pipelines --json` | `pipelines.json` | account | +| 10 | `heroku pipelines:info --json` | `pipeline-.json` | per pipeline from row 9 | +| 11 | `heroku spaces --json` | `spaces.json` | account (empty result is normal — most startups have no spaces) | +| 12 | `heroku spaces:info -s --json` | `space-.json` | per space from row 11 | +| 13 | `heroku spaces:peerings -s --json` | `space-peerings-.json` | per space from row 11 | + +Text captures use the `.out` extension (not `.txt`) so they can never collide with +the phase's `_forbids_files: "*.txt"` scope boundary, regardless of how a host +scopes that glob. + +**Row 5 fallback:** if `jq` is unavailable, use +`heroku config -a --json | python3 -c "import json,sys; print(json.dumps(sorted(json.load(sys.stdin))))"`. +If neither filter runtime exists, skip row 5 for all apps and record `skipped` in the +manifest — never capture unfiltered config output. + +**3c. Per-command error handling:** + +| Error | Behavior | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| 403 / forbidden on a team app | Record the capture as `failed` with the reason; continue with remaining apps | +| Command not found (missing CLI plugin, row 8b) | Record `skipped`; continue | +| Timeout / transient network error | Retry once; on second failure record `failed`; continue | +| 401 / token expired mid-run | STOP capturing. Hand off as in Step 2.2. On resume, re-run Step 3 from the top (captures overwrite) | +| Rate limited (429) | Unexpected at this call volume — wait 60s, retry once, then record `failed` | + +## Step 4: Write the Manifest + +Write `$MIGRATION_DIR/live-capture/manifest.json`: + +```json +{ + "captured_at": "", + "cli_version": "", + "account": "", + "apps_selected": ["my-web-app"], + "captures": [ + { "command": "heroku apps --all --json", "file": "apps.json", "status": "ok", "note": null }, + { + "command": "heroku config -a my-web-app --json | jq 'keys'", + "file": "config-keys-my-web-app.json", + "status": "ok", + "note": null + } + ] +} +``` + +`status` ∈ `ok | failed | skipped`. Every attempted or deliberately skipped row gets +an entry. The manifest is the fragment's index — its existence is also the `live` +fragment's `_trigger`. + +## Step 5: Return to `discover.md` + +Tell the user in one line how many apps were captured and whether any captures +failed or were skipped, then continue the phase per `discover.md` (the dispatched +worker will parse `live-capture/` via the `live` fragment). Do NOT parse captures +here, do NOT write inventory entries, and do NOT update `.phase-status.json`. diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live.md new file mode 100644 index 00000000..49f181a1 --- /dev/null +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover-live.md @@ -0,0 +1,176 @@ +--- +_fragment: live +_of_phase: discover +_contributes: + - heroku-resource-inventory.json (resource entries, apps, metadata, live_metadata sections) +--- + +# Discover Phase: Live Discovery (Parse Captured CLI Output) + +> Self-contained live-discovery fragment. Reads the raw CLI captures that +> `discover-live-capture.md` wrote to `$MIGRATION_DIR/live-capture/` and maps them +> to inventory entries. **Parse-only**: this fragment runs inside the dispatched +> worker — it has no shell and MUST NOT run any `heroku` command, prompt the user, +> or re-capture anything. If `live-capture/manifest.json` does not exist, exit +> cleanly with no output. + +**Execute ALL steps in order. Do not skip or optimize.** + +--- + +## Step 0: Read the Manifest + +Read `$MIGRATION_DIR/live-capture/manifest.json`. It indexes every capture file with +a `status` of `ok`, `failed`, or `skipped`. Process only `ok` captures. Carry every +`failed`/`skipped` entry forward into `live_metadata.capture_warnings`. + +## Step 1: Map Apps — `apps.json` + `app-.json` + +For each app in `apps.json` that is in the manifest's `apps_selected`: + +| Capture field | Inventory field (`apps[]` entry) | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `name` | `app_name` | +| `id` | `app_id` (real UUID — live discovery always has this) | +| `space.name` | `space` (or `null`) | +| `stack.name` | feeds `heroku_generation`: `heroku-20`/`heroku-22`/`heroku-24` → `"cedar"`; contains `fir` or `cnb` → `"fir"`; else `"unknown"` | + +Set `generation_action: "detect_only"`, `discovery_status: "success"`. An app whose +per-app captures all failed (manifest `failed`) gets `discovery_status: +"discovery_failed"` with `failure_reason` from the manifest note — it still gets an +`apps[]` entry and still counts in `metadata.total_apps_discovered` (it was +discovered; its details were not). + +## Step 2: Map Formations — `ps-.json` + +Group the dyno list by `type`: + +| Derivation | Inventory field (`formation` config) | +| ------------------------------------------------------- | -------------------------------------------------- | +| dyno `type` | `process_type` | +| count of dynos of that type | `quantity` | +| dyno `size`, lowercased (`Standard-1X` → `standard-1x`) | `dyno_type` — normalize to match the sizing tables | +| dyno `command` | `command` | + +Resource entry: `resource_id: "formation:{app_name}:{process_type}"`, +`resource_type: "formation"`, `source: "live"`. + +**Known limitation:** `heroku ps` shows running dynos only — a process type scaled +to zero is invisible to live discovery. Record the limitation string +`"formations scaled to zero are not visible to live discovery"` once in +`live_metadata.limitations`. (When Terraform or a Procfile also ran, the assembler +recovers those process types from that source.) + +## Step 3: Map Add-ons — `addons.json` (+ `pg/redis/kafka` info captures) + +For each add-on attached to a selected app: + +| Capture field | Inventory field (`addon` config) | +| ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `plan.name` (e.g. `heroku-postgresql:standard-0`) | split into `addon_service` and `plan` (same rule as Terraform discovery) | +| `addon_service.name` | `provider` — `"heroku"` when the service name starts with `heroku-` (first-party data add-ons), otherwise the service name itself (matches the Terraform fragment's convention) | +| `plan.price` (`cents`, `unit`) | `monthly_price_usd` = `cents / 100` when `unit == "month"` (optional field) | + +Resource entry: `resource_id: "addon:{app_name}:{addon_service}:{plan}"`, +`resource_type: "addon"`, `source: "live"`. + +**Enrich from info captures** (parse the `Key: Value` text lines; on unparseable +text, keep the plan-only entry and record a warning): + +- `pg-.out` → `pg_version`, `data_size_gb` (convert e.g. `10.5 GB`; MB → GB), + `table_count`, `connection_pooling` (`true` if a Connection Pooling line is + present, `false` if absent — never omit the field). The `data_size_gb` value + feeds downstream database-migration tool selection. +- `redis-.out` → `redis_version`, `maxmemory_policy`; set `ha_enabled: true` + when the plan tier starts with `premium` or `private`, and + `encryption_in_transit: true` for those tiers. +- `kafka-.out` → `topic_count`, `partitions_per_topic`, `replication_factor` + when present. + +## Step 4: Map Domains, Config, Pipelines, Spaces + +**Domains** (`domains-.json`): one entry per custom domain — +`resource_id: "domain:{app_name}:{hostname}"`, `resource_type: "domain"`, config +`{ "hostname": ..., "sni_endpoint": }`. +Skip default `*.herokuapp.com` hostnames — record their count as +`live_metadata.default_heroku_domains_skipped` (exact field name), not as resources. + +**Config** (`config-keys-.json`): `resource_id: "config:{app_name}"`, +`resource_type: "config"`, config `{ "config_var_keys": [...] }`. Keys only — if a +capture file unexpectedly contains values (objects, not a string array), DISCARD it, +do not copy any part into the inventory, and record a warning. + +**Pipelines** (`pipelines.json` + `pipeline-.json`): +`resource_id: "pipeline:{pipeline_name}"`, `resource_type: "pipeline"`, config +`{ "pipeline_name": ..., "stages": [{ "stage": ..., "app": ... }], "review_apps_enabled": false, "detection_status": "detect-only" }`. +Populate `stages` from the per-pipeline capture; if `review_apps_enabled` is not +derivable, set it `false` and append a note to `live_metadata.capture_warnings`. + +**Spaces** (`spaces.json` + `space-.json` + `space-peerings-.json`): +`resource_id: "space:{space_name}"`, `resource_type: "space"`, config per the schema — +`space_name`, `region`, `shield`, and `peering` filled from the peerings capture +(`detected: true` with `vpc_id`/`peer_cidr` when an active peering exists; this is +data Terraform discovery usually cannot see). + +All entries: `heroku_app` = owning app name, or `"unassociated"` for spaces and +pipelines. `source: "live"` on every entry this fragment contributes. + +## Step 5: Output Contribution for the Assembler + +The assembler (`discover-assemble.md`) owns the inventory's structure and the merge +with Terraform-sourced entries. This fragment contributes: + +- **Resources:** all entries from Steps 2–4, each with `source: "live"`. +- **Apps:** the `apps[]` entries from Step 1. +- **Discovery sources:** contribute `"live"` to `metadata.discovery_sources`. +- **Confidence:** `"full"` when every capture for the selected apps has manifest + status `ok`; `"reduced"` otherwise (with `confidence_note` naming what failed or + was skipped). +- **`live_metadata`:** + +```json +{ + "live_metadata": { + "found": true, + "captured_at": "2026-07-15T18:20:00Z", + "apps_captured": 3, + "apps_failed": 0, + "capture_warnings": [], + "limitations": ["formations scaled to zero are not visible to live discovery"] + } +} +``` + +--- + +## Error Handling + +| Error Category | Behavior | +| --------------------------------------------- | ----------------------------------------------------------------------- | +| `manifest.json` missing | Exit cleanly with no output (capture never ran) | +| Capture file named in manifest is missing | Record warning, skip that capture, continue | +| Malformed JSON in a capture file | Record warning, skip that file, continue | +| Unparseable `pg/redis/kafka` info text | Keep the plan-only addon entry, record warning, continue | +| Config capture contains values (not key list) | Discard the file entirely, record warning, continue — never copy values | + +**Key principle:** partial results are better than no results. Any single capture +failure degrades confidence; it never halts the fragment. + +--- + +## Scope Boundary + +**This fragment covers parsing of `live-capture/` files ONLY.** + +FORBIDDEN — Do NOT include ANY of: + +- Running `heroku` (or any shell) commands — capture already happened in the main window +- Prompting the user +- AWS service names, recommendations, or equivalents +- Migration strategies, timelines, cost estimates, or effort estimates +- Merging or de-duplicating against Terraform-sourced entries — the assembler owns the merge + +**Your ONLY job: turn raw CLI captures into inventory entries. Nothing else.** + +After producing entries, the assembler handles merging into the final inventory; do +NOT update `.phase-status.json` from this fragment. diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover.md index 348b16dd..37ec7c23 100644 --- a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover.md +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/phases/discover/discover.md @@ -7,6 +7,9 @@ _fragments: - _id: terraform _trigger: { _always: true } _file: phases/discover/discover-terraform.md + - _id: live + _trigger: { _when: "$MIGRATION_DIR/live-capture/manifest.json exists (the live-capture pre-work ran — see Orientation § Live capture)" } + _file: phases/discover/discover-live.md - _id: billing _trigger: { _glob: "**/*{billing,invoice}*.{csv,json}" } _file: phases/discover/discover-billing.md @@ -26,7 +29,7 @@ _re_entry_guard: _preconditions: - _check_single_active_phase: true _on_failure: _halt_and_inform - - _assert: "at least one .tf file containing a heroku_* resource exists in the workspace" + - _assert: "at least one Heroku source is available: a .tf file containing a heroku_* resource exists in the workspace, OR $MIGRATION_DIR/live-capture/manifest.json exists. Evaluating this check is where live capture is offered: if no heroku_* Terraform is found, load references/phases/discover/discover-live-capture.md in the MAIN window and run it (consent-gated); fail this check only after the user declines live capture or capture cannot run" _on_failure: _unrecoverable _postconditions: - _check_file_exists: heroku-resource-inventory.json @@ -39,7 +42,11 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "no forbidden clustering fields are present (cluster_id, creation_order_depth, edges, dependencies, must_migrate_together)" _on_failure: _halt_and_inform - - _assert: "metadata.discovery_sources reflects which sub-discoveries actually ran; if the terraform sub-discovery ran, resources[] contains at least one Terraform-sourced resource" + - _assert: "metadata.discovery_sources reflects which sub-discoveries actually produced data; if heroku_* Terraform files were FOUND in the workspace (not merely that the terraform fragment ran — it always runs and may exit empty), resources[] contains at least one Terraform-sourced resource" + _on_failure: _halt_and_inform + - _assert: "if the live fragment ran ($MIGRATION_DIR/live-capture/manifest.json exists), resources[] contains at least one live-sourced resource, a live_metadata section is present, and 'live' appears in metadata.discovery_sources" + _on_failure: _halt_and_inform + - _assert: "no config var VALUES appear anywhere in the inventory — config entries carry key names only" _on_failure: _halt_and_inform - _assert: "if a billing/invoice file was present in the workspace, heroku-resource-inventory.json has a billing_profile section" _on_failure: _halt_and_inform @@ -64,11 +71,43 @@ owns only lifecycle + the cross-cutting `_postconditions`. Two facts the contract can't express: Procfile/app.json parsing is integrated into the terraform fragment (there is no standalone Procfile fragment) — when present alongside Terraform, they supplement resource data with commands, buildpacks, and -declared add-ons. And Platform API discovery is NOT supported in v1: no API calls -are made, discovery is entirely file-based. Billing data, when present, is embedded -in `heroku-resource-inventory.json` (not a separate file); all user communication +declared add-ons. Billing data, when present, is embedded in +`heroku-resource-inventory.json` (not a separate file); all user communication is via output messages only (no report/log files). +### Live capture (main-window pre-work) + +Live discovery reads the user's Heroku account through their authenticated Heroku +CLI — read-only, consent-gated, key-names-only for config vars. It is split in two +because the dispatched `rw` worker has no shell and cannot converse with the user: + +1. **Capture** (`discover-live-capture.md`) — runs in the MAIN window, after + `_init` and before the phase's work is dispatched. It asks for consent, preflights + the CLI (`heroku auth:whoami`), runs an exact-command whitelist of list/info + commands, and writes raw output to `$MIGRATION_DIR/live-capture/` plus a + `manifest.json` index. It writes NO inventory entries. +2. **Parse** (`discover-live.md`, the `live` fragment) — runs in the worker with the + other fragments. Its `_trigger` is the manifest's existence; it maps captures to + inventory entries with `source: "live"`. + +**Explicit ordering (cold start):** run `_init` state setup FIRST (create +`$MIGRATION_DIR`, write `.phase-status.json`), THEN evaluate the source +`_precondition` — offering and running capture as part of that evaluation — then +dispatch the phase's work. Capture writes into `$MIGRATION_DIR/live-capture/`, so +it cannot run before `_init` has created the run directory. + +**When to offer capture:** while evaluating the source `_precondition`, scan the +workspace first (free). If NO `heroku_*` Terraform is found, offer live capture as +the primary source — load `discover-live-capture.md` — instead of failing the check. +If Terraform IS found, still offer capture once as an optional live cross-check +("catch resources managed outside Terraform"); a decline is fine and is not +re-asked. Never run capture without explicit consent. + +**Source-of-truth rule (for the assembler):** when both Terraform and live entries +exist, live is authoritative for current state (config values, plans, quantities); +Terraform supplements structure and provenance. Disagreements are surfaced as drift, +never silently resolved — see `discover-assemble.md` § Merge & Drift Rules. + --- ## Handoff @@ -78,6 +117,8 @@ completion message from the inventory contents: - "Discovered X total resources across Y apps." - If billing data available: "Parsed billing data ($Z/month)." +- If live discovery ran: "Live discovery captured N apps via the Heroku CLI." +- If both live and Terraform ran: "Drift check: N resources live but not in Terraform, M in Terraform but not live, K config conflicts (live values used)." - If Terraform secondary: "Supplemented with Terraform-sourced resources (N conflicts resolved)." - If Pipeline detected: "Detected N pipeline(s) (detect-only)." - If Cedar/Fir mixed: "Generation detection: N Cedar, M Fir, P unknown." @@ -90,12 +131,14 @@ Format: "Discover phase complete. [artifact summaries] Next required step: Phase Non-fatal discovery errors and their handling (fatal source/gate failures are handled by `_preconditions`/`_postconditions` + `INTERPRETER.md` § `_on_error`): -| Error Category | Behavior | -| ------------------------------------------------- | ---------------------------------------------- | -| Terraform parse error (malformed HCL) | Log warning, skip malformed blocks, continue | -| Procfile/app.json parse error | Record warning per-app, continue | -| Generation detection unresolvable (no stack attr) | Set `heroku_generation` to `unknown`, continue | -| Pipeline detection from Terraform incomplete | Record with available data, continue | +| Error Category | Behavior | +| -------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| Terraform parse error (malformed HCL) | Log warning, skip malformed blocks, continue | +| Procfile/app.json parse error | Record warning per-app, continue | +| Generation detection unresolvable (no stack attr) | Set `heroku_generation` to `unknown`, continue | +| Pipeline detection from Terraform incomplete | Record with available data, continue | +| Live capture partially failed (some apps 403 etc.) | Parse the `ok` captures, mark failed apps `discovery_failed`, confidence `reduced`, continue | +| Live capture declined or CLI unavailable | Skip the `live` fragment (no manifest → trigger never fires), continue with file-based sources | --- diff --git a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/shared/schema-discover-heroku.md b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/shared/schema-discover-heroku.md index ff997640..0362b880 100644 --- a/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/shared/schema-discover-heroku.md +++ b/migrate/plugins/migration-to-aws/skills/heroku-to-aws/references/shared/schema-discover-heroku.md @@ -42,7 +42,8 @@ Complete inventory of discovered Heroku resources. Uses a **flat resource model* } ], "billing_profile": {}, - "terraform_metadata": {} + "terraform_metadata": {}, + "live_metadata": {} } ``` @@ -54,13 +55,13 @@ Complete inventory of discovered Heroku resources. Uses a **flat resource model* Report-level information about the discovery run. -| Field | Type | Required | Description | -| ----------------------- | ----------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `discovery_timestamp` | string (ISO 8601) | ✅ | When discovery was executed | -| `total_apps_discovered` | integer | ✅ | Count of Heroku apps found | -| `discovery_sources` | string[] | ✅ | Sources used: `"terraform"`, `"procfile"`, `"billing"` | -| `confidence` | string | ✅ | `"full"` (Terraform files present and parsed successfully) or `"reduced"` (Partial data, e.g., Terraform parse errors on some files, missing expected resources) | -| `confidence_note` | string | ❌ | Explanation when confidence is `"reduced"` | +| Field | Type | Required | Description | +| ----------------------- | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `discovery_timestamp` | string (ISO 8601) | ✅ | When discovery was executed | +| `total_apps_discovered` | integer | ✅ | Count of Heroku apps found | +| `discovery_sources` | string[] | ✅ | Sources used: `"terraform"`, `"procfile"`, `"billing"`, `"live"` | +| `confidence` | string | ✅ | `"full"` (primary source(s) parsed/captured successfully) or `"reduced"` (partial data, e.g., Terraform parse errors, failed/skipped live captures, missing expected resources) | +| `confidence_note` | string | ❌ | Explanation when confidence is `"reduced"` | ### `apps[]` (REQUIRED) @@ -83,12 +84,15 @@ Per-app metadata entries. One entry per discovered Heroku app. Flat array of all discovered resources. **No nesting, no clustering.** -| Field | Type | Required | Description | -| --------------- | ------ | -------- | --------------------------------------------------------- | -| `resource_id` | string | ✅ | Unique identifier (format below) | -| `resource_type` | string | ✅ | One of: `"formation"`, `"addon"`, `"space"`, `"pipeline"` | -| `heroku_app` | string | ✅ | App name this resource belongs to, or `"unassociated"` | -| `config` | object | ✅ | Type-specific configuration (see per-type schemas below) | +| Field | Type | Required | Description | +| ------------------------ | ------- | -------- | ------------------------------------------------------------------------------------------- | +| `resource_id` | string | ✅ | Unique identifier (format below) | +| `resource_type` | string | ✅ | One of: `"formation"`, `"addon"`, `"space"`, `"pipeline"`, `"domain"`, `"config"` | +| `heroku_app` | string | ✅ | App name this resource belongs to, or `"unassociated"` | +| `config` | object | ✅ | Type-specific configuration (see per-type schemas below) | +| `source` | string | ❌ | Discovery provenance: `"terraform"`, `"live"`, or `"live+terraform"` (merged) | +| `unmanaged_by_terraform` | boolean | ❌ | Set `true` when live discovery found the resource but Terraform does not manage it (drift) | +| `not_found_live` | boolean | ❌ | Set `true` when Terraform declares the resource but live discovery did not find it deployed | ### `billing_profile` (OPTIONAL — present when billing data available) @@ -117,6 +121,19 @@ Flat array of all discovered resources. **No nesting, no clustering.** | `resource_types_extracted` | string[] | ✅ | List of extracted resource types (e.g., `"heroku_app"`, `"heroku_addon"`) | | `parse_warnings` | string[] | ✅ | Any parse warnings encountered during extraction | +### `live_metadata` (OPTIONAL — present when live CLI discovery ran) + +| Field | Type | Required | Description | +| -------------------------------- | -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `found` | boolean | ✅ | Whether live capture produced usable data | +| `captured_at` | string | ✅ | ISO 8601 timestamp of the CLI capture run | +| `apps_captured` | integer | ✅ | Number of apps successfully captured | +| `apps_failed` | integer | ✅ | Number of apps whose captures failed (e.g., 403 on team apps) | +| `capture_warnings` | string[] | ✅ | Failed/skipped capture notes carried from `live-capture/manifest.json` | +| `limitations` | string[] | ✅ | Known live-discovery blind spots (e.g., formations scaled to zero) | +| `default_heroku_domains_skipped` | integer | ❌ | Count of default `*.herokuapp.com` hostnames skipped (not recorded as domain resources) | +| `drift` | object | ❌ | Present only when Terraform AND live both ran: `resources_live_only` (int), `resources_terraform_only` (int), `config_conflicts[]` (`{resource_id, field, terraform_value, live_value}`) | + --- ## Resource ID Formats @@ -129,6 +146,8 @@ Deterministic ID format per resource type: | `addon` | `addon:{app_name}:{addon_service}:{plan}` | `addon:my-web-app:heroku-postgresql:standard-0` | | `space` | `space:{space_name}` | `space:my-private-space` | | `pipeline` | `pipeline:{pipeline_name}` | `pipeline:my-pipeline` | +| `domain` | `domain:{app_name}:{hostname}` | `domain:my-web-app:www.example.com` | +| `config` | `config:{app_name}` | `config:my-web-app` | --- @@ -176,6 +195,12 @@ Deterministic ID format per resource type: - **heroku-kafka**: `topic_count` (integer), `partitions_per_topic` (integer), `replication_factor` (integer) - **Other add-ons**: No additional required fields +**Optional live-enrichment fields** (present only when live discovery ran): + +- Any addon: `monthly_price_usd` (number — from the add-on's plan price) +- **heroku-postgresql**: `pg_version` (string), `data_size_gb` (number — feeds database migration tool selection), `table_count` (integer) +- **heroku-redis**: `maxmemory_policy` (string) + ### `space` config ```json @@ -251,10 +276,10 @@ The following fields MUST NOT appear anywhere in `heroku-resource-inventory.json ## Confidence Levels -| Level | Meaning | When Used | -| --------- | ----------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `full` | Terraform files present and parsed successfully | Terraform discovery completed without parse errors | -| `reduced` | Partial data — Terraform had parse errors or missing expected resources | Some `.tf` files could not be parsed, or expected resource types were not found | +| Level | Meaning | When Used | +| --------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `full` | Every source that ran produced complete data | Terraform parsed without errors and/or every live capture for the selected apps succeeded | +| `reduced` | Partial data from at least one source | Terraform parse errors, failed/skipped live captures (e.g., 403 on team apps, missing CLI plugin), or missing expected resources | --- @@ -419,3 +444,5 @@ The following fields MUST NOT appear anywhere in `heroku-resource-inventory.json 9. ✅ If Terraform discovery ran → resources include Terraform-sourced entries 10. ✅ If Terraform had parse errors → `metadata.confidence` is `"reduced"` 11. ✅ If billing discovery ran → `billing_profile` section present with `available: true` +12. ✅ If live discovery ran → resources include live-sourced entries, `live_metadata` present, and `"live"` in `metadata.discovery_sources` +13. ✅ No config var VALUES anywhere in the document — `config` entries carry key names only From fbbe273e4b567dc396c4707d0f699105f98ba9ef Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 18 Jul 2026 16:14:17 -0700 Subject: [PATCH 2/2] test(heroku-to-aws): port the GCP drift asserter for fixture parity check_expected_drift.py (stdlib-only) machine-checks a scenario-B replay's heroku-resource-inventory.json against expected-drift.json: metadata, apps, merged resources, conflict fields, drift counts, forbidden clustering fields, and secret hygiene (no config-var value payloads, no raw config_vars, no secret-shaped strings). Validated PASS against a fresh fresh-agent replay of scenario B on this branch. Same pattern as the gcp-live-capture asserter (#149) so both live-discovery skills regress identically. --- .../fixtures/heroku-live-capture/README.md | 5 +- .../check_expected_drift.py | 140 ++++++++++++++++++ 2 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/check_expected_drift.py diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md index 18b2cf22..879e1fbd 100644 --- a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/README.md @@ -36,8 +36,9 @@ designed-for behavior: merge rule it exercises (config conflicts, plan change, terraform-only, live-only, scaled-to-zero gap-fill). 3. Check the assembled `heroku-resource-inventory.json` against - `expected-drift.json` — it lists the assertions (merged sources, conflict - fields, drift counts, and `must_not_exist` entries). + `expected-drift.json` — machine-checkable via + `python3 check_expected_drift.py ` (exits non-zero on any failed + assertion, including secret-hygiene checks for config-var values). **What a run must never produce** (from either scenario): diff --git a/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/check_expected_drift.py b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/check_expected_drift.py new file mode 100644 index 00000000..f7a130e9 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/heroku-live-capture/check_expected_drift.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Assert a Discover run's output against expected-drift.json (scenario B). + +Usage: + python3 check_expected_drift.py + +Where contains the heroku-resource-inventory.json produced +by a replay of this fixture's scenario B (live-capture/ + workspace-terraform/ +heroku.tf). Exits 0 on PASS, 1 on FAIL with one line per failed assertion. +Stdlib only. (Same pattern as the gcp-live-capture asserter.) +""" + +import json +import sys +from pathlib import Path + +FAILS: list[str] = [] + + +def check(cond: bool, msg: str) -> None: + if not cond: + FAILS.append(msg) + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__) + return 2 + run_dir = Path(sys.argv[1]) + fixture_dir = Path(__file__).resolve().parent + + inv = json.loads((run_dir / "heroku-resource-inventory.json").read_text()) + exp = json.loads((fixture_dir / "expected-drift.json").read_text()) + + # Metadata + meta = inv["metadata"] + check(meta["total_apps_discovered"] == exp["metadata"]["total_apps_discovered"], "total_apps_discovered") + for s in exp["metadata"]["discovery_sources_must_include"]: + check(s in meta.get("discovery_sources", []), f"discovery_sources missing {s}") + check(meta.get("confidence") == exp["metadata"]["confidence"], f"confidence={meta.get('confidence')}") + + # Apps + apps = {a["app_name"]: a for a in inv["apps"]} + for name, e in exp["apps"].items(): + a = apps.get(name) + if a is None: + check(False, f"missing app {name}") + continue + for k in ("discovery_status", "heroku_generation", "app_id"): + if k in e: + check(a.get(k) == e[k], f"app {name} {k}={a.get(k)} want {e[k]}") + if "failure_reason_contains" in e: + check(e["failure_reason_contains"] in (a.get("failure_reason") or ""), f"app {name} failure_reason") + + # Resources + res = {r["resource_id"]: r for r in inv["resources"]} + for rid, e in exp["merged_resources"].items(): + r = res.get(rid) + if r is None: + check(False, f"missing resource {rid}") + continue + if "source" in e: + check(r.get("source") == e["source"], f"{rid} source={r.get('source')} want {e['source']}") + if e.get("unmanaged_by_terraform"): + check(r.get("unmanaged_by_terraform") is True, f"{rid} unmanaged flag") + if e.get("not_found_live"): + check(r.get("not_found_live") is True, f"{rid} not_found_live flag") + for flag in e.get("must_not_have", []): + check(r.get(flag) is not True, f"{rid} must not have {flag}") + for k, v in e.get("config", {}).items(): + if k == "config_var_keys_count": + check(len(r["config"].get("config_var_keys", [])) == v, f"{rid} key count") + elif k == "stages": + check(r["config"].get("stages") == v, f"{rid} stages") + else: + check(r["config"].get(k) == v, f"{rid} config.{k}={r['config'].get(k)} want {v}") + if "expected_config_conflicts" in e: + conflict_fields = [ + c["field"] for c in inv.get("live_metadata", {}).get("drift", {}).get("config_conflicts", []) + if c.get("resource_id") == rid + ] + for f in e["expected_config_conflicts"]: + check(f in conflict_fields, f"{rid} missing config conflict on {f}") + + # live_metadata + drift + lm = inv["live_metadata"] + check(lm.get("apps_captured") == exp["live_metadata"]["apps_captured"], "apps_captured") + check(lm.get("apps_failed") == exp["live_metadata"]["apps_failed"], "apps_failed") + lim = json.dumps(lm.get("limitations", [])) + check(exp["live_metadata"]["limitations_must_include"] in lim, "limitations missing scaled-to-zero note") + drift = lm.get("drift", {}) + check( + drift.get("resources_live_only", 0) >= exp["live_metadata"]["drift"]["resources_live_only_min"], + f"resources_live_only={drift.get('resources_live_only')}", + ) + check( + drift.get("resources_terraform_only") == exp["live_metadata"]["drift"]["resources_terraform_only"], + f"resources_terraform_only={drift.get('resources_terraform_only')}", + ) + conflict_fields = sorted(c["field"] for c in drift.get("config_conflicts", [])) + check( + conflict_fields == sorted(exp["live_metadata"]["drift"]["config_conflicts_expected_fields"]), + f"conflict fields {conflict_fields}", + ) + + # Must-not-exist / secret hygiene + doc = json.dumps(inv) + check("domain:acme-web:acme-web-1a2b3c4d5e6f" not in doc, "default herokuapp domain leaked as resource") + for bad in ("cluster_id", "creation_order_depth", "must_migrate_together", '"edges"', '"dependencies"'): + check(bad not in doc, f"forbidden clustering field {bad}") + for bad in ("sk_live", "postgres://", "rediss://", "AKIA", "Bearer "): + check(bad not in doc, f"possible secret value: {bad}") + + def walk(node, path="$"): + if isinstance(node, dict): + if "name" in node and ("value" in node or "valueFrom" in node): + check(False, f"env-like object with a value payload at {path}") + for k, v in node.items(): + if k == "config_vars": + check(False, f"raw config_vars at {path} — only config_var_keys (names) allowed") + walk(v, f"{path}.{k}") + elif isinstance(node, list): + for i, v in enumerate(node): + walk(v, f"{path}[{i}]") + + walk(inv) + for key_name in ("STRIPE_SECRET_KEY", "DATABASE_URL", "SESSION_SECRET"): + check(f'"{key_name}": ' not in doc, f"config var name {key_name} appears as a KEY (value paired)") + + if FAILS: + print(f"FAIL ({len(FAILS)}):") + for f in FAILS: + print(f" - {f}") + return 1 + print("PASS — expected-drift.json assertions hold") + return 0 + + +if __name__ == "__main__": + sys.exit(main())