diff --git a/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/README.md b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/README.md index 8f2dcf0c..ec879231 100644 --- a/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/README.md +++ b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/README.md @@ -54,6 +54,22 @@ and exercises the designed-for behaviors: `python3 check_expected_drift.py ` (exits non-zero on any failed assertion). +**Scenario C — derived GCP baseline from live sizing (no billing export):** + +1. Create `.migration//` containing only + `seed-baseline/gcp-resource-inventory.json` (the scenario-B resources in + schema shape, capture-shaped config paths, NO `billing-profile.json`). +2. Run the Estimate phase's Part 1 (Calculate Current GCP Costs). Rung 2 must + derive the baseline from sizing via + `references/shared/gcp-infra-pricing-cache.md`: Cloud SQL + `db-custom-2-8192` ZONAL + 50 GB SSD + backup upper bound = $113.68, + Memorystore BASIC 1 GB = $35.77 → **$149.45/month**, source + `inventory_estimate`, ±20-30%, mandatory not-a-bill caveat; Cloud Run ×2 + and the bucket excluded as `usage_based` (warned), the VPC as + `no_standing_charge`. +3. Check with `python3 check_expected_baseline.py ` (accepts a full + `estimation-infra.json` or a Part-1-only `current-costs-preview.json`). + **What a run must never produce** (either scenario): env var or secret values anywhere; any mutating `gcloud` command (including `gcloud services enable`); AWS service names in discover artifacts; a halt caused by the failed diff --git a/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/check_expected_baseline.py b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/check_expected_baseline.py new file mode 100644 index 00000000..a4e6f675 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/check_expected_baseline.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Assert a derived GCP baseline against expected-baseline.json (scenario C). + +Usage: + python3 check_expected_baseline.py + +Reads current_costs from /estimation-infra.json, or from +/current-costs-preview.json when only Part 1 was exercised (the +targeted-replay harness). Exits 0 on PASS, 1 on FAIL with one line per failed +assertion. Stdlib only. +""" + +import json +import math +import sys +from pathlib import Path + +FAILS: list = [] + + +def check(cond, msg): + if not cond: + FAILS.append(msg) + + +def is_num(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def find_number_near(node, target, tol): + """True if a number within tol of target appears anywhere under node.""" + if isinstance(node, dict): + return any(find_number_near(v, target, tol) for v in node.values()) + if isinstance(node, list): + return any(find_number_near(v, target, tol) for v in node) + return is_num(node) and math.isclose(node, target, abs_tol=tol) + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__) + return 2 + run_dir = Path(sys.argv[1]) + fixture_dir = Path(__file__).resolve().parent + exp = json.loads((fixture_dir / "expected-baseline.json").read_text()) + + src = None + for name in ("estimation-infra.json", "current-costs-preview.json"): + if (run_dir / name).is_file(): + src = json.loads((run_dir / name).read_text()) + break + if src is None: + print("FAIL: neither estimation-infra.json nor current-costs-preview.json in run dir") + return 1 + cur = src.get("current_costs", src) # preview file may BE the current_costs object + doc = json.dumps(src) + + e = exp["current_costs"] + check(cur.get("source") == e["source"], f"source={cur.get('source')} want {e['source']}") + check(e["accuracy_contains"] in str(cur.get("accuracy", "")), f"accuracy={cur.get('accuracy')}") + check( + find_number_near(cur, e["monthly_total"], e["tolerance"]), + f"no numeric total ≈ {e['monthly_total']} (±{e['tolerance']}) in current_costs — rate-card math broken", + ) + note = str(cur.get("baseline_note") or "") + check(bool(note.strip()), "baseline_note missing — the derived-baseline caveat is mandatory") + for phrase in e["baseline_note_must_mention"]: + check(phrase.lower() in note.lower(), f"baseline_note does not mention '{phrase}'") + + for rid, spec in exp["per_resource"].items(): + check( + find_number_near(cur, spec["monthly"], spec["tolerance"]), + f"per-resource figure for {rid} ≈ {spec['monthly']} not found in the breakdown", + ) + + warn_text = json.dumps(src.get("warnings", [])) + json.dumps(cur.get("warnings", [])) + json.dumps( + cur.get("excluded", []) + ) + for name in exp["excluded_must_be_warned"]: + check(name in warn_text or name in json.dumps(cur), f"excluded resource '{name}' not surfaced in warnings") + + check(cur.get("source") != "billing_data", "claims billing_data with no billing-profile.json") + check("billing-profile.json" not in doc or "unavailable" in doc or True, "") # provenance is via source field + + if FAILS: + print(f"FAIL ({len(FAILS)}):") + for f in FAILS: + print(f" - {f}") + return 1 + print("PASS — expected-baseline.json assertions hold") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/expected-baseline.json b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/expected-baseline.json new file mode 100644 index 00000000..9c5413c7 --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/expected-baseline.json @@ -0,0 +1,19 @@ +{ + "_comment": "Expected Part 1 rung-2 derived baseline for seed-baseline/ (no billing-profile.json). Math from gcp-infra-pricing-cache.md: Cloud SQL db-custom-2-8192 ZONAL = (2 x 0.0413 + 8 x 0.0070) x 730 = 101.18 + 50 GB SSD x 0.17 = 8.50 + backup upper bound 50 x 0.08 = 4.00 -> 113.68; Memorystore BASIC 1 GB = 1 x 0.049 x 730 = 35.77. Derivable total 149.45. Cloud Run x2 and the bucket are usage-based -> EXCLUDED with warnings; the VPC network is excluded as no_standing_charge. Checked by check_expected_baseline.py.", + "current_costs": { + "source": "inventory_estimate", + "accuracy_contains": "20-30", + "monthly_total": 149.45, + "tolerance": 0.75, + "baseline_note_must_mention": ["not a bill", "usage-based"] + }, + "per_resource": { + "google_sql_database_instance.db": { "monthly": 113.68, "tolerance": 0.6 }, + "google_redis_instance.cache": { "monthly": 35.77, "tolerance": 0.25 } + }, + "excluded_must_be_warned": ["orders_api", "web_frontend", "acme_prod_uploads"], + "must_not_exist": [ + "a billing_data source claim (no billing-profile.json in this scenario)", + "any priced Cloud Run / bucket line inside the baseline total" + ] +} diff --git a/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/seed-baseline/gcp-resource-inventory.json b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/seed-baseline/gcp-resource-inventory.json new file mode 100644 index 00000000..7e5db6ca --- /dev/null +++ b/migrate/plugins/migration-to-aws/fixtures/gcp-live-capture/seed-baseline/gcp-resource-inventory.json @@ -0,0 +1,134 @@ +{ + "_comment": "Seed for the derived-baseline validation (scenario C in README.md): the scenario-B acme-prod resources in schema shape, NO billing-profile.json on purpose — estimate-infra.md Part 1 rung 2 must derive the GCP baseline from sizing via gcp-infra-pricing-cache.md. Config uses capture-shaped field paths (settings.tier etc.) exactly as the live parse emits them.", + "metadata": { + "discovery_timestamp": "2026-07-20T18:30:00Z", + "discovery_sources": ["terraform", "live"], + "clustering_mode": "simplified_live", + "project": "acme-prod" + }, + "resources": [ + { + "address": "google_sql_database_instance.db", + "type": "google_sql_database_instance", + "name": "db", + "classification": "PRIMARY", + "tier": "data", + "confidence": 0.99, + "source": "live+terraform", + "config": { + "name": "orders-db", + "region": "us-central1", + "databaseVersion": "POSTGRES_16", + "settings.tier": "db-custom-2-8192", + "settings.availabilityType": "ZONAL", + "settings.dataDiskSizeGb": "50", + "settings.backupConfiguration.enabled": true + }, + "depth": 3, + "cluster_id": "data_cloudsql_us-central1_001" + }, + { + "address": "google_redis_instance.cache", + "type": "google_redis_instance", + "name": "cache", + "classification": "PRIMARY", + "tier": "data", + "confidence": 0.95, + "source": "live", + "unmanaged_by_terraform": true, + "config": { + "name": "cache", + "tier": "BASIC", + "memorySizeGb": 1, + "redisVersion": "REDIS_7_2", + "locationId": "us-central1-a" + }, + "depth": 3, + "cluster_id": "data_redis_us-central1_001" + }, + { + "address": "google_cloud_run_service.orders_api", + "type": "google_cloud_run_service", + "name": "orders_api", + "classification": "PRIMARY", + "tier": "compute", + "confidence": 0.99, + "source": "live+terraform", + "config": { + "live_type": "google_cloud_run_v2_service", + "image": "us-docker.pkg.dev/acme-prod/app/orders-api:v42", + "memory_mb": 512, + "concurrency": 100, + "timeoutSeconds": 60 + }, + "depth": 3, + "cluster_id": "compute_cloudrun_us-central1_001" + }, + { + "address": "google_cloud_run_v2_service.web_frontend", + "type": "google_cloud_run_v2_service", + "name": "web_frontend", + "classification": "PRIMARY", + "tier": "compute", + "confidence": 0.95, + "source": "live", + "unmanaged_by_terraform": true, + "config": { + "image": "us-docker.pkg.dev/acme-prod/app/web-frontend:v18", + "memory_mb": 256, + "concurrency": 80 + }, + "depth": 3, + "cluster_id": "compute_cloudrun_us-central1_001" + }, + { + "address": "google_storage_bucket.acme_prod_uploads", + "type": "google_storage_bucket", + "name": "acme_prod_uploads", + "classification": "SECONDARY", + "tier": "storage", + "confidence": 0.95, + "source": "live", + "unmanaged_by_terraform": true, + "secondary_role": "storage", + "serves": ["google_cloud_run_service.orders_api"], + "config": { + "name": "acme-prod-uploads", + "location": "US-CENTRAL1", + "storageClass": "STANDARD" + }, + "depth": 2, + "cluster_id": "compute_cloudrun_us-central1_001" + }, + { + "address": "google_compute_network.main", + "type": "google_compute_network", + "name": "main", + "classification": "SECONDARY", + "tier": "networking", + "confidence": 0.99, + "source": "live+terraform", + "secondary_role": "networking", + "serves": ["google_sql_database_instance.db", "google_redis_instance.cache"], + "config": { "name": "main", "autoCreateSubnetworks": false }, + "depth": 0, + "cluster_id": "networking_vpc_global_000" + } + ], + "edges": [ + { + "from": "google_cloud_run_service.orders_api", + "to": "google_sql_database_instance.db", + "relationship_type": "data_dependency", + "evidence": "run.googleapis.com/cloudsql-instances annotation" + } + ], + "live_metadata": { + "found": true, + "captured_at": "2026-07-20T18:20:00Z", + "project": "acme-prod", + "method": "per_service", + "capture_warnings": [], + "unmapped_asset_types": {} + } +} diff --git a/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/phases/estimate/estimate-infra.md b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/phases/estimate/estimate-infra.md index 7b88c5bb..2ee43ac6 100644 --- a/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/phases/estimate/estimate-infra.md +++ b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/phases/estimate/estimate-infra.md @@ -69,7 +69,43 @@ Do NOT call get_pricing_service_codes, get_pricing_service_attributes, or get_pr Determine the current GCP monthly infrastructure costs. Use the best available source: 1. **`billing-profile.json` (preferred)** — Use actual billing data as the GCP baseline. Highest confidence (±5%). -2. **`gcp-resource-inventory.json` (fallback)** — Estimate costs from discovered resource configurations. Wider range (±20-30%). +2. **`gcp-resource-inventory.json` (fallback)** — Derive costs from discovered + resource sizing using `references/shared/gcp-infra-pricing-cache.md` — never + from remembered GCP prices. Wider range (±20-30%). Procedure: + - For each inventory resource, apply the matching rate-card derivation + (Cloud SQL tier decoding × HA multiplier + disk; Memorystore GB × tier + band; GCE/GKE-node machine types × 730; GKE cluster fee). Config field + names may be capture-shaped (`settings.tier`, `settings.dataDiskSizeGb`, + `memorySizeGb`) or Terraform-shaped (`tier`, `disk_size`) — resolve + either. + - Resources the rate card marks NOT derivable (Cloud Run, Functions, + Pub/Sub, buckets, Autopilot, egress) are EXCLUDED from the total with + reason `"usage_based"`; name each in `warnings[]` with: "usage-based — + not derivable from sizing; provide a billing export for the full + baseline." (Exclusion reason enum: `usage_based`, `no_standing_charge`, + `unpriced_gcp` — nothing else.) + - A sized resource whose rate is missing from the card → `"unpriced_gcp"`, + excluded, warned. Never guess a rate. When `settings.dataDiskType` is + absent, assume `PD_SSD` (the Cloud SQL default) and say so in the + derivation entry. When `settings.backupConfiguration.enabled` is true, + add a backup-storage line using the card's backup rate against the + provisioned disk size as an upper bound, labeled "(backup upper bound)". + - No-standing-charge resources (VPC networks/subnets, service accounts, + IAM, secrets at trivial volume) are excluded with reason + `"no_standing_charge"` — NOT the usage-based warning. + - Set `current_costs.source: "inventory_estimate"` (the value + `schema-estimate-infra.md` already enumerates) and + `current_costs.accuracy: "±20-30%"`. + - Shape: `current_costs.breakdown` stays the schema's category-keyed map + (`compute` / `database` / `cache` / `storage` / ...); record the + per-resource arithmetic in an additive `current_costs.derivation[]` + array (address, resolved config, calculation string, monthly), and the + exclusions in `current_costs.excluded_resources[]` + + `current_costs.warnings[]`. + - `baseline_note` (mandatory): "Derived from discovered resource sizing × + GCP list rates — not a bill. Excludes usage-based services (Cloud Run, + bandwidth, storage volume), sustained-use/committed-use discounts, and + credits; your invoice may differ materially." 3. **`preferences.json` → `gcp_monthly_spend`** — User-provided monthly spend from clarification. 4. **Ask the user** — If none of the above are available, ask: "I need your current GCP monthly spend to produce a meaningful cost comparison. What is your approximate GCP monthly infrastructure cost?" Use the user's answer. If the user declines or is unsure, present AWS costs without a GCP comparison and note: "GCP baseline unavailable — AWS costs shown without comparison." @@ -350,16 +386,34 @@ This section covers **GCP vendor/network charges** for outbound data during migr Set `billing_data_available: true` in the output `migration_cost_considerations` object. -### IF billing data is NOT available (`billing-profile.json` does not exist): +### IF billing data is NOT available but the inventory carries database/disk sizes: -**Omit GCP data transfer fee estimates.** Without billing data, there is no grounding for egress projections. Instead, include only this note in the output: +Provisioned sizes are an UPPER BOUND on one-time migration egress (actual data +≤ provisioned; #149's sizing caveat applies). Present a bounded estimate, never +a point figure: Set `migration_cost_considerations` to: +```json +{ + "categories": [ + "GCP data transfer egress (upper bound): ≤ GB × $0.12/GB ≈ ≤ $ one-time" + ], + "billing_data_available": false, + "baseline_available": true, + "note": "Upper bound from PROVISIONED sizes (actual data is typically smaller). Object-storage volume is unknown without billing data — a billing export tightens this to measured volumes." +} +``` + +### IF neither billing data nor any sized inventory resource exists: + +**Omit GCP data transfer fee estimates.** There is no grounding for egress projections. Set `migration_cost_considerations` to: + ```json { "categories": [], "billing_data_available": false, + "baseline_available": false, "note": "Data transfer cost estimates require GCP billing data. Re-run discovery with a GCP billing export to see GCP egress fee projections." } ``` diff --git a/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/gcp-infra-pricing-cache.md b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/gcp-infra-pricing-cache.md new file mode 100644 index 00000000..d0ff5c07 --- /dev/null +++ b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/gcp-infra-pricing-cache.md @@ -0,0 +1,89 @@ +# GCP Infrastructure Pricing Cache (source-side rates) + +**Last updated:** 2026-07-19 +**Region basis:** us-central1 (GCP list prices vary by region — note the region when the inventory is elsewhere) +**Sources:** cloud.google.com/sql/pricing, cloud.google.com/memorystore/docs/redis/pricing, cloud.google.com/compute/vm-instance-pricing, cloud.google.com/kubernetes-engine/pricing +**Currency:** USD · **Accuracy:** ±15% per rate; a baseline derived from these rates carries the Part 1 rung-2 label (±20–30%) + +> Use this cache to derive the **current GCP monthly baseline** from discovered +> resource sizing when no billing export exists (estimate-infra.md Part 1 +> rung 2). These are SOURCE-side (GCP) rates — never use them to price AWS +> services. Hours/month: 730. + +--- + +## Cloud SQL (`google_sql_database_instance`) — Enterprise edition + +| Component | Rate | +| ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| vCPU | $0.0413 /vCPU/hour | +| RAM | $0.0070 /GB/hour | +| SSD storage (`PD_SSD`, the default — assume it when `dataDiskType` is absent) | $0.170 /GB/month | +| HDD storage (`PD_HDD`) | $0.090 /GB/month | +| Backup storage | $0.080 /GB/month — backup volume is not in the capture; when backups are enabled, use provisioned disk GB as an upper bound, labeled "(backup upper bound)" | + +**Tier decoding:** + +- `db-custom-C-M` → C vCPUs, M MB of RAM (e.g. `db-custom-2-8192` = 2 vCPU + 8 GB) +- Shared-core flat rates: `db-f1-micro` ≈ $0.0105/hr, `db-g1-small` ≈ $0.0350/hr +- `availabilityType: REGIONAL` (HA) → **2×** the vCPU + RAM + storage subtotal + (GCP bills the standby); `ZONAL` → 1× + +Monthly = (C × 0.0413 + (M/1024) × 0.0070) × 730 × HA-multiplier + disk_GB × storage rate × HA-multiplier. The backup upper-bound line is NOT multiplied by the HA multiplier (backups are taken once, not per replica). + +## Memorystore for Redis (`google_redis_instance`) — per GB-hour by capacity band + +| Capacity (GB) | Basic | Standard (HA) | +| ------------- | ------ | ------------- | +| 1–4 (M1) | $0.049 | $0.077 | +| 5–10 (M2) | $0.039 | $0.062 | +| 11–35 (M3) | $0.031 | $0.050 | +| 36–100 (M4) | $0.024 | $0.039 | +| >100 (M5) | $0.021 | $0.034 | + +Monthly = `memorySizeGb` × band rate (by `tier`) × 730. + +## Compute Engine (`google_compute_instance`) — on-demand, us-central1 + +| Machine type | $/hour | +| ------------- | ------- | +| e2-micro | 0.00838 | +| e2-small | 0.01675 | +| e2-medium | 0.03351 | +| e2-standard-2 | 0.06701 | +| e2-standard-4 | 0.13402 | +| e2-standard-8 | 0.26805 | +| n1-standard-1 | 0.0475 | +| n1-standard-2 | 0.0950 | +| n1-standard-4 | 0.1900 | +| n2-standard-2 | 0.0971 | +| n2-standard-4 | 0.1942 | + +Persistent disk: pd-standard $0.040/GB-mo, pd-ssd $0.170/GB-mo, pd-balanced +$0.100/GB-mo. Machine type not in this table → do NOT guess; mark that resource +`"unpriced_gcp"` and add a warning. + +## GKE (`google_container_cluster`) + +- Cluster management fee: $0.10/cluster/hour (≈ $73/month; first zonal/autopilot + cluster's fee is covered by the GCP free tier — note it, still count it) +- Standard mode nodes: price each node pool as `machineType` × node count via + the Compute Engine table +- **Autopilot mode** (`autopilot.enabled: true`): pod-resource usage-based — + NOT derivable from sizing; exclude (see below) + +## NOT derivable from sizing — always exclude from a derived baseline + +Usage-based services have no meaningful list-price-from-sizing derivation. +Exclude them from the rung-2 baseline total, name each in `warnings[]`, and +state that a billing export (rung 1) is the upgrade input: + +- Cloud Run / Cloud Functions (request + vCPU-second billing) +- Pub/Sub, egress/network data processing +- Cloud Storage buckets (capture has no size), BigQuery, Spanner processing +- GKE Autopilot workloads + +> **Staleness:** if today is more than 30 days after **Last updated** above, +> treat these rates as potentially stale and note it in the estimate output +> (same convention as `pricing-cache.md`); GCP list prices change rarely, but +> verify via cloud.google.com/products/calculator when precision matters. diff --git a/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/schema-estimate-infra.md b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/schema-estimate-infra.md index 3fa50727..ce338f13 100644 --- a/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/schema-estimate-infra.md +++ b/migrate/plugins/migration-to-aws/skills/gcp-to-aws/references/shared/schema-estimate-infra.md @@ -47,10 +47,14 @@ The fields **`aws_monthly_premium`**, **`aws_monthly_balanced`**, **`aws_monthly "current_costs": { "source": "billing_data|inventory_estimate|preferences|user_provided|unavailable", + "accuracy": "±5% (billing) | ±20-30% (inventory_estimate) — states the SOURCE's confidence; distinct from top-level accuracy_confidence, which covers AWS pricing mode", "gcp_monthly": 300, "gcp_annual": 3600, - "baseline_note": "From billing-profile.json actual spend data", - "breakdown": { "compute": 75, "database": 50, "storage": 40, "networking": 20, "other": 15 } + "baseline_note": "From billing-profile.json actual spend data — or the mandatory derived-baseline caveat for inventory_estimate", + "breakdown": { "compute": 75, "database": 50, "storage": 40, "networking": 20, "other": 15 }, + "derivation": [], + "excluded_resources": [], + "warnings": [] }, "projected_costs": {