direct: new 'migrate' impl that looks up values from tfstate without reading resources

acceptance/bundle/deploy/snapshot-comparison/output.txt

@@ -8,8 +8,6 @@ Deployment complete!
 
 === Run migrate on bundle 1
 >>> [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/bundle1/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt

@@ -3,6 +3,4 @@
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/yaml-sync-empty-grants/default/files...
 Deploying resources...
 Updating deployment state...
-Warn: Failed to create config snapshot: state conversion failed
-Warn: Config snapshot: state entry not found for "resources.schemas.schema1.grants"
 Deployment complete!

acceptance/bundle/help/bundle-deployment-migrate/output.txt

@@ -13,7 +13,7 @@ Usage:
 
 Flags:
   -h, --help          help for migrate
-      --noplancheck   Skip running bundle plan before migration.
+      --noplancheck   No-op (kept for compatibility).
 
 Global Flags:
       --debug            enable debug logging

acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    trigger_job:
+      name: test-trigger-$UNIQUE_NAME
+      # 9007199254740993 = 2^53+1, the smallest integer float64 cannot represent
+      # exactly (it rounds to 2^53 = ...992). Stored as a literal via PrepareState,
+      # so trigger_job keeps the exact value; watcher_job references it to force the
+      # migration to resolve it from TF state, where naive float64 parsing of JSON
+      # numbers would corrupt it. Local-only: deliberately out of range for the
+      # real backend (see no_run_job_ref_on_cloud in the parent test.toml).
+      max_concurrent_runs: 9007199254740993
+      tasks:
+        - task_key: notebook
+          notebook_task:
+            notebook_path: /Shared/notebook
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            instance_pool_id: $TEST_INSTANCE_POOL_ID
+            num_workers: 1
+
+    watcher_job:
+      name: test-watcher-$UNIQUE_NAME
+      # Resolved from trigger_job's TF state during migration. Exercises the
+      # json.Number parsing path end-to-end: with the fix the migrated state keeps
+      # 9007199254740993; without it the value is silently truncated to ...992.
+      max_concurrent_runs: ${resources.jobs.trigger_job.max_concurrent_runs}
+      tasks:
+        - task_key: run_trigger
+          run_job_task:
+            job_id: ${resources.jobs.trigger_job.id}

acceptance/bundle/invariant/test.toml

@@ -36,6 +36,7 @@ EnvMatrix.INPUT_CONFIG = [
     "job_pydabs_1000_tasks.yml.tmpl",
     "job_cross_resource_ref.yml.tmpl",
     "job_permission_ref.yml.tmpl",
+    "job_run_job_ref.yml.tmpl",
     "job_with_depends_on.yml.tmpl",
     "job_with_permissions.yml.tmpl",
     "job_with_task.yml.tmpl",
@@ -71,6 +72,11 @@ EnvMatrix.INPUT_CONFIG = [
 [EnvMatrixExclude]
 no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"]
 
+# job_run_job_ref sets a health rule value to 2^53+1 to exercise large-integer
+# precision in state migration. The value is out of range for the real backend,
+# so this config is local-only (the mock server stores it verbatim).
+no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"]
+
 # Postgres resources only work on AWS
 no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"]
 no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"]

acceptance/bundle/migrate/added/output.txt

@@ -5,14 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> musterr [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-create jobs.job_b
-
-Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
-Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option
-
->>> [CLI] bundle deployment migrate --noplancheck
+>>> [CLI] bundle deployment migrate
 Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/added/script

@@ -6,11 +6,8 @@ trace $CLI bundle deploy
 # Uncomment job_b (add it to config without deploying)
 update_file.py databricks.yml "#job_b" "job_b"
 
-# Should fail at plan check: job_b is "1 to add"
-trace musterr $CLI bundle deployment migrate
-
-# Should succeed: job_b skipped, will be created on next deploy
-trace $CLI bundle deployment migrate --noplancheck
+# job_b skipped, will be created on next deploy
+trace $CLI bundle deployment migrate
 
 # After migration: plan shows job_b as "to add"
 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to add"

acceptance/bundle/migrate/basic/output.txt

@@ -11,8 +11,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
 Success! Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/dashboards/output.txt

@@ -6,8 +6,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/default-python/output.txt

@@ -22,16 +22,7 @@ Deployment complete!
 
 >>> print_state.py
 
->>> musterr [CLI] bundle deployment migrate
-Building python_artifact...
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Building python_artifact...
-update jobs.sample_job
-
-Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged
-Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option
-
->>> [CLI] bundle deployment migrate --noplancheck
+>>> [CLI] bundle deployment migrate
 Building python_artifact...
 Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/resources.json
 

acceptance/bundle/migrate/default-python/script

@@ -5,8 +5,7 @@ cd my_default_python
 trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
 trace print_state.py > ../out.state_original.json
 
-trace musterr $CLI bundle deployment migrate
-trace $CLI bundle deployment migrate --noplancheck
+trace $CLI bundle deployment migrate
 
 trace print_state.py > ../out.state_after_migration.json
 trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json

acceptance/bundle/migrate/grants/output.txt

@@ -6,8 +6,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged
 Success! Migrated 6 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/permissions/output.txt

@@ -6,8 +6,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged
 Success! Migrated 4 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/profile_arg/output.txt

@@ -6,8 +6,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate -p non_existent321
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
 
 Validate the migration by running "databricks bundle plan -p non_existent321", there should be no actions planned.
@@ -24,8 +22,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate -p non_existent321 -t prod
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json
 
 Validate the migration by running "databricks bundle plan -t prod -p non_existent321", there should be no actions planned.

acceptance/bundle/migrate/removed/output.txt

@@ -5,14 +5,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> musterr [CLI] bundle deployment migrate
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-delete jobs.job_b
-
-Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
-Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option
-
->>> [CLI] bundle deployment migrate --noplancheck
+>>> [CLI] bundle deployment migrate
 Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/removed/script

@@ -6,11 +6,8 @@ trace $CLI bundle deploy
 # Remove job_b from config without deploying the deletion
 grep -v job_b databricks.yml > databricks_tmp.yml && mv databricks_tmp.yml databricks.yml
 
-# Should fail at plan check: job_b is "1 to delete"
-trace musterr $CLI bundle deployment migrate
-
-# Should succeed: job_b's ID preserved in direct state for deletion on next deploy
-trace $CLI bundle deployment migrate --noplancheck
+# job_b's ID preserved in direct state for deletion on next deploy
+trace $CLI bundle deployment migrate
 
 # After migration: plan shows job_b as "to delete"
 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to delete"

acceptance/bundle/migrate/runas/output.txt

@@ -81,20 +81,6 @@ Consider using a adding a top-level permissions section such as the following:
 See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration.
   in databricks.yml:5:3
 
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups
-If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy.
-
-Consider using a adding a top-level permissions section such as the following:
-
-  permissions:
-    - user_name: [USERNAME]
-      level: CAN_MANAGE
-
-See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration.
-  in databricks.yml:5:3
-
-Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/production/resources.json
 
 Validate the migration by running "databricks bundle plan", there should be no actions planned.

acceptance/bundle/migrate/var_arg/output.txt

@@ -6,8 +6,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate --var=job_name=Custom Job Name
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
 
 Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned.
@@ -39,8 +37,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] bundle deployment migrate --var job_name=Custom Job Name
-Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json
 
 Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned.

bundle/deploy/terraform/util.go

@@ -51,6 +51,15 @@ type stateInstanceAttributes struct {
 }
 
 // Returns a mapping resourceKey -> stateInstanceAttributes
+// ParseResourcesStateFromBytes parses a terraform state file from already-read bytes.
+func ParseResourcesStateFromBytes(ctx context.Context, raw []byte) (ExportedResourcesMap, error) {
+	var state resourcesState
+	if err := json.Unmarshal(raw, &state); err != nil {
+		return nil, err
+	}
+	return resourcesStateToMap(ctx, &state)
+}
+
 func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap, error) {
 	rawState, err := os.ReadFile(path)
 	if err != nil {
@@ -59,12 +68,10 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap
 		}
 		return nil, err
 	}
-	var state resourcesState
-	err = json.Unmarshal(rawState, &state)
-	if err != nil {
-		return nil, err
-	}
+	return ParseResourcesStateFromBytes(ctx, rawState)
+}
 
+func resourcesStateToMap(ctx context.Context, state *resourcesState) (ExportedResourcesMap, error) {
 	if state.Version != SupportedStateVersion {
 		return nil, fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version)
 	}
@@ -131,5 +138,10 @@ func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (ExportedResourc
 		return nil, err
 	}
 	filename, _ := b.StateFilenameTerraform(ctx)
-	return parseResourcesState(ctx, filepath.Join(cacheDir, filename))
+	return ParseResourcesStateFromPath(ctx, filepath.Join(cacheDir, filename))
+}
+
+// ParseResourcesStateFromPath parses a terraform state file at a known path.
+func ParseResourcesStateFromPath(ctx context.Context, path string) (ExportedResourcesMap, error) {
+	return parseResourcesState(ctx, path)
 }

bundle/direct/bundle_apply.go

@@ -15,9 +15,7 @@ import (
 	"github.com/databricks/databricks-sdk-go"
 )
 
-type MigrateMode bool
-
-func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, migrateMode MigrateMode) {
+func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) {
 	if plan == nil {
 		panic("Planning is not done")
 	}
@@ -52,9 +50,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 
 		action := entry.Action
 		errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey)
-		if migrateMode {
-			errorPrefix = "cannot migrate " + resourceKey
-		}
 
 		if action == deployplan.Undefined {
 			logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action))
@@ -82,20 +77,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 		}
 
 		if action == deployplan.Delete {
-			if migrateMode {
-				// Resource is in terraform state but not in config. Preserve its ID in
-				// direct state so the next direct deploy will plan and execute deletion.
-				id := b.StateDB.GetResourceID(resourceKey)
-				if id == "" {
-					logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no ID in state", errorPrefix))
-					return false
-				}
-				if err = b.StateDB.SaveState(resourceKey, id, json.RawMessage("{}"), entry.DependsOn); err != nil {
-					logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
-					return false
-				}
-				return true
-			}
 			err = d.Destroy(ctx, &b.StateDB)
 			if err != nil {
 				logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
@@ -123,19 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 				return false
 			}
 
-			if migrateMode {
-				// In migration mode we're reading resources in DAG order so that we have fully resolved config snapshots stored
-				id := b.StateDB.GetResourceID(resourceKey)
-				if id == "" {
-					logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey))
-					return false
-				}
-				err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn)
-			} else {
-				// TODO: redo calcDiff to downgrade planned action if possible (?)
-				err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry)
-			}
-
+			// TODO: redo calcDiff to downgrade planned action if possible (?)
+			err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry)
 			if err != nil {
 				logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
 				return false