From 3bae48a806a4e2304ac170bb3c8fd9df2e609f78 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 15:56:04 +0200 Subject: [PATCH 01/38] bundle migrate: add no-API-call migration from TF to direct engine Implement a new `bundle migrate` command that creates the direct state file from local config + Terraform state attributes, without making any API calls. Unlike `bundle deployment migrate` which calls DoRead for each resource, this command reads resolved field values directly from the local TF state file. Cross-resource references (e.g. ${resources.jobs.src.git_source[0].branch}) are resolved using two independent methods and the results reconciled: - Method A: look up the field in the TF state of the resource that contains the reference (e.g. read name from databricks_job.dst TF attributes). - Method B: evaluate the template by reading each ${resources.*} reference from the TF state of the referenced resource and interpolating. If both methods agree, the value is used silently. If only one succeeds, that value is used. If both succeed but disagree, the longer string is used with a warning. If both fail, an error is returned. The bundle/migrate package provides: - ParseTFStateAttrs: parses the full TF state file (all resource attributes) - LookupTFField: looks up a field value using DABsPathToTerraform translation - ResolveFieldRef: reconciles Methods A and B for a single field Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 + bundle/migrate/resolve.go | 91 ++++++++ bundle/migrate/tf_state.go | 107 +++++++++ cmd/bundle/bundle.go | 1 + cmd/bundle/deployment/migrate.go | 12 + cmd/bundle/migrate.go | 372 +++++++++++++++++++++++++++++++ 6 files changed, 589 insertions(+) create mode 100644 bundle/migrate/resolve.go create mode 100644 bundle/migrate/tf_state.go create mode 100644 cmd/bundle/migrate.go diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d890b8d5d7b..ca07b477f12 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1029,6 +1029,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } +// ExtractReferences extracts all variable references from the config subtree rooted at node. +// Returns a map from structpath string (field path within the resource) to template string. +func ExtractReferences(root dyn.Value, node string) (map[string]string, error) { + return extractReferences(root, node) +} + func extractReferences(root dyn.Value, node string) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go new file mode 100644 index 00000000000..8d5f6bc6b35 --- /dev/null +++ b/bundle/migrate/resolve.go @@ -0,0 +1,91 @@ +package migrate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" +) + +// evaluateTemplate evaluates a template string like "${resources.pipelines.bar.cluster[0].label}" +// by looking up each ${...} reference from TF state. +func evaluateTemplate(state TFStateAttrs, template string) (string, error) { + ref, ok := dynvar.NewRef(dyn.V(template)) + if !ok { + return template, nil + } + + result := template + for _, pathString := range ref.References() { + path, err := structpath.ParsePath(pathString) + if err != nil { + return "", fmt.Errorf("cannot parse reference path %q: %w", pathString, err) + } + // Expect resources... + if path.Len() < 4 { + return "", fmt.Errorf("unexpected reference format (too short): %q", pathString) + } + // Check first component is "resources" + firstNode := path.Prefix(1) + if firstNode.String() != "resources" { + return "", fmt.Errorf("unexpected reference format (expected resources.*): %q", pathString) + } + + group := path.SkipPrefix(1).Prefix(1).String() + name := path.SkipPrefix(2).Prefix(1).String() + fieldPath := path.SkipPrefix(3) + + value, err := LookupTFField(state, group, name, fieldPath) + if err != nil { + return "", fmt.Errorf("cannot look up %q: %w", pathString, err) + } + + result = strings.ReplaceAll(result, "${"+pathString+"}", fmt.Sprintf("%v", value)) + } + return result, nil +} + +// ResolveFieldRef resolves a single reference for a field in resource (srcGroup, srcName). +// fieldPath is the path of the field within the source resource (in DABs naming, from sv.Refs key). +// refTemplate is the template string for that field, e.g. "${resources.pipelines.bar.cluster[0].label}". +// +// Two methods are tried: +// - Method A: read the field from the source resource's own TF state. +// - Method B: evaluate the template by reading each referenced field from TF state. +// +// Returns the reconciled value or an error if both methods fail. +func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string) (any, error) { + // Method A: read field from source resource's TF state. + valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) + + // Method B: evaluate the template by looking up each reference. + valueB, errB := evaluateTemplate(state, refTemplate) + + switch { + case errA == nil && errB == nil: + aStr := fmt.Sprintf("%v", valueA) + if aStr == valueB { + return valueA, nil + } + // Both succeeded but disagree: prefer longer string and warn. + if len(valueB) > len(aStr) { + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueB, nil + } + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueA, nil + case errA == nil: + return valueA, nil + case errB == nil: + return valueB, nil + default: + return nil, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", + srcGroup, srcName, fieldPath, errA, errB) + } +} diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go new file mode 100644 index 00000000000..642f7a26b74 --- /dev/null +++ b/bundle/migrate/tf_state.go @@ -0,0 +1,107 @@ +package migrate + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "sync" + + "github.com/databricks/cli/bundle/deploy/terraform" + tfschema "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + tfjson "github.com/hashicorp/terraform-json" +) + +// TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). +type TFStateAttrs map[string]map[string]json.RawMessage + +// ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. +func ParseTFStateAttrs(path string) (TFStateAttrs, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var state struct { + Version int `json:"version"` + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []struct { + Attributes json.RawMessage `json:"attributes"` + } `json:"instances"` + } `json:"resources"` + } + + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + + result := make(TFStateAttrs) + for _, r := range state.Resources { + if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { + continue + } + if result[r.Type] == nil { + result[r.Type] = make(map[string]json.RawMessage) + } + result[r.Type][r.Name] = r.Instances[0].Attributes + } + return result, nil +} + +// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). +var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { + t := reflect.TypeOf(tfschema.AllResources{}) + m := make(map[string]reflect.Type, t.NumField()) + for i := range t.NumField() { + f := t.Field(i) + tag := strings.Split(f.Tag.Get("json"), ",")[0] + if tag != "" && tag != "-" { + m[tag] = f.Type + } + } + return m +}) + +// LookupTFField looks up a field from TF state attributes for a bundle resource. +// group is the DABs group (e.g. "pipelines"), name is the resource name. +// fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). +// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. +func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { + tfType, ok := terraform.GroupToTerraformName[group] + if !ok { + return nil, fmt.Errorf("unknown resource group %q", group) + } + + // Translate field path to TF naming. + // DABsPathToTerraform handles both DABs names (renames) and TF names (pass-through for unknowns). + // Returns error for known DABs-only fields that have no TF equivalent. + tfFieldPath, err := terraform_dabs_map.DABsPathToTerraform(group, fieldPath) + if err != nil { + return nil, err + } + + attrsJSON, ok := state[tfType][name] + if !ok { + return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) + } + + schemaType, ok := tfSchemaTypeMap()[tfType] + if !ok { + return nil, fmt.Errorf("no schema type registered for %q", tfType) + } + + // Unmarshal attributes into a new instance of the schema struct. + ptr := reflect.New(schemaType) + if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) + } + + return structaccess.Get(ptr.Interface(), tfFieldPath) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 7189b1d431d..84666eec689 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,6 +38,7 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) + cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 2d54aafb1ed..2803a0d2712 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -30,6 +30,12 @@ import ( const backupSuffix = ".backup" +// RunPlanCheck runs bundle plan and checks if there are any actions planned. +// Returns error if plan fails or if there are actions planned. +func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { + return runPlanCheck(cmd, extraArgs, extraArgsStr) +} + // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -78,6 +84,12 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } +// GetCommonArgs extracts common flags (target, profile, var) from the command into +// argument slices suitable for forwarding to a subprocess. +func GetCommonArgs(cmd *cobra.Command) ([]string, string) { + return getCommonArgs(cmd) +} + func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go new file mode 100644 index 00000000000..3579658d3fe --- /dev/null +++ b/cmd/bundle/migrate.go @@ -0,0 +1,372 @@ +package bundle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/spf13/cobra" +) + +func newMigrateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrate from Terraform to Direct deployment engine (no API calls)", + Long: `Creates a Direct deployment state file from the local config and Terraform state, +without making API calls. Cross-resource references are resolved from TF state.`, + Args: root.NoArgs, + } + + var noPlanCheck bool + cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // Clear engine env var: we read TF state and produce a direct state. + cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) + + opts := utils.ProcessOptions{ + AlwaysPull: true, + FastValidate: true, + Build: true, + PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { + if b.Config.Bundle.Engine == engine.EngineTerraform { + return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) + } + return nil + }, + } + + b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) + if err != nil { + return err + } + ctx := cmd.Context() + + if stateDesc.Lineage == "" { + cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) + return root.ErrAlreadyPrinted + } + if stateDesc.Engine.IsDirect() { + return fmt.Errorf("already using direct engine: %s", stateDesc.String()) + } + + _, localTerraformPath := b.StateFilenameTerraform(ctx) + if _, err := os.Stat(localTerraformPath); err != nil { + return fmt.Errorf("reading %s: %w", localTerraformPath, err) + } + + // Run plan check unless --noplancheck is set. + if !noPlanCheck { + cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") + extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) + if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { + return err + } + } + + // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). + tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) + if err != nil { + return fmt.Errorf("failed to parse terraform state: %w", err) + } + for key, entry := range tfResourceIDs { + if entry.ID == "" { + return fmt.Errorf("missing ID for %s in terraform state", key) + } + } + + cacheDir, err := terraform.Dir(ctx, b) + if err != nil { + return err + } + tfStateFilename, _ := b.StateFilenameTerraform(ctx) + tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) + + tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + + _, localPath := b.StateFilenameDirect(ctx) + tempPath := localPath + ".temp-migration" + + if _, err := os.Stat(tempPath); err == nil { + return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) + } + if _, err := os.Stat(localPath); err == nil { + return fmt.Errorf("state file %s already exists", localPath) + } + + // Apply SecretScopeFixups so the config matches what the direct engine expects. + // This adds MANAGE ACL for the current user to all secret scopes, ensuring + // the migrated state and config agree on .permissions entries. + bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + // Build initial state with IDs and optional ETags. + etags := map[string]string{} + state := make(map[string]dstate.ResourceEntry) + for key, resourceEntry := range tfResourceIDs { + state[key] = dstate.ResourceEntry{ + ID: resourceEntry.ID, + State: json.RawMessage("{}"), + } + if resourceEntry.ETag != "" { + etags[key] = resourceEntry.ETag + } + } + + migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) + migratedDB.State = state + + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempPath, migratedDB) + + removeTempPath := true + defer func() { + if removeTempPath { + _ = os.Remove(tempPath) + } + }() + + // Initialize adapters. + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + if err != nil { + return err + } + + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for write: %w", err) + } + + // Process each resource: prepare state, resolve refs from TF state, save. + if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + return err + } + + if _, err := stateDB.Finalize(ctx); err != nil { + return fmt.Errorf("finalizing state: %w", err) + } + if logdiag.HasError(ctx) { + return errors.New("migration encountered errors") + } + + if err := os.Rename(tempPath, localPath); err != nil { + return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) + } + removeTempPath = false + + localTerraformBackupPath := localTerraformPath + ".backup" + err = os.Rename(localTerraformPath, localTerraformBackupPath) + if err != nil { + // Not fatal since we've already incremented the serial. + logdiag.LogError(ctx, err) + } + + extraArgsStr := "" + if flag := cmd.Flag("target"); flag != nil && flag.Changed { + extraArgsStr = " -t " + flag.Value.String() + } + + cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s + +Validate the migration by running "databricks bundle plan%s", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". + +To undo the migration, remove %s and rename %s to %s +`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) + return nil + } + + return cmd +} + +// buildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +func buildStateFromTF( + ctx context.Context, + b *bundle.Bundle, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs migrate.TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + configRoot := &b.Config + + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // We need to extract the resource name for Method A (looking up in the source resource's TF state). + parts := strings.SplitN(node, ".", 4) + // node format: "resources.." or "resources...permissions" + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} From 6abb64db9fb07f0b1288ee80a8c561ad907b6580 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:12:10 +0200 Subject: [PATCH 02/38] bundle migrate: remove plan check, make --noplancheck a no-op The new migrate command reads only from the local TF state file and never invokes the Terraform engine, so a pre-migration plan check has no place here. The --noplancheck flag is kept but ignored to avoid breaking callers. Co-authored-by: Isaac --- cmd/bundle/migrate.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index 3579658d3fe..ffc79ddda18 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -20,7 +20,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" @@ -43,8 +42,9 @@ without making API calls. Cross-resource references are resolved from TF state.` Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + // --noplancheck is kept for compatibility but has no effect: this command reads + // only from the local TF state file and never invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") cmd.RunE = func(cmd *cobra.Command, args []string) error { // Clear engine env var: we read TF state and produce a direct state. @@ -81,15 +81,6 @@ without making API calls. Cross-resource references are resolved from TF state.` return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - // Run plan check unless --noplancheck is set. - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") - extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) - if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) if err != nil { From f72f85a9cdd52568d7edb6d6ee2a5be80120a1fa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:31:10 +0200 Subject: [PATCH 03/38] bundle/direct: drop MigrateMode, consolidate state-building in bundle/migrate MigrateMode was a bool parameter on Apply that forked between "save state without deploying" and "actually deploy". The two MigrateMode(true) callers (bundle deployment migrate and upload_state_for_yaml_sync) now use migrate.BuildStateFromTF directly, reading from the local TF state file without any API calls. Changes: - New bundle/migrate/build_state.go: public BuildStateFromTF extracted from cmd/bundle/migrate.go, taking *config.Root so callers can pass an un-interpolated config (needed by upload_state_for_yaml_sync). - bundle/direct/bundle_apply.go: drop MigrateMode type and parameter; Apply now only handles real deployments. - bundle/phases/{deploy,destroy}.go: drop MigrateMode(false) argument. - upload_state_for_yaml_sync.go: replace CalculatePlan+Apply with BuildStateFromTF; keep reverseInterpolate since config is TF-interpolated. - cmd/bundle/deployment/migrate.go: replace CalculatePlan+Apply with BuildStateFromTF; drop exported RunPlanCheck/GetCommonArgs wrappers. - cmd/bundle/migrate.go: delegate to migrate.BuildStateFromTF. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 35 +--- bundle/migrate/build_state.go | 179 ++++++++++++++++++ bundle/phases/deploy.go | 3 +- bundle/phases/destroy.go | 3 +- .../statemgmt/upload_state_for_yaml_sync.go | 45 ++--- cmd/bundle/deployment/migrate.go | 74 ++------ cmd/bundle/migrate.go | 173 +---------------- 7 files changed, 216 insertions(+), 296 deletions(-) create mode 100644 bundle/migrate/build_state.go diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a9981ee63d8..a5849ffbd1a 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -15,9 +15,7 @@ import ( "github.com/databricks/databricks-sdk-go" ) -type MigrateMode bool - -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, migrateMode MigrateMode) { +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { if plan == nil { panic("Planning is not done") } @@ -52,9 +50,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa action := entry.Action errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) - if migrateMode { - errorPrefix = "cannot migrate " + resourceKey - } if action == deployplan.Undefined { logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) @@ -82,20 +77,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - if migrateMode { - // Resource is in terraform state but not in config. Preserve its ID in - // direct state so the next direct deploy will plan and execute deletion. - id := b.StateDB.GetResourceID(resourceKey) - if id == "" { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no ID in state", errorPrefix)) - return false - } - if err = b.StateDB.SaveState(resourceKey, id, json.RawMessage("{}"), entry.DependsOn); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } - return true - } err = d.Destroy(ctx, &b.StateDB) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) @@ -123,18 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - if migrateMode { - // In migration mode we're reading resources in DAG order so that we have fully resolved config snapshots stored - id := b.StateDB.GetResourceID(resourceKey) - if id == "" { - logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) - return false - } - err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) - } else { - // TODO: redo calcDiff to downgrade planned action if possible (?) - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - } + // TODO: redo calcDiff to downgrade planned action if possible (?) + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go new file mode 100644 index 00000000000..af2b3449086 --- /dev/null +++ b/bundle/migrate/build_state.go @@ -0,0 +1,179 @@ +package migrate + +import ( + "context" + "fmt" + "maps" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" +) + +// BuildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +// configRoot should be an un-interpolated config (with ${resources.*} references). +func BuildStateFromTF( + ctx context.Context, + configRoot *config.Root, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // node format: "resources.." or "resources...permissions" + parts := strings.SplitN(node, ".", 4) + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index fd76151483c..840c7d821f1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -83,7 +82,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta err error ) if targetEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) state, err = b.DeploymentBundle.StateDB.Finalize(ctx) // Capture the finalized state for deploy telemetry. It carries each // resource's state-size in bytes (from the WAL replay Finalize just diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 74049f26f42..fd580f5e971 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -78,7 +77,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { // Core destructive mutators for destroy. These require informed user consent. bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 6573d1dc56f..ee750203d96 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -14,18 +14,16 @@ import ( "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" ) type uploadStateForYamlSync struct { @@ -131,6 +129,11 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to read terraform state: %w", err) } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return false, fmt.Errorf("failed to read terraform state attributes: %w", err) + } + state := make(map[string]dstate.ResourceEntry) etags := map[string]string{} @@ -155,8 +158,8 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(snapshotPath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(snapshotPath, migratedDB) // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring @@ -166,9 +169,9 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, errors.New("failed to apply secret scope fixups") } - // Get the dynamic value from b.Config and reverse the interpolation. // b.Config has been modified by terraform.Interpolate which converts bundle-style // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). + // BuildStateFromTF expects ${resources.*} references, so reverse the interpolation first. interpolatedRoot := b.Config.Value() uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) if err != nil { @@ -183,36 +186,20 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &uninterpolatedConfig) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return false, err } - for _, entry := range plan.Plan { - entry.Action = deployplan.Update - } - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - continue - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - log.Warnf(ctx, "Failed to set etag on %q: %v", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return false, fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return false, err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 2803a0d2712..1c59616fcc9 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -14,28 +14,20 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" "github.com/spf13/cobra" ) const backupSuffix = ".backup" -// RunPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - return runPlanCheck(cmd, extraArgs, extraArgsStr) -} - // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -84,12 +76,6 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } -// GetCommonArgs extracts common flags (target, profile, var) from the command into -// argument slices suitable for forwarding to a subprocess. -func GetCommonArgs(cmd *cobra.Command) ([]string, string) { - return getCommonArgs(cmd) -} - func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string @@ -221,6 +207,11 @@ To start using direct engine, set "engine: direct" under bundle in your databric } } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + etags := map[string]string{} state := make(map[string]dstate.ResourceEntry) @@ -238,8 +229,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(tempStatePath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempStatePath, migratedDB) tempStatePathAutoRemove := true @@ -257,55 +248,20 @@ To start using direct engine, set "engine: direct" under bundle in your databric return root.ErrAlreadyPrinted } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return err } - for _, entry := range plan.Plan { - switch entry.Action { - case deployplan.Create: - // Resource is in config but not in terraform state; skip it during migration. - // It will be created on the first direct deploy. - entry.Action = deployplan.Skip - case deployplan.Delete: - // Resource is in terraform state but not in config. Keep as Delete so the - // apply migrate path can preserve its ID in direct state, allowing the next - // direct deploy to remove it. - default: - // Force existing resources to Update so migration reads their remote state - // and writes a full config snapshot. - entry.Action = deployplan.Update - } - } - - // We need to copy ETag into new state. - // For most resources state consists of fully resolved local config snapshot + id. - // Dashboards are special in that they also store "etag" in state which is not provided by user but - // comes from remote state. If we don't store "etag" in state, we won't detect remote drift, because - // local=nil, remote="" which will be classified as a backend default and skipped. - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - return fmt.Errorf("failed to read state for %q", key) - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - return fmt.Errorf("failed to set etag on %q: %w", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { logdiag.LogError(ctx, err) } if logdiag.HasError(ctx) { diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index ffc79ddda18..8a33fe85f55 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -5,31 +5,21 @@ import ( "encoding/json" "errors" "fmt" - "maps" "os" "path/filepath" - "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structvar" "github.com/spf13/cobra" ) @@ -159,7 +149,7 @@ without making API calls. Cross-resource references are resolved from TF state.` } // Process each resource: prepare state, resolve refs from TF state, save. - if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { return err } @@ -200,164 +190,3 @@ To undo the migration, remove %s and rename %s to %s return cmd } - -// buildStateFromTF iterates over bundle resources, resolves cross-resource -// references using TF state attributes, and writes each resource's state entry. -func buildStateFromTF( - ctx context.Context, - b *bundle.Bundle, - adapters map[string]*dresources.Adapter, - stateDB *dstate.DeploymentState, - tfAttrs migrate.TFStateAttrs, - tfIDs terraform.ExportedResourcesMap, - etags map[string]string, -) error { - configRoot := &b.Config - - // Collect all resource nodes (same patterns as makePlan). - var nodes []string - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - } - for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), - pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nodes = append(nodes, p.String()) - return dyn.InvalidValue, nil - }, - ) - if err != nil { - return err - } - } - - for _, node := range nodes { - idEntry, ok := tfIDs[node] - if !ok { - // Resource is in config but not in TF state (new resource); skip. - continue - } - - group := config.GetResourceTypeFromKey(node) - if group == "" { - return fmt.Errorf("cannot determine resource type for %q", node) - } - - adapter, ok := adapters[group] - if !ok { - log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) - continue - } - - inputConfig, err := configRoot.GetResourceConfig(node) - if err != nil { - return fmt.Errorf("%s: getting config: %w", node, err) - } - - baseRefs := map[string]string{} - - switch { - case strings.HasSuffix(node, ".permissions"): - var sv *structvar.StructVar - if strings.HasPrefix(node, "resources.secret_scopes.") { - typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) - if !ok { - return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) - } - sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) - } - } else { - sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing permissions config: %w", node, err) - } - } - inputConfig = sv.Value - baseRefs = sv.Refs - - case strings.HasSuffix(node, ".grants"): - sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing grants config: %w", node, err) - } - inputConfig = sv.Value - baseRefs = sv.Refs - } - - newStateValue, err := adapter.PrepareState(inputConfig) - if err != nil { - return fmt.Errorf("%s: PrepareState: %w", node, err) - } - - refs, err := direct.ExtractReferences(configRoot.Value(), node) - if err != nil { - return fmt.Errorf("%s: extracting references: %w", node, err) - } - maps.Copy(refs, baseRefs) - - sv := structvar.NewStructVar(newStateValue, refs) - - // Resolve each reference using TF state. - // We need to extract the resource name for Method A (looking up in the source resource's TF state). - parts := strings.SplitN(node, ".", 4) - // node format: "resources.." or "resources...permissions" - var srcGroup, srcName string - if len(parts) >= 3 { - srcGroup = parts[1] - srcName = parts[2] - } - - // Collect all field paths that need resolution (avoid modifying map during iteration). - type refEntry struct { - fieldPathStr string - refTemplate string - } - var pendingRefs []refEntry - for fieldPathStr, refTemplate := range sv.Refs { - pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) - } - - for _, pending := range pendingRefs { - fieldPath, err := structpath.ParsePath(pending.fieldPathStr) - if err != nil { - return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) - } - - // ResolveFieldRef returns the fully resolved value for this field, - // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) - if err != nil { - return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) - } - - // Set the resolved value directly and remove the ref entry. - if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { - return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) - } - delete(sv.Refs, pending.fieldPathStr) - } - - if len(sv.Refs) > 0 { - return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) - } - - // Handle etag for dashboards. - if etag := etags[node]; etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) - } - } - - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { - return fmt.Errorf("%s: SaveState: %w", node, err) - } - } - - return nil -} From 78eb0f8d9a4c3ec7c2c72b711cb0ef8fbc33bc27 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 17:30:22 +0200 Subject: [PATCH 04/38] bundle migrate: remove new top-level command, keep deployment migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new approach (BuildStateFromTF, no API calls) has been folded into the existing 'bundle deployment migrate' command. The separate 'bundle migrate' command is removed — it was redundant. Also add resolve_test.go: verify that int and bool cross-resource references work correctly when Method B returns a string value. structaccess.Set already handles string→int and string→bool conversion via strconv, so no bug exists. Co-authored-by: Isaac --- bundle/migrate/resolve_test.go | 78 ++++++++++++++ cmd/bundle/bundle.go | 1 - cmd/bundle/migrate.go | 192 --------------------------------- 3 files changed, 78 insertions(+), 193 deletions(-) create mode 100644 bundle/migrate/resolve_test.go delete mode 100644 cmd/bundle/migrate.go diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go new file mode 100644 index 00000000000..8a2ab06d13b --- /dev/null +++ b/bundle/migrate/resolve_test.go @@ -0,0 +1,78 @@ +package migrate_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// state with src job having int and bool fields set. +func testState() migrate.TFStateAttrs { + return migrate.TFStateAttrs{ + "databricks_job": { + "src": json.RawMessage(`{ + "id": "111", + "max_concurrent_runs": 4, + "always_running": true + }`), + "dst": json.RawMessage(`{ + "id": "222", + "max_concurrent_runs": 4, + "always_running": true + }`), + }, + } +} + +// TestResolveFieldRefInt proves that when Method B (template evaluation) wins for +// an int field, the returned string value is still usable: structaccess.Set must +// parse it back to int and not error. +func TestResolveFieldRefInt(t *testing.T) { + state := testState() + // Remove dst from state so Method A fails and Method B must be used. + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("max_concurrent_runs") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}") + require.NoError(t, err) + + // Method B succeeds: returns string "4". Verify Set converts it to int. + type target struct { + MaxConcurrentRuns int `json:"max_concurrent_runs"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into int field", value) + assert.Equal(t, 4, s.MaxConcurrentRuns) +} + +// TestResolveFieldRefBool is the same for a bool field. +func TestResolveFieldRefBool(t *testing.T) { + state := testState() + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("always_running") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}") + require.NoError(t, err) + + type target struct { + AlwaysRunning bool `json:"always_running"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into bool field", value) + assert.Equal(t, true, s.AlwaysRunning) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 84666eec689..7189b1d431d 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,7 +38,6 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) - cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go deleted file mode 100644 index 8a33fe85f55..00000000000 --- a/cmd/bundle/migrate.go +++ /dev/null @@ -1,192 +0,0 @@ -package bundle - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/utils" - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/logdiag" - "github.com/spf13/cobra" -) - -func newMigrateCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "migrate", - Short: "Migrate from Terraform to Direct deployment engine (no API calls)", - Long: `Creates a Direct deployment state file from the local config and Terraform state, -without making API calls. Cross-resource references are resolved from TF state.`, - Args: root.NoArgs, - } - - // --noplancheck is kept for compatibility but has no effect: this command reads - // only from the local TF state file and never invokes the Terraform engine. - cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - // Clear engine env var: we read TF state and produce a direct state. - cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) - - opts := utils.ProcessOptions{ - AlwaysPull: true, - FastValidate: true, - Build: true, - PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { - if b.Config.Bundle.Engine == engine.EngineTerraform { - return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) - } - return nil - }, - } - - b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) - if err != nil { - return err - } - ctx := cmd.Context() - - if stateDesc.Lineage == "" { - cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) - return root.ErrAlreadyPrinted - } - if stateDesc.Engine.IsDirect() { - return fmt.Errorf("already using direct engine: %s", stateDesc.String()) - } - - _, localTerraformPath := b.StateFilenameTerraform(ctx) - if _, err := os.Stat(localTerraformPath); err != nil { - return fmt.Errorf("reading %s: %w", localTerraformPath, err) - } - - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). - tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) - if err != nil { - return fmt.Errorf("failed to parse terraform state: %w", err) - } - for key, entry := range tfResourceIDs { - if entry.ID == "" { - return fmt.Errorf("missing ID for %s in terraform state", key) - } - } - - cacheDir, err := terraform.Dir(ctx, b) - if err != nil { - return err - } - tfStateFilename, _ := b.StateFilenameTerraform(ctx) - tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) - - tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) - if err != nil { - return fmt.Errorf("failed to read terraform state attributes: %w", err) - } - - _, localPath := b.StateFilenameDirect(ctx) - tempPath := localPath + ".temp-migration" - - if _, err := os.Stat(tempPath); err == nil { - return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) - } - if _, err := os.Stat(localPath); err == nil { - return fmt.Errorf("state file %s already exists", localPath) - } - - // Apply SecretScopeFixups so the config matches what the direct engine expects. - // This adds MANAGE ACL for the current user to all secret scopes, ensuring - // the migrated state and config agree on .permissions entries. - bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted - } - - // Build initial state with IDs and optional ETags. - etags := map[string]string{} - state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range tfResourceIDs { - state[key] = dstate.ResourceEntry{ - ID: resourceEntry.ID, - State: json.RawMessage("{}"), - } - if resourceEntry.ETag != "" { - etags[key] = resourceEntry.ETag - } - } - - migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) - migratedDB.State = state - - var stateDB dstate.DeploymentState - stateDB.OpenWithData(tempPath, migratedDB) - - removeTempPath := true - defer func() { - if removeTempPath { - _ = os.Remove(tempPath) - } - }() - - // Initialize adapters. - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) - if err != nil { - return err - } - - if err := stateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for write: %w", err) - } - - // Process each resource: prepare state, resolve refs from TF state, save. - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { - return err - } - - if _, err := stateDB.Finalize(ctx); err != nil { - return fmt.Errorf("finalizing state: %w", err) - } - if logdiag.HasError(ctx) { - return errors.New("migration encountered errors") - } - - if err := os.Rename(tempPath, localPath); err != nil { - return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) - } - removeTempPath = false - - localTerraformBackupPath := localTerraformPath + ".backup" - err = os.Rename(localTerraformPath, localTerraformBackupPath) - if err != nil { - // Not fatal since we've already incremented the serial. - logdiag.LogError(ctx, err) - } - - extraArgsStr := "" - if flag := cmd.Flag("target"); flag != nil && flag.Changed { - extraArgsStr = " -t " + flag.Value.String() - } - - cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s - -Validate the migration by running "databricks bundle plan%s", there should be no actions planned. - -The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". - -To undo the migration, remove %s and rename %s to %s -`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) - return nil - } - - return cmd -} From 72267445e62a8a2c8934336d6329b0f1ec9f9fae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 2 Jun 2026 14:11:14 +0200 Subject: [PATCH 05/38] deployment migrate: drop plan check and --noplancheck flag The command now reads from the local TF state file without invoking the Terraform binary, so there is nothing for the plan check to verify. Also simplify getCommonArgs to return only the display string (the args slice was only needed to forward to the plan subprocess). Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 1 - bundle/migrate/resolve_test.go | 2 +- bundle/migrate/tf_state.go | 5 +- cmd/bundle/deployment/migrate.go | 99 ++++---------------------------- 4 files changed, 14 insertions(+), 93 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a5849ffbd1a..a63d70aee13 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -106,7 +106,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 8a2ab06d13b..221c873a73d 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -74,5 +74,5 @@ func TestResolveFieldRefBool(t *testing.T) { s := &target{} err = structaccess.Set(s, fieldPath, value) assert.NoError(t, err, "Set should parse string %q into bool field", value) - assert.Equal(t, true, s.AlwaysRunning) + assert.True(t, s.AlwaysRunning) } diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 642f7a26b74..f4733c152eb 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -57,10 +57,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { // tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeOf(tfschema.AllResources{}) + t := reflect.TypeFor[tfschema.AllResources]() m := make(map[string]reflect.Type, t.NumField()) - for i := range t.NumField() { - f := t.Field(i) + for f := range t.Fields() { tag := strings.Split(f.Tag.Get("json"), ",")[0] if tag != "" && tag != "-" { m[tag] = f.Type diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 1c59616fcc9..fb2179145d3 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,13 +1,11 @@ package deployment import ( - "bytes" "context" "encoding/json" "errors" "fmt" "os" - "os/exec" "strings" "github.com/databricks/cli/bundle" @@ -28,95 +26,31 @@ import ( const backupSuffix = ".backup" -// runPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - ctx := cmd.Context() - - executable, err := os.Executable() - if err != nil { - return fmt.Errorf("failed to get executable path: %w", err) - } - - args := []string{"bundle", "plan"} - args = append(args, extraArgs...) - - planCmd := exec.CommandContext(ctx, executable, args...) - var stdout bytes.Buffer - planCmd.Stdout = &stdout - planCmd.Stderr = cmd.ErrOrStderr() - - // Use the engine encoded in the state - planCmd.Env = append(os.Environ(), "DATABRICKS_BUNDLE_ENGINE=terraform") - - err = planCmd.Run() - - // Output the plan stdout as is - output := stdout.String() - fmt.Fprint(cmd.OutOrStdout(), output) - - if err != nil { - msg := "" - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - msg = fmt.Sprintf("exit code %d", exitErr.ExitCode()) - } else { - msg = err.Error() - } - return fmt.Errorf("bundle plan failed with %s, aborting migration. To proceed with migration anyway, re-run the command with --noplancheck option", msg) - } - - if !strings.Contains(output, "Plan:") { - return fmt.Errorf("cannot parse 'databricks bundle plan%s' output, aborting migration. Skip plan check with --noplancheck option", extraArgsStr) - } - - if !strings.Contains(output, "Plan: 0 to add, 0 to change, 0 to delete") { - return fmt.Errorf("'databricks bundle plan%s' shows actions planned, aborting migration. Please run 'databricks bundle deploy%s' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option", extraArgsStr, extraArgsStr) - } - - return nil -} - -func getCommonArgs(cmd *cobra.Command) ([]string, string) { - var args []string +func getCommonArgs(cmd *cobra.Command) string { var quotedArgs []string if flag := cmd.Flag("target"); flag != nil && flag.Changed { - target := flag.Value.String() - if target != "" { - args = append(args, "-t") - args = append(args, target) - quotedArgs = append(quotedArgs, "-t") - quotedArgs = append(quotedArgs, shellquote.BashArg(target)) + if target := flag.Value.String(); target != "" { + quotedArgs = append(quotedArgs, "-t", shellquote.BashArg(target)) } } if flag := cmd.Flag("profile"); flag != nil && flag.Changed { - profile := flag.Value.String() - if profile != "" { - args = append(args, "-p") - args = append(args, profile) - quotedArgs = append(quotedArgs, "-p") - quotedArgs = append(quotedArgs, shellquote.BashArg(profile)) + if profile := flag.Value.String(); profile != "" { + quotedArgs = append(quotedArgs, "-p", shellquote.BashArg(profile)) } } if flag := cmd.Flag("var"); flag != nil && flag.Changed { - varValues, err := cmd.Flags().GetStringSlice("var") - if err == nil { + if varValues, err := cmd.Flags().GetStringSlice("var"); err == nil { for _, v := range varValues { - args = append(args, "--var") - args = append(args, v) - quotedArgs = append(quotedArgs, "--var") - quotedArgs = append(quotedArgs, shellquote.BashArg(v)) + quotedArgs = append(quotedArgs, "--var", shellquote.BashArg(v)) } } } - argsStr := "" - - if len(quotedArgs) > 0 { - argsStr = " " + strings.Join(quotedArgs, " ") + if len(quotedArgs) == 0 { + return "" } - - return args, argsStr + return " " + strings.Join(quotedArgs, " ") } func newMigrateCommand() *cobra.Command { @@ -134,11 +68,8 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") - cmd.RunE = func(cmd *cobra.Command, args []string) error { - extraArgs, extraArgsStr := getCommonArgs(cmd) + extraArgsStr := getCommonArgs(cmd) // Clear the engine env var so migrate always uses terraform engine to read existing state, // regardless of what the user may have set in their environment. @@ -199,14 +130,6 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - // Run plan check unless --noplancheck is set - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:") - if err = runPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) if err != nil { return fmt.Errorf("failed to read terraform state attributes: %w", err) From cc8b19e31418d88de2dff8cc7f536b9a1e615fc3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 11:50:57 +0200 Subject: [PATCH 06/38] acceptance: update bundle-deployment-migrate help output Remove --noplancheck from expected output after dropping that flag. Co-authored-by: Isaac --- acceptance/bundle/help/bundle-deployment-migrate/output.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index da2f95707bc..36ee7d481e7 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,8 +12,7 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate - --noplancheck Skip running bundle plan before migration. + -h, --help help for migrate Global Flags: --debug enable debug logging From 799d9789c4853dc69b246c8d8183ba8526c710d3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:08:36 +0200 Subject: [PATCH 07/38] migrate: fix TF list-block unmarshaling, restore --noplancheck no-op, update outputs TF state stores single-block fields (e.g. continuous, deployment) as single-element arrays [{}], not plain objects. json.Unmarshal into the generated Go schema structs fails with type mismatch. Switch LookupTFField to navigate via map[string]any + custom navigateTFState that auto-unwraps single-element lists when a string-key step follows. Also: - Restore --noplancheck as a no-op flag (backward compat; used by invariant tests for job_with_depends_on config). - Remove the plan-check lines from acceptance test output.txt files. - Update help output to include --noplancheck. Co-authored-by: Isaac --- .../help/bundle-deployment-migrate/output.txt | 3 +- .../bundle/migrate/dashboards/output.txt | 2 - .../bundle/migrate/default-python/output.txt | 1 - acceptance/bundle/migrate/grants/output.txt | 2 - .../bundle/migrate/permissions/output.txt | 2 - .../bundle/migrate/profile_arg/output.txt | 4 - acceptance/bundle/migrate/runas/output.txt | 1 - acceptance/bundle/migrate/var_arg/output.txt | 4 - bundle/migrate/tf_state.go | 83 ++++++++++++------- cmd/bundle/deployment/migrate.go | 4 + 10 files changed, 61 insertions(+), 45 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index 36ee7d481e7..9312fb85ffc 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,7 +12,8 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate + -h, --help help for migrate + --noplancheck No-op (kept for compatibility). Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/migrate/dashboards/output.txt b/acceptance/bundle/migrate/dashboards/output.txt index 19a4f1c7bb5..cfda4350ceb 100644 --- a/acceptance/bundle/migrate/dashboards/output.txt +++ b/acceptance/bundle/migrate/dashboards/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 15d49805b47..92efb3066d6 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -24,7 +24,6 @@ Deployment complete! >>> musterr [CLI] bundle deployment migrate Building python_artifact... -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Building python_artifact... update jobs.sample_job diff --git a/acceptance/bundle/migrate/grants/output.txt b/acceptance/bundle/migrate/grants/output.txt index 146787d549a..86acb6398f5 100644 --- a/acceptance/bundle/migrate/grants/output.txt +++ b/acceptance/bundle/migrate/grants/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged Success! Migrated 6 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/permissions/output.txt b/acceptance/bundle/migrate/permissions/output.txt index f85c8d7bdbf..51caca51104 100644 --- a/acceptance/bundle/migrate/permissions/output.txt +++ b/acceptance/bundle/migrate/permissions/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged Success! Migrated 4 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/profile_arg/output.txt b/acceptance/bundle/migrate/profile_arg/output.txt index a6def38363a..082feee15b1 100644 --- a/acceptance/bundle/migrate/profile_arg/output.txt +++ b/acceptance/bundle/migrate/profile_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan -p non_existent321", there should be no actions planned. @@ -24,8 +22,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -t prod -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json Validate the migration by running "databricks bundle plan -t prod -p non_existent321", there should be no actions planned. diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index 74b9a0217f3..f7034d74ec0 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -81,7 +81,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. diff --git a/acceptance/bundle/migrate/var_arg/output.txt b/acceptance/bundle/migrate/var_arg/output.txt index a7f8c0e5b2e..3c20819211d 100644 --- a/acceptance/bundle/migrate/var_arg/output.txt +++ b/acceptance/bundle/migrate/var_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var=job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. @@ -39,8 +37,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index f4733c152eb..52799dd79b8 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -4,14 +4,9 @@ import ( "encoding/json" "fmt" "os" - "reflect" - "strings" - "sync" "github.com/databricks/cli/bundle/deploy/terraform" - tfschema "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" tfjson "github.com/hashicorp/terraform-json" ) @@ -55,23 +50,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { return result, nil } -// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). -var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeFor[tfschema.AllResources]() - m := make(map[string]reflect.Type, t.NumField()) - for f := range t.Fields() { - tag := strings.Split(f.Tag.Get("json"), ",")[0] - if tag != "" && tag != "-" { - m[tag] = f.Type - } - } - return m -}) - // LookupTFField looks up a field from TF state attributes for a bundle resource. // group is the DABs group (e.g. "pipelines"), name is the resource name. // fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). -// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { tfType, ok := terraform.GroupToTerraformName[group] if !ok { @@ -91,16 +72,62 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) } - schemaType, ok := tfSchemaTypeMap()[tfType] - if !ok { - return nil, fmt.Errorf("no schema type registered for %q", tfType) - } - - // Unmarshal attributes into a new instance of the schema struct. - ptr := reflect.New(schemaType) - if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + // Unmarshal into map[string]any to handle TF list-blocks: in TF state, single-block + // fields are stored as single-element arrays [{"field": "value"}], not as plain objects. + // Navigating via map avoids the json.Unmarshal type mismatch between []T in JSON and + // struct-typed schema fields. + var attrs map[string]any + if err := json.Unmarshal(attrsJSON, &attrs); err != nil { return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - return structaccess.Get(ptr.Interface(), tfFieldPath) + return navigateTFState(attrs, tfFieldPath) +} + +// navigateTFState walks the TF state map using the given path. +// TF stores single-block fields as single-element arrays ([{…}]). When a string-key +// step encounters a []any, it auto-descends into element [0] so callers can use plain +// paths like "continuous.pause_status" even though TF stores them as [{"pause_status":…}]. +func navigateTFState(data map[string]any, path *structpath.PathNode) (any, error) { + var current any = data + for _, node := range path.AsSlice() { + if current == nil { + return nil, nil + } + + if key, ok := node.StringKey(); ok { + // Auto-unwrap TF list-blocks: if the current value is a single-element + // array and the next step wants a map key, descend into element 0. + if arr, isArr := current.([]any); isArr { + if len(arr) == 0 { + return nil, nil + } + current = arr[0] + } + m, ok := current.(map[string]any) + if !ok { + return nil, fmt.Errorf("expected map at %q, got %T", key, current) + } + val, ok := m[key] + if !ok { + return nil, fmt.Errorf("%q: key not found", key) + } + current = val + } else if idx, ok := node.Index(); ok { + switch v := current.(type) { + case []any: + if idx < 0 || idx >= len(v) { + return nil, fmt.Errorf("index %d out of range (len %d)", idx, len(v)) + } + current = v[idx] + default: + // TF [0] on a non-slice (already unwrapped) is a no-op. + if idx == 0 { + continue + } + return nil, fmt.Errorf("index %d: not a slice (%T)", idx, current) + } + } + } + return current, nil } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index fb2179145d3..9b217062fc8 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -68,6 +68,10 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } + // --noplancheck kept for backward compatibility; the plan check was removed + // because the command no longer invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") + cmd.RunE = func(cmd *cobra.Command, args []string) error { extraArgsStr := getCommonArgs(cmd) From adeba72f66d3ef5638b5b6be89f914fdada6c077 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:55:02 +0200 Subject: [PATCH 08/38] migrate: compute depends_on from refs before resolving, restore --noplancheck The refs map is shared with sv.Refs and gets mutated (entries deleted) during reference resolution. depends_on must be computed before that loop runs. Also restore --noplancheck as a no-op flag kept for backward compatibility (used by the invariant test suite for job_with_depends_on config). Co-authored-by: Isaac --- bundle/migrate/build_state.go | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index af2b3449086..785c325bfad 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "maps" + "slices" "strings" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -120,6 +123,45 @@ func BuildStateFromTF( sv := structvar.NewStructVar(newStateValue, refs) + // Compute depends_on from cross-resource references before resolving them + // (resolution deletes entries from the refs map). + // Same logic as makePlan in bundle/direct/bundle_plan.go. + var dependsOn []deployplan.DependsOnEntry //nolint:prealloc + for _, refTemplate := range refs { + ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + if !ok { + continue + } + for _, targetPath := range ref.References() { + targetPathParsed, err := dyn.NewPathFromString(targetPath) + if err != nil { + continue + } + targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) + targetNode := targetNodeDP.String() + fullRef := "${" + targetPath + "}" + found := false + for _, dep := range dependsOn { + if dep.Node == targetNode && dep.Label == fullRef { + found = true + break + } + } + if !found { + dependsOn = append(dependsOn, deployplan.DependsOnEntry{ + Node: targetNode, + Label: fullRef, + }) + } + } + } + slices.SortFunc(dependsOn, func(a, b deployplan.DependsOnEntry) int { + if a.Node != b.Node { + return strings.Compare(a.Node, b.Node) + } + return strings.Compare(a.Label, b.Label) + }) + // Resolve each reference using TF state. // node format: "resources.." or "resources...permissions" parts := strings.SplitN(node, ".", 4) @@ -170,7 +212,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, dependsOn); err != nil { return fmt.Errorf("%s: SaveState: %w", node, err) } } From 07874867625326885776727ed7c42a261b9e3d79 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 15:01:45 +0200 Subject: [PATCH 09/38] migrate: fix TF field lookup for postgres spec wrapper and model_id alias Two categories of field lookup failures in LookupTFField: 1. Postgres resources (postgres_projects, postgres_branches, etc.) use a 'spec' wrapper via DABsToTerraformWrappers, but some fields like 'name' are at the TF state root, not under spec. When the spec-prefixed path fails, retry with the original unwrapped path. 2. Model permissions reference 'model_id' (the numeric model ID) which TF stores as 'registered_model_id'. Add a tfStateFieldAliases map for such state-only field name mismatches. Also update acceptance output files: - default-python: remove musterr/--noplancheck pattern; the plan check that made the first migrate call fail no longer exists, so just call migrate once. - runas: remove plan-check output lines that appeared before 'Success!'. Co-authored-by: Isaac --- .../bundle/migrate/default-python/output.txt | 10 +---- .../bundle/migrate/default-python/script | 3 +- acceptance/bundle/migrate/runas/output.txt | 13 ------ bundle/migrate/tf_state.go | 44 ++++++++++++++++++- 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 92efb3066d6..fb554bc258d 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -22,15 +22,7 @@ Deployment complete! >>> print_state.py ->>> musterr [CLI] bundle deployment migrate -Building python_artifact... -Building python_artifact... -update jobs.sample_job - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged -Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option - ->>> [CLI] bundle deployment migrate --noplancheck +>>> [CLI] bundle deployment migrate Building python_artifact... Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/resources.json diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index c9b585dbea7..e3a0b9b3e05 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -5,8 +5,7 @@ cd my_default_python trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_state.py > ../out.state_original.json -trace musterr $CLI bundle deployment migrate -trace $CLI bundle deployment migrate --noplancheck +trace $CLI bundle deployment migrate trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index f7034d74ec0..f8a9b475bdc 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -81,19 +81,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups -If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. - -Consider using a adding a top-level permissions section such as the following: - - permissions: - - user_name: [USERNAME] - level: CAN_MANAGE - -See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. - in databricks.yml:5:3 - -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/production/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 52799dd79b8..90a5d06bb39 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -11,6 +11,15 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) +// tfStateFieldAliases maps DABs group → DABs field name → TF state field name for +// cases where a DABs state-computed field has a different name in TF state. +// These fields are not captured by DABsToTerraformRenameMap because they are +// state-only (not part of the bundle config struct). +var tfStateFieldAliases = map[string]map[string]string{ + // models.model_id is the numeric model ID; TF stores it as registered_model_id. + "models": {"model_id": "registered_model_id"}, +} + // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage @@ -81,7 +90,40 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - return navigateTFState(attrs, tfFieldPath) + value, err := navigateTFState(attrs, tfFieldPath) + if err == nil { + return value, nil + } + + // Some DABs fields are top-level in TF state but DABsPathToTerraform added a + // wrapper prefix (e.g. "spec" for postgres resources). When the wrapped path + // fails, retry with the original unwrapped path. + if _, hasWrapper := terraform_dabs_map.DABsToTerraformWrappers[group]; hasWrapper { + if v, e := navigateTFState(attrs, fieldPath); e == nil { + return v, nil + } + } + + // Apply state-only field aliases for fields whose DABs name differs from TF state name. + if aliases, ok := tfStateFieldAliases[group]; ok { + // Replace the first path segment if it matches a known alias. + if head, ok := fieldPath.StringKey(); ok { + if tfName, ok := aliases[head]; ok { + aliasPath := structpath.NewStringKey(nil, tfName) + if rest := fieldPath.SkipPrefix(1); rest != nil { + _ = rest // navigate through the alias root + } + // Translate aliased path with full DABsToTerraform for the renamed field. + if aliasFieldPath, e := terraform_dabs_map.DABsPathToTerraform(group, aliasPath); e == nil { + if v, e := navigateTFState(attrs, aliasFieldPath); e == nil { + return v, nil + } + } + } + } + } + + return nil, err } // navigateTFState walks the TF state map using the given path. From b021302a5d981bcebec43f5144fc85f2fc1e6363 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 17:08:20 +0200 Subject: [PATCH 10/38] acceptance: remove plan check output from snapshot-comparison Co-authored-by: Isaac --- acceptance/bundle/deploy/snapshot-comparison/output.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index 1db9fa5024d..b5bb597e264 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -8,8 +8,6 @@ Deployment complete! === Run migrate on bundle 1 >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/bundle1/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. From a9ffc181578530799a3eb15bcc4b1849ec60fdfd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 21:30:21 +0200 Subject: [PATCH 11/38] migrate: add unit tests for BuildStateFromTF Tests cover: - Basic job stored with correct ID - Resource absent from TF state is skipped - Cross-resource string ref: depends_on computed, field resolved from TF state - Cross-resource numeric ref: int value stored as number not string - Dashboard etag stored from etags map Co-authored-by: Isaac --- bundle/migrate/build_state_test.go | 223 +++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 bundle/migrate/build_state_test.go diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go new file mode 100644 index 00000000000..2a673f7fe75 --- /dev/null +++ b/bundle/migrate/build_state_test.go @@ -0,0 +1,223 @@ +package migrate_test + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// rootFromYAML builds a config.Root from a YAML snippet. +// Template strings like "${resources.jobs.src.name}" are preserved in the +// internal dyn.Value so BuildStateFromTF can find them via ExtractReferences. +func rootFromYAML(t *testing.T, yaml string) config.Root { + t.Helper() + v, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yaml)) + require.NoError(t, err) + var root config.Root + require.NoError(t, convert.ToTyped(&root, v)) + require.NoError(t, root.Mutate(func(_ dyn.Value) (dyn.Value, error) { return v, nil })) + return root +} + +// runBuildStateFromTF is a helper that runs BuildStateFromTF, finalizes the +// state, then reloads it so callers can inspect ResourceEntry (State + DependsOn). +func runBuildStateFromTF( + t *testing.T, + yaml string, + tfAttrs migrate.TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) map[string]dstate.ResourceEntry { + t.Helper() + + root := rootFromYAML(t, yaml) + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + statePath := filepath.Join(t.TempDir(), "resources.json") + + var db dstate.DeploymentState + db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) + require.NoError(t, db.UpgradeToWrite()) + + err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, etags) + require.NoError(t, err) + + _, err = db.Finalize(t.Context()) + require.NoError(t, err) + + // Reload from disk to access the full ResourceEntry (State JSON + DependsOn). + raw, err := os.ReadFile(statePath) + require.NoError(t, err) + var data dstate.Database + require.NoError(t, json.Unmarshal(raw, &data)) + return data.State +} + +func TestBuildStateFromTF_BasicJob(t *testing.T) { + bundleYAML := ` +resources: + jobs: + my_job: + name: "hello" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "my_job": json.RawMessage(`{"id": "123", "name": "hello"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.my_job": {ID: "123"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + entry, ok := state["resources.jobs.my_job"] + require.True(t, ok) + assert.Equal(t, "123", entry.ID) + assert.Empty(t, entry.DependsOn) +} + +func TestBuildStateFromTF_ResourceNotInTFState_Skipped(t *testing.T) { + bundleYAML := ` +resources: + jobs: + new_job: + name: "new" + existing_job: + name: "existing" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "existing_job": json.RawMessage(`{"id": "456", "name": "existing"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.existing_job": {ID: "456"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + assert.Contains(t, state, "resources.jobs.existing_job") + assert.NotContains(t, state, "resources.jobs.new_job") +} + +func TestBuildStateFromTF_DependsOnComputedFromRefs(t *testing.T) { + bundleYAML := ` +resources: + pipelines: + src: + name: "source-pipeline" + jobs: + dst: + name: "${resources.pipelines.src.name}" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_pipeline": { + "src": json.RawMessage(`{"id": "p1", "name": "source-pipeline"}`), + }, + "databricks_job": { + "dst": json.RawMessage(`{"id": "j1", "name": "source-pipeline"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.pipelines.src": {ID: "p1"}, + "resources.jobs.dst": {ID: "j1"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + entry, ok := state["resources.jobs.dst"] + require.True(t, ok) + + // depends_on must point back to the referenced pipeline + require.Len(t, entry.DependsOn, 1) + assert.Equal(t, deployplan.DependsOnEntry{ + Node: "resources.pipelines.src", + Label: "${resources.pipelines.src.name}", + }, entry.DependsOn[0]) + + // resolved field value + var jobState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &jobState)) + assert.Equal(t, "source-pipeline", jobState["name"]) +} + +func TestBuildStateFromTF_NumericFieldReference(t *testing.T) { + // dst_job.max_concurrent_runs references src_job's int field. + // Verifies that the resolved value is stored as a number (not a string) + // and that depends_on is recorded. + bundleYAML := ` +resources: + jobs: + src_job: + name: "source" + max_concurrent_runs: 4 + dst_job: + name: "dest" + max_concurrent_runs: "${resources.jobs.src_job.max_concurrent_runs}" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "src_job": json.RawMessage(`{"id": "111", "name": "source", "max_concurrent_runs": 4}`), + "dst_job": json.RawMessage(`{"id": "222", "name": "dest", "max_concurrent_runs": 4}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.src_job": {ID: "111"}, + "resources.jobs.dst_job": {ID: "222"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + + entry, ok := state["resources.jobs.dst_job"] + require.True(t, ok) + + // depends_on must point to src_job + require.Len(t, entry.DependsOn, 1) + assert.Equal(t, "resources.jobs.src_job", entry.DependsOn[0].Node) + + // max_concurrent_runs must be stored as a number, not a string + var jobState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &jobState)) + assert.EqualValues(t, 4, jobState["max_concurrent_runs"]) +} + +func TestBuildStateFromTF_EtagStoredForDashboard(t *testing.T) { + bundleYAML := ` +resources: + dashboards: + my_dash: + display_name: "My Dashboard" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_dashboard": { + "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.dashboards.my_dash": {ID: "d1"}, + } + etags := map[string]string{ + "resources.dashboards.my_dash": "etag-abc123", + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, etags) + entry, ok := state["resources.dashboards.my_dash"] + require.True(t, ok) + + var dashState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &dashState)) + assert.Equal(t, "etag-abc123", dashState["etag"]) +} From b0df22c5e3d95a7536f6a64254afd5b3aa1e87ad Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 20:39:22 +0200 Subject: [PATCH 12/38] =?UTF-8?q?migrate:=20pass=20nil=20client=20to=20Ini?= =?UTF-8?q?tAll=20=E2=80=94=20PrepareState=20never=20uses=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Isaac --- bundle/statemgmt/upload_state_for_yaml_sync.go | 2 +- cmd/bundle/deployment/migrate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index ee750203d96..4663f69f583 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -186,7 +186,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + adapters, err := dresources.InitAll(nil) if err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 9b217062fc8..57ca89f6f6f 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -175,7 +175,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return root.ErrAlreadyPrinted } - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + adapters, err := dresources.InitAll(nil) if err != nil { return err } From b38eb49c652937919518f9ef356622c6434daf27 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:22:18 +0200 Subject: [PATCH 13/38] migrate: eliminate duplicate TF state reads; etag from attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both callers were reading and parsing the same .tfstate file twice — once for resource IDs, once for full attributes. Now a single ParseTFStateFull reads and unmarshals the file once, returning attrs, IDs, and lineage/serial together. Also drop the separate `etags map[string]string` parameter from BuildStateFromTF. The dashboard etag is a regular attribute in the TF state JSON ("etag" field), so LookupTFField finds it directly without any special-case plumbing. Implementation: - terraform: expose ParseResourcesStateFromBytes so callers can pass already-read bytes - migrate: add ParseTFStateFull / parseTFStateAttrsFromBytes - migrate: remove etags param; look up "etag" via LookupTFField - tests: etag test now puts the etag in the TF attributes JSON Co-authored-by: Isaac --- bundle/deploy/terraform/util.go | 24 ++++++++--- bundle/migrate/build_state.go | 13 +++--- bundle/migrate/build_state_test.go | 19 ++++----- bundle/migrate/tf_state.go | 40 ++++++++++++++++++- .../statemgmt/upload_state_for_yaml_sync.go | 33 +++------------ cmd/bundle/deployment/migrate.go | 16 +------- 6 files changed, 79 insertions(+), 66 deletions(-) diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 07384d6b262..3e078e8ae73 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -51,6 +51,15 @@ type stateInstanceAttributes struct { } // Returns a mapping resourceKey -> stateInstanceAttributes +// ParseResourcesStateFromBytes parses a terraform state file from already-read bytes. +func ParseResourcesStateFromBytes(ctx context.Context, raw []byte) (ExportedResourcesMap, error) { + var state resourcesState + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + return resourcesStateToMap(ctx, &state) +} + func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap, error) { rawState, err := os.ReadFile(path) if err != nil { @@ -59,12 +68,10 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap } return nil, err } - var state resourcesState - err = json.Unmarshal(rawState, &state) - if err != nil { - return nil, err - } + return ParseResourcesStateFromBytes(ctx, rawState) +} +func resourcesStateToMap(ctx context.Context, state *resourcesState) (ExportedResourcesMap, error) { if state.Version != SupportedStateVersion { return nil, fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version) } @@ -131,5 +138,10 @@ func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (ExportedResourc return nil, err } filename, _ := b.StateFilenameTerraform(ctx) - return parseResourcesState(ctx, filepath.Join(cacheDir, filename)) + return ParseResourcesStateFromPath(ctx, filepath.Join(cacheDir, filename)) +} + +// ParseResourcesStateFromPath parses a terraform state file at a known path. +func ParseResourcesStateFromPath(ctx context.Context, path string) (ExportedResourcesMap, error) { + return parseResourcesState(ctx, path) } diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 785c325bfad..1accf0e576e 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -32,7 +32,6 @@ func BuildStateFromTF( stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, tfIDs terraform.ExportedResourcesMap, - etags map[string]string, ) error { // Collect all resource nodes (same patterns as makePlan). var nodes []string @@ -205,10 +204,14 @@ func BuildStateFromTF( return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } - // Handle etag for dashboards. - if etag := etags[node]; etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) + // Handle etag for dashboards: look it up directly from TF state attributes. + // The "etag" field is a computed TF attribute not present in the bundle config, + // so it does not flow through PrepareState/ExtractReferences. + if etag, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil && etag != nil { + if etagStr, ok := etag.(string); ok && etagStr != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etagStr); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } } } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 2a673f7fe75..a7db542e09e 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -40,7 +40,6 @@ func runBuildStateFromTF( yaml string, tfAttrs migrate.TFStateAttrs, tfIDs terraform.ExportedResourcesMap, - etags map[string]string, ) map[string]dstate.ResourceEntry { t.Helper() @@ -54,7 +53,7 @@ func runBuildStateFromTF( db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) - err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, etags) + err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs) require.NoError(t, err) _, err = db.Finalize(t.Context()) @@ -84,7 +83,7 @@ resources: "resources.jobs.my_job": {ID: "123"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.my_job"] require.True(t, ok) assert.Equal(t, "123", entry.ID) @@ -109,7 +108,7 @@ resources: "resources.jobs.existing_job": {ID: "456"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) assert.Contains(t, state, "resources.jobs.existing_job") assert.NotContains(t, state, "resources.jobs.new_job") } @@ -137,7 +136,7 @@ resources: "resources.jobs.dst": {ID: "j1"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.dst"] require.True(t, ok) @@ -179,7 +178,7 @@ resources: "resources.jobs.dst_job": {ID: "222"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.dst_job"] require.True(t, ok) @@ -195,6 +194,7 @@ resources: } func TestBuildStateFromTF_EtagStoredForDashboard(t *testing.T) { + // Etag is a top-level attribute in the TF dashboard state JSON; no separate map needed. bundleYAML := ` resources: dashboards: @@ -203,17 +203,14 @@ resources: ` tfAttrs := migrate.TFStateAttrs{ "databricks_dashboard": { - "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard"}`), + "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard", "etag": "etag-abc123"}`), }, } tfIDs := terraform.ExportedResourcesMap{ "resources.dashboards.my_dash": {ID: "d1"}, } - etags := map[string]string{ - "resources.dashboards.my_dash": "etag-abc123", - } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, etags) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.dashboards.my_dash"] require.True(t, ok) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 90a5d06bb39..217e021f637 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -1,6 +1,7 @@ package migrate import ( + "context" "encoding/json" "fmt" "os" @@ -23,13 +24,50 @@ var tfStateFieldAliases = map[string]map[string]string{ // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage +// TFStateMeta holds the top-level metadata from a terraform state file. +type TFStateMeta struct { + Lineage string + Serial int +} + +// ParseTFStateFull reads the terraform state file once and returns the full +// attribute map, the resource ID map, and the state metadata (lineage/serial). +// Avoids reading and unmarshaling the file multiple times. +func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform.ExportedResourcesMap, TFStateMeta, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + + var meta struct { + Lineage string `json:"lineage"` + Serial int `json:"serial"` + } + if err := json.Unmarshal(raw, &meta); err != nil { + return nil, nil, TFStateMeta{}, err + } + + attrs, err := parseTFStateAttrsFromBytes(raw) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + ids, err := terraform.ParseResourcesStateFromBytes(ctx, raw) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil +} + // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) if err != nil { return nil, err } + return parseTFStateAttrsFromBytes(raw) +} +func parseTFStateAttrsFromBytes(raw []byte) (TFStateAttrs, error) { var state struct { Version int `json:"version"` Resources []struct { @@ -41,11 +79,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { } `json:"instances"` } `json:"resources"` } - if err := json.Unmarshal(raw, &state); err != nil { return nil, err } - result := make(TFStateAttrs) for _, r := range state.Resources { if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 4663f69f583..8072f76adae 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -112,50 +112,27 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { } func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, error) { - terraformResources, err := terraform.ParseResourcesState(ctx, b) + _, localTerraformPath := b.StateFilenameTerraform(ctx) + tfAttrs, terraformResources, tfMeta, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return false, fmt.Errorf("failed to parse terraform state: %w", err) } - // ParseResourcesState returns nil when the terraform state file doesn't exist + // ParseTFStateFull returns nil IDs when the terraform state file doesn't exist // (e.g. first deploy with no resources). if terraformResources == nil { return false, nil } - _, localTerraformPath := b.StateFilenameTerraform(ctx) - data, err := os.ReadFile(localTerraformPath) - if err != nil { - return false, fmt.Errorf("failed to read terraform state: %w", err) - } - - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) - if err != nil { - return false, fmt.Errorf("failed to read terraform state attributes: %w", err) - } - state := make(map[string]dstate.ResourceEntry) - etags := map[string]string{} - for key, resourceEntry := range terraformResources { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), } - if resourceEntry.ETag != "" { - etags[key] = resourceEntry.ETag - } - } - - var tfState struct { - Lineage string `json:"lineage"` - Serial int `json:"serial"` - } - if err := json.Unmarshal(data, &tfState); err != nil { - return false, err } - migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) + migratedDB := dstate.NewDatabase(tfMeta.Lineage, tfMeta.Serial+1) migratedDB.State = state var stateDB dstate.DeploymentState @@ -195,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 57ca89f6f6f..3c17e5c4be5 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" @@ -114,7 +113,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - terraformResources, err := terraform.ParseResourcesState(ctx, b) + tfAttrs, terraformResources, _, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return fmt.Errorf("failed to parse terraform state: %w", err) } @@ -134,23 +133,12 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) - if err != nil { - return fmt.Errorf("failed to read terraform state attributes: %w", err) - } - - etags := map[string]string{} - state := make(map[string]dstate.ResourceEntry) for key, resourceEntry := range terraformResources { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), } - if resourceEntry.ETag != "" { - // dashboard: - etags[key] = resourceEntry.ETag - } } migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) @@ -184,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources); err != nil { return err } From 14fb29d074260e785753ced1041825b984d3032f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:27:02 +0200 Subject: [PATCH 14/38] migrate: mark ParseTFStateAttrs as deadcode:allow Still useful as a standalone API; suppress the dead-code checker. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 217e021f637..cd7ba7d380d 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -58,6 +58,7 @@ func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil } +//deadcode:allow retained as standalone API for callers that only need attributes. // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) From ad157c0fe15649b23205734e334a56aa626588f6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:53:12 +0200 Subject: [PATCH 15/38] migrate: fix gofmt in tf_state.go Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index cd7ba7d380d..a512446fa47 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -58,8 +58,9 @@ func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil } -//deadcode:allow retained as standalone API for callers that only need attributes. // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. +// +//deadcode:allow retained as standalone API for callers that only need attributes. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) if err != nil { From 3bfc8f6ac03db6d0661729156e0834d2822f4dd3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 22:22:36 +0200 Subject: [PATCH 16/38] migrate: handle missing TF state file in ParseTFStateFull Return (nil, nil, ...) when the state file doesn't exist, matching the old parseResourcesState behaviour. Empty bundles with no resources don't create a terraform.tfstate file. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index a512446fa47..72b64e77f19 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -3,6 +3,7 @@ package migrate import ( "context" "encoding/json" + "errors" "fmt" "os" @@ -36,6 +37,9 @@ type TFStateMeta struct { func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform.ExportedResourcesMap, TFStateMeta, error) { raw, err := os.ReadFile(path) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil, TFStateMeta{}, nil + } return nil, nil, TFStateMeta{}, err } From cd626815d02b1a0115adb36436401c27457653ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 09:20:58 +0200 Subject: [PATCH 17/38] acceptance: update added/removed migrate tests for no-plan-check Remove the musterr + --noplancheck pattern; the plan check was removed so migration succeeds on the first call. Regenerate output files. Co-authored-by: Isaac --- acceptance/bundle/migrate/added/output.txt | 9 +-------- acceptance/bundle/migrate/added/script | 7 ++----- acceptance/bundle/migrate/removed/output.txt | 9 +-------- acceptance/bundle/migrate/removed/script | 7 ++----- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/migrate/added/output.txt b/acceptance/bundle/migrate/added/output.txt index 70e6f0bc192..9a77ac41c1c 100644 --- a/acceptance/bundle/migrate/added/output.txt +++ b/acceptance/bundle/migrate/added/output.txt @@ -5,14 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> musterr [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -create jobs.job_b - -Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged -Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option - ->>> [CLI] bundle deployment migrate --noplancheck +>>> [CLI] bundle deployment migrate Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/added/script b/acceptance/bundle/migrate/added/script index 0adc713936a..ecf7fec95b6 100644 --- a/acceptance/bundle/migrate/added/script +++ b/acceptance/bundle/migrate/added/script @@ -6,11 +6,8 @@ trace $CLI bundle deploy # Uncomment job_b (add it to config without deploying) update_file.py databricks.yml "#job_b" "job_b" -# Should fail at plan check: job_b is "1 to add" -trace musterr $CLI bundle deployment migrate - -# Should succeed: job_b skipped, will be created on next deploy -trace $CLI bundle deployment migrate --noplancheck +# job_b skipped, will be created on next deploy +trace $CLI bundle deployment migrate # After migration: plan shows job_b as "to add" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to add" diff --git a/acceptance/bundle/migrate/removed/output.txt b/acceptance/bundle/migrate/removed/output.txt index 8e4bcdf555c..9896acb4247 100644 --- a/acceptance/bundle/migrate/removed/output.txt +++ b/acceptance/bundle/migrate/removed/output.txt @@ -5,14 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> musterr [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -delete jobs.job_b - -Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged -Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option - ->>> [CLI] bundle deployment migrate --noplancheck +>>> [CLI] bundle deployment migrate Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/removed/script b/acceptance/bundle/migrate/removed/script index 99c6f5716ec..c6cea1e2c04 100644 --- a/acceptance/bundle/migrate/removed/script +++ b/acceptance/bundle/migrate/removed/script @@ -6,11 +6,8 @@ trace $CLI bundle deploy # Remove job_b from config without deploying the deletion grep -v job_b databricks.yml > databricks_tmp.yml && mv databricks_tmp.yml databricks.yml -# Should fail at plan check: job_b is "1 to delete" -trace musterr $CLI bundle deployment migrate - -# Should succeed: job_b's ID preserved in direct state for deletion on next deploy -trace $CLI bundle deployment migrate --noplancheck +# job_b's ID preserved in direct state for deletion on next deploy +trace $CLI bundle deployment migrate # After migration: plan shows job_b as "to delete" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to delete" From ff3a7e5548da3e8612316cf54b3d174e9cce0174 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 16:54:32 +0200 Subject: [PATCH 18/38] migrate: collapse ParseTFStateFull return values into TFState struct Co-authored-by: Isaac --- .claude/scheduled_tasks.lock | 1 + bundle/migrate/tf_state.go | 28 +++++++++++-------- .../statemgmt/upload_state_for_yaml_sync.go | 10 +++---- cmd/bundle/deployment/migrate.go | 8 +++--- 4 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 .claude/scheduled_tasks.lock diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 00000000000..c4cf2f00e31 --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"025a4936-8674-457f-90c1-142ba70800fb","pid":67869,"procStart":"Mon Jun 1 09:03:38 2026","acquiredAt":1780566124629} \ No newline at end of file diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 72b64e77f19..75238d5e3f0 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -25,22 +25,26 @@ var tfStateFieldAliases = map[string]map[string]string{ // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage -// TFStateMeta holds the top-level metadata from a terraform state file. -type TFStateMeta struct { +// TFState holds everything parsed from a single terraform state file read. +type TFState struct { + // Attrs maps (tfResourceType → resourceName → raw JSON attributes). + Attrs TFStateAttrs + // IDs maps bundle resource key → {ID, ETag} (same as terraform.ParseResourcesState). + IDs terraform.ExportedResourcesMap + // Lineage and Serial are the top-level state metadata used to seed the direct state. Lineage string Serial int } -// ParseTFStateFull reads the terraform state file once and returns the full -// attribute map, the resource ID map, and the state metadata (lineage/serial). -// Avoids reading and unmarshaling the file multiple times. -func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform.ExportedResourcesMap, TFStateMeta, error) { +// ParseTFStateFull reads the terraform state file once and returns all parsed data. +// Returns nil without error when the file does not exist (first deploy with no resources). +func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { raw, err := os.ReadFile(path) if err != nil { if errors.Is(err, os.ErrNotExist) { - return nil, nil, TFStateMeta{}, nil + return nil, nil } - return nil, nil, TFStateMeta{}, err + return nil, err } var meta struct { @@ -48,18 +52,18 @@ func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform Serial int `json:"serial"` } if err := json.Unmarshal(raw, &meta); err != nil { - return nil, nil, TFStateMeta{}, err + return nil, err } attrs, err := parseTFStateAttrsFromBytes(raw) if err != nil { - return nil, nil, TFStateMeta{}, err + return nil, err } ids, err := terraform.ParseResourcesStateFromBytes(ctx, raw) if err != nil { - return nil, nil, TFStateMeta{}, err + return nil, err } - return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil + return &TFState{Attrs: attrs, IDs: ids, Lineage: meta.Lineage, Serial: meta.Serial}, nil } // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 8072f76adae..f7179c51e2b 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -113,26 +113,26 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, error) { _, localTerraformPath := b.StateFilenameTerraform(ctx) - tfAttrs, terraformResources, tfMeta, err := migrate.ParseTFStateFull(ctx, localTerraformPath) + tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return false, fmt.Errorf("failed to parse terraform state: %w", err) } // ParseTFStateFull returns nil IDs when the terraform state file doesn't exist // (e.g. first deploy with no resources). - if terraformResources == nil { + if tfState == nil { return false, nil } state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range terraformResources { + for key, resourceEntry := range tfState.IDs { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), } } - migratedDB := dstate.NewDatabase(tfMeta.Lineage, tfMeta.Serial+1) + migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) migratedDB.State = state var stateDB dstate.DeploymentState @@ -172,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources); err != nil { + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 3c17e5c4be5..2f00cadbb8a 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -113,12 +113,12 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - tfAttrs, terraformResources, _, err := migrate.ParseTFStateFull(ctx, localTerraformPath) + tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return fmt.Errorf("failed to parse terraform state: %w", err) } - for key, resourceEntry := range terraformResources { + for key, resourceEntry := range tfState.IDs { if resourceEntry.ID == "" { return fmt.Errorf("failed to intepret terraform state for %s: missing ID", key) } @@ -134,7 +134,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric } state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range terraformResources { + for key, resourceEntry := range tfState.IDs { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), @@ -172,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { return err } From 1802f3d57304f585f2223c47d1ae22f62e074761 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 16:55:43 +0200 Subject: [PATCH 19/38] migrate: convert build_state_test.go to table-driven test YAML + TF JSON as input, expected state fields as output. Co-authored-by: Isaac --- bundle/migrate/build_state_test.go | 230 ++++++++++++++--------------- 1 file changed, 113 insertions(+), 117 deletions(-) diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index a7db542e09e..85a3b18ca93 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -33,8 +33,6 @@ func rootFromYAML(t *testing.T, yaml string) config.Root { return root } -// runBuildStateFromTF is a helper that runs BuildStateFromTF, finalizes the -// state, then reloads it so callers can inspect ResourceEntry (State + DependsOn). func runBuildStateFromTF( t *testing.T, yaml string, @@ -59,7 +57,6 @@ func runBuildStateFromTF( _, err = db.Finalize(t.Context()) require.NoError(t, err) - // Reload from disk to access the full ResourceEntry (State JSON + DependsOn). raw, err := os.ReadFile(statePath) require.NoError(t, err) var data dstate.Database @@ -67,54 +64,54 @@ func runBuildStateFromTF( return data.State } -func TestBuildStateFromTF_BasicJob(t *testing.T) { - bundleYAML := ` +func TestBuildStateFromTF(t *testing.T) { + tests := []struct { + name string + yaml string + tfAttrs migrate.TFStateAttrs + tfIDs terraform.ExportedResourcesMap + wantKey string // primary key to assert on + absentKey string // key that must NOT be in state + wantID string // expected entry.ID + wantState map[string]any // expected fields in the state JSON + wantDeps []deployplan.DependsOnEntry + }{ + { + name: "basic job stored with ID", + yaml: ` resources: jobs: my_job: name: "hello" -` - tfAttrs := migrate.TFStateAttrs{ - "databricks_job": { - "my_job": json.RawMessage(`{"id": "123", "name": "hello"}`), +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_job": {"my_job": json.RawMessage(`{"id": "123", "name": "hello"}`)}, + }, + tfIDs: terraform.ExportedResourcesMap{"resources.jobs.my_job": {ID: "123"}}, + wantKey: "resources.jobs.my_job", + wantID: "123", }, - } - tfIDs := terraform.ExportedResourcesMap{ - "resources.jobs.my_job": {ID: "123"}, - } - - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) - entry, ok := state["resources.jobs.my_job"] - require.True(t, ok) - assert.Equal(t, "123", entry.ID) - assert.Empty(t, entry.DependsOn) -} - -func TestBuildStateFromTF_ResourceNotInTFState_Skipped(t *testing.T) { - bundleYAML := ` + { + name: "resource not in TF state is skipped", + yaml: ` resources: jobs: new_job: name: "new" existing_job: name: "existing" -` - tfAttrs := migrate.TFStateAttrs{ - "databricks_job": { - "existing_job": json.RawMessage(`{"id": "456", "name": "existing"}`), +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_job": {"existing_job": json.RawMessage(`{"id": "456", "name": "existing"}`)}, + }, + tfIDs: terraform.ExportedResourcesMap{"resources.jobs.existing_job": {ID: "456"}}, + wantKey: "resources.jobs.existing_job", + absentKey: "resources.jobs.new_job", + wantID: "456", }, - } - tfIDs := terraform.ExportedResourcesMap{ - "resources.jobs.existing_job": {ID: "456"}, - } - - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) - assert.Contains(t, state, "resources.jobs.existing_job") - assert.NotContains(t, state, "resources.jobs.new_job") -} - -func TestBuildStateFromTF_DependsOnComputedFromRefs(t *testing.T) { - bundleYAML := ` + { + name: "cross-resource ref: depends_on computed, field resolved", + yaml: ` resources: pipelines: src: @@ -122,42 +119,25 @@ resources: jobs: dst: name: "${resources.pipelines.src.name}" -` - tfAttrs := migrate.TFStateAttrs{ - "databricks_pipeline": { - "src": json.RawMessage(`{"id": "p1", "name": "source-pipeline"}`), - }, - "databricks_job": { - "dst": json.RawMessage(`{"id": "j1", "name": "source-pipeline"}`), +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_pipeline": {"src": json.RawMessage(`{"id": "p1", "name": "source-pipeline"}`)}, + "databricks_job": {"dst": json.RawMessage(`{"id": "j1", "name": "source-pipeline"}`)}, + }, + tfIDs: terraform.ExportedResourcesMap{ + "resources.pipelines.src": {ID: "p1"}, + "resources.jobs.dst": {ID: "j1"}, + }, + wantKey: "resources.jobs.dst", + wantID: "j1", + wantState: map[string]any{"name": "source-pipeline"}, + wantDeps: []deployplan.DependsOnEntry{ + {Node: "resources.pipelines.src", Label: "${resources.pipelines.src.name}"}, + }, }, - } - tfIDs := terraform.ExportedResourcesMap{ - "resources.pipelines.src": {ID: "p1"}, - "resources.jobs.dst": {ID: "j1"}, - } - - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) - entry, ok := state["resources.jobs.dst"] - require.True(t, ok) - - // depends_on must point back to the referenced pipeline - require.Len(t, entry.DependsOn, 1) - assert.Equal(t, deployplan.DependsOnEntry{ - Node: "resources.pipelines.src", - Label: "${resources.pipelines.src.name}", - }, entry.DependsOn[0]) - - // resolved field value - var jobState map[string]any - require.NoError(t, json.Unmarshal(entry.State, &jobState)) - assert.Equal(t, "source-pipeline", jobState["name"]) -} - -func TestBuildStateFromTF_NumericFieldReference(t *testing.T) { - // dst_job.max_concurrent_runs references src_job's int field. - // Verifies that the resolved value is stored as a number (not a string) - // and that depends_on is recorded. - bundleYAML := ` + { + name: "numeric field reference stored as number not string", + yaml: ` resources: jobs: src_job: @@ -166,55 +146,71 @@ resources: dst_job: name: "dest" max_concurrent_runs: "${resources.jobs.src_job.max_concurrent_runs}" -` - tfAttrs := migrate.TFStateAttrs{ - "databricks_job": { - "src_job": json.RawMessage(`{"id": "111", "name": "source", "max_concurrent_runs": 4}`), - "dst_job": json.RawMessage(`{"id": "222", "name": "dest", "max_concurrent_runs": 4}`), +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_job": { + "src_job": json.RawMessage(`{"id": "111", "name": "source", "max_concurrent_runs": 4}`), + "dst_job": json.RawMessage(`{"id": "222", "name": "dest", "max_concurrent_runs": 4}`), + }, + }, + tfIDs: terraform.ExportedResourcesMap{ + "resources.jobs.src_job": {ID: "111"}, + "resources.jobs.dst_job": {ID: "222"}, + }, + wantKey: "resources.jobs.dst_job", + wantID: "222", + wantState: map[string]any{"max_concurrent_runs": float64(4)}, // JSON numbers decode as float64 + wantDeps: []deployplan.DependsOnEntry{{Node: "resources.jobs.src_job"}}, }, - } - tfIDs := terraform.ExportedResourcesMap{ - "resources.jobs.src_job": {ID: "111"}, - "resources.jobs.dst_job": {ID: "222"}, - } - - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) - - entry, ok := state["resources.jobs.dst_job"] - require.True(t, ok) - - // depends_on must point to src_job - require.Len(t, entry.DependsOn, 1) - assert.Equal(t, "resources.jobs.src_job", entry.DependsOn[0].Node) - - // max_concurrent_runs must be stored as a number, not a string - var jobState map[string]any - require.NoError(t, json.Unmarshal(entry.State, &jobState)) - assert.EqualValues(t, 4, jobState["max_concurrent_runs"]) -} - -func TestBuildStateFromTF_EtagStoredForDashboard(t *testing.T) { - // Etag is a top-level attribute in the TF dashboard state JSON; no separate map needed. - bundleYAML := ` + { + name: "dashboard etag stored from TF attributes", + yaml: ` resources: dashboards: my_dash: display_name: "My Dashboard" -` - tfAttrs := migrate.TFStateAttrs{ - "databricks_dashboard": { - "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard", "etag": "etag-abc123"}`), +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_dashboard": { + "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard", "etag": "etag-abc123"}`), + }, + }, + tfIDs: terraform.ExportedResourcesMap{"resources.dashboards.my_dash": {ID: "d1"}}, + wantKey: "resources.dashboards.my_dash", + wantID: "d1", + wantState: map[string]any{"etag": "etag-abc123"}, }, } - tfIDs := terraform.ExportedResourcesMap{ - "resources.dashboards.my_dash": {ID: "d1"}, - } - - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) - entry, ok := state["resources.dashboards.my_dash"] - require.True(t, ok) - var dashState map[string]any - require.NoError(t, json.Unmarshal(entry.State, &dashState)) - assert.Equal(t, "etag-abc123", dashState["etag"]) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + state := runBuildStateFromTF(t, tc.yaml, tc.tfAttrs, tc.tfIDs) + + if tc.absentKey != "" { + assert.NotContains(t, state, tc.absentKey) + } + + entry, ok := state[tc.wantKey] + require.True(t, ok, "key %q not in state", tc.wantKey) + assert.Equal(t, tc.wantID, entry.ID) + + if len(tc.wantState) > 0 { + var got map[string]any + require.NoError(t, json.Unmarshal(entry.State, &got)) + for k, v := range tc.wantState { + assert.Equal(t, v, got[k], "state[%q]", k) + } + } + + if tc.wantDeps != nil { + require.Len(t, entry.DependsOn, len(tc.wantDeps)) + for i, want := range tc.wantDeps { + assert.Equal(t, want.Node, entry.DependsOn[i].Node) + if want.Label != "" { + assert.Equal(t, want.Label, entry.DependsOn[i].Label) + } + } + } + }) + } } From 10fa11b416ef6fd30deaf2dc57cb15931b720487 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 17:09:24 +0200 Subject: [PATCH 20/38] chore: untrack .claude/scheduled_tasks.lock Co-authored-by: Isaac --- .claude/scheduled_tasks.lock | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .claude/scheduled_tasks.lock diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock deleted file mode 100644 index c4cf2f00e31..00000000000 --- a/.claude/scheduled_tasks.lock +++ /dev/null @@ -1 +0,0 @@ -{"sessionId":"025a4936-8674-457f-90c1-142ba70800fb","pid":67869,"procStart":"Mon Jun 1 09:03:38 2026","acquiredAt":1780566124629} \ No newline at end of file From c06fa65e32ab658b4fa22cf7e6ff4f6f9366209c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 17:21:57 +0200 Subject: [PATCH 21/38] migrate: add ETagFor method on TFStateAttrs; simplify etag handling Replace the verbose LookupTFField call for etag with a direct JSON read via TFStateAttrs.ETagFor(group, name). Co-authored-by: Isaac --- bundle/migrate/build_state.go | 10 ++++------ bundle/migrate/tf_state.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 1accf0e576e..34fa21a935b 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -204,14 +204,12 @@ func BuildStateFromTF( return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } - // Handle etag for dashboards: look it up directly from TF state attributes. + // Handle etag for dashboards: read it directly from TF state attributes. // The "etag" field is a computed TF attribute not present in the bundle config, // so it does not flow through PrepareState/ExtractReferences. - if etag, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil && etag != nil { - if etagStr, ok := etag.(string); ok && etagStr != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etagStr); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) - } + if etag := tfAttrs.ETagFor(group, srcName); etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) } } diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 75238d5e3f0..0ab5482871f 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -25,6 +25,26 @@ var tfStateFieldAliases = map[string]map[string]string{ // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage +// ETagFor returns the "etag" attribute for a bundle resource, or "" if absent. +// Reads directly from the raw JSON without full path translation. +func (a TFStateAttrs) ETagFor(group, name string) string { + tfType, ok := terraform.GroupToTerraformName[group] + if !ok { + return "" + } + raw, ok := a[tfType][name] + if !ok { + return "" + } + var v struct { + Etag string `json:"etag,omitempty"` + } + if err := json.Unmarshal(raw, &v); err != nil { + return "" + } + return v.Etag +} + // TFState holds everything parsed from a single terraform state file read. type TFState struct { // Attrs maps (tfResourceType → resourceName → raw JSON attributes). From 70123e5dcc9b2acfa40938bc087b5b39067469a4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 17:28:43 +0200 Subject: [PATCH 22/38] migrate: unify raw TF state parsing into rawTFState struct - Introduce rawTFState that captures lineage/serial alongside resources in one unmarshal, eliminating the separate meta struct parse. - parseTFStateAttrsFromBytes and parseTFStateAttrsFromRaw now share the same struct instead of defining an anonymous one. - Move Lineage/Serial after Attrs/IDs in TFState struct. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 73 +++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 0ab5482871f..d76ef2dee04 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -51,6 +51,7 @@ type TFState struct { Attrs TFStateAttrs // IDs maps bundle resource key → {ID, ETag} (same as terraform.ParseResourcesState). IDs terraform.ExportedResourcesMap + // Lineage and Serial are the top-level state metadata used to seed the direct state. Lineage string Serial int @@ -67,23 +68,49 @@ func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { return nil, err } - var meta struct { - Lineage string `json:"lineage"` - Serial int `json:"serial"` - } - if err := json.Unmarshal(raw, &meta); err != nil { + // Parse once: lineage/serial live at the top level alongside the resources array, + // so a single unmarshal captures everything needed for both attrs and IDs. + var parsed rawTFState + if err := json.Unmarshal(raw, &parsed); err != nil { return nil, err } - attrs, err := parseTFStateAttrsFromBytes(raw) - if err != nil { - return nil, err - } + attrs := parseTFStateAttrsFromRaw(&parsed) + ids, err := terraform.ParseResourcesStateFromBytes(ctx, raw) if err != nil { return nil, err } - return &TFState{Attrs: attrs, IDs: ids, Lineage: meta.Lineage, Serial: meta.Serial}, nil + return &TFState{Attrs: attrs, IDs: ids, Lineage: parsed.Lineage, Serial: parsed.Serial}, nil +} + +// rawTFState is the on-disk terraform state format; it captures everything we need in one parse. +type rawTFState struct { + Version int `json:"version"` + Lineage string `json:"lineage"` + Serial int `json:"serial"` + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []struct { + Attributes json.RawMessage `json:"attributes"` + } `json:"instances"` + } `json:"resources"` +} + +func parseTFStateAttrsFromRaw(s *rawTFState) TFStateAttrs { + result := make(TFStateAttrs) + for _, r := range s.Resources { + if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { + continue + } + if result[r.Type] == nil { + result[r.Type] = make(map[string]json.RawMessage) + } + result[r.Type][r.Name] = r.Instances[0].Attributes + } + return result } // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. @@ -98,31 +125,11 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { } func parseTFStateAttrsFromBytes(raw []byte) (TFStateAttrs, error) { - var state struct { - Version int `json:"version"` - Resources []struct { - Type string `json:"type"` - Name string `json:"name"` - Mode tfjson.ResourceMode `json:"mode"` - Instances []struct { - Attributes json.RawMessage `json:"attributes"` - } `json:"instances"` - } `json:"resources"` - } - if err := json.Unmarshal(raw, &state); err != nil { + var s rawTFState + if err := json.Unmarshal(raw, &s); err != nil { return nil, err } - result := make(TFStateAttrs) - for _, r := range state.Resources { - if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { - continue - } - if result[r.Type] == nil { - result[r.Type] = make(map[string]json.RawMessage) - } - result[r.Type][r.Name] = r.Instances[0].Attributes - } - return result, nil + return parseTFStateAttrsFromRaw(&s), nil } // LookupTFField looks up a field from TF state attributes for a bundle resource. From 98a8b97501f5fb6734dcce888a40081e7098928a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 17:53:32 +0200 Subject: [PATCH 23/38] migrate: inline parseTFStateAttrsFromBytes to fix dead-code check Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index d76ef2dee04..46a5d002924 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -121,10 +121,6 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { if err != nil { return nil, err } - return parseTFStateAttrsFromBytes(raw) -} - -func parseTFStateAttrsFromBytes(raw []byte) (TFStateAttrs, error) { var s rawTFState if err := json.Unmarshal(raw, &s); err != nil { return nil, err From 18fe414af448d3b3bc3263c58dab4395135c6906 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 14 Jun 2026 08:35:28 -0700 Subject: [PATCH 24/38] acceptance: remove remaining plan check output from migrate/basic Co-authored-by: Isaac --- acceptance/bundle/migrate/basic/output.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt index 85cdfebcc48..cc70cdaee02 100644 --- a/acceptance/bundle/migrate/basic/output.txt +++ b/acceptance/bundle/migrate/basic/output.txt @@ -11,8 +11,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged Success! Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. From 70be57331737fbd60b59253d7ce46519e84b5306 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 09:13:25 -0700 Subject: [PATCH 25/38] migrate: remove wrapper fallback in LookupTFField DABsPathToTerraform now correctly handles root-level TF fields for wrapped groups (e.g. postgres) via DABsToTerraformWrapperFields, so the retry-with-unwrapped-path fallback is no longer needed. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 46a5d002924..fe37f1ae869 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -164,15 +164,6 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return value, nil } - // Some DABs fields are top-level in TF state but DABsPathToTerraform added a - // wrapper prefix (e.g. "spec" for postgres resources). When the wrapped path - // fails, retry with the original unwrapped path. - if _, hasWrapper := terraform_dabs_map.DABsToTerraformWrappers[group]; hasWrapper { - if v, e := navigateTFState(attrs, fieldPath); e == nil { - return v, nil - } - } - // Apply state-only field aliases for fields whose DABs name differs from TF state name. if aliases, ok := tfStateFieldAliases[group]; ok { // Replace the first path segment if it matches a known alias. From 58f23c1ade3cb9aecc81e1bd48effd3956c84b1b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 09:46:28 -0700 Subject: [PATCH 26/38] migrate: remove dead code from LookupTFField and ParseTFStateAttrs - Remove the unused `rest` variable in the alias resolution block (SkipPrefix(1) on a single-segment path always returns nil) - Remove ParseTFStateAttrs which has no callers; the deadcode:allow annotation was papering over the linter Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index fe37f1ae869..61a51e29bca 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -113,21 +113,6 @@ func parseTFStateAttrsFromRaw(s *rawTFState) TFStateAttrs { return result } -// ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. -// -//deadcode:allow retained as standalone API for callers that only need attributes. -func ParseTFStateAttrs(path string) (TFStateAttrs, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var s rawTFState - if err := json.Unmarshal(raw, &s); err != nil { - return nil, err - } - return parseTFStateAttrsFromRaw(&s), nil -} - // LookupTFField looks up a field from TF state attributes for a bundle resource. // group is the DABs group (e.g. "pipelines"), name is the resource name. // fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). @@ -164,16 +149,11 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return value, nil } - // Apply state-only field aliases for fields whose DABs name differs from TF state name. + // Apply state-only field aliases for single-segment fields whose DABs name differs from TF state name. if aliases, ok := tfStateFieldAliases[group]; ok { - // Replace the first path segment if it matches a known alias. if head, ok := fieldPath.StringKey(); ok { if tfName, ok := aliases[head]; ok { aliasPath := structpath.NewStringKey(nil, tfName) - if rest := fieldPath.SkipPrefix(1); rest != nil { - _ = rest // navigate through the alias root - } - // Translate aliased path with full DABsToTerraform for the renamed field. if aliasFieldPath, e := terraform_dabs_map.DABsPathToTerraform(group, aliasPath); e == nil { if v, e := navigateTFState(attrs, aliasFieldPath); e == nil { return v, nil From d1b93a1b0dbafce55a09203f2bb3839d6b693a88 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 10:30:20 -0700 Subject: [PATCH 27/38] acceptance: update yaml-sync-empty-grants output BuildStateFromTF silently skips resources not in TF state (like schema grants in terraform mode), so the old "state entry not found" warning no longer fires. Remove the two spurious warning lines. Co-authored-by: Isaac --- acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt b/acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt index fa2643665d8..e037b25fd29 100644 --- a/acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt +++ b/acceptance/bundle/deploy/yaml-sync-empty-grants/output.txt @@ -3,6 +3,4 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/yaml-sync-empty-grants/default/files... Deploying resources... Updating deployment state... -Warn: Failed to create config snapshot: state conversion failed -Warn: Config snapshot: state entry not found for "resources.schemas.schema1.grants" Deployment complete! From e007339894250ecaf6c546815900848b54b69d9f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 10:41:05 -0700 Subject: [PATCH 28/38] migrate: simplify TFState.IDs to map[string]string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ETag field is no longer needed in the migration path — etags are read directly from TFStateAttrs.ETagFor. Replace ExportedResourcesMap with map[string]string throughout migrate, build_state, and callers. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 7 +++--- bundle/migrate/build_state_test.go | 23 +++++++++---------- bundle/migrate/tf_state.go | 10 +++++--- .../statemgmt/upload_state_for_yaml_sync.go | 4 ++-- cmd/bundle/deployment/migrate.go | 8 +++---- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 34fa21a935b..09275b328c7 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" @@ -31,7 +30,7 @@ func BuildStateFromTF( adapters map[string]*dresources.Adapter, stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, - tfIDs terraform.ExportedResourcesMap, + tfIDs map[string]string, ) error { // Collect all resource nodes (same patterns as makePlan). var nodes []string @@ -55,7 +54,7 @@ func BuildStateFromTF( } for _, node := range nodes { - idEntry, ok := tfIDs[node] + id, ok := tfIDs[node] if !ok { // Resource is in config but not in TF state (new resource); skip. continue @@ -213,7 +212,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, dependsOn); err != nil { + if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { return fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 85a3b18ca93..16cd8b45207 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" @@ -37,7 +36,7 @@ func runBuildStateFromTF( t *testing.T, yaml string, tfAttrs migrate.TFStateAttrs, - tfIDs terraform.ExportedResourcesMap, + tfIDs map[string]string, ) map[string]dstate.ResourceEntry { t.Helper() @@ -69,7 +68,7 @@ func TestBuildStateFromTF(t *testing.T) { name string yaml string tfAttrs migrate.TFStateAttrs - tfIDs terraform.ExportedResourcesMap + tfIDs map[string]string wantKey string // primary key to assert on absentKey string // key that must NOT be in state wantID string // expected entry.ID @@ -87,7 +86,7 @@ resources: tfAttrs: migrate.TFStateAttrs{ "databricks_job": {"my_job": json.RawMessage(`{"id": "123", "name": "hello"}`)}, }, - tfIDs: terraform.ExportedResourcesMap{"resources.jobs.my_job": {ID: "123"}}, + tfIDs: map[string]string{"resources.jobs.my_job": "123"}, wantKey: "resources.jobs.my_job", wantID: "123", }, @@ -104,7 +103,7 @@ resources: tfAttrs: migrate.TFStateAttrs{ "databricks_job": {"existing_job": json.RawMessage(`{"id": "456", "name": "existing"}`)}, }, - tfIDs: terraform.ExportedResourcesMap{"resources.jobs.existing_job": {ID: "456"}}, + tfIDs: map[string]string{"resources.jobs.existing_job": "456"}, wantKey: "resources.jobs.existing_job", absentKey: "resources.jobs.new_job", wantID: "456", @@ -124,9 +123,9 @@ resources: "databricks_pipeline": {"src": json.RawMessage(`{"id": "p1", "name": "source-pipeline"}`)}, "databricks_job": {"dst": json.RawMessage(`{"id": "j1", "name": "source-pipeline"}`)}, }, - tfIDs: terraform.ExportedResourcesMap{ - "resources.pipelines.src": {ID: "p1"}, - "resources.jobs.dst": {ID: "j1"}, + tfIDs: map[string]string{ + "resources.pipelines.src": "p1", + "resources.jobs.dst": "j1", }, wantKey: "resources.jobs.dst", wantID: "j1", @@ -153,9 +152,9 @@ resources: "dst_job": json.RawMessage(`{"id": "222", "name": "dest", "max_concurrent_runs": 4}`), }, }, - tfIDs: terraform.ExportedResourcesMap{ - "resources.jobs.src_job": {ID: "111"}, - "resources.jobs.dst_job": {ID: "222"}, + tfIDs: map[string]string{ + "resources.jobs.src_job": "111", + "resources.jobs.dst_job": "222", }, wantKey: "resources.jobs.dst_job", wantID: "222", @@ -175,7 +174,7 @@ resources: "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard", "etag": "etag-abc123"}`), }, }, - tfIDs: terraform.ExportedResourcesMap{"resources.dashboards.my_dash": {ID: "d1"}}, + tfIDs: map[string]string{"resources.dashboards.my_dash": "d1"}, wantKey: "resources.dashboards.my_dash", wantID: "d1", wantState: map[string]any{"etag": "etag-abc123"}, diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 61a51e29bca..6af73b96699 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -49,8 +49,8 @@ func (a TFStateAttrs) ETagFor(group, name string) string { type TFState struct { // Attrs maps (tfResourceType → resourceName → raw JSON attributes). Attrs TFStateAttrs - // IDs maps bundle resource key → {ID, ETag} (same as terraform.ParseResourcesState). - IDs terraform.ExportedResourcesMap + // IDs maps bundle resource key → resource ID. + IDs map[string]string // Lineage and Serial are the top-level state metadata used to seed the direct state. Lineage string @@ -77,10 +77,14 @@ func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { attrs := parseTFStateAttrsFromRaw(&parsed) - ids, err := terraform.ParseResourcesStateFromBytes(ctx, raw) + exportedIDs, err := terraform.ParseResourcesStateFromBytes(ctx, raw) if err != nil { return nil, err } + ids := make(map[string]string, len(exportedIDs)) + for k, v := range exportedIDs { + ids[k] = v.ID + } return &TFState{Attrs: attrs, IDs: ids, Lineage: parsed.Lineage, Serial: parsed.Serial}, nil } diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index f7179c51e2b..163c9fb4fdb 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -125,9 +125,9 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun } state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range tfState.IDs { + for key, id := range tfState.IDs { state[key] = dstate.ResourceEntry{ - ID: resourceEntry.ID, + ID: id, State: json.RawMessage("{}"), } } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 2f00cadbb8a..39d9a0454d5 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -118,8 +118,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("failed to parse terraform state: %w", err) } - for key, resourceEntry := range tfState.IDs { - if resourceEntry.ID == "" { + for key, id := range tfState.IDs { + if id == "" { return fmt.Errorf("failed to intepret terraform state for %s: missing ID", key) } } @@ -134,9 +134,9 @@ To start using direct engine, set "engine: direct" under bundle in your databric } state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range tfState.IDs { + for key, id := range tfState.IDs { state[key] = dstate.ResourceEntry{ - ID: resourceEntry.ID, + ID: id, State: json.RawMessage("{}"), } } From 5a8459b3be25f5c6452ac8ccb786ec995b68d0b6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 11:03:44 -0700 Subject: [PATCH 29/38] migrate: use json.Number to preserve large integer precision in TF state json.Unmarshal into map[string]any decodes all JSON numbers as float64. Integers beyond 2^53 (~9e15) lose precision: 9007199254740993 becomes 9007199254740992. run_job_task.job_id is a JSON number in TF state and realistic job IDs can exceed this threshold. Fix: use json.NewDecoder.UseNumber() when parsing TF state attributes so numbers are preserved as their original decimal string. Add a unit test case with job_id = 2^53+1 that asserts the raw state JSON contains the exact value, and an invariant config (job_run_job_ref.yml.tmpl) that exercises the run_job_task.job_id cross-reference in cloud tests. Co-authored-by: Isaac --- .../configs/job_run_job_ref.yml.tmpl | 23 ++++++++ acceptance/bundle/invariant/test.toml | 1 + bundle/migrate/build_state_test.go | 58 +++++++++++++++---- bundle/migrate/tf_state.go | 7 ++- 4 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl b/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl new file mode 100644 index 00000000000..bc128eee9db --- /dev/null +++ b/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + trigger_job: + name: test-trigger-$UNIQUE_NAME + tasks: + - task_key: notebook + notebook_task: + notebook_path: /Shared/notebook + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + + watcher_job: + name: test-watcher-$UNIQUE_NAME + tasks: + - task_key: run_trigger + run_job_task: + job_id: ${resources.jobs.trigger_job.id} diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index f0971991b6a..59ed1abc35b 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -37,6 +37,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 16cd8b45207..c3f6f5174fb 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -65,15 +65,16 @@ func runBuildStateFromTF( func TestBuildStateFromTF(t *testing.T) { tests := []struct { - name string - yaml string - tfAttrs migrate.TFStateAttrs - tfIDs map[string]string - wantKey string // primary key to assert on - absentKey string // key that must NOT be in state - wantID string // expected entry.ID - wantState map[string]any // expected fields in the state JSON - wantDeps []deployplan.DependsOnEntry + name string + yaml string + tfAttrs migrate.TFStateAttrs + tfIDs map[string]string + wantKey string // primary key to assert on + absentKey string // key that must NOT be in state + wantID string // expected entry.ID + wantState map[string]any // expected fields in the state JSON (parsed via json.Unmarshal) + wantStateRaw string // expected substring in raw state JSON bytes (use for large integers) + wantDeps []deployplan.DependsOnEntry }{ { name: "basic job stored with ID", @@ -158,9 +159,42 @@ resources: }, wantKey: "resources.jobs.dst_job", wantID: "222", - wantState: map[string]any{"max_concurrent_runs": float64(4)}, // JSON numbers decode as float64 + wantState: map[string]any{"max_concurrent_runs": float64(4)}, wantDeps: []deployplan.DependsOnEntry{{Node: "resources.jobs.src_job"}}, }, + { + // 2^53+1 = 9007199254740993 cannot be represented exactly as float64. + // json.Unmarshal into map[string]any would produce 9007199254740992 (off by one). + // UseNumber preserves the original decimal string, so the value is exact. + name: "large integer run_job_task.job_id preserved exactly", + yaml: ` +resources: + jobs: + trigger_job: + name: "trigger" + watcher_job: + name: "watcher" + tasks: + - task_key: run_trigger + run_job_task: + job_id: "${resources.jobs.trigger_job.id}" +`, + tfAttrs: migrate.TFStateAttrs{ + "databricks_job": { + "trigger_job": json.RawMessage(`{"id": "9007199254740993", "name": "trigger", "max_concurrent_runs": 1}`), + "watcher_job": json.RawMessage(`{"id": "100", "name": "watcher", "task": [{"task_key": "run_trigger", "run_job_task": [{"job_id": 9007199254740993}]}]}`), + }, + }, + tfIDs: map[string]string{ + "resources.jobs.trigger_job": "9007199254740993", + "resources.jobs.watcher_job": "100", + }, + wantKey: "resources.jobs.watcher_job", + wantID: "100", + // job_id must be stored as 9007199254740993, not 9007199254740992 (float64 truncation). + // Check raw bytes because json.Unmarshal would silently re-truncate when reading back. + wantStateRaw: `"job_id": 9007199254740993`, + }, { name: "dashboard etag stored from TF attributes", yaml: ` @@ -201,6 +235,10 @@ resources: } } + if tc.wantStateRaw != "" { + assert.Contains(t, string(entry.State), tc.wantStateRaw, "raw state JSON") + } + if tc.wantDeps != nil { require.Len(t, entry.DependsOn, len(tc.wantDeps)) for i, want := range tc.wantDeps { diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 6af73b96699..f97f9507a21 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -1,6 +1,7 @@ package migrate import ( + "bytes" "context" "encoding/json" "errors" @@ -143,8 +144,12 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath // fields are stored as single-element arrays [{"field": "value"}], not as plain objects. // Navigating via map avoids the json.Unmarshal type mismatch between []T in JSON and // struct-typed schema fields. + // UseNumber preserves large integers exactly; plain Unmarshal into interface{} uses + // float64 which loses precision for integers beyond 2^53. var attrs map[string]any - if err := json.Unmarshal(attrsJSON, &attrs); err != nil { + dec := json.NewDecoder(bytes.NewReader(attrsJSON)) + dec.UseNumber() + if err := dec.Decode(&attrs); err != nil { return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } From 4e39c05440502953d6b602eceb56d10401d12069 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 17 Jun 2026 12:08:51 -0700 Subject: [PATCH 30/38] acceptance: exercise large-int reference in migrate invariant config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend job_run_job_ref so watcher_job references trigger_job's max_concurrent_runs literal of 2^53+1 (9007199254740993). This drives the migration to resolve a large integer from TF state end-to-end; with the json.Number fix the migrated state keeps the exact value, without it the value is truncated to 2^53. The value is out of range for the real backend, so the config is local-only (no_run_job_ref_on_cloud). Note: the unit test (TestBuildStateFromTF/large_integer) is the deterministic regression guard, since the "no drift" plan check cannot detect this — the plan diff also collapses ints to float64. Co-authored-by: Isaac --- .../bundle/invariant/configs/job_run_job_ref.yml.tmpl | 11 +++++++++++ acceptance/bundle/invariant/test.toml | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl b/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl index bc128eee9db..c84f7cd5145 100644 --- a/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job_run_job_ref.yml.tmpl @@ -5,6 +5,13 @@ resources: jobs: trigger_job: name: test-trigger-$UNIQUE_NAME + # 9007199254740993 = 2^53+1, the smallest integer float64 cannot represent + # exactly (it rounds to 2^53 = ...992). Stored as a literal via PrepareState, + # so trigger_job keeps the exact value; watcher_job references it to force the + # migration to resolve it from TF state, where naive float64 parsing of JSON + # numbers would corrupt it. Local-only: deliberately out of range for the + # real backend (see no_run_job_ref_on_cloud in the parent test.toml). + max_concurrent_runs: 9007199254740993 tasks: - task_key: notebook notebook_task: @@ -17,6 +24,10 @@ resources: watcher_job: name: test-watcher-$UNIQUE_NAME + # Resolved from trigger_job's TF state during migration. Exercises the + # json.Number parsing path end-to-end: with the fix the migrated state keeps + # 9007199254740993; without it the value is silently truncated to ...992. + max_concurrent_runs: ${resources.jobs.trigger_job.max_concurrent_runs} tasks: - task_key: run_trigger run_job_task: diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 59ed1abc35b..6a4f46c5f17 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -73,6 +73,11 @@ EnvMatrix.INPUT_CONFIG = [ [EnvMatrixExclude] no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] +# job_run_job_ref sets a health rule value to 2^53+1 to exercise large-integer +# precision in state migration. The value is out of range for the real backend, +# so this config is local-only (the mock server stores it verbatim). +no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] + # Postgres resources only work on AWS no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] From 418e02eec73191ba09977a8092ae9c3461dab20f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 23 Jun 2026 14:48:28 +0200 Subject: [PATCH 31/38] migrate: drop model_id alias, rely on DABsPathToTerraform main's DABsToTerraformRenameMap now maps models.model_id -> registered_model_id (PR #5621), so the manual tfStateFieldAliases map and the alias-fallback branch in LookupTFField are redundant. LookupTFField now translates once and navigates. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index f97f9507a21..e4d2d24328e 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -14,15 +14,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) -// tfStateFieldAliases maps DABs group → DABs field name → TF state field name for -// cases where a DABs state-computed field has a different name in TF state. -// These fields are not captured by DABsToTerraformRenameMap because they are -// state-only (not part of the bundle config struct). -var tfStateFieldAliases = map[string]map[string]string{ - // models.model_id is the numeric model ID; TF stores it as registered_model_id. - "models": {"model_id": "registered_model_id"}, -} - // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage @@ -153,26 +144,7 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - value, err := navigateTFState(attrs, tfFieldPath) - if err == nil { - return value, nil - } - - // Apply state-only field aliases for single-segment fields whose DABs name differs from TF state name. - if aliases, ok := tfStateFieldAliases[group]; ok { - if head, ok := fieldPath.StringKey(); ok { - if tfName, ok := aliases[head]; ok { - aliasPath := structpath.NewStringKey(nil, tfName) - if aliasFieldPath, e := terraform_dabs_map.DABsPathToTerraform(group, aliasPath); e == nil { - if v, e := navigateTFState(attrs, aliasFieldPath); e == nil { - return v, nil - } - } - } - } - } - - return nil, err + return navigateTFState(attrs, tfFieldPath) } // navigateTFState walks the TF state map using the given path. From fa109f636111ffa9f96b73a8e2a7ed6946b88734 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 23 Jun 2026 18:30:49 +0200 Subject: [PATCH 32/38] migrate: move rawTFState type up with the other type declarations Group all type/struct declarations (TFStateAttrs, TFState, rawTFState) at the top of the file, with methods and functions below. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index e4d2d24328e..665c970a55e 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -17,6 +17,33 @@ import ( // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage +// TFState holds everything parsed from a single terraform state file read. +type TFState struct { + // Attrs maps (tfResourceType → resourceName → raw JSON attributes). + Attrs TFStateAttrs + // IDs maps bundle resource key → resource ID. + IDs map[string]string + + // Lineage and Serial are the top-level state metadata used to seed the direct state. + Lineage string + Serial int +} + +// rawTFState is the on-disk terraform state format; it captures everything we need in one parse. +type rawTFState struct { + Version int `json:"version"` + Lineage string `json:"lineage"` + Serial int `json:"serial"` + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []struct { + Attributes json.RawMessage `json:"attributes"` + } `json:"instances"` + } `json:"resources"` +} + // ETagFor returns the "etag" attribute for a bundle resource, or "" if absent. // Reads directly from the raw JSON without full path translation. func (a TFStateAttrs) ETagFor(group, name string) string { @@ -37,18 +64,6 @@ func (a TFStateAttrs) ETagFor(group, name string) string { return v.Etag } -// TFState holds everything parsed from a single terraform state file read. -type TFState struct { - // Attrs maps (tfResourceType → resourceName → raw JSON attributes). - Attrs TFStateAttrs - // IDs maps bundle resource key → resource ID. - IDs map[string]string - - // Lineage and Serial are the top-level state metadata used to seed the direct state. - Lineage string - Serial int -} - // ParseTFStateFull reads the terraform state file once and returns all parsed data. // Returns nil without error when the file does not exist (first deploy with no resources). func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { @@ -80,21 +95,6 @@ func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { return &TFState{Attrs: attrs, IDs: ids, Lineage: parsed.Lineage, Serial: parsed.Serial}, nil } -// rawTFState is the on-disk terraform state format; it captures everything we need in one parse. -type rawTFState struct { - Version int `json:"version"` - Lineage string `json:"lineage"` - Serial int `json:"serial"` - Resources []struct { - Type string `json:"type"` - Name string `json:"name"` - Mode tfjson.ResourceMode `json:"mode"` - Instances []struct { - Attributes json.RawMessage `json:"attributes"` - } `json:"instances"` - } `json:"resources"` -} - func parseTFStateAttrsFromRaw(s *rawTFState) TFStateAttrs { result := make(TFStateAttrs) for _, r := range s.Resources { From 0a3ac581f7af7ad0cea79c8ba5ad767e3073b339 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 07:24:17 +0200 Subject: [PATCH 33/38] migrate: drop unused rawTFState.Version, order serial before lineage Version was never read (ParseResourcesStateFromBytes validates the state version separately). Order the serial/lineage fields to match their on-disk order in the terraform state file. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 665c970a55e..9c966e71b68 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -24,16 +24,15 @@ type TFState struct { // IDs maps bundle resource key → resource ID. IDs map[string]string - // Lineage and Serial are the top-level state metadata used to seed the direct state. - Lineage string + // Serial and Lineage are the top-level state metadata used to seed the direct state. Serial int + Lineage string } // rawTFState is the on-disk terraform state format; it captures everything we need in one parse. type rawTFState struct { - Version int `json:"version"` - Lineage string `json:"lineage"` Serial int `json:"serial"` + Lineage string `json:"lineage"` Resources []struct { Type string `json:"type"` Name string `json:"name"` From 2b9f8e57cbf70a208cb596b756a58e715a8ab493 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 07:30:37 +0200 Subject: [PATCH 34/38] migrate: read etag via LookupTFField, drop bespoke ETagFor ETagFor did its own json.Unmarshal of the attrs blob; the general LookupTFField path already reads any TF-state field. Use it for "etag" too (a resource without an etag returns an error, treated as "no etag"), removing the one-off parser. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 11 +++++++---- bundle/migrate/tf_state.go | 20 -------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 09275b328c7..5ec8c877eed 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -205,10 +205,13 @@ func BuildStateFromTF( // Handle etag for dashboards: read it directly from TF state attributes. // The "etag" field is a computed TF attribute not present in the bundle config, - // so it does not flow through PrepareState/ExtractReferences. - if etag := tfAttrs.ETagFor(group, srcName); etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) + // so it does not flow through PrepareState/ExtractReferences. Resources without + // an etag return an error from LookupTFField, which we treat as "no etag". + if v, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil { + if etag, ok := v.(string); ok && etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } } } diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 9c966e71b68..151b9ff5633 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -43,26 +43,6 @@ type rawTFState struct { } `json:"resources"` } -// ETagFor returns the "etag" attribute for a bundle resource, or "" if absent. -// Reads directly from the raw JSON without full path translation. -func (a TFStateAttrs) ETagFor(group, name string) string { - tfType, ok := terraform.GroupToTerraformName[group] - if !ok { - return "" - } - raw, ok := a[tfType][name] - if !ok { - return "" - } - var v struct { - Etag string `json:"etag,omitempty"` - } - if err := json.Unmarshal(raw, &v); err != nil { - return "" - } - return v.Etag -} - // ParseTFStateFull reads the terraform state file once and returns all parsed data. // Returns nil without error when the file does not exist (first deploy with no resources). func ParseTFStateFull(ctx context.Context, path string) (*TFState, error) { From bc3bee1c08628fdc25d4bfb2a11b669ea187abe0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 09:27:50 +0200 Subject: [PATCH 35/38] migrate: order TFState metadata fields first Put Serial/Lineage at the top of TFState to match rawTFState's field order (metadata before resource data). Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 151b9ff5633..55634234051 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -19,14 +19,14 @@ type TFStateAttrs map[string]map[string]json.RawMessage // TFState holds everything parsed from a single terraform state file read. type TFState struct { + // Serial and Lineage are the top-level state metadata used to seed the direct state. + Serial int + Lineage string + // Attrs maps (tfResourceType → resourceName → raw JSON attributes). Attrs TFStateAttrs // IDs maps bundle resource key → resource ID. IDs map[string]string - - // Serial and Lineage are the top-level state metadata used to seed the direct state. - Serial int - Lineage string } // rawTFState is the on-disk terraform state format; it captures everything we need in one parse. From 62eed6f65da3d028ea81ba6f47d279084bc15242 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 11:21:41 +0200 Subject: [PATCH 36/38] migrate: log resources skipped because they are absent from TF state Co-authored-by: Isaac --- bundle/migrate/build_state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 5ec8c877eed..315d1791666 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -57,6 +57,7 @@ func BuildStateFromTF( id, ok := tfIDs[node] if !ok { // Resource is in config but not in TF state (new resource); skip. + log.Infof(ctx, "%s: not found in terraform state, skipping", node) continue } From f16f9fa4aa5d267a7aaf980826e0ed8a3f2b2b6b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 13:27:52 +0200 Subject: [PATCH 37/38] update NEXT_CHANGELOG --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 989a1f273b0..d72fba264f6 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -16,6 +16,7 @@ * Fix missing field descriptions in the bundle JSON schema for fields whose upstream API docs arrived after the field was first annotated (e.g. `vector_search_endpoints.*.target_qps`); stale placeholder markers no longer hide them ([#5588](https://github.com/databricks/cli/pull/5588)). * Fix `bundle deploy --plan` dropping a `postgres_role`'s `role_id`, which caused the role to be recreated on the next deploy ([#5672](https://github.com/databricks/cli/pull/5672)). * direct: Fix spurious cluster recreate when `apply_policy_default_values: true` is set ([#5693](https://github.com/databricks/cli/pull/5693)). +* direct: New 'deployment migrate' implementation that parses terraform state instead of fetching resources state from the backend ([#5399](https://github.com/databricks/cli/pull/5399)). ### Dependency updates * Bump `github.com/databricks/databricks-sdk-go` from v0.141.0 to v0.147.0 ([#5636](https://github.com/databricks/cli/pull/5636)). From 730b2568edabc2b33460e9832243a0bfeed0a1d6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 24 Jun 2026 13:38:39 +0200 Subject: [PATCH 38/38] fix out.test.toml --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/migrate/out.test.toml | 1 + acceptance/bundle/invariant/no_drift/out.test.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index d9bdc5b830b..5c601542bce 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index d9bdc5b830b..5c601542bce 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index c4fb7e5bb95..e39444592d5 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl",