Expose computed volume_path during initialize

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### CLI
 
 ### Bundles
+* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes.<key>.volume_path}` instead of hardcoding `/Volumes/<catalog>/<schema>/<name>` ([#5550](https://github.com/databricks/cli/pull/5550)). Derived from `catalog_name`/`schema_name`/`name` and resolved at initialize, so the reference depends on those underlying resources rather than the volume itself. The field is computed by the CLI and never sent to the API (dropped before Terraform apply). References resolve on both engines when the path components are known at initialize; components that are only known after deploy (for example a remote `creator_user_name`) are supported on the direct engine (the default) but not on Terraform.
 
  * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)).
 

acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl

@@ -0,0 +1,24 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# A job parameter references a volume's computed volume_path, and the volume's
+# schema_name/name are references to another job's remote creator_user_name. So
+# volume_path embeds a reference at initialize that only resolves at deploy,
+# exercising the deferred-resolution chain (foo -> data.volume_path -> process)
+# end to end with no drift afterwards.
+resources:
+  jobs:
+    foo:
+      name: test-job-foo-$UNIQUE_NAME
+
+    process:
+      name: test-job-process-$UNIQUE_NAME
+      parameters:
+        - name: data_path
+          default: ${resources.volumes.data.volume_path}
+
+  volumes:
+    data:
+      catalog_name: main
+      schema_name: ${resources.jobs.foo.creator_user_name}
+      name: ${resources.jobs.foo.creator_user_name}

acceptance/bundle/invariant/continue_293/test.toml

@@ -16,6 +16,10 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
 
+# Computed volume_path is not supported on v0.293.0, so the seed deploy cannot
+# resolve ${resources.volumes.*.volume_path}. Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]
+
 # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
 # per variant (two full deploys at 1000 tasks) without incremental coverage.
 EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]

acceptance/bundle/invariant/migrate/test.toml

@@ -30,3 +30,10 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml
 # migrate deploys via Terraform first, and the TF provider rejects an uppercase
 # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct.
 EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"]
+
+# volume_path embeds ${resources.jobs.foo.creator_user_name}; the terraform
+# interpolator converts it to ${databricks_job.foo.creator_user_name}, but the
+# terraform databricks_job resource does not export creator_user_name, so the
+# terraform seed deploy fails. Same class as no_cross_resource_ref above.
+# Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -67,6 +67,7 @@ EnvMatrix.INPUT_CONFIG = [
     "vector_search_index.yml.tmpl",
     "volume.yml.tmpl",
     "volume_external.yml.tmpl",
+    "volume_path_job_ref.yml.tmpl",
     "volume_uppercase_name.yml.tmpl",
 ]
 

acceptance/bundle/python/volumes-support/output.txt

@@ -17,12 +17,14 @@
         "catalog_name": "my_catalog",
         "name": "My Volume (updated)",
         "schema_name": "my_schema",
+        "volume_path": "/Volumes/my_catalog/my_schema/My Volume (updated)",
         "volume_type": "MANAGED"
       },
       "my_volume_2": {
         "catalog_name": "my_catalog_2",
         "name": "My Volume (2) (updated)",
         "schema_name": "my_schema_2",
+        "volume_path": "/Volumes/my_catalog_2/my_schema_2/My Volume (2) (updated)",
         "volume_type": "MANAGED"
       }
     }

acceptance/bundle/refschema/out.fields.txt

@@ -3476,6 +3476,7 @@ resources.volumes.*.updated_at	int64	REMOTE
 resources.volumes.*.updated_by	string	REMOTE
 resources.volumes.*.url	string	INPUT
 resources.volumes.*.volume_id	string	REMOTE
+resources.volumes.*.volume_path	string	INPUT
 resources.volumes.*.volume_type	catalog.VolumeType	ALL
 resources.volumes.*.grants.full_name	string	ALL
 resources.volumes.*.grants.securable_type	string	ALL

acceptance/bundle/resource_deps/bad_syntax/output.txt

@@ -6,6 +6,7 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {

acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+  name: testbundle-${UNIQUE_NAME}
+
+resources:
+  schemas:
+    my:
+      catalog_name: main
+      name: myschema-${UNIQUE_NAME}
+
+  volumes:
+    bar:
+      catalog_name: main
+      schema_name: myschema-${UNIQUE_NAME}
+      name: volumebar-${UNIQUE_NAME}
+
+    foo:
+      catalog_name: main
+      schema_name: myschema-${UNIQUE_NAME}
+      name: volumefoo-${UNIQUE_NAME}
+      comment: ${resources.volumes.bar.volume_path}

acceptance/bundle/resource_deps/computed_volume_path/output.txt

@@ -0,0 +1,36 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "schemas": {
+    "my": {
+      "catalog_name": "main",
+      "name": "myschema-[UNIQUE_NAME]"
+    }
+  },
+  "volumes": {
+    "bar": {
+      "catalog_name": "main",
+      "name": "volumebar-[UNIQUE_NAME]",
+      "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
+      "volume_type": "MANAGED"
+    },
+    "foo": {
+      "catalog_name": "main",
+      "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
+      "name": "volumefoo-[UNIQUE_NAME]",
+      "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]",
+      "volume_type": "MANAGED"
+    }
+  }
+}
+
+>>> [CLI] bundle plan
+create schemas.my
+create volumes.bar
+create volumes.foo
+
+Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> print_requests.py --get //unity

acceptance/bundle/resource_deps/computed_volume_path/script

@@ -0,0 +1,4 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+trace $CLI bundle validate -o json | jq .resources
+trace $CLI bundle plan
+trace print_requests.py --get //unity

acceptance/bundle/resource_deps/implicit_deps_volume/output.txt

@@ -17,6 +17,7 @@
       "catalog_name": "${resources.catalogs.my_catalog.name}",
       "name": "myvolume",
       "schema_name": "${resources.schemas.my_schema.name}",
+      "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myvolume",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/non_existent_field/output.txt

@@ -6,12 +6,14 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "${resources.volumes.bar.non_existent}",
       "name": "myname",
       "schema_name": "${resources.volumes.bar.schema_name}",
+      "volume_path": "/Volumes/${resources.volumes.bar.non_existent}/myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/remote_field_storage_location/output.txt

@@ -12,13 +12,15 @@
       "catalog_name": "main",
       "name": "volumebar-[UNIQUE_NAME]",
       "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "main",
       "comment": "${resources.volumes.bar.storage_location}",
       "name": "volumefoo-[UNIQUE_NAME]",
       "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/resources_var/output.txt

@@ -21,12 +21,14 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "${resources.volumes.bar.catalog_name}",
       "name": "myname",
       "schema_name": "${resources.volumes.bar.schema_name}",
+      "volume_path": "/Volumes/mycatalog/myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt

@@ -27,6 +27,7 @@
       "catalog_name": "${resources.schemas.bar.catalog_name}",
       "name": "myname",
       "schema_name": "${resources.schemas.bar.name}",
+      "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml

@@ -0,0 +1,29 @@
+bundle:
+  name: test-bundle
+
+# volume_path is a CLI-only computed field (not API state). When a component
+# (catalog_name, schema_name, name) is an unresolved ${...} reference, it stays
+# embedded in volume_path so it resolves later at plan/deploy.
+#
+# Chain under test: bar.name is a remote value (baz.id), so bar.volume_path is
+# computed at initialize with that reference intact; foo.comment picks up that
+# string and resolves at deploy to the fully-resolved path.
+#
+# baz.id is used instead of baz.storage_location because the resolved value
+# becomes bar's name, and the testserver can't route a name containing the
+# slashes of a storage_location URL.
+resources:
+  volumes:
+    baz:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: bazname
+    bar:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: ${resources.volumes.baz.id}
+    foo:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: fooname
+      comment: ${resources.volumes.bar.volume_path}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json

@@ -0,0 +1,31 @@
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname",
+    "name": "fooname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "mycatalog.myschema.bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json

@@ -0,0 +1,43 @@
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname"
+}
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname"
+}
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname"
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname",
+    "name": "fooname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "mycatalog.myschema.bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt

@@ -0,0 +1,17 @@
+The following resources will be deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt

@@ -0,0 +1,17 @@
+The following resources will be deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!