diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/databricks.yml b/acceptance/bundle/deploy/readplan/grants-remove-principal/databricks.yml new file mode 100644 index 00000000000..be0581c3702 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +resources: + schemas: + myschema: + name: myschema + catalog_name: main + grants: + - principal: ${workspace.current_user.userName} + privileges: + - USE_SCHEMA + - principal: extra@example.test # TO_REMOVE + privileges: # TO_REMOVE + - CREATE_TABLE # TO_REMOVE diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.plan.grants.json b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.plan.grants.json new file mode 100644 index 00000000000..0b90f0b19e6 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.plan.grants.json @@ -0,0 +1,58 @@ +{ + "depends_on": [ + { + "node": "resources.schemas.myschema", + "label": "${resources.schemas.myschema.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "securable_type": "schema", + "full_name": "main.myschema", + "__embed__": [ + { + "principal": "[USERNAME]", + "privileges": [ + "USE_SCHEMA" + ] + } + ] + } + }, + "remote_state": { + "securable_type": "schema", + "full_name": "main.myschema", + "__embed__": [ + { + "principal": "extra@example.test", + "privileges": [ + "CREATE_TABLE" + ] + }, + { + "principal": "[USERNAME]", + "privileges": [ + "USE_SCHEMA" + ] + } + ] + }, + "changes": { + "[principal='extra@example.test']": { + "action": "update", + "old": { + "principal": "extra@example.test", + "privileges": [ + "CREATE_TABLE" + ] + }, + "remote": { + "principal": "extra@example.test", + "privileges": [ + "CREATE_TABLE" + ] + } + } + } +} diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml new file mode 100644 index 00000000000..71970b719d4 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/output.txt b/acceptance/bundle/deploy/readplan/grants-remove-principal/output.txt new file mode 100644 index 00000000000..f7926fa9fa9 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/output.txt @@ -0,0 +1,81 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog/permissions --sort +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/schema/main.myschema", + "body": { + "changes": [ + { + "add": [ + "USE_SCHEMA" + ], + "principal": "[USERNAME]", + "remove": [ + "ALL_PRIVILEGES" + ] + }, + { + "add": [ + "CREATE_TABLE" + ], + "principal": "extra@example.test", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} + +>>> [CLI] bundle plan -o json + +>>> print_requests.py --get //unity-catalog/permissions --sort +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/permissions/schema/main.myschema" +} +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog/permissions --sort +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/schema/main.myschema", + "body": { + "changes": [ + { + "add": [ + "USE_SCHEMA" + ], + "principal": "[USERNAME]", + "remove": [ + "ALL_PRIVILEGES" + ] + }, + { + "principal": "extra@example.test", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} +The following resources will be deleted: + delete resources.schemas.myschema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.myschema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/script b/acceptance/bundle/deploy/readplan/grants-remove-principal/script new file mode 100644 index 00000000000..df94954d5a9 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/script @@ -0,0 +1,25 @@ +cleanup() { + $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Deploy schema with two grant principals. +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/permissions --sort + +# Remove extra@example.test from config. +grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.yml + +# Generate update plan: entry.RemoteState captures both principals as GrantsState. +trace $CLI bundle plan -o json > tmp.plan.json +jq '.plan["resources.schemas.myschema.grants"]' tmp.plan.json > out.plan.grants.json +trace print_requests.py --get //unity-catalog/permissions --sort + +# Deploy from serialized plan (READPLAN=1) or without (READPLAN=""). +# Without the fix, entry.RemoteState.(*GrantsState) fails for JSON-loaded plan, +# removedGrantPrincipals returns nil, and the revocation is silently skipped. +$CLI bundle deploy $(readplanarg tmp.plan.json) + +# Verify the PATCH call included a REMOVE for extra@example.test. +trace print_requests.py //unity-catalog/permissions --sort | contains.py 'extra@example.test' diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/test.toml new file mode 100644 index 00000000000..75904d6e3da --- /dev/null +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/test.toml @@ -0,0 +1,4 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] +RecordRequests = true +Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/deploy/readplan/large-int/databricks.yml b/acceptance/bundle/deploy/readplan/large-int/databricks.yml new file mode 100644 index 00000000000..5d9e396c5bd --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +# Test that large int64 values (> 2^53, not representable by float64 without loss) +# are correctly saved to and read from the plan JSON. +# job_id here is not a real job reference — it's a field that happens to be int64. +resources: + jobs: + job: + name: test-job + tasks: + - task_key: max_int64 + run_job_task: + job_id: 9223372036854775807 + - task_key: max_int64_minus_1 + run_job_task: + job_id: 9223372036854775806 + - task_key: min_int64 + run_job_task: + job_id: -9223372036854775808 diff --git a/acceptance/bundle/deploy/readplan/large-int/out.plan_create.json b/acceptance/bundle/deploy/readplan/large-int/out.plan_create.json new file mode 100644 index 00000000000..25eb30d9ff5 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/out.plan_create.json @@ -0,0 +1,44 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.job": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "test-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max_int64" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max_int64_minus_1" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min_int64" + } + ] + } + } + } + } +} diff --git a/acceptance/bundle/deploy/readplan/large-int/out.plan_skip.json b/acceptance/bundle/deploy/readplan/large-int/out.plan_skip.json new file mode 100644 index 00000000000..684e78cbd45 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/out.plan_skip.json @@ -0,0 +1,122 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.jobs.job": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "test-job", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tasks": [ + { + "email_notifications": {}, + "run_if": "ALL_SUCCESS", + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max_int64", + "timeout_seconds": 0 + }, + { + "email_notifications": {}, + "run_if": "ALL_SUCCESS", + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max_int64_minus_1", + "timeout_seconds": 0 + }, + { + "email_notifications": {}, + "run_if": "ALL_SUCCESS", + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min_int64", + "timeout_seconds": 0 + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='max_int64'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='max_int64'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='max_int64'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "tasks[task_key='max_int64_minus_1'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='max_int64_minus_1'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='max_int64_minus_1'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "tasks[task_key='min_int64'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='min_int64'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='min_int64'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} diff --git a/acceptance/bundle/deploy/readplan/large-int/out.test.toml b/acceptance/bundle/deploy/readplan/large-int/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/large-int/output.txt b/acceptance/bundle/deploy/readplan/large-int/output.txt new file mode 100644 index 00000000000..a29ec21cae9 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle plan -o json + +>>> print_requests.py --get //jobs + +>>> [CLI] bundle deploy --plan out.plan_create.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --get //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "test-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max_int64" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max_int64_minus_1" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min_int64" + } + ] + } +} + +>>> [CLI] bundle plan -o json + +>>> print_requests.py --get //jobs +{ + "method": "GET", + "path": "/api/2.2/jobs/get", + "q": { + "job_id": "[NUMID]" + } +} + +>>> [CLI] bundle deploy --plan out.plan_skip.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --get //jobs + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/readplan/large-int/script b/acceptance/bundle/deploy/readplan/large-int/script new file mode 100644 index 00000000000..6d4e36f2c67 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/script @@ -0,0 +1,18 @@ +# Generate initial plan +trace $CLI bundle plan -o json > out.plan_create.json +trace print_requests.py --get //jobs | contains.py "!GET" "!POST" + +# First deploy to create the resource +trace $CLI bundle deploy --plan out.plan_create.json +trace print_requests.py --get //jobs | contains.py "!GET" "POST" + +# Generate a skip plan — remote_state.job_id must be an integer, not float64. +# If float conversion happens, [MAX_INT_64] replacements in test.toml won't match. +trace $CLI bundle plan -o json > out.plan_skip.json +trace print_requests.py --get //jobs | contains.py "GET" + +trace $CLI bundle deploy --plan out.plan_skip.json +trace print_requests.py --get //jobs | contains.py "!GET" "!POST" + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/deploy/readplan/large-int/test.toml b/acceptance/bundle/deploy/readplan/large-int/test.toml new file mode 100644 index 00000000000..d30b40218cc --- /dev/null +++ b/acceptance/bundle/deploy/readplan/large-int/test.toml @@ -0,0 +1,16 @@ +# Terraform panics on int64 values that don't fit in int32: +# panic: Error reading level state: strconv.ParseInt: parsing "[NUMID]": value out of range +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true + +[[Repls]] +Old = '9223372036854775807' +New = '[MAX_INT_64]' + +[[Repls]] +Old = '9223372036854775806' +New = '[MAX_INT_64_MINUS_1]' + +[[Repls]] +Old = '-9223372036854775808' +New = '[MIN_INT_64]' diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/app/app.py b/acceptance/bundle/resources/apps/readplan-lifecycle/app/app.py new file mode 100644 index 00000000000..ad1e64f9fb0 --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/app/app.py @@ -0,0 +1,5 @@ +import http.server, os + +http.server.HTTPServer( + ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler +).serve_forever() diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/databricks.yml b/acceptance/bundle/resources/apps/readplan-lifecycle/databricks.yml new file mode 100644 index 00000000000..0ee14d5371f --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + apps: + myapp: + name: myapp + description: initial + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml new file mode 100644 index 00000000000..71970b719d4 --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt b/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt new file mode 100644 index 00000000000..5f2234fb675 --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt @@ -0,0 +1,25 @@ + +=== Initial deploy: app created and started +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //apps/myapp/start + +=== Change description and deploy from plan: no extra start +>>> update_file.py databricks.yml description: initial description: updated +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //apps/myapp/start +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/script b/acceptance/bundle/resources/apps/readplan-lifecycle/script new file mode 100644 index 00000000000..f3e11eced2d --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/script @@ -0,0 +1,15 @@ +cleanup() { + $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deploy: app created and started" +trace $CLI bundle deploy +trace print_requests.py //apps/myapp/start + +title "Change description and deploy from plan: no extra start" +trace update_file.py databricks.yml "description: initial" "description: updated" +$CLI bundle plan -o json > tmp.plan.json +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace print_requests.py //apps/myapp/start diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml new file mode 100644 index 00000000000..4138449c6d7 --- /dev/null +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = false +Ignore = [".databricks", "tmp.plan.json"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] + +[[Repls]] +Old = '(?m)^✓ [^\n]*\n' +New = "" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index 4bbf20c6d67..2907af3b8e8 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -130,8 +130,12 @@ "cluster_name": "test-cluster-[UNIQUE_NAME]", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "lifecycle": { + "started": false + }, "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "13.3.x-snapshot-scala2.12" + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "TERMINATED" }, "changes": { "autoscale.max_workers": { diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/databricks.yml b/acceptance/bundle/resources/clusters/readplan-lifecycle/databricks.yml new file mode 100644 index 00000000000..800e2f2c2de --- /dev/null +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + clusters: + mycluster: + cluster_name: test-cluster + spark_version: 13.3.x-scala2.12 + num_workers: 1 + custom_tags: + env: original + lifecycle: + started: true diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml new file mode 100644 index 00000000000..71970b719d4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt new file mode 100644 index 00000000000..d0651891f34 --- /dev/null +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt @@ -0,0 +1,25 @@ + +=== Initial deploy: cluster created and running +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //clusters/start + +=== Change tags and deploy from plan: no extra start +>>> update_file.py databricks.yml env: original env: updated +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //clusters/start +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/script b/acceptance/bundle/resources/clusters/readplan-lifecycle/script new file mode 100644 index 00000000000..cfd5f75de30 --- /dev/null +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/script @@ -0,0 +1,15 @@ +cleanup() { + $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deploy: cluster created and running" +trace $CLI bundle deploy +trace print_requests.py //clusters/start + +title "Change tags and deploy from plan: no extra start" +trace update_file.py databricks.yml "env: original" "env: updated" +$CLI bundle plan -o json > tmp.plan.json +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace print_requests.py //clusters/start diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml new file mode 100644 index 00000000000..3be928738cb --- /dev/null +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = false +Ignore = [".databricks", "tmp.plan.json"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/models/readplan-permissions/databricks.yml b/acceptance/bundle/resources/models/readplan-permissions/databricks.yml new file mode 100644 index 00000000000..1d4e2f53e6a --- /dev/null +++ b/acceptance/bundle/resources/models/readplan-permissions/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + models: + mymodel: + name: test-model + description: initial # TO_REMOVE + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE # TO_REMOVE + user_name: manager@example.test # TO_REMOVE diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml new file mode 100644 index 00000000000..71970b719d4 --- /dev/null +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/output.txt b/acceptance/bundle/resources/models/readplan-permissions/output.txt new file mode 100644 index 00000000000..307313f6059 --- /dev/null +++ b/acceptance/bundle/resources/models/readplan-permissions/output.txt @@ -0,0 +1,61 @@ + +=== Initial deploy: model created with permissions +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //permissions/registered-models +{ + "method": "PUT", + "path": "/api/2.0/permissions/registered-models/[MODEL_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "manager@example.test" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} + +=== Remove manager permission and description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //permissions/registered-models +{ + "method": "PUT", + "path": "/api/2.0/permissions/registered-models/[MODEL_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.models.mymodel + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script new file mode 100644 index 00000000000..52d8975afe5 --- /dev/null +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -0,0 +1,23 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +register_model_id() { + # Register the model's numeric ID so it appears as [MODEL_ID] in output + MODEL_ID=$($CLI model-registry get-model test-model | jq -r '.registered_model_databricks.id') + add_repl.py "$MODEL_ID" "MODEL_ID" +} + +title "Initial deploy: model created with permissions" +trace $CLI bundle deploy +register_model_id +trace print_requests.py //permissions/registered-models + +title "Remove manager permission and description, deploy from plan" +grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml +$CLI bundle plan -o json > tmp.plan.json +$CLI bundle deploy $(readplanarg tmp.plan.json) +register_model_id +trace print_requests.py //permissions/registered-models diff --git a/acceptance/bundle/resources/models/readplan-permissions/test.toml b/acceptance/bundle/resources/models/readplan-permissions/test.toml new file mode 100644 index 00000000000..0f0ae403587 --- /dev/null +++ b/acceptance/bundle/resources/models/readplan-permissions/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +Ignore = [".databricks", "tmp.plan.json"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index c7fe8a3c989..a718cae165c 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -55,6 +55,7 @@ func LoadPlanFromFile(path string) (*Plan, error) { var plan Plan decoder := json.NewDecoder(file) decoder.DisallowUnknownFields() + decoder.UseNumber() if err := decoder.Decode(&plan); err != nil { return nil, fmt.Errorf("parsing plan JSON: %w", err) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d890b8d5d7b..fcb9d1b9299 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -68,11 +68,16 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return err } - // Eagerly parse all StructVarJSON entries to catch parsing errors early. - // When the plan is read from JSON, Value contains raw JSON bytes. - // We parse them into typed structs and cache them for later use. + // Eagerly parse plan entries loaded from JSON: + // - NewState.Value contains raw JSON bytes; parse into typed structs and cache. + // - RemoteState is decoded as map[string]interface{}; round-trip through the + // adapter's StateType to recover the correct typed struct so that type + // assertions in resource adapters (e.g. entry.RemoteState.(*GrantsState)) + // work identically whether the plan came from a file or from memory. for resourceKey, entry := range plan.Plan { - if entry.NewState == nil || len(entry.NewState.Value) == 0 { + hasNewState := entry.NewState != nil && len(entry.NewState.Value) > 0 + hasRemoteState := entry.RemoteState != nil + if !hasNewState && !hasRemoteState { continue } @@ -81,12 +86,27 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return fmt.Errorf("converting plan entry %s: %w", resourceKey, err) } - sv, err := entry.NewState.ToStructVar(adapter.StateType()) - if err != nil { - return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) + if hasNewState { + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) + } + b.StateCache.Store(resourceKey, sv) } - b.StateCache.Store(resourceKey, sv) + if hasRemoteState { + data, err := json.Marshal(entry.RemoteState) + if err != nil { + return fmt.Errorf("re-serializing remote state for %s: %w", resourceKey, err) + } + // RemoteType() returns a pointer type (e.g. *AppRemote); Elem() gives + // the struct type so reflect.New produces a single pointer, not double. + typed := reflect.New(adapter.RemoteType().Elem()).Interface() + if err := json.Unmarshal(data, typed); err != nil { + return fmt.Errorf("loading remote state for %s: %w", resourceKey, err) + } + entry.RemoteState = typed + } } b.Plan = plan diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 596346f1614..f436a331590 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -220,6 +220,9 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName } pageToken = resp.NextPageToken } + slices.SortFunc(assignments, func(a, b catalog.PrivilegeAssignment) int { + return strings.Compare(a.Principal, b.Principal) + }) return assignments, nil } diff --git a/bundle/direct/dresources/model.go b/bundle/direct/dresources/model.go index ad8a9cca5a3..2a7554818bf 100644 --- a/bundle/direct/dresources/model.go +++ b/bundle/direct/dresources/model.go @@ -24,14 +24,14 @@ type MlflowModelRemote struct { ModelId string `json:"model_id"` } -// Custom marshalers needed because embedded ml.ModelDatabricks has its own -// MarshalJSON which would otherwise take over and ignore model_id. -func (s *MlflowModelRemote) UnmarshalJSON(b []byte) error { - return marshal.Unmarshal(b, s) +// Custom marshalers needed because embedded ml.ModelDatabricks has its own MarshalJSON +// that otherwise shadows the outer struct's fields (model_id gets dropped without this). +func (r *MlflowModelRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) } -func (s MlflowModelRemote) MarshalJSON() ([]byte, error) { - return marshal.Marshal(s) +func (r MlflowModelRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) } func (*ResourceMlflowModel) New(client *databricks.WorkspaceClient) *ResourceMlflowModel { diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index b6e1c6c1710..02ae7da7b26 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -75,11 +75,14 @@ func (s *FakeWorkspace) ClustersEdit(req Request) any { } defer s.LockUnlock()() - _, ok := s.Clusters[request.ClusterId] + existing, ok := s.Clusters[request.ClusterId] if !ok { return Response{StatusCode: 404} } + // Preserve runtime-only fields that the Edit API request doesn't include. + request.State = existing.State + request.ClusterId = existing.ClusterId clusterFixUps(&request) s.Clusters[request.ClusterId] = request