Add hidden --local flag to bundle plan and deploy

acceptance/bundle/invariant/no_drift/script

@@ -42,11 +42,13 @@ cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null
 # Any failures after this point will be considered as "bug detected" by fuzzer.
 echo INPUT_CONFIG_OK
 
-# Check both text and JSON plan for no changes
+# Check both text and JSON plan for no changes.
+# LOCAL_DIFF is either empty or "--local"; with --local the plan ignores the remote
+# state of resources, so no drift means the local state saved by deploy matches config.
 # Note, expect that there maybe more than one resource unchanged
-$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err
+$CLI bundle plan $LOCAL_DIFF -o json > LOG.planjson 2>LOG.planjson.err
 cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null
 verify_no_drift.py LOG.planjson
 
-$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+$CLI bundle plan $LOCAL_DIFF 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
 cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null

acceptance/bundle/invariant/no_drift/test.toml

@@ -1 +1,17 @@
 EnvMatrix.READPLAN = ["", "1"]
+
+# Run every config twice: once planning normally and once with --local, which plans
+# using only the local state. The no-drift invariant must hold either way: after a
+# deploy the local state saved by the engine must already match the config.
+EnvMatrix.LOCAL_DIFF = ["", "--local"]
+
+# The configs below show drift under --local and are excluded from that variant only.
+# They still run in the normal (LOCAL_DIFF="") variant.
+
+# dashboard, genie_space and vector_search_index persist a remote-sourced value in
+# state for drift detection (etag, endpoint_uuid) that the config never carries.
+# A normal plan reconciles it against the freshly-read remote; --local has no remote
+# to compare against, so the persisted value reads as drift. This is inherent to --local.
+EnvMatrixExclude.no_dashboard_local = ["LOCAL_DIFF=--local", "INPUT_CONFIG=dashboard.yml.tmpl"]
+EnvMatrixExclude.no_genie_space_local = ["LOCAL_DIFF=--local", "INPUT_CONFIG=genie_space.yml.tmpl"]
+EnvMatrixExclude.no_vector_search_index_local = ["LOCAL_DIFF=--local", "INPUT_CONFIG=vector_search_index.yml.tmpl"]

acceptance/bundle/local/basic/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: local-$UNIQUE_NAME
+
+resources:
+  jobs:
+    bar:
+      name: bar-$UNIQUE_NAME
+
+    foo:
+      name: foo-$UNIQUE_NAME
+      tasks:
+        - task_key: run_bar
+          run_job_task:
+            job_id: ${resources.jobs.bar.id}

acceptance/bundle/local/basic/output.txt

@@ -0,0 +1,58 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan --local
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+=== Job reads during 'bundle plan --local':
+
+>>> print_requests.py --get //jobs/get
+
+=== Plan --local as JSON:
+
+>>> [CLI] bundle plan --local -o json
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "local_only": true,
+  "plan": {
+    "resources.jobs.bar": {
+      "action": "skip"
+    },
+    "resources.jobs.foo": {
+      "depends_on": [
+        {
+          "node": "resources.jobs.bar",
+          "label": "${resources.jobs.bar.id}"
+        }
+      ],
+      "action": "skip"
+    }
+  }
+}
+
+>>> [CLI] bundle deploy --local
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Telemetry:
+local_used true
+
+=== Destroy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.bar
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/local-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/local/basic/script

@@ -0,0 +1,31 @@
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    title "Destroy"
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Deploy both jobs so that they are recorded in the local state. foo references
+# ${resources.jobs.bar.id}.
+trace $CLI bundle deploy
+rm -f out.requests.txt
+
+# A --local plan reports no drift without reading the remote state of the jobs:
+# no GET is issued. The ${resources.jobs.bar.id} reference still resolves because
+# it comes from the local state, not a remote read.
+trace $CLI bundle plan --local
+title "Job reads during 'bundle plan --local':\n"
+trace print_requests.py --get //jobs/get
+
+# The JSON plan is self-describing: local_only is true and no entry carries
+# remote_state (nothing was fetched), so deploy --plan can warn before applying it.
+title "Plan --local as JSON:\n"
+trace $CLI bundle plan --local -o json
+rm -f out.requests.txt
+
+# A --local deploy likewise does not read remote state up front and reports its use via telemetry.
+trace $CLI bundle deploy --local
+title "Telemetry:\n"
+print_telemetry_bool_values | grep '^local_used '

acceptance/bundle/local/basic/test.toml

@@ -0,0 +1,7 @@
+Local = true
+Cloud = true
+RecordRequests = true
+# --local skips the per-resource remote read, which only the direct engine performs.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+# databricks.yml is generated at runtime from the template.
+Ignore = [".databricks", ".gitignore", "databricks.yml"]

acceptance/bundle/local/rejected/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: local-rejected
+
+resources:
+  jobs:
+    my_job:
+      name: my-job

acceptance/bundle/local/rejected/output.txt

@@ -0,0 +1,12 @@
+
+>>> [CLI] bundle plan --local
+Error: --local is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct
+
+
+Exit code: 1
+
+>>> [CLI] bundle deploy --local
+Error: --local is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct
+
+
+Exit code: 1

acceptance/bundle/local/rejected/script

@@ -0,0 +1,3 @@
+# --local is only supported by the direct engine; both plan and deploy reject it.
+errcode trace $CLI bundle plan --local
+errcode trace $CLI bundle deploy --local

acceptance/bundle/local/rejected/test.toml

@@ -0,0 +1,2 @@
+# --local is rejected on the terraform engine with an actionable error.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/jobs/num_workers/script

@@ -1,6 +1,6 @@
 envsubst < databricks.yml.tmpl > databricks.yml
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 trace print_requests.py //jobs
 
-trace $CLI bundle plan
+trace $CLI bundle plan $LOCAL
 rm out.requests.txt

acceptance/bundle/resources/jobs/num_workers/test.toml

@@ -0,0 +1,12 @@
+# Also run plan/deploy with --local (direct engine only; rejected on terraform).
+# Deploy issues the same mutating requests with or without --local — only the
+# remote reads (GETs, excluded from print_requests.py) differ — so the recorded
+# output is identical across variants. The " --local" token is stripped from the
+# command echo so the >>> lines match the non-local variant.
+EnvMatrix.LOCAL = ["", "--local"]
+EnvMatrixExclude.no_local_on_terraform = ["LOCAL=--local", "DATABRICKS_BUNDLE_ENGINE=terraform"]
+EnvRepl.LOCAL = false
+
+[[Repls]]
+Old = " --local"
+New = ""

acceptance/bundle/resources/schemas/update/script

@@ -1,20 +1,20 @@
 echo "*" > .gitignore
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 
 trace print_requests.py //unity
 read_state.py schemas schema1 id name catalog_name comment
 
 title "Update comment and re-deploy"
 trace update_file.py databricks.yml COMMENT1 COMMENT2
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 # Why the first time request match for direct & terraform, the requests from second deploy no longer match:
 # Terraform also sends "enable_predictive_optimization": "INHERIT" which is remote value that it stored in the state.
 trace print_requests.py //unity | gron.py | grep -v enable_predictive_optimization
 read_state.py schemas schema1 id name catalog_name comment
 
 title "Restore comment to original value and re-deploy"
 trace update_file.py databricks.yml COMMENT2 COMMENT1
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 trace print_requests.py //unity | gron.py | grep -v enable_predictive_optimization
 read_state.py schemas schema1 id name catalog_name comment
 

acceptance/bundle/resources/schemas/update/test.toml

@@ -0,0 +1,12 @@
+# Also run plan/deploy with --local (direct engine only; rejected on terraform).
+# Deploy issues the same mutating requests with or without --local — only the
+# remote reads (GETs, excluded from print_requests.py) differ — so the recorded
+# output is identical across variants. The " --local" token is stripped from the
+# command echo so the >>> lines match the non-local variant.
+EnvMatrix.LOCAL = ["", "--local"]
+EnvMatrixExclude.no_local_on_terraform = ["LOCAL=--local", "DATABRICKS_BUNDLE_ENGINE=terraform"]
+EnvRepl.LOCAL = false
+
+[[Repls]]
+Old = " --local"
+New = ""

acceptance/bundle/resources/volumes/change-name/script

@@ -1,4 +1,4 @@
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 
 trace print_requests.py //unity
 
@@ -8,10 +8,10 @@ $CLI bundle summary -o json | jq .resources
 title "Update name"
 trace update_file.py databricks.yml myvolume mynewvolume
 
-trace $CLI bundle plan
+trace $CLI bundle plan $LOCAL
 # terraform marks this as "update", direct marks this as "update_with_id"
 $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
-trace $CLI bundle deploy
+trace $CLI bundle deploy $LOCAL
 trace print_requests.py //unity
 
 trace $CLI volumes read main.myschema.mynewvolume

acceptance/bundle/resources/volumes/change-name/test.toml

@@ -1,3 +1,17 @@
 Ignore = [
     ".databricks",
 ]
+
+# Also run plan/deploy with --local (direct engine only; rejected on terraform).
+# Deploy issues the same mutating requests with or without --local — only the
+# remote reads (GETs, excluded from print_requests.py) differ — so the recorded
+# output is identical across variants. The " --local" token is stripped from the
+# command echo so the >>> lines match the non-local variant. Note: the JSON plan
+# (`bundle plan -o json`) keeps no --local: its remote_state would differ.
+EnvMatrix.LOCAL = ["", "--local"]
+EnvMatrixExclude.no_local_on_terraform = ["LOCAL=--local", "DATABRICKS_BUNDLE_ENGINE=terraform"]
+EnvRepl.LOCAL = false
+
+[[Repls]]
+Old = " --local"
+New = ""

bundle/bundle.go

@@ -156,6 +156,10 @@ type Bundle struct {
 	// When non-empty, only the specified resources are included in deployment.
 	Select []string
 
+	// Local, when set via the --local flag, plans and deploys using only the local
+	// state. The remote state of resources is neither fetched nor considered.
+	Local bool
+
 	// SkipLocalFileValidation makes path translation tolerant of missing local files.
 	// When set, TranslatePaths computes workspace paths without verifying files exist.
 	// Used by config-remote-sync: a user may modify resource paths remotely (e.g.,

bundle/deployplan/plan.go

@@ -16,11 +16,17 @@ import (
 const currentPlanVersion = 2
 
 type Plan struct {
-	PlanVersion int                   `json:"plan_version,omitempty"`
-	CLIVersion  string                `json:"cli_version,omitempty"`
-	Lineage     string                `json:"lineage,omitempty"`
-	Serial      int                   `json:"serial,omitempty"`
-	Plan        map[string]*PlanEntry `json:"plan,omitzero"`
+	PlanVersion int    `json:"plan_version,omitempty"`
+	CLIVersion  string `json:"cli_version,omitempty"`
+	Lineage     string `json:"lineage,omitempty"`
+	Serial      int    `json:"serial,omitempty"`
+
+	// LocalOnly is set when the plan was computed with --local, i.e. without
+	// fetching the remote state of resources. Such a plan can miss out-of-band
+	// drift, so consumers like "deploy --plan" warn before applying it.
+	LocalOnly bool `json:"local_only,omitempty"`
+
+	Plan map[string]*PlanEntry `json:"plan,omitzero"`
 
 	mutex   sync.Mutex `json:"-"`
 	lockmap lockmap    `json:"-"`

bundle/direct/bind.go

@@ -114,7 +114,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
 		os.Remove(tmpStatePath)
 		return nil, err
 	}
-	plan, err := b.CalculatePlan(ctx, client, configRoot)
+	plan, err := b.CalculatePlan(ctx, client, configRoot, false)
 	if err != nil {
 		os.Remove(tmpStatePath)
 		return nil, err
@@ -170,7 +170,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
 		os.Remove(tmpStatePath)
 		return nil, err
 	}
-	plan, err = b.CalculatePlan(ctx, client, configRoot)
+	plan, err = b.CalculatePlan(ctx, client, configRoot, false)
 	if _, ferr := b.StateDB.Finalize(ctx); ferr != nil {
 		log.Warnf(ctx, "failed to finalize state: %v", ferr)
 	}