direct: store serialized_dashboard/serialized_space in state as content hashes

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json

@@ -12,7 +12,7 @@
     "etag": [ETAG],
     "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}",
+    "serialized_dashboard": "sha256:[HASH]",
     "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
    }
   }

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -12,7 +12,7 @@
     "etag": "[NUMID]",
     "parent_path": "/Workspace/Users/[USERNAME]",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+    "serialized_dashboard": "sha256:[HASH]",
     "warehouse_id": "123456"
    }
   }

acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json

@@ -30,9 +30,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
-          "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
-          "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
+          "old": "sha256:[HASH]",
+          "new": "sha256:[HASH]",
+          "remote": "sha256:[HASH]"
         }
       }
     }

acceptance/bundle/resources/dashboard-state-sha/dashboard.lvdash.json

@@ -0,0 +1 @@
+{"pages":[{"name":"main","displayName":"Sales Overview"}]}

acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: dashboard-state-sha-$UNIQUE_NAME
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: $DASHBOARD_DISPLAY_NAME
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      embed_credentials: true
+      file_path: "dashboard.lvdash.json"
+      parent_path: /Users/$CURRENT_USER_NAME

acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json

@@ -0,0 +1,3 @@
+[
+  "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}"
+]

acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json

@@ -0,0 +1,19 @@
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "display_name": "dashboard-state-sha [UUID]",
+          "embed_credentials": true,
+          "parent_path": "/Workspace/Users/[USERNAME]",
+          "published": true,
+          "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}",
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json

@@ -0,0 +1,40 @@
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "skip",
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "dashboard-state-sha [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json",
+        "published": true,
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "etag": {
+          "action": "skip",
+          "reason": "custom",
+          "old": [ETAG],
+          "remote": [ETAG]
+        },
+        "serialized_dashboard": {
+          "action": "skip",
+          "reason": "etag_based",
+          "old": "sha256:[HASH]",
+          "new": "sha256:[HASH]",
+          "remote": "sha256:[HASH]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt

@@ -0,0 +1 @@
+json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256:[HASH]";

acceptance/bundle/resources/dashboard-state-sha/output.txt

@@ -0,0 +1,17 @@
+
+>>> [CLI] bundle plan -o json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan -o json
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboard-state-sha/script

@@ -0,0 +1,38 @@
+DASHBOARD_DISPLAY_NAME="dashboard-state-sha $(uuid)"
+if [ -z "$CLOUD_ENV" ]; then
+    export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234"
+    echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS
+fi
+export DASHBOARD_DISPLAY_NAME
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+rm -f out.requests.txt
+
+# The direct-engine plan keeps the FULL serialized_dashboard in new_state (so it can be
+# deployed), and reports the diff against state as a content hash.
+trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+
+# Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The plan
+# and the persisted state keep only the hash, but the create API call must still send the
+# FULL serialized_dashboard. out.create.serialized.json is identical for READPLAN="" and
+# READPLAN=1, which proves the saved plan applies the real content rather than the hash.
+# Not traced: the deploy command line differs by READPLAN (--plan vs none).
+$CLI bundle deploy $(readplanarg out.plan_create.direct.json)
+
+DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
+echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+
+jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json
+rm -f out.requests.txt
+
+# Persisted state holds ONLY the content hash, never the dashboard JSON.
+print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt
+
+# Re-plan with no local change: the server normalizes serialized_dashboard, but it is
+# ignore_remote_changes (etag_based), so the resource is unchanged (no spurious update).
+trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/dashboard-state-sha/test.toml

@@ -0,0 +1,22 @@
+# Direct-engine hashed-state behaviour for dashboards. Kept outside dashboards/ so it can run
+# direct-only: terraform does not hash state, and the saved-plan path (deploy --plan, i.e.
+# READPLAN=1) is direct-only. RecordRequests is inherited from resources/test.toml.
+Local = true
+Cloud = true
+RequiresWarehouse = true
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.READPLAN = ["", "1"]
+
+[Env]
+# Prevent MSYS2 from rewriting /Users/... paths on Windows.
+MSYS_NO_PATHCONV = "1"
+
+# Etag can be both negative and positive.
+[[Repls]]
+Old = "\"[-0-9]{8,}\""
+New = "[ETAG]"
+
+[[Repls]]
+Old = "\"[0-9]{8,}\""
+New = "[ETAG]"

acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json

@@ -39,9 +39,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\n  \"pages\": [\n    {\n      \"displayName\": \"New Page\",\n      \"layout\": [\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"widget\": {\n            \"name\": \"82eb9107\",\n            \"textbox_spec\": \"# I'm a title\"\n          }\n        },\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"widget\": {\n            \"name\": \"ffa6de4f\",\n            \"textbox_spec\": \"Text\"\n          }\n        }\n      ],\n      \"name\": \"fdd21a3c\"\n    }\n  ]\n}\n",
-          "new": "{\n  \"pages\": [\n    {\n      \"displayName\": \"New Page\",\n      \"layout\": [\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"widget\": {\n            \"name\": \"82eb9107\",\n            \"textbox_spec\": \"# I'm a title\"\n          }\n        },\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"widget\": {\n            \"name\": \"ffa6de4f\",\n            \"textbox_spec\": \"Text\"\n          }\n        }\n      ],\n      \"name\": \"fdd21a3c\"\n    }\n  ]\n}\n",
-          "remote": "{}\n"
+          "old": "sha256:[HASH]",
+          "new": "sha256:[HASH]",
+          "remote": "sha256:[HASH]"
         }
       }
     }

acceptance/bundle/resources/dashboards/simple/out.plan.direct.json

@@ -30,9 +30,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{ }\n",
-          "new": "{ }\n",
-          "remote": "{}\n"
+          "old": "sha256:[HASH]",
+          "new": "sha256:[HASH]",
+          "remote": "sha256:[HASH]"
         }
       }
     }

acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json

@@ -46,9 +46,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}",
-          "new": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}",
-          "remote": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}"
+          "old": "sha256:[HASH]",
+          "new": "sha256:[HASH]",
+          "remote": "sha256:[HASH]"
         }
       }
     }

acceptance/bundle/test.toml

@@ -22,3 +22,12 @@ New = 'os/[OS]'
 [[Repls]]
 Old = ' cicd/github'
 New = ''
+
+# serialized_dashboard is persisted in direct-engine state as a content hash. Mask the
+# 64-char digest with a hash-specific token (not the generic [ALPHANUMID]) so the output
+# reads clearly as a sha256 hash, at a low Order so it runs before the generic [NUMID] /
+# [DASHBOARD_ID] rules that would otherwise split the hex into pieces.
+[[Repls]]
+Old = 'sha256:[0-9a-f]{64}'
+New = 'sha256:[HASH]'
+Order = -1

bundle/direct/apply.go

@@ -50,6 +50,16 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState,
 	}
 }
 
+// saveState compacts the state (replacing fields declared hashed_in_state with content
+// hashes, see dresources.CompactState) before persisting it.
+func (d *DeploymentUnit) saveState(db *dstate.DeploymentState, id string, newState any) error {
+	compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), newState)
+	if err != nil {
+		return fmt.Errorf("compacting state: %w", err)
+	}
+	return db.SaveState(d.ResourceKey, id, compacted, d.DependsOn)
+}
+
 func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error {
 	var newID string
 	var remoteState any
@@ -75,7 +85,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState,
 		return err
 	}
 
-	err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn)
+	err = d.saveState(db, newID, newState)
 	if err != nil {
 		return fmt.Errorf("saving state after creating id=%s: %w", newID, err)
 	}
@@ -146,7 +156,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState,
 		return err
 	}
 
-	err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn)
+	err = d.saveState(db, id, newState)
 	if err != nil {
 		return fmt.Errorf("saving state id=%s: %w", id, err)
 	}
@@ -190,7 +200,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment
 		return err
 	}
 
-	err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn)
+	err = d.saveState(db, newID, newState)
 	if err != nil {
 		return fmt.Errorf("saving state id=%s: %w", oldID, err)
 	}
@@ -252,7 +262,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState,
 		return fmt.Errorf("resizing id=%s: %w", id, err)
 	}
 
-	err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn)
+	err = d.saveState(db, id, newState)
 	if err != nil {
 		return fmt.Errorf("saving state id=%s: %w", id, err)
 	}

bundle/direct/bind.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/deployplan"
+	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/bundle/direct/dstate"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/structs/structaccess"
@@ -145,13 +146,24 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
 			}
 		}
 
+		adapter, err := b.getAdapterForKey(resourceKey)
+		if err != nil {
+			os.Remove(tmpStatePath)
+			return nil, err
+		}
+		compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value)
+		if err != nil {
+			os.Remove(tmpStatePath)
+			return nil, fmt.Errorf("compacting state: %w", err)
+		}
+
 		err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true))
 		if err != nil {
 			os.Remove(tmpStatePath)
 			return nil, err
 		}
 
-		err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn)
+		err = b.StateDB.SaveState(resourceKey, resourceID, compacted, dependsOn)
 		if err != nil {
 			os.Remove(tmpStatePath)
 			return nil, err

bundle/direct/bundle_apply.go

@@ -130,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 					logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey))
 					return false
 				}
-				err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn)
+				err = d.saveState(&b.StateDB, id, sv.Value)
 			} else {
 				// TODO: redo calcDiff to downgrade planned action if possible (?)
 				err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry)