From 55902586c920f827e7d5883f940ea243d2a1a7b6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 01:41:10 +0200 Subject: [PATCH 01/56] bundle: record and read deployment state via DMS with server-generated IDs Wire the direct engine into the Deployment Metadata Service (DMS) so that a `record_deployment_history`-enabled bundle records each deploy/destroy as a version and can read its resource state back from DMS. The deployment ID is now assigned by the server: the first deploy calls CreateDeployment with an empty ID, reads the assigned ID back from the response, and persists it in the direct-engine state header (Header.DeploymentID). Later deploys pass the stored ID back, so a bundle maps one-to-one to a DMS deployment even after the local cache is deleted (the ID rides along in the workspace-synced state file). - libs/dms: Recorder creates the deployment (server-assigned ID) + version, heartbeats the lease, completes it, and deletes the deployment on destroy. - bundle/direct: operationRecorder reports each applied resource operation; the wire resource_key drops the CLI-internal "resources." prefix. - bundle/direct/dstate: Open takes a DMS client and overlays DMS resource state when DMS holds a successful version; deployment ID persisted in the header. - bundle/phases: create the version after plan approval, complete it under the lock, record operations during apply. - libs/testserver: stateful fake DMS (deployments/versions/operations/resources) with server-generated IDs; acceptance test covers deploy, cache-loss redeploy, and destroy. Co-authored-by: Isaac --- acceptance/bundle/dms/record/databricks.yml | 10 + acceptance/bundle/dms/record/out.test.toml | 3 + acceptance/bundle/dms/record/output.txt | 140 +++++++++++ acceptance/bundle/dms/record/script | 15 ++ acceptance/bundle/dms/test.toml | 13 + bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +- bundle/direct/bundle_apply.go | 13 + bundle/direct/dstate/dms.go | 104 ++++++++ bundle/direct/dstate/state.go | 43 +++- bundle/direct/dstate/state_test.go | 45 +++- bundle/direct/oprecorder.go | 107 ++++++++ bundle/direct/oprecorder_test.go | 84 +++++++ bundle/direct/pkg.go | 5 + bundle/phases/deploy.go | 32 +++ bundle/phases/destroy.go | 23 ++ bundle/phases/dms.go | 37 +++ cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 12 +- libs/dms/recorder.go | 255 ++++++++++++++++++++ libs/dms/recorder_test.go | 165 +++++++++++++ libs/testserver/bundle.go | 228 +++++++++++++++++ libs/testserver/fake_workspace.go | 5 + libs/testserver/handlers.go | 29 +++ 26 files changed, 1363 insertions(+), 25 deletions(-) create mode 100644 acceptance/bundle/dms/record/databricks.yml create mode 100644 acceptance/bundle/dms/record/out.test.toml create mode 100644 acceptance/bundle/dms/record/output.txt create mode 100644 acceptance/bundle/dms/record/script create mode 100644 acceptance/bundle/dms/test.toml create mode 100644 bundle/direct/dstate/dms.go create mode 100644 bundle/direct/oprecorder.go create mode 100644 bundle/direct/oprecorder_test.go create mode 100644 bundle/phases/dms.go create mode 100644 libs/dms/recorder.go create mode 100644 libs/dms/recorder_test.go create mode 100644 libs/testserver/bundle.go diff --git a/acceptance/bundle/dms/record/databricks.yml b/acceptance/bundle/dms/record/databricks.yml new file mode 100644 index 00000000000..b20e6274310 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..5c0317f38cc --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,140 @@ + +=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The server-assigned deployment ID is persisted in the local state file +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DESTROY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_key": "jobs.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..ab59d38afb4 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,15 @@ +title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The server-assigned deployment ID is persisted in the local state file" +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +rm -rf .databricks +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..24ce9756629 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,13 @@ +Local = true +Cloud = false + +# Deployment Metadata Service (DMS) recording is only meaningful in the direct +# engine, where the deployment ID is stored in and read from the direct-engine +# state. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = true + +Ignore = [ + '.databricks', +] diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 055a47dc934..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c4178c4e601..afef2367e5b 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -88,6 +88,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + // Record the delete with DMS. State is nil: the resource is gone. + if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } return true } @@ -116,6 +121,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + + // Record the operation with DMS. The resource ID and applied config + // (sv.Value) come from the write just performed; GetResourceID reads + // the ID assigned by Deploy. + if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..1d19d1fe214 --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,104 @@ +package dstate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// overlayDMSState replaces the file-derived resource state with the state +// recorded in the deployment metadata service (DMS), when DMS owns this +// deployment. Once DMS is authoritative its resource set is trusted even when +// empty (a successful deploy with no resources); the file's resources are only +// used when DMS has no successful version, or when the user opts out of +// recording deployment history. The caller holds db.mu and has already +// populated db.Data from the file, including the DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + if !authoritative { + // DMS has no completed version for this deployment: a prior direct deploy + // that has not yet successfully recorded to DMS. Keep the file state. + return nil + } + + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + + db.Data.State = resources + db.stateIDs = make(map[string]string, len(resources)) + for key, entry := range resources { + db.stateIDs[key] = entry.ID + } + return nil +} + +// deploymentHasSuccessfulVersion reports whether DMS holds a successfully +// completed version for the deployment. It is the signal that DMS owns the +// state: if the deployment was never recorded to DMS, or its initial DMS deploy +// did not complete successfully, DMS state is absent or partial and Open keeps +// the local file's resources instead. +func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { + // Versions are listed newest-first and fetched page by page, and we stop at + // the first successful one, so a deployment with a long version history does + // not require reading the whole list (typically just the first page). + it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ + Parent: "deployments/" + deploymentID, + }) + for it.HasNext(ctx) { + v, err := it.Next(ctx) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) + } + if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && + v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + return true, nil + } + } + return false, nil +} + +// fetchDeploymentResources lists every resource recorded for the deployment in +// DMS and maps them into state entries keyed by the fully-qualified resource key. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { + it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: "deployments/" + deploymentID, + }) + + out := make(map[string]ResourceEntry) + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + + // DMS reports resource keys without the "resources." prefix (e.g. + // "jobs.foo"), but the state DB keys are fully qualified + // ("resources.jobs.foo"), so prepend it here. + key := "resources." + res.ResourceKey + + var state json.RawMessage + if res.State != nil { + state = *res.State + } + + out[key] = ResourceEntry{ + ID: res.ResourceId, + State: state, + } + } + return out, nil +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f6c8fc8ba3c..64fc050bdc0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -80,6 +81,13 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` + // DeploymentID is the ID the deployment metadata service (DMS) assigned to + // this deployment. Unlike Lineage (a locally generated identifier for the + // state file), it is minted server-side by CreateDeployment and stored here so + // later deploys can find the same DMS deployment record and read its state. + // Empty/omitted until the bundle first records to DMS. + DeploymentID string `json:"deployment_id,omitempty"` + // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -209,6 +217,25 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } +// GetDeploymentID returns the DMS deployment ID recorded in the state, or an +// empty string if this bundle has not yet recorded a deployment to DMS. +func (db *DeploymentState) GetDeploymentID() string { + db.mu.Lock() + defer db.mu.Unlock() + return db.Data.DeploymentID +} + +// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state +// header. It is set during deploy, after CreateDeployment returns the +// server-generated ID, and persisted to the state file by Finalize. Storing it +// on db.Data (not the WAL header, which is written before the ID is known) +// means the subsequent state write carries it forward. +func (db *DeploymentState) SetDeploymentID(id string) { + db.mu.Lock() + defer db.mu.Unlock() + db.Data.DeploymentID = id +} + type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -218,7 +245,15 @@ type ( WithWrite bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk (and recovers the WAL when +// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// service is the source of truth for resource state: if DMS holds a +// successfully completed version for this deployment, the resources read from +// the file are replaced with the ones recorded in DMS. The local identity +// (lineage, serial, and deployment ID) always comes from the file, since that +// is what the write path increments and carries forward. A nil dmsClient keeps +// the behavior file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { db.mu.Lock() defer db.mu.Unlock() @@ -266,6 +301,12 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } + if dmsClient != nil && db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient); err != nil { + return err + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 11589944472..e95ad1b0224 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,24 +20,43 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } +func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + assert.Empty(t, db.GetDeploymentID()) + + // The deployment ID is set during deploy (after CreateDeployment) and + // persisted by Finalize even though it is not part of the WAL header. + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + mustFinalize(t, &reopened) +} + func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -93,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -107,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -128,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -171,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -193,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -210,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go new file mode 100644 index 00000000000..467f8ac648c --- /dev/null +++ b/bundle/direct/oprecorder.go @@ -0,0 +1,107 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// opRecorder records a resource operation with the deployment metadata service +// (DMS) after it has been applied to the workspace. state is the serialized +// local config after the operation and must be nil for delete operations. +type opRecorder interface { + record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +} + +// recordOperation reports an applied resource operation to DMS. It is a no-op +// unless the bundle opted into recording deployment history (OpRec is set). +// state is the serialized local config after the operation and must be nil for +// delete operations. +func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if b.OpRec == nil { + return nil + } + return b.OpRec.record(ctx, resourceKey, action, resourceID, state) +} + +// operationRecorder records operations via the DMS CreateOperation API. +type operationRecorder struct { + client bundledeployments.BundleDeploymentsInterface + // parent is the version the operations are recorded under, formatted as + // "deployments/{deployment_id}/versions/{version_id}". + parent string +} + +// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { + return &operationRecorder{ + client: client, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + } +} + +func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + actionType, err := deployActionToSDK(action) + if err != nil { + return err + } + + // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state + // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on + // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). + dmsKey := strings.TrimPrefix(resourceKey, "resources.") + + op := bundledeployments.Operation{ + ActionType: actionType, + ResourceId: resourceID, + ResourceKey: dmsKey, + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + if state != nil { + raw, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + msg := json.RawMessage(raw) + op.State = &msg + } + + _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + Parent: r.parent, + ResourceKey: dmsKey, + Operation: op, + }) + return err +} + +// deployActionToSDK maps a deployplan action to its DMS operation action type. +// Only actions that mutate a resource are recordable; Skip and Undefined never +// reach a recorder and are rejected rather than silently coerced. +func deployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go new file mode 100644 index 00000000000..56d860de3e6 --- /dev/null +++ b/bundle/direct/oprecorder_test.go @@ -0,0 +1,84 @@ +package direct + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeOpClient struct { + bundledeployments.BundleDeploymentsInterface + requests []bundledeployments.CreateOperationRequest +} + +func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.requests = append(f.requests, req) + return &bundledeployments.Operation{}, nil +} + +func TestOperationRecorderStripsResourcePrefix(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + req := f.requests[0] + // The wire key drops the CLI-internal "resources." prefix, both in the query + // param and the operation body. + assert.Equal(t, "jobs.foo", req.ResourceKey) + assert.Equal(t, "jobs.foo", req.Operation.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", req.Parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, req.Operation.ActionType) + assert.Equal(t, "job-123", req.Operation.ResourceId) + require.NotNil(t, req.Operation.State) +} + +func TestOperationRecorderDeleteHasNoState(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 3) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) + // Delete operations carry no serialized state. + assert.Nil(t, f.requests[0].Operation.State) +} + +func TestDeployActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := deployActionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Skip and Undefined never reach a recorder and are rejected. + _, err := deployActionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = deployActionToSDK(deployplan.Undefined) + assert.Error(t, err) +} + +func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { + b := &DeploymentBundle{} + // No OpRec set: recording is a no-op. + assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) +} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..f95b515f726 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -44,6 +44,11 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // OpRec records each applied resource operation with the deployment metadata + // service (DMS). It is nil unless the bundle opts into recording deployment + // history, in which case the phases package sets it after CreateVersion. + OpRec opRecorder } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..792c016f963 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -24,6 +25,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -161,7 +163,17 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // Set up DMS recording of this deployment as a version. The version is not + // created until the plan is approved (below), so a cancelled deploy records + // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. + // CompleteVersion is deferred before lock.Release so it runs while the lock + // is still held (defers run last-in-first-out). + recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -255,6 +267,26 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { + // Record the DMS version now that the plan is approved and the state WAL + // has been opened. CreateVersion requests version_id == last_version_id + 1; + // the server returns ABORTED if a concurrent deploy advanced the deployment + // since the plan was computed, so a stale plan is not applied. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + // On a first deploy the server assigned the deployment ID; persist it in + // state (Finalize writes it to disk) so later deploys reuse the record. + // Record operations under the version just created so DMS holds the + // deployed resource state. + b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } deployCore(ctx, b, plan, stateEngine, requestedEngine) } else { cmdio.LogString(ctx, "Deployment cancelled!") diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..244f593476f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -13,8 +13,10 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -131,7 +133,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // Set up DMS recording of this destroy as a version. The version is not + // created until the destroy is approved (below), so a cancelled destroy + // records nothing; the deferred CompleteVersion is a no-op until then. It is + // deferred before lock.Release so it runs while the lock is still held. + recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() @@ -188,6 +198,19 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record the DMS version now that the destroy is approved and the state WAL + // has been opened, then record each delete operation under it. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..667ef8627aa --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,37 @@ +package phases + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/dms" +) + +// newDeploymentRecorder returns a dms.Recorder for the current deployment, or +// nil when DMS recording does not apply. A nil recorder is a no-op, so callers +// do not need to branch on it. +// +// Recording is enabled only when experimental.record_deployment_history is set +// AND the engine is direct: DMS resource state is tracked per direct-engine +// deployment, and only the direct engine opens the state DB where the +// deployment ID is stored. Returning nil for terraform leaves those deployments +// untouched. +// +// The deployment ID passed to the recorder is the one persisted in state from a +// previous deploy; it is empty on a bundle's first recorded deploy, in which +// case the recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if !eng.IsDirect() { + return nil + } + return dms.NewRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + b.DeploymentBundle.StateDB.GetDeploymentID(), + b.Config.Bundle.Target, + versionType, + ) +} diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e4f232605ce..2815f591b22 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -211,7 +212,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + // When the bundle records deployment history, the deployment metadata + // service owns resource state, so hand Open its client to overlay DMS + // state on top of the local identity (lineage/serial/deployment ID). + // Reads open the state write-disabled, so no lineage is minted here. + var dmsClient bundledeployments.BundleDeploymentsInterface + if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { + dmsClient = b.WorkspaceClient(ctx).BundleDeployments + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go new file mode 100644 index 00000000000..eed8485f2c2 --- /dev/null +++ b/libs/dms/recorder.go @@ -0,0 +1,255 @@ +// Package dms records bundle deployments as versions with the Deployment +// Metadata Service (DMS). +// +// It is intentionally independent of the deployment lock: a Recorder does not +// acquire or hold any lock. Callers are responsible for serializing concurrent +// deployments (today via the workspace-filesystem lock). The server-side +// version counter — CreateVersion only succeeds when the requested version is +// last_version_id + 1 — provides the concurrency control for the records +// themselves. +package dms + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The server expires a version's lease if it does not receive a heartbeat +// within a 2-minute TTL; we heartbeat well inside that window. +const defaultHeartbeatInterval = 30 * time.Second + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Recorder records a single deploy/destroy as a version with DMS. +// +// The deployment ID is assigned by the server on the first deploy: NewRecorder +// is given the ID persisted in state (empty on a bundle's first-ever recorded +// deploy), and CreateVersion creates the deployment record when that ID is +// empty and exposes the server-assigned ID via DeploymentID so the caller can +// persist it. Later deploys pass the stored ID back in and reuse the record. +type Recorder struct { + svc bundledeployments.BundleDeploymentsInterface + deploymentID string + targetName string + versionType VersionType + + // populated by CreateVersion + versionNum int64 + stopHeartbeat context.CancelFunc +} + +// NewRecorder returns a Recorder for the given deployment. deploymentID is the +// DMS deployment ID persisted in state, or empty if this bundle has not yet +// recorded a deployment (the server assigns one during CreateVersion). +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { + return &Recorder{ + svc: svc, + deploymentID: deploymentID, + targetName: targetName, + versionType: versionType, + } +} + +// DeploymentID returns the DMS deployment ID this recorder is bound to. It is +// empty until CreateVersion has created the deployment record (on a first +// deploy) and non-empty afterwards, so callers persist it once CreateVersion +// succeeds. +func (r *Recorder) DeploymentID() string { + if r == nil { + return "" + } + return r.deploymentID +} + +// Version returns the version number claimed by CreateVersion. It is zero until +// CreateVersion has run; callers use it to parent operations under the version. +func (r *Recorder) Version() int64 { + if r == nil { + return 0 + } + return r.versionNum +} + +// CreateVersion registers a new version with DMS, claiming it for the duration +// of the deployment. A nil Recorder is a no-op, so callers can leave it nil +// when recording is disabled. +func (r *Recorder) CreateVersion(ctx context.Context) error { + if r == nil { + return nil + } + + versionID, err := r.createDeploymentVersion(ctx) + if err != nil { + return err + } + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + return nil +} + +// CompleteVersion finalizes the version created by CreateVersion. A nil +// Recorder, or one whose CreateVersion never ran, is a no-op. +func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { + if r == nil || r.stopHeartbeat == nil { + return nil + } + + r.stopHeartbeat() + + versionIDStr := strconv.FormatInt(r.versionNum, 10) + versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + + _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName, + CompletionReason: reason, + }) + if err != nil { + return err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%s reason=%s", r.deploymentID, versionIDStr, reason) + + // For destroy operations, delete the deployment record after the version + // completes successfully. + if success && r.versionType == VersionTypeDestroy { + err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if err != nil { + return fmt.Errorf("failed to delete deployment: %w", err) + } + } + + return nil +} + +// createDeploymentVersion ensures the deployment record exists, then creates a +// new version under it. On a first deploy (no stored deployment ID) it creates +// the deployment and lets the server assign the ID; otherwise it reads the +// existing deployment to compute the next version number. +func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID == "" { + // First deploy: create the deployment with an empty ID so the server + // assigns one, then start at version 1. + dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + Deployment: bundledeployments.Deployment{ + TargetName: r.targetName, + }, + }) + if createErr != nil { + return "", fmt.Errorf("failed to create deployment: %w", createErr) + } + id, idErr := deploymentIDFromName(dep.Name) + if idErr != nil { + return "", idErr + } + r.deploymentID = id + versionID = "1" + } else { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if getErr != nil { + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } + + // The server validates that versionID equals last_version_id + 1 and returns + // ABORTED otherwise (e.g. a concurrent deploy already created this version). + version, versionErr := r.svc.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: "deployments/" + r.deploymentID, + VersionId: versionID, + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.targetName, + }, + }) + if versionErr != nil { + return "", fmt.Errorf("failed to create deployment version: %w", versionErr) + } + + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) + return versionID, nil +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, "deployments/") + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// startHeartbeat starts a background goroutine that sends heartbeats to keep +// the deployment version's lease alive. Returns a cancel function to stop it. +func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeploymentsInterface, deploymentID, versionID string) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + versionName := fmt.Sprintf("deployments/%s/versions/%s", deploymentID, versionID) + + go func() { + ticker := time.NewTicker(defaultHeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := svc.Heartbeat(ctx, bundledeployments.HeartbeatRequest{Name: versionName}) + if err != nil { + // A 409 ABORTED is expected if the version was completed + // between the ticker firing and the heartbeat. + if isAbortedErr(err) { + log.Debugf(ctx, "Heartbeat stopped: version already completed") + return + } + log.Warnf(ctx, "Failed to send deployment heartbeat: %v", err) + } else { + log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%s", deploymentID, versionID) + } + } + } + }() + + return cancel +} + +// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. +func isAbortedErr(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" +} diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go new file mode 100644 index 00000000000..91848f74a70 --- /dev/null +++ b/libs/dms/recorder_test.go @@ -0,0 +1,165 @@ +package dms + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeDMS records the calls the recorder makes and lets a test script the +// server-side responses. It embeds the SDK interface so it satisfies it while +// only overriding the methods the recorder uses. +type fakeDMS struct { + bundledeployments.BundleDeploymentsInterface + + // scripted behavior + getDeployment func(id string) (*bundledeployments.Deployment, error) + + // assigned deployment ID for CreateDeployment (server-generated flow) + assignedID string + + // captured requests + created []bundledeployments.CreateDeploymentRequest + versions []bundledeployments.CreateVersionRequest + completed []bundledeployments.CompleteVersionRequest + deleted []string +} + +func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { + f.created = append(f.created, req) + id := req.DeploymentId + if id == "" { + id = f.assignedID + } + return &bundledeployments.Deployment{Name: "deployments/" + id}, nil +} + +func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { + id := req.Name[len("deployments/"):] + return f.getDeployment(id) +} + +func (f *fakeDMS) CreateVersion(ctx context.Context, req bundledeployments.CreateVersionRequest) (*bundledeployments.Version, error) { + f.versions = append(f.versions, req) + return &bundledeployments.Version{VersionId: req.VersionId}, nil +} + +func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { + f.completed = append(f.completed, req) + return &bundledeployments.Version{}, nil +} + +func (f *fakeDMS) DeleteDeployment(ctx context.Context, req bundledeployments.DeleteDeploymentRequest) error { + f.deleted = append(f.deleted, req.Name) + return nil +} + +func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.HeartbeatRequest) (*bundledeployments.HeartbeatResponse, error) { + return &bundledeployments.HeartbeatResponse{}, nil +} + +func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + // A first deploy has no stored deployment ID. + r := NewRecorder(f, "", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // The deployment was created with an empty ID so the server assigns one, and + // the recorder exposes the assigned ID for the caller to persist. + require.Len(t, f.created, 1) + assert.Empty(t, f.created[0].DeploymentId) + assert.Equal(t, "server-generated-id", r.DeploymentID()) + + // The first version is 1, parented under the assigned deployment. + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/server-generated-id", f.versions[0].Parent) + assert.Equal(t, int64(1), r.Version()) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.Len(t, f.completed, 1) + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) + assert.Empty(t, f.deleted) +} + +func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + // A subsequent deploy passes the stored deployment ID. + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // No new deployment is created; the version increments to last_version_id + 1. + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "5", f.versions[0].VersionId) + assert.Equal(t, "stored-id", r.DeploymentID()) +} + +func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + // A successful destroy deletes the deployment record. + require.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + +func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), false)) + + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) + // A failed destroy leaves the deployment in place. + assert.Empty(t, f.deleted) +} + +func TestNilRecorderIsNoOp(t *testing.T) { + var r *Recorder + assert.NoError(t, r.CreateVersion(t.Context())) + assert.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, r.DeploymentID()) + assert.Zero(t, r.Version()) +} + +func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { + f := &fakeDMS{} + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + // CompleteVersion before CreateVersion is a no-op (nothing was claimed). + require.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, f.completed) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go new file mode 100644 index 00000000000..5f2c7e0cfd1 --- /dev/null +++ b/libs/testserver/bundle.go @@ -0,0 +1,228 @@ +package testserver + +import ( + "encoding/json" + "slices" + "strconv" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. +// State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. + +// dmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type dmsDeployment struct { + deployment bundledeployments.Deployment + versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + resources map[string]bundledeployments.Resource +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + // The client either supplies the deployment ID or, in the server-generated + // flow, leaves it empty for the server to mint one. + deploymentID := req.URL.Query().Get("deployment_id") + if deploymentID == "" { + deploymentID = nextUUID() + } + + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + return Response{Body: d.deployment} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + delete(s.dmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Mirror the server-side optimistic concurrency check: the new version must + // be exactly last_version_id + 1. + want := "1" + if d.deployment.LastVersionId != "" { + last, _ := strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + want = strconv.FormatInt(last+1, 10) + } + if versionID != want { + return dmsAborted("expected version " + want + ", got " + versionID) + } + + d.deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.versions[versionID] = &version + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID string) Response { + resourceKey := req.URL.Query().Get("resource_key") + + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + op.ResourceKey = resourceKey + + // Reflect the operation onto the deployment-level resource set the way the + // backend does: a delete removes the resource, anything else upserts it. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: op.ResourceId, + ResourceType: op.ResourceType, + LastActionType: op.ActionType, + LastVersionId: versionID, + State: op.State, + } + } + return Response{Body: op} +} + +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // The API returns versions newest-first (descending version_id). + ids := make([]int64, 0, len(d.versions)) + for id := range d.versions { + n, _ := strconv.ParseInt(id, 10, 64) + ids = append(ids, n) + } + slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) + + versions := make([]bundledeployments.Version, 0, len(ids)) + for _, id := range ids { + versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) + } + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.resources)) + for key := range d.resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8d6e8ee0dd3..a3c4519ccc4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -227,6 +227,10 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + dmsDeployments map[string]*dmsDeployment } func (s *FakeWorkspace) LockUnlock() func() { @@ -378,6 +382,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + dmsDeployments: map[string]*dmsDeployment{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 1d534c47431..39fe6fbb057 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -266,6 +266,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { From da1ed9dfc00a9d6171b2fbeb60d9f17a3e900b26 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 16:49:32 +0200 Subject: [PATCH 02/56] bundle: read DMS authority from last_successful_version_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read overlay decided whether DMS owns a deployment's state by listing versions and scanning for a successful one. The deployment now exposes last_successful_version_id directly, so a single GetDeployment answers the same question — no version listing. The field is still stage:DEVELOPMENT in the proto and therefore stripped from the generated SDK, so this reads the deployment via a raw GET into a local struct as a temporary stub. Once the field is promoted to PRIVATE_PREVIEW and regenerated, the raw call collapses to client.GetDeployment(...). LastSuccessfulVersionId and the threaded config argument goes away (see the TODO in deploymentHasSuccessfulVersion). The testserver's GetDeployment now serves last_successful_version_id (tracked on version completion), and the now-unused ListVersions fake is removed. Co-authored-by: Isaac --- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +++--- bundle/direct/dstate/dms.go | 64 +++++++++++++++++++----------- bundle/direct/dstate/state.go | 9 ++++- bundle/direct/dstate/state_test.go | 30 +++++++------- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 7 +++- libs/testserver/bundle.go | 45 ++++++++++----------- libs/testserver/handlers.go | 3 -- 11 files changed, 100 insertions(+), 78 deletions(-) diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ca5b2c9410b..17ed3b30d5e 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 433b607a037..be3e536f37f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ec910b2734e..ccfbcf788ab 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 1d19d1fe214..659d7c9dd0b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -5,8 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -17,8 +21,11 @@ import ( // used when DMS has no successful version, or when the user opts out of // recording deployment history. The caller holds db.mu and has already // populated db.Data from the file, including the DeploymentID. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) +// +// cfg is threaded in only for the temporary raw read in +// deploymentHasSuccessfulVersion; see the TODO there. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) if err != nil { return err } @@ -46,29 +53,40 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // state: if the deployment was never recorded to DMS, or its initial DMS deploy // did not complete successfully, DMS state is absent or partial and Open keeps // the local file's resources instead. -func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { - // Versions are listed newest-first and fetched page by page, and we stop at - // the first successful one, so a deployment with a long version history does - // not require reading the whole list (typically just the first page). - it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ - Parent: "deployments/" + deploymentID, - }) - for it.HasNext(ctx) { - v, err := it.Next(ctx) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) - } - if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && - v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { - return true, nil +// +// The deployment carries last_successful_version_id, which the server advances +// only when a version completes successfully (unlike last_version_id, which +// also advances on failure). So a non-empty value is exactly the "DMS owns the +// state" signal, readable in a single GetDeployment. +// +// TODO(DMS): this reads the deployment via a raw GET into a local struct +// because last_successful_version_id is still stage:DEVELOPMENT in the proto +// and therefore stripped from the generated SDK. Once the field is promoted to +// PRIVATE_PREVIEW and regenerated, replace the raw call with +// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument +// (revert overlayDMSState/Open back to taking only the typed client). +func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { + apiClient, err := client.New(cfg) + if err != nil { + return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) + } + + // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with + // name=deployments/{id}); we unmarshal into a local struct so we can read + // last_successful_version_id, which the typed SDK response drops. + var dep struct { + LastSuccessfulVersionID string `json:"last_successful_version_id"` + } + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil } + return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) } - return false, nil + return dep.LastSuccessfulVersionID != "", nil } // fetchDeploymentResources lists every resource recorded for the deployment in diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 64fc050bdc0..2c969667c9e 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -253,7 +254,11 @@ type ( // (lineage, serial, and deployment ID) always comes from the file, since that // is what the write path increments and carries forward. A nil dmsClient keeps // the behavior file-only. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { +// +// dmsCfg accompanies dmsClient (both come from the same workspace client) and +// is used only for a temporary raw read of last_successful_version_id; see the +// TODO in deploymentHasSuccessfulVersion. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { db.mu.Lock() defer db.mu.Unlock() @@ -302,7 +307,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient); err != nil { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { return err } } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index e95ad1b0224..16066bf81f8 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,14 +20,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -37,7 +37,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Empty(t, db.GetDeploymentID()) // The deployment ID is set during deploy (after CreateDeployment) and @@ -47,7 +47,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) mustFinalize(t, &reopened) } @@ -56,7 +56,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -112,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) }) mustFinalize(t, &db) } @@ -126,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -147,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -190,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -212,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -229,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 2b286bcad3d..4866f27c5b3 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 48ecc92a6cd..b5dbeed6c56 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2815f591b22..f556c2b3450 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -217,11 +218,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // service owns resource state, so hand Open its client to overlay DMS // state on top of the local identity (lineage/serial/deployment ID). // Reads open the state write-disabled, so no lineage is minted here. + // dmsCfg accompanies the client for a temporary raw read (see the TODO + // in dstate.deploymentHasSuccessfulVersion). var dmsClient bundledeployments.BundleDeploymentsInterface + var dmsCfg *sdkconfig.Config if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { dmsClient = b.WorkspaceClient(ctx).BundleDeployments + dmsCfg = b.WorkspaceClient(ctx).Config } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 5f2c7e0cfd1..34003a507a5 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -20,6 +20,12 @@ type dmsDeployment struct { // resources is the latest resource state per resource key, updated as // operations are recorded. resources map[string]bundledeployments.Resource + // lastSuccessfulVersionID is the highest version that completed + // successfully. The server advances last_successful_version_id only on + // success (unlike last_version_id), and the read path treats a non-empty + // value as "DMS owns the state". Tracked separately because the SDK + // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). + lastSuccessfulVersionID string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -54,7 +60,18 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { if !ok { return dmsNotFound("deployment " + deploymentID) } - return Response{Body: d.deployment} + + // The SDK Deployment struct does not yet carry last_successful_version_id + // (still stage:DEVELOPMENT, so stripped from generation), but the read path + // reads it off the raw JSON. Serve it as an extra field alongside the typed + // deployment so the overlay behaves as it will against the real server. + return Response{Body: struct { + bundledeployments.Deployment + LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` + }{ + Deployment: d.deployment, + LastSuccessfulVersionID: d.lastSuccessfulVersionID, + }} } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { @@ -117,6 +134,9 @@ func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID str v.Status = bundledeployments.VersionStatusVersionStatusCompleted v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.lastSuccessfulVersionID = versionID + } return Response{Body: *v} } @@ -160,29 +180,6 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return Response{Body: op} } -func (s *FakeWorkspace) ListVersions(deploymentID string) Response { - defer s.LockUnlock()() - - d, ok := s.dmsDeployments[deploymentID] - if !ok { - return dmsNotFound("deployment " + deploymentID) - } - - // The API returns versions newest-first (descending version_id). - ids := make([]int64, 0, len(d.versions)) - for id := range d.versions { - n, _ := strconv.ParseInt(id, 10, 64) - ids = append(ids, n) - } - slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) - - versions := make([]bundledeployments.Version, 0, len(ids)) - for _, id := range ids { - versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) - } - return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} -} - func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 39fe6fbb057..3cbfa16c7e1 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -276,9 +276,6 @@ func AddDefaultHandlers(server *Server) { server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) }) - server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { - return req.Workspace.ListVersions(req.Vars["deployment_id"]) - }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) From 011c0928c7962098840aa370c1e13508b80447a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 00:41:17 +0000 Subject: [PATCH 03/56] bundle: fix DMS deployment recording bugs found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five fixes to the DMS state recording added in #6052: 1. The fake DMS server dropped last_successful_version_id. GetDeployment serialized the response through a struct embedding bundledeployments.Deployment, whose promoted MarshalJSON silently discards sibling fields. The CLI reads a missing value as "DMS does not own the state", so the entire read/overlay path (overlayDMSState, fetchDeploymentResources, deploymentHasSuccessfulVersion) never ran in any test. Serialize through a map instead, and unit-test the shape. 2. A bundle with no resources leaked a deployment record per deploy. Such a deploy writes no WAL entries, and the state file was only persisted when the WAL carried entries, so the server-assigned deployment ID was dropped and the next deploy created a second deployment. Track a dirty header so Finalize persists an ID change on its own. A header-only WAL that changed nothing still skips the write, keeping the serial in step (acceptance/bundle/deploy/wal/header-only-wal). 3. Deploy after destroy failed permanently. A successful destroy deletes the deployment record but leaves its ID in local state, so the next deploy's GetDeployment 404'd and the error was fatal — unrecoverable on retry. Treat a missing deployment as "create a new one"; any other read error stays fatal. 4. The overlay dropped depends_on. DMS does not record dependency edges, so replacing local state with DMS resources lost them, affecting delete ordering, the apply graph, and --select expansion. Carry depends_on over from the local entry. Masked by (1) until now. 5. Recording bypassed secret redaction. dstate.SaveState redacts bundle:"sensitive" fields before writing state, but the operation recorder marshalled raw, so a secret would be sent to DMS in plaintext and read back into local state. Route through structwalk.RedactSensitiveFields. Latent today: no resource state type carries a sensitive field yet. Adds acceptance coverage for deploy-destroy-deploy and for a bundle with no resources, both of which now exercise the read path (visible as GET .../resources in the recorded requests). Co-authored-by: Isaac --- .../bundle/dms/no-resources/databricks.yml | 5 + .../bundle/dms/no-resources/out.test.toml | 3 + acceptance/bundle/dms/no-resources/output.txt | 78 ++++++++++++++++ acceptance/bundle/dms/no-resources/script | 8 ++ .../dms/redeploy-after-destroy/databricks.yml | 10 ++ .../dms/redeploy-after-destroy/out.test.toml | 3 + .../dms/redeploy-after-destroy/output.txt | 92 +++++++++++++++++++ .../bundle/dms/redeploy-after-destroy/script | 15 +++ bundle/direct/dstate/dms.go | 14 ++- bundle/direct/dstate/dms_test.go | 74 +++++++++++++++ bundle/direct/dstate/state.go | 42 +++++++-- bundle/direct/dstate/state_test.go | 42 +++++++++ bundle/direct/oprecorder.go | 9 +- bundle/direct/oprecorder_test.go | 22 +++++ libs/dms/recorder.go | 44 +++++---- libs/dms/recorder_test.go | 38 ++++++++ libs/testserver/bundle.go | 44 ++++++--- libs/testserver/bundle_test.go | 51 ++++++++++ 18 files changed, 552 insertions(+), 42 deletions(-) create mode 100644 acceptance/bundle/dms/no-resources/databricks.yml create mode 100644 acceptance/bundle/dms/no-resources/out.test.toml create mode 100644 acceptance/bundle/dms/no-resources/output.txt create mode 100644 acceptance/bundle/dms/no-resources/script create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/databricks.yml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/out.test.toml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/output.txt create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/script create mode 100644 bundle/direct/dstate/dms_test.go create mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/no-resources/databricks.yml b/acceptance/bundle/dms/no-resources/databricks.yml new file mode 100644 index 00000000000..78fad3a292e --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: dms-no-resources + +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..e774fd7546c --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,78 @@ + +=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy: the persisted ID is reused, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..4bc97c5864d --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,8 @@ +title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy: the persisted ID is reused, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml new file mode 100644 index 00000000000..8f79a2c0381 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-redeploy-after-destroy + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt new file mode 100644 index 00000000000..5326653519a --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -0,0 +1,92 @@ + +=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default + +Deleting files... +Destroy complete! + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[DESTROYED_DEPLOYMENT_ID]" + +=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The new deployment ID replaces the stale one in state +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script new file mode 100644 index 00000000000..628cd7bf494 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -0,0 +1,15 @@ +title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve +print_requests.py //api/2.0/bundle --sort --get > /dev/null + +destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) +add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get + +title "The new deployment ID replaces the stale one in state" +trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 659d7c9dd0b..e09cb859221 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -35,7 +35,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) if err != nil { return err } @@ -91,7 +91,12 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { +// +// DMS does not record dependency edges, so depends_on is carried over from the +// local state entry for the same key. It is derived from the local config on +// every deploy and is only consumed for delete ordering, so falling back to an +// empty list when the local state has no entry is safe. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -114,8 +119,9 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } out[key] = ResourceEntry{ - ID: res.ResourceId, - State: state, + ID: res.ResourceId, + State: state, + DependsOn: local[key].DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..df1084b9de7 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,74 @@ +package dstate + +import ( + "context" + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeResourceLister serves a fixed set of resources from ListResources. It +// embeds the SDK interface so it satisfies it while only overriding the one +// method the read path uses. +type fakeResourceLister struct { + bundledeployments.BundleDeploymentsInterface + resources []bundledeployments.Resource +} + +func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeployments.ListResourcesRequest) listing.Iterator[bundledeployments.Resource] { + return listing.NewIterator( + &req, + func(ctx context.Context, r bundledeployments.ListResourcesRequest) (*bundledeployments.ListResourcesResponse, error) { + return &bundledeployments.ListResourcesResponse{Resources: f.resources}, nil + }, + func(resp *bundledeployments.ListResourcesResponse) []bundledeployments.Resource { + return resp.Resources + }, + func(resp *bundledeployments.ListResourcesResponse) *bundledeployments.ListResourcesRequest { + return nil + }, + ) +} + +func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { + state := json.RawMessage(`{"name":"foo"}`) + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "pipelines.bar", ResourceId: "456"}, + }} + + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} + local := map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, + } + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + require.NoError(t, err) + + // DMS owns the ID and state, but it does not record dependency edges, so + // depends_on must survive from the local entry. Losing it breaks delete + // ordering and --select expansion. + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.pipelines.bar": {ID: "456"}, + }, got) +} + +func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123"}, + }} + + // A bundle whose local state was wiped has no entry to carry depends_on from; + // the resource is still recovered from DMS. + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) + require.NoError(t, err) + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123"}, + }, got) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2c969667c9e..f554af3c9b6 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,6 +74,11 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // headerDirty records that a header field changed in memory during this + // deployment (today only the DMS deployment ID), so the state file must be + // written on Finalize even when the WAL carried no resource entries. + headerDirty bool } type Header struct { @@ -231,10 +236,18 @@ func (db *DeploymentState) GetDeploymentID() string { // server-generated ID, and persisted to the state file by Finalize. Storing it // on db.Data (not the WAL header, which is written before the ID is known) // means the subsequent state write carries it forward. +// +// The header is marked dirty so Finalize persists it even when the deploy wrote +// no resource entries; otherwise a bundle with no resources would mint a fresh +// deployment record on every deploy, leaking one orphan per run. func (db *DeploymentState) SetDeploymentID(id string) { db.mu.Lock() defer db.mu.Unlock() + if db.Data.DeploymentID == id { + return + } db.Data.DeploymentID = id + db.headerDirty = true } type ( @@ -353,7 +366,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - hasEntries, err := db.mergeWalIntoState(ctx) + persist, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -362,7 +375,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if hasEntries { + if persist { if err := db.unlockedSave(); err != nil { return err } @@ -373,6 +386,9 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } +// mergeWalIntoState replays the WAL into db.Data and reports whether the caller +// must persist the state file: either the WAL carried resource entries, or a +// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -450,17 +466,23 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) hasEntries := lineNumber > 1 - // Only advance the serial when the WAL carried entries, because the caller - // (replayWAL) persists the new state file only in that case. A header-only - // WAL is a deploy that started but committed nothing; advancing the serial - // for it leaves the in-memory serial ahead of the persisted one, so the - // next deploy writes its WAL header at serial+2 and recovery rejects it as - // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. - if hasEntries { + // A header-only WAL still has to be persisted when a header field changed in + // memory during this deployment (the DMS deployment ID): dropping the write + // would lose the ID and make the next deploy create a second deployment. + persist := hasEntries || db.headerDirty + + // Only advance the serial when the state file is actually written, because + // the caller (replayWAL) persists it only in that case. A header-only WAL + // that changed nothing is a deploy that started but committed nothing; + // advancing the serial for it leaves the in-memory serial ahead of the + // persisted one, so the next deploy writes its WAL header at serial+2 and + // recovery rejects it as "ahead of expected". + // See acceptance/bundle/deploy/wal/header-only-wal. + if persist { db.Data.Serial = newSerial } - return hasEntries, nil + return persist, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 16066bf81f8..6530ca049d9 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -63,6 +63,48 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // A bundle with no resources writes no WAL entries, but the deployment ID + // still has to be persisted: otherwise the next deploy sees no ID and creates + // a second deployment record, leaking one per deploy. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + assert.Equal(t, 1, reopened.Data.Serial) + mustFinalize(t, &reopened) +} + +func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + before, err := os.ReadFile(path) + require.NoError(t, err) + + // Re-setting the same ID is not a header change, so a deploy that commits + // nothing must not bump the serial (see mergeWalIntoState). + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) + reopened.SetDeploymentID("server-assigned-id") + mustFinalize(t, &reopened) + + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) +} + func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 467f8ac648c..033aefb5f2f 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,8 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -67,8 +69,13 @@ func (r *operationRecorder) record(ctx context.Context, resourceKey string, acti // The DMS Operation.State field carries the serialized config so the backend // can serve it as resource state. It is intentionally left unset for delete, // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. if state != nil { - raw, err := json.Marshal(state) + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 56d860de3e6..4c1bbcacda6 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,6 +53,27 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } +func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + state := struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + }{Name: "foo", Token: "super-secret"} + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + require.NotNil(t, f.requests[0].Operation.State) + // Sensitive fields are redacted before leaving the CLI, matching what + // dstate.SaveState writes to the local state file. + assert.JSONEq(t, + `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + string(*f.requests[0].Operation.State)) +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index eed8485f2c2..0d9563dd052 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -149,10 +149,35 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. On a first deploy (no stored deployment ID) it creates -// the deployment and lets the server assign the ID; otherwise it reads the -// existing deployment to compute the next version number. +// new version under it. When no deployment ID is stored, or the stored one no +// longer exists in DMS, it creates the deployment and lets the server assign the +// ID; otherwise it reads the existing deployment to compute the next version +// number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID != "" { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound): + // The record the state points at is gone: a successful destroy deletes + // it (leaving the ID behind in the local state file), and it can also be + // deleted out of band. Recording must not dead-end on it, so fall back to + // creating a new deployment; the caller persists the new ID. + log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) + r.deploymentID = "" + default: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + } + if r.deploymentID == "" { // First deploy: create the deployment with an empty ID so the server // assigns one, then start at version 1. @@ -170,19 +195,6 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } r.deploymentID = id versionID = "1" - } else { - // Existing deployment: read it to compute the next version number. - dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: "deployments/" + r.deploymentID, - }) - if getErr != nil { - return "", fmt.Errorf("failed to get deployment: %w", getErr) - } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } // The server validates that versionID equals last_version_id + 1 and returns diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 91848f74a70..9b60078635f 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -2,8 +2,11 @@ package dms import ( "context" + "errors" + "fmt" "testing" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -104,6 +107,41 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } +func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { + f := &fakeDMS{ + assignedID: "fresh-id", + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) + }, + } + // A deploy after a destroy still has the destroyed deployment's ID in state, + // but the record is gone. Recording must recover rather than fail the deploy. + r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + require.Len(t, f.created, 1) + assert.Equal(t, "fresh-id", r.DeploymentID()) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) +} + +func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + // Only a missing deployment is recovered from; any other read failure is fatal + // rather than silently forking a second deployment record. + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 34003a507a5..a1b0cba24a9 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "slices" "strconv" @@ -61,17 +62,38 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { return dmsNotFound("deployment " + deploymentID) } - // The SDK Deployment struct does not yet carry last_successful_version_id - // (still stage:DEVELOPMENT, so stripped from generation), but the read path - // reads it off the raw JSON. Serve it as an extra field alongside the typed - // deployment so the overlay behaves as it will against the real server. - return Response{Body: struct { - bundledeployments.Deployment - LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` - }{ - Deployment: d.deployment, - LastSuccessfulVersionID: d.lastSuccessfulVersionID, - }} + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment the way the real server does: the typed +// fields plus last_successful_version_id, which the generated SDK struct does +// not carry yet (still stage:DEVELOPMENT) but the read path reads off the raw +// JSON. +// +// The extra field cannot be added by embedding Deployment in a wrapper struct: +// Deployment has its own MarshalJSON, which is promoted to the wrapper and +// silently drops any sibling field. +func deploymentBody(d *dmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.lastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.lastSuccessfulVersionID + } + return body, nil } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go new file mode 100644 index 00000000000..4d28624ba3e --- /dev/null +++ b/libs/testserver/bundle_test.go @@ -0,0 +1,51 @@ +package testserver + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against +// serializing the deployment through a struct that embeds +// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is +// promoted to the embedding struct and silently drops last_successful_version_id. +// The CLI read path treats a missing value as "DMS does not own the state", so +// losing the field here makes the whole overlay path untestable. +func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{lastSuccessfulVersionID: "2"} + d.deployment.Name = "deployments/abc" + d.deployment.LastVersionId = "3" + d.deployment.TargetName = "default" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.Equal(t, "deployments/abc", body["name"]) + assert.Equal(t, "3", body["last_version_id"]) + assert.Equal(t, "default", body["target_name"]) + assert.Equal(t, "2", body["last_successful_version_id"]) + + // The response must round-trip as JSON the same way, since that is what the + // client actually reads. + raw, err := json.Marshal(body) + require.NoError(t, err) + assert.JSONEq(t, + `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, + string(raw)) +} + +// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment +// with no successful version does not advertise one: the read path must keep +// using the local state file in that case. +func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{} + d.deployment.Name = "deployments/abc" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.NotContains(t, body, "last_successful_version_id") +} From 38dc0823b4e97cf6caac784d8041f8a35339b51e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:09:47 +0000 Subject: [PATCH 04/56] bundle: upload DMS operations asynchronously Recording an operation with the deployment metadata service used to happen inline on the apply worker, so every resource paid a CreateOperation round trip before the worker moved on to the next one. Queue the operations instead and upload them from a small pool of background workers (operationQueue, in the new opqueue.go). The queue holds resource keys rather than operations, so an operation recorded for a resource that is still waiting replaces the queued one: DMS keeps one state per resource key, so the later operation supersedes the earlier one and a single request records both. This is best effort - only operations that no worker has picked up yet are coalesced. Uploads are not fire-and-forget. Apply drains the queue before returning and reports the first failure, because a version that completes successfully makes DMS authoritative for resource state; dropping an operation would leave DMS with an incomplete resource set and the next deploy would plan to create resources that already exist. At most one upload per resource key runs at a time, so the last operation recorded for a resource is also the last one the service sees. Co-authored-by: Isaac --- .../dms/multiple-resources/databricks.yml | 18 ++ .../dms/multiple-resources/out.test.toml | 3 + .../bundle/dms/multiple-resources/output.txt | 25 +++ .../bundle/dms/multiple-resources/script | 7 + bundle/direct/bundle_apply.go | 16 +- bundle/direct/opqueue.go | 192 ++++++++++++++++++ bundle/direct/opqueue_test.go | 192 ++++++++++++++++++ bundle/direct/oprecorder.go | 103 ++++++---- bundle/direct/oprecorder_test.go | 42 ++-- bundle/direct/pkg.go | 6 +- 10 files changed, 537 insertions(+), 67 deletions(-) create mode 100644 acceptance/bundle/dms/multiple-resources/databricks.yml create mode 100644 acceptance/bundle/dms/multiple-resources/out.test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/output.txt create mode 100644 acceptance/bundle/dms/multiple-resources/script create mode 100644 bundle/direct/opqueue.go create mode 100644 bundle/direct/opqueue_test.go diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml b/acceptance/bundle/dms/multiple-resources/databricks.yml new file mode 100644 index 00000000000..30f2f433b81 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: dms-multiple-resources + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..4bacec0c317 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,25 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --del-body state --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..e9b9339607e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,7 @@ +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index afef2367e5b..b26861128b7 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -34,6 +34,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // Operations are recorded with DMS from background workers so a resource's + // deploy is not held up by the CreateOperation round trip. The queue is + // drained below, once every apply worker has finished recording. + opQueue := newOperationQueue(ctx, b.OpRec) + g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -89,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -125,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads // the ID assigned by Deploy. - if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -152,6 +157,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the queued operations before returning: the caller completes the + // DMS version right after, and a version must not be completed with uploads + // still in flight. + if err := opQueue.close(); err != nil { + logdiag.LogError(ctx, err) + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go new file mode 100644 index 00000000000..0804ad2d5b3 --- /dev/null +++ b/bundle/direct/opqueue.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "fmt" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" +) + +const ( + // operationQueueSize bounds how many recorded operations wait for upload. + // Apply deploys at most defaultParallelism resources at a time, so a queue + // this deep means an apply worker practically never blocks on a free slot. + operationQueueSize = 10 + + // operationUploadWorkers is how many uploads run at a time. It is below + // operationQueueSize so a burst of operations is absorbed by the queue rather + // than by one request per resource. + operationUploadWorkers = 4 +) + +// operationQueue uploads recorded operations from background workers, so an apply +// worker does not wait for the CreateOperation round trip before moving on to the +// next resource. +// +// It guarantees at most one upload in flight per resource key: within a key the +// worker that owns it uploads sequentially, so the last operation recorded for a +// resource is also the last one the service sees. +// +// Uploads are not fire-and-forget: close drains the queue and returns the first +// failure, which fails the deploy. That matters because a successfully completed +// version makes DMS the source of truth for resource state (see +// dstate.overlayDMSState); silently dropping an operation would leave DMS with an +// incomplete resource set, and the next deploy would plan to create resources +// that already exist. +type operationQueue struct { + uploader operationUploader + + // queue carries resource keys, not the operations themselves: a worker looks + // the operation up in pending when it picks the key up, which is what lets + // record collapse repeated writes to the same resource. + queue chan string + wg sync.WaitGroup + + // mu guards the fields below. + mu sync.Mutex + + // pending is the latest operation recorded per resource key that no worker has + // picked up yet. + pending map[string]recordedOperation + + // inflight holds the resource keys a worker currently owns. A key that is + // in flight is not queued again: the owning worker re-checks pending after its + // upload and picks up anything recorded in the meantime. + inflight map[string]bool + + err error + closed bool +} + +// newOperationQueue starts the upload workers. It returns nil when uploader is +// nil (recording disabled), and every method is a no-op on a nil queue so callers +// do not have to branch. +// +// ctx is used for the uploads, so it must stay valid until close returns. +func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { + if uploader == nil { + return nil + } + + q := &operationQueue{ + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + inflight: make(map[string]bool), + } + + q.wg.Add(operationUploadWorkers) + for range operationUploadWorkers { + go q.work(ctx) + } + + return q +} + +// record serializes an operation and queues it for upload. It performs no API +// call, so upload failures surface from close rather than here; the error +// returned is only about turning the applied resource into a payload. +// +// When an operation for the same resource is already waiting it is replaced +// instead of queued again: DMS keeps one state per resource key, so the later +// operation supersedes the earlier one and a single upload records both. This is +// best effort - only operations that have not been picked up yet are collapsed. +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if q == nil { + return nil + } + + op, err := newRecordedOperation(action, resourceID, state) + if err != nil { + return err + } + + q.mu.Lock() + _, waiting := q.pending[resourceKey] + owned := waiting || q.inflight[resourceKey] + q.pending[resourceKey] = op + q.mu.Unlock() + + if owned { + log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + return nil + } + + q.queue <- resourceKey + return nil +} + +// close drains the queue and returns the first upload error. All callers of +// record must have returned first: record on a closed queue panics. Calling close +// more than once is safe, so callers can defer it and still check the error at a +// specific point. +func (q *operationQueue) close() error { + if q == nil { + return nil + } + + q.mu.Lock() + closed := q.closed + q.closed = true + q.mu.Unlock() + + if !closed { + close(q.queue) + q.wg.Wait() + } + + q.mu.Lock() + defer q.mu.Unlock() + return q.err +} + +func (q *operationQueue) work(ctx context.Context) { + defer q.wg.Done() + + for resourceKey := range q.queue { + // Keep uploading this key until nothing new was recorded for it, instead of + // putting it back on the queue: a worker sending to the channel it consumes + // from can deadlock once the queue is full. + for { + op, ok := q.take(resourceKey) + if !ok { + break + } + + if err := q.uploader.upload(ctx, resourceKey, op); err != nil { + q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) + } + } + } +} + +// take claims the operation waiting for resourceKey, marking the key in flight so +// record does not queue it a second time. It reports false, and releases the key, +// when nothing is waiting. +func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + op, ok := q.pending[resourceKey] + if !ok { + delete(q.inflight, resourceKey) + return recordedOperation{}, false + } + + delete(q.pending, resourceKey) + q.inflight[resourceKey] = true + return op, true +} + +// setErr keeps the first upload error; later ones are dropped because one failure +// is enough to fail the deploy. +func (q *operationQueue) setErr(err error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.err == nil { + q.err = err + } +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go new file mode 100644 index 00000000000..5b267c4d8fa --- /dev/null +++ b/bundle/direct/opqueue_test.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeUploader records the uploads it receives and optionally blocks until +// release is closed, so a test can hold operations in the queue and observe +// coalescing. +type fakeUploader struct { + block chan struct{} + started chan string + err error + + mu sync.Mutex + uploads []string +} + +func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + if f.started != nil { + f.started <- resourceKey + } + if f.block != nil { + <-f.block + } + + f.mu.Lock() + defer f.mu.Unlock() + f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + return f.err +} + +func (f *fakeUploader) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.uploads...) +} + +func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { + t.Helper() + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) +} + +func TestOperationQueueUploadsEachOperation(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + for i := range 20 { + recordState(t, q, "resources.jobs.job"+strconv.Itoa(i), "n") + } + require.NoError(t, q.close()) + + assert.Len(t, f.recorded(), 20) +} + +func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { + // Hold the first upload so later operations for the same resource pile up in + // the queue and are collapsed into one. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + // Wait until a worker owns the key, so the operations below are queued behind + // an in-flight upload rather than racing it. + assert.Equal(t, "resources.jobs.foo", <-f.started) + + recordState(t, q, "resources.jobs.foo", "v2") + recordState(t, q, "resources.jobs.foo", "v3") + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, not three: v2 was superseded by v3 while both were queued, and + // the last recorded state is the one the service ends up with. + assert.Equal(t, []string{ + `resources.jobs.foo={"name":"v1"}`, + `resources.jobs.foo={"name":"v3"}`, + }, f.recorded()) +} + +func TestOperationQueueReturnsUploadError(t *testing.T) { + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + err := q.close() + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + assert.Contains(t, err.Error(), "resources.jobs.foo") +} + +func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + // Serialization failures surface at record time, on the resource that caused + // them, rather than from the drain at the end of apply. + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + require.Error(t, err) + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + +func TestOperationQueueCloseIsIdempotent(t *testing.T) { + f := &fakeUploader{err: errors.New("boom")} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + require.Error(t, q.close()) + // A second close reports the same error instead of panicking on the already + // closed channel, so callers can both defer close and check it explicitly. + require.Error(t, q.close()) +} + +// serialUploader fails if two uploads for the same resource key ever overlap. +type serialUploader struct { + mu sync.Mutex + live map[string]bool + last map[string]string + uneven bool +} + +func (s *serialUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + s.mu.Lock() + if s.live[resourceKey] { + s.uneven = true + } + s.live[resourceKey] = true + s.mu.Unlock() + + s.mu.Lock() + defer s.mu.Unlock() + s.live[resourceKey] = false + s.last[resourceKey] = string(op.state) + return nil +} + +func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { + // Concurrent apply workers repeatedly record overlapping resource keys, the + // case where a coalesced key can be handed to a second worker while the first + // is still uploading it. The service keeps one state per key, so overlapping + // uploads for a key could land out of order and leave a stale state behind. + const ( + workers = 10 + perWorker = 5 + distinctKeyMod = 12 + ) + + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(t.Context(), u) + + var wg sync.WaitGroup + for w := range workers { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + recordState(t, q, key, strconv.Itoa(w)) + } + }() + } + wg.Wait() + require.NoError(t, q.close()) + + assert.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + assert.Len(t, u.last, distinctKeyMod) + assert.Empty(t, q.pending) + assert.Empty(t, q.inflight) +} + +func TestNilOperationQueueIsNoOp(t *testing.T) { + // Recording is disabled: newOperationQueue returns nil and every method is a + // no-op, so Apply does not have to branch. + q := newOperationQueue(t.Context(), nil) + require.Nil(t, q) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.close()) +} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 033aefb5f2f..cdd99966f15 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,25 +12,57 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// opRecorder records a resource operation with the deployment metadata service -// (DMS) after it has been applied to the workspace. state is the serialized -// local config after the operation and must be nil for delete operations. -type opRecorder interface { - record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +// recordedOperation is an applied resource operation, serialized and waiting to be +// uploaded to the deployment metadata service (DMS). +// +// The payload is built on the apply worker rather than in the uploader so the +// queue does not hold on to the live resource struct, and so a malformed state +// fails the resource that produced it instead of the drain at the end of apply. +type recordedOperation struct { + action bundledeployments.OperationActionType + resourceID string + + // state is the serialized local config after the operation. It is nil for a + // delete, where the resource no longer exists. + state json.RawMessage } -// recordOperation reports an applied resource operation to DMS. It is a no-op -// unless the bundle opted into recording deployment history (OpRec is set). -// state is the serialized local config after the operation and must be nil for -// delete operations. -func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - if b.OpRec == nil { - return nil +// newRecordedOperation serializes an applied operation for upload. state is the +// local config after the operation and must be nil for delete operations. +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + op := recordedOperation{action: actionType, resourceID: resourceID} + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. + if state != nil { + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + op.state = raw } - return b.OpRec.record(ctx, resourceKey, action, resourceID, state) + + return op, nil +} + +// operationUploader records an applied resource operation with DMS. Uploads run +// on the operationQueue workers, off the apply path. +type operationUploader interface { + upload(ctx context.Context, resourceKey string, op recordedOperation) error } -// operationRecorder records operations via the DMS CreateOperation API. +// operationRecorder uploads operations via the DMS CreateOperation API. type operationRecorder struct { client bundledeployments.BundleDeploymentsInterface // parent is the version the operations are recorded under, formatted as @@ -38,55 +70,36 @@ type operationRecorder struct { parent string } -// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation -// API. deploymentID and version identify the deployment version assigned by DMS -// that the operations are recorded under. -func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { +// NewOperationRecorder returns an operationUploader backed by the DMS +// CreateOperation API. deploymentID and version identify the deployment version +// assigned by DMS that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) operationUploader { return &operationRecorder{ client: client, parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), } } -func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - actionType, err := deployActionToSDK(action) - if err != nil { - return err - } - +func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). dmsKey := strings.TrimPrefix(resourceKey, "resources.") - op := bundledeployments.Operation{ - ActionType: actionType, - ResourceId: resourceID, + operation := bundledeployments.Operation{ + ActionType: op.action, + ResourceId: op.resourceID, ResourceKey: dmsKey, Status: bundledeployments.OperationStatusOperationStatusSucceeded, } - - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. - // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. - if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) - if err != nil { - return fmt.Errorf("serializing state: %w", err) - } - msg := json.RawMessage(raw) - op.State = &msg + if op.state != nil { + operation.State = &op.state } - _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ Parent: r.parent, ResourceKey: dmsKey, - Operation: op, + Operation: operation, }) return err } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 4c1bbcacda6..aaf3243ec60 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "sync" "testing" "github.com/databricks/cli/bundle/deployplan" @@ -13,20 +14,32 @@ import ( type fakeOpClient struct { bundledeployments.BundleDeploymentsInterface + + mu sync.Mutex requests []bundledeployments.CreateOperationRequest } func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.mu.Lock() + defer f.mu.Unlock() f.requests = append(f.requests, req) return &bundledeployments.Operation{}, nil } +// uploadOne records a single operation through the given uploader, mirroring what +// an operationQueue worker does. +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { + t.Helper() + op, err := newRecordedOperation(action, resourceID, state) + require.NoError(t, err) + require.NoError(t, u.upload(t.Context(), resourceKey, op)) +} + func TestOperationRecorderStripsResourcePrefix(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 2) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) require.Len(t, f.requests, 1) req := f.requests[0] @@ -44,8 +57,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 3) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) require.Len(t, f.requests, 1) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) @@ -53,25 +65,25 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 2) - +func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state) require.NoError(t, err) - require.Len(t, f.requests, 1) - require.NotNil(t, f.requests[0].Operation.State) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, - string(*f.requests[0].Operation.State)) + string(op.state)) +} + +func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + assert.Error(t, err) } func TestDeployActionToSDK(t *testing.T) { @@ -98,9 +110,3 @@ func TestDeployActionToSDK(t *testing.T) { _, err = deployActionToSDK(deployplan.Undefined) assert.Error(t, err) } - -func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { - b := &DeploymentBundle{} - // No OpRec set: recording is a no-op. - assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) -} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index f95b515f726..03864af5da2 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -45,10 +45,12 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache - // OpRec records each applied resource operation with the deployment metadata + // OpRec uploads each applied resource operation to the deployment metadata // service (DMS). It is nil unless the bundle opts into recording deployment // history, in which case the phases package sets it after CreateVersion. - OpRec opRecorder + // Apply queues the operations and drains them before returning, so the + // uploads do not block the resources being deployed. + OpRec operationUploader } // SetRemoteState updates the remote state with type validation and marks as fresh. From 19467dc2425fd093f06e9d11773ca724ea3cb94c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:52:34 +0000 Subject: [PATCH 05/56] bundle: gate experimental.record_deployment_history behind an env var Recording deployment history is implemented end to end, but it cannot be exposed to users yet: enabling it makes the deployment metadata service the source of truth for resource state, and there is no upgrade path from an existing direct-engine state file to a DMS-owned one. Setting the flag is now an error. DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error so the CLI's own acceptance tests and DMS development can exercise the feature until the direct state upgrade lands. Co-authored-by: Isaac --- .../bundle/dms/not-supported/databricks.yml | 10 ++++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++++++ acceptance/bundle/dms/not-supported/script | 5 ++ acceptance/bundle/dms/not-supported/test.toml | 5 ++ acceptance/bundle/dms/test.toml | 6 ++ bundle/config/experimental.go | 4 ++ .../validate_record_deployment_history.go | 48 ++++++++++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 +++++++ bundle/internal/schema/annotations.yml | 2 + bundle/phases/initialize.go | 5 ++ bundle/schema/jsonschema.json | 2 +- 13 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..ff5758d574d --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden opt-in lifts the error +>>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..68c0a3c5f3e --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" +trace musterr $CLI bundle validate + +title "The hidden opt-in lifts the error" +trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..c6daad089b2 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,5 @@ +# Unset the opt-in inherited from the parent: this test asserts the error users see. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..8c12d014fea 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,3 +11,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected until the direct engine has a +# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests +# exercise the feature itself, so they opt in through the same escape hatch DMS +# development uses. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..56cf3486ed3 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,10 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Setting this is currently an error: the direct engine needs a state upgrade + // path before DMS can own resource state. See + // validate.ValidateRecordDeploymentHistory. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..10144d9c796 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,48 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but it is not usable yet: +// enabling it makes the deployment metadata service the source of truth for resource +// state, and there is no upgrade path from an existing direct-engine state file to a +// DMS-owned one. A bundle that flips the flag on today would have its local state +// silently overlaid by an empty DMS resource set, and the next deploy would try to +// create resources that already exist. The direct state upgrade has to land before +// this flag can be exposed; until then it errors, and +// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own +// tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.EnableRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..06f71d10afa --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + optIn string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, + {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, + {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go new file mode 100644 index 00000000000..e17fdeb9f8d --- /dev/null +++ b/bundle/env/record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the +// error on experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" + +// EnableRecordDeploymentHistory reports whether the environment opts into +// experimental.record_deployment_history despite it being gated off. +func EnableRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + EnableRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..ed4420cbe15 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,6 +178,8 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. + + This setting is not supported yet and enabling it is an error. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..02e03837603 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Rejects experimental.record_deployment_history until the direct state upgrade lands. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..1752e643769 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", "$ref": "#/$defs/bool" }, "scripts": { From 22ec66a4cc6b3a792b4943e7360bc787144edf47 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:08:45 +0000 Subject: [PATCH 06/56] bundle: rename the record_deployment_history escape hatch to force_allow DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY read as if it were the switch that turns the feature on. It is not: the flag in databricks.yml does that, and this variable only permits the flag to be set while the feature is gated off. Name it DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY. Co-authored-by: Isaac --- .../bundle/dms/not-supported/output.txt | 4 ++-- acceptance/bundle/dms/not-supported/script | 4 ++-- acceptance/bundle/dms/not-supported/test.toml | 5 +++-- acceptance/bundle/dms/test.toml | 6 +++--- .../validate_record_deployment_history.go | 6 +++--- ...validate_record_deployment_history_test.go | 16 ++++++++-------- .../force_allow_record_deployment_history.go | 19 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 ------------------- bundle/phases/initialize.go | 2 +- 9 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 bundle/env/force_allow_record_deployment_history.go delete mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt index ff5758d574d..0237a81da27 100644 --- a/acceptance/bundle/dms/not-supported/output.txt +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -13,8 +13,8 @@ Workspace: Found 1 error -=== The hidden opt-in lifts the error ->>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate Name: dms-not-supported Target: default Workspace: diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script index 68c0a3c5f3e..3bf017b50a0 100644 --- a/acceptance/bundle/dms/not-supported/script +++ b/acceptance/bundle/dms/not-supported/script @@ -1,5 +1,5 @@ title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" trace musterr $CLI bundle validate -title "The hidden opt-in lifts the error" -trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml index c6daad089b2..4617ff88f20 100644 --- a/acceptance/bundle/dms/not-supported/test.toml +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -1,5 +1,6 @@ -# Unset the opt-in inherited from the parent: this test asserts the error users see. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" # This test only checks validation output; no DMS request is made either way. RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 8c12d014fea..9942a441539 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -14,6 +14,6 @@ Ignore = [ # experimental.record_deployment_history is rejected until the direct engine has a # state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they opt in through the same escape hatch DMS -# development uses. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" +# exercise the feature itself, so they force allow it the same way DMS development +# does. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go index 10144d9c796..4ebb681d595 100644 --- a/bundle/config/validate/validate_record_deployment_history.go +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -30,13 +30,13 @@ func (v *validateRecordDeploymentHistory) Name() string { // silently overlaid by an empty DMS resource set, and the next deploy would try to // create resources that already exist. The direct state upgrade has to land before // this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own -// tests and for DMS development. +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil } - if env.EnableRecordDeploymentHistory(ctx) { + if env.ForceAllowRecordDeploymentHistory(ctx) { return nil } return diag.Diagnostics{{ diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go index 06f71d10afa..1bb172766f2 100644 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -14,16 +14,16 @@ import ( func TestValidateRecordDeploymentHistory(t *testing.T) { tests := []struct { - name string - enabled bool - optIn string - wantError bool + name string + enabled bool + forceAllow string + wantError bool }{ {name: "flag unset", enabled: false, wantError: false}, {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, - {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, - {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, } for _, tc := range tests { @@ -34,7 +34,7 @@ func TestValidateRecordDeploymentHistory(t *testing.T) { }, } - ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) diags := ValidateRecordDeploymentHistory().Apply(ctx, b) if !tc.wantError { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go deleted file mode 100644 index e17fdeb9f8d..00000000000 --- a/bundle/env/record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the -// error on experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" - -// EnableRecordDeploymentHistory reports whether the environment opts into -// experimental.record_deployment_history despite it being gated off. -func EnableRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - EnableRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 02e03837603..60ea68fab82 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -178,7 +178,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.ValidateDeploymentFields(), // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) // Rejects experimental.record_deployment_history until the direct state upgrade lands. validate.ValidateRecordDeploymentHistory(), From d7441e43355eae51c355c07f10b52f84fdf00984 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:45:13 +0000 Subject: [PATCH 07/56] bundle: only record net-new deployments in DMS Replace the blanket gate on experimental.record_deployment_history with a narrower check in dstate.DeploymentState.Open: recording is refused only when the state file already tracks deployed resources that DMS does not know about. Once DMS holds a successful version it is authoritative for resource state even when its resource set is empty, so adopting a state file written by a CLI that predates DMS would make already-owned resources look absent and create them a second time. A state file that DMS already owns is fine, and so is one with no resources (e.g. left behind by a destroy), which is what makes the error's destroy-and-redeploy advice work. The check keys off len(State) rather than the file existing because destroy leaves resources.json in place with an empty resource set. This drops validate.ValidateRecordDeploymentHistory and the DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, which are no longer needed. Upgrading an existing state in place (state v3 with a feature flag plus per-resource tombstones so older clients refuse the state) is left as a TODO. Co-authored-by: Isaac --- .../databricks.yml | 4 +- .../out.test.toml | 0 .../bundle/dms/existing-state/output.txt | 45 +++++++++++++++ acceptance/bundle/dms/existing-state/script | 17 ++++++ .../bundle/dms/not-supported/output.txt | 24 -------- acceptance/bundle/dms/not-supported/script | 5 -- acceptance/bundle/dms/not-supported/test.toml | 6 -- acceptance/bundle/dms/test.toml | 6 -- bundle/config/experimental.go | 6 +- .../validate_record_deployment_history.go | 48 ---------------- ...validate_record_deployment_history_test.go | 55 ------------------- bundle/direct/dstate/state.go | 30 +++++++++- .../force_allow_record_deployment_history.go | 19 ------- bundle/internal/schema/annotations.yml | 2 +- bundle/phases/initialize.go | 5 -- bundle/schema/jsonschema.json | 2 +- 16 files changed, 96 insertions(+), 178 deletions(-) rename acceptance/bundle/dms/{not-supported => existing-state}/databricks.yml (52%) rename acceptance/bundle/dms/{not-supported => existing-state}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/existing-state/output.txt create mode 100644 acceptance/bundle/dms/existing-state/script delete mode 100644 acceptance/bundle/dms/not-supported/output.txt delete mode 100644 acceptance/bundle/dms/not-supported/script delete mode 100644 acceptance/bundle/dms/not-supported/test.toml delete mode 100644 bundle/config/validate/validate_record_deployment_history.go delete mode 100644 bundle/config/validate/validate_record_deployment_history_test.go delete mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/existing-state/databricks.yml similarity index 52% rename from acceptance/bundle/dms/not-supported/databricks.yml rename to acceptance/bundle/dms/existing-state/databricks.yml index c6edca465b6..cfd64979342 100644 --- a/acceptance/bundle/dms/not-supported/databricks.yml +++ b/acceptance/bundle/dms/existing-state/databricks.yml @@ -1,8 +1,8 @@ bundle: - name: dms-not-supported + name: dms-existing-state experimental: - record_deployment_history: true + record_deployment_history: false resources: jobs: diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml similarity index 100% rename from acceptance/bundle/dms/not-supported/out.test.toml rename to acceptance/bundle/dms/existing-state/out.test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..d0cae5fb0ec --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,45 @@ + +=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +=== No deployment was created in DMS +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Destroy clears the tracked resources, so recording can be enabled afterwards +>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default + +Deleting files... +Destroy complete! + +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..7bc296465d5 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,17 @@ +title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace musterr $CLI bundle deploy + +title "No deployment was created in DMS" +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Destroy clears the tracked resources, so recording can be enabled afterwards" +trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" +trace $CLI bundle destroy --auto-approve +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt deleted file mode 100644 index 0237a81da27..00000000000 --- a/acceptance/bundle/dms/not-supported/output.txt +++ /dev/null @@ -1,24 +0,0 @@ - -=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path ->>> musterr [CLI] bundle validate -Error: experimental.record_deployment_history is not supported yet - at experimental.record_deployment_history - in databricks.yml:5:30 - -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Found 1 error - -=== The hidden force-allow variable permits it ->>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script deleted file mode 100644 index 3bf017b50a0..00000000000 --- a/acceptance/bundle/dms/not-supported/script +++ /dev/null @@ -1,5 +0,0 @@ -title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" -trace musterr $CLI bundle validate - -title "The hidden force-allow variable permits it" -trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml deleted file mode 100644 index 4617ff88f20..00000000000 --- a/acceptance/bundle/dms/not-supported/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# Unset the force-allow variable inherited from the parent: this test asserts the -# error users see. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" - -# This test only checks validation output; no DMS request is made either way. -RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 9942a441539..24ce9756629 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,9 +11,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# experimental.record_deployment_history is rejected until the direct engine has a -# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they force allow it the same way DMS development -# does. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 56cf3486ed3..c3f1465d880 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,9 +54,9 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Setting this is currently an error: the direct engine needs a state upgrade - // path before DMS can own resource state. See - // validate.ValidateRecordDeploymentHistory. + // Only supported for a bundle with no deployed resources yet: DMS becomes the + // source of truth for resource state, and resources tracked in an existing + // state file cannot be handed over to it yet. See dstate.DeploymentState.Open. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go deleted file mode 100644 index 4ebb681d595..00000000000 --- a/bundle/config/validate/validate_record_deployment_history.go +++ /dev/null @@ -1,48 +0,0 @@ -package validate - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -const recordDeploymentHistoryPath = "experimental.record_deployment_history" - -func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { - return &validateRecordDeploymentHistory{} -} - -type validateRecordDeploymentHistory struct{ bundle.RO } - -func (v *validateRecordDeploymentHistory) Name() string { - return "validate:validate_record_deployment_history" -} - -// Apply rejects experimental.record_deployment_history. -// -// Recording deployment history is implemented end to end, but it is not usable yet: -// enabling it makes the deployment metadata service the source of truth for resource -// state, and there is no upgrade path from an existing direct-engine state file to a -// DMS-owned one. A bundle that flips the flag on today would have its local state -// silently overlaid by an empty DMS resource set, and the next deploy would try to -// create resources that already exist. The direct state upgrade has to land before -// this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's -// own tests and for DMS development. -func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil - } - if env.ForceAllowRecordDeploymentHistory(ctx) { - return nil - } - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: recordDeploymentHistoryPath + " is not supported yet", - Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, - Locations: b.Config.GetLocations(recordDeploymentHistoryPath), - }} -} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go deleted file mode 100644 index 1bb172766f2..00000000000 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - bundleenv "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/env" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValidateRecordDeploymentHistory(t *testing.T) { - tests := []struct { - name string - enabled bool - forceAllow string - wantError bool - }{ - {name: "flag unset", enabled: false, wantError: false}, - {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, - {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, - {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, - }, - } - - ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) - diags := ValidateRecordDeploymentHistory().Apply(ctx, b) - - if !tc.wantError { - assert.Empty(t, diags) - return - } - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) - assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) - }) - } -} - -func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { - b := &bundle.Bundle{Config: config.Root{}} - assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) -} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f554af3c9b6..e5eb44df24a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -319,9 +319,33 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { - return err + if dmsClient != nil { + // Only deployments that start out empty are recorded in DMS. Resources + // tracked in a state file that DMS does not know about are not in DMS and + // never will be: the first recorded deploy would create a deployment whose + // resource set covers only what that deploy touched, and DMS would then be + // authoritative for everything (see overlayDMSState). Resources this bundle + // already owns would look absent and be created a second time. + // + // A state file that DMS already owns (it carries a deployment ID) is fine — + // that is a bundle that opted in while it was still empty. So is a state file + // with no resources, e.g. one left behind by a destroy. + // + // TODO(DMS): lift this restriction by upgrading an existing state in place. + // That means writing the state at featureStateVersion (3) with a feature flag + // recording that DMS owns it, plus a tombstone entry per resource so a CLI + // that predates DMS refuses the state instead of silently deploying against a + // resource set it cannot see. The feature-flag scaffolding for this already + // exists (see featureStateVersion and Header.Features); once it is written, + // this check goes away and record_deployment_history becomes usable on + // existing bundles. + if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + } + if db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + return err + } } } diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go deleted file mode 100644 index 297ccb6f6e3..00000000000 --- a/bundle/env/force_allow_record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force -// allows experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" - -// ForceAllowRecordDeploymentHistory reports whether the environment force allows -// experimental.record_deployment_history despite it being gated off. -func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - ForceAllowRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ed4420cbe15..100d33356fd 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -179,7 +179,7 @@ experimental: "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - This setting is not supported yet and enabling it is an error. + Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 60ea68fab82..bfa2af4124b 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,11 +177,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) - // Rejects experimental.record_deployment_history until the direct state upgrade lands. - validate.ValidateRecordDeploymentHistory(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 1752e643769..c52e96d5dc9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", "$ref": "#/$defs/bool" }, "scripts": { From 026a5213943453ad2a35f47420cefa7503e8f34a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:04:51 +0000 Subject: [PATCH 08/56] bundle: fix lint errors in the operation queue test The concurrent-producer test tripped three linters: - modernize/revive want wg.Go instead of wg.Add + go func + defer wg.Done. - testifylint's go-require flags recordState, which calls require inside the spawned goroutines. testify assertions may only run on the goroutine running the test function. Record inline and send each error to a buffered channel that the test goroutine drains after wg.Wait, so the assertions stay on the test goroutine. Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 5b267c4d8fa..1b818eb3ba1 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -158,21 +158,27 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { distinctKeyMod = 12 ) + ctx := t.Context() u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(t.Context(), u) + q := newOperationQueue(ctx, u) + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) var wg sync.WaitGroup for w := range workers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - recordState(t, q, key, strconv.Itoa(w)) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) } - }() + }) } wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } require.NoError(t, q.close()) assert.False(t, u.uneven, "two uploads overlapped for the same resource key") From 481259bd95019be6463ef77c2a34134767d1b428 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:24:50 +0000 Subject: [PATCH 09/56] bundle: keep the create action when coalescing DMS operations The operation queue collapses repeated writes to the same resource key by replacing the queued operation wholesale, so a create followed by an update was uploaded as an update. That tells DMS the resource already existed before this deploy, when in fact this deploy created it. Merge the actions instead: the state uploaded is still the later one, but a queued create or recreate wins over a subsequent update. A delete still wins over anything queued before it, since the resource is gone. This is not reachable from Apply today - each resource is recorded once per deploy, because there is one record call per graph node and dagrun visits each node exactly once - so this is about the queue being correct for any caller that records a resource more than once. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 11 ++++++--- bundle/direct/opqueue_test.go | 41 ++++++++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 18 ++++++++++++++ bundle/direct/oprecorder_test.go | 30 +++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 0804ad2d5b3..72a2fa9c39e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -91,8 +91,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // // When an operation for the same resource is already waiting it is replaced // instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. This is -// best effort - only operations that have not been picked up yet are collapsed. +// operation supersedes the earlier one and a single upload records both. The +// merged operation keeps the action of a queued create (see mergeAction), so +// collapsing a create and a later update still records a create. This is best +// effort - only operations that have not been picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -104,8 +106,11 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - _, waiting := q.pending[resourceKey] + queued, waiting := q.pending[resourceKey] owned := waiting || q.inflight[resourceKey] + if waiting { + op.action = mergeAction(queued.action, op.action) + } q.pending[resourceKey] = op q.mu.Unlock() diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 1b818eb3ba1..a8e05141fc7 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,6 +23,7 @@ type fakeUploader struct { mu sync.Mutex uploads []string + actions map[string]bundledeployments.OperationActionType } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -35,6 +37,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.mu.Lock() defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + if f.actions == nil { + f.actions = map[string]bundledeployments.OperationActionType{} + } + f.actions[resourceKey] = op.action return f.err } @@ -44,6 +50,12 @@ func (f *fakeUploader) recorded() []string { return append([]string(nil), f.uploads...) } +func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { + f.mu.Lock() + defer f.mu.Unlock() + return f.actions[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) @@ -86,6 +98,35 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } +func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { + // Hold the first upload so the create below stays queued and the update + // coalesces into it. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.hold", "v1") + assert.Equal(t, "resources.jobs.hold", <-f.started) + + // Occupy the remaining workers so nothing drains the key under test. + for i := range operationUploadWorkers - 1 { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + + close(f.block) + require.NoError(t, q.close()) + + // The state is the later one, but the action stays CREATE: recording an update + // would tell DMS the resource already existed before this deploy. + assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeCreate, + f.actionFor("resources.jobs.foo")) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index cdd99966f15..63d0c84a621 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -56,6 +56,24 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } +// mergeAction returns the action to record when a later operation coalesces into +// one still queued for the same resource (see operationQueue.record). The state +// uploaded is the later one, but the action must not be downgraded: Create and +// Recreate tell DMS the resource ID is new, and a subsequent Update only refines +// the state of that same new resource. Recording the pair as an Update would +// claim the resource already existed. A Delete is the exception - the resource is +// gone, so nothing earlier is worth reporting. +func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { + if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { + return next + } + if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || + queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { + return queued + } + return next +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index aaf3243ec60..70afb788cd3 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -86,6 +86,36 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } +func TestMergeAction(t *testing.T) { + const ( + create = bundledeployments.OperationActionTypeOperationActionTypeCreate + recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate + update = bundledeployments.OperationActionTypeOperationActionTypeUpdate + resize = bundledeployments.OperationActionTypeOperationActionTypeResize + del = bundledeployments.OperationActionTypeOperationActionTypeDelete + ) + + cases := []struct { + queued, next, want bundledeployments.OperationActionType + }{ + // A queued create is not downgraded: the resource is still new. + {create, update, create}, + {create, resize, create}, + {recreate, update, recreate}, + {create, create, create}, + // A delete wins: the resource is gone, so the earlier action is moot. + {create, del, del}, + {update, del, del}, + // Neither side is a create, so the later action stands. + {update, resize, resize}, + {resize, update, update}, + {del, create, create}, + } + for _, c := range cases { + assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) + } +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 309a1a4fe5098bd122103ba880b862ac5d65f864 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 22:32:16 +0000 Subject: [PATCH 10/56] bundle: drop the dms package comment Co-authored-by: Isaac --- libs/dms/recorder.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 0d9563dd052..1a113491cf9 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -1,12 +1,3 @@ -// Package dms records bundle deployments as versions with the Deployment -// Metadata Service (DMS). -// -// It is intentionally independent of the deployment lock: a Recorder does not -// acquire or hold any lock. Callers are responsible for serializing concurrent -// deployments (today via the workspace-filesystem lock). The server-side -// version counter — CreateVersion only succeeds when the requested version is -// last_version_id + 1 — provides the concurrency control for the records -// themselves. package dms import ( From 211aa94a2b5e00107b2bc6231192f1d9c3c7df88 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 23:33:49 +0000 Subject: [PATCH 11/56] bundle: resolve the DMS deployment ID from the workspace The deployment ID was persisted in the local state file header, which made the CLI the source of truth for a value the service mints. DMS registers each deployment as a workspace node named resources.deployment.json under initial_parent_path, and the node's ID *is* the deployment ID, so it can be resolved from the workspace instead. ResolveDeploymentID does a get-status on /resources.deployment.json and returns the node ID, or empty when the node is absent. Deploy, destroy, and the read path all resolve the ID that way and pass it down; the read path then constructs state from GetDeployment + ListResources as before. Consequences: - Header.DeploymentID, GetDeploymentID, and SetDeploymentID are gone, along with the headerDirty machinery that only existed to persist the ID on a resource-less deploy. - Open takes a *DMSSource instead of a (client, config) pair, since the resolved ID now has to be threaded in too. - CreateDeployment sets initial_parent_path, which the service requires and the CLI never set. - createDeploymentVersion no longer recovers from a 404 on GetDeployment by creating a second deployment. A destroy trashes the node, so a resolved ID whose record is missing means the two are out of sync, and creating another deployment would collide on the same node path. The testserver models the real derivation: CreateDeployment creates the workspace node and uses its object ID as the deployment ID, so the acceptance tests exercise get-status resolution end to end. dms/record now wipes the local cache before redeploying and records zero operations, which is the read path reconstructing state entirely from DMS. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 8 +- .../bundle/dms/multiple-resources/output.txt | 14 +-- acceptance/bundle/dms/no-resources/output.txt | 28 +++--- acceptance/bundle/dms/no-resources/script | 6 +- acceptance/bundle/dms/record/output.txt | 33 ++++--- acceptance/bundle/dms/record/script | 7 +- .../dms/redeploy-after-destroy/output.txt | 31 +++---- .../bundle/dms/redeploy-after-destroy/script | 12 +-- acceptance/bundle/dms/test.toml | 5 +- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +-- bundle/direct/dstate/dms.go | 16 ++-- bundle/direct/dstate/state.go | 91 ++++++------------- bundle/direct/dstate/state_test.go | 87 +++--------------- bundle/phases/deploy.go | 12 ++- bundle/phases/destroy.go | 6 +- bundle/phases/dms.go | 29 +++--- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 29 +++--- libs/dms/recorder.go | 73 ++++++++------- libs/dms/recorder_test.go | 83 ++++++++--------- libs/dms/resolve.go | 45 +++++++++ libs/dms/resolve_test.go | 67 ++++++++++++++ libs/testserver/bundle.go | 47 ++++++++-- 26 files changed, 406 insertions(+), 343 deletions(-) create mode 100644 libs/dms/resolve.go create mode 100644 libs/dms/resolve_test.go diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index d0cae5fb0ec..6aeff83c04b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -39,7 +39,7 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 4bacec0c317..75086ceb5f7 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -7,11 +7,11 @@ Updating deployment state... Deployment complete! >>> print_requests.py //versions/1/operations --sort --del-body state --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index e774fd7546c..73bd3adfa11 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -1,9 +1,8 @@ -=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get @@ -11,12 +10,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,38 +28,40 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" +} -=== Redeploy: the persisted ID is reused, so no second deployment is created +=== Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -71,7 +73,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 4bc97c5864d..847935af5d1 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ -title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace jq .deployment_id .databricks/bundle/default/resources.json +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' -title "Redeploy: the persisted ID is reused, so no second deployment is created" +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 5c0317f38cc..ed10f06e699 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -11,12 +11,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,14 +29,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -60,11 +61,17 @@ Deployment complete! } } -=== The server-assigned deployment ID is persisted in the local state file ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +=== The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" +} + +>>> jq has("deployment_id") .databricks/bundle/default/resources.json +false -=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +=== Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... Deploying resources... @@ -74,7 +81,7 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -86,7 +93,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -105,11 +112,11 @@ Destroy complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "3" }, @@ -121,14 +128,14 @@ Destroy complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", "q": { "resource_key": "jobs.foo" }, diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index ab59d38afb4..63eaa323b1b 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -2,10 +2,11 @@ title "Deploy: the server assigns the deployment ID, and a version + create oper trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort -title "The server-assigned deployment ID is persisted in the local state file" -trace jq .deployment_id .databricks/bundle/default/resources.json +title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 5326653519a..ed7b6ede989 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -1,5 +1,5 @@ -=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +=== Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... @@ -15,35 +15,34 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[DESTROYED_DEPLOYMENT_ID]" +>>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. -=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +=== Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" -} +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } + +>>> print_requests.py //api/2.0/bundle --sort --get { "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -55,14 +54,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -86,7 +85,3 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } - -=== The new deployment ID replaces the stale one in state ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 628cd7bf494..04c639019c9 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,15 +1,11 @@ -title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) -add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID -trace jq .deployment_id .databricks/bundle/default/resources.json +trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" -title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get - -title "The new deployment ID replaces the stale one in state" -trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -1,9 +1,8 @@ Local = true Cloud = false -# Deployment Metadata Service (DMS) recording is only meaningful in the direct -# engine, where the deployment ID is stored in and read from the direct-engine -# state. +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 17ed3b30d5e..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index be3e536f37f..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ccfbcf788ab..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e09cb859221..0941d87d35c 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -19,13 +19,10 @@ import ( // deployment. Once DMS is authoritative its resource set is trusted even when // empty (a successful deploy with no resources); the file's resources are only // used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu and has already -// populated db.Data from the file, including the DeploymentID. -// -// cfg is threaded in only for the temporary raw read in -// deploymentHasSuccessfulVersion; see the TODO there. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) +// recording deployment history. The caller holds db.mu, has already populated +// db.Data from the file, and has resolved src.DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { return err } @@ -35,7 +32,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) if err != nil { return err } @@ -63,8 +60,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // because last_successful_version_id is still stage:DEVELOPMENT in the proto // and therefore stripped from the generated SDK. Once the field is promoted to // PRIVATE_PREVIEW and regenerated, replace the raw call with -// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument -// (revert overlayDMSState/Open back to taking only the typed client). +// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) if err != nil { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e5eb44df24a..c27de3ca44d 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,11 +74,6 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string - - // headerDirty records that a header field changed in memory during this - // deployment (today only the DMS deployment ID), so the state file must be - // written on Finalize even when the WAL carried no resource entries. - headerDirty bool } type Header struct { @@ -87,13 +82,6 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` - // DeploymentID is the ID the deployment metadata service (DMS) assigned to - // this deployment. Unlike Lineage (a locally generated identifier for the - // state file), it is minted server-side by CreateDeployment and stored here so - // later deploys can find the same DMS deployment record and read its state. - // Empty/omitted until the bundle first records to DMS. - DeploymentID string `json:"deployment_id,omitempty"` - // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -223,33 +211,6 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } -// GetDeploymentID returns the DMS deployment ID recorded in the state, or an -// empty string if this bundle has not yet recorded a deployment to DMS. -func (db *DeploymentState) GetDeploymentID() string { - db.mu.Lock() - defer db.mu.Unlock() - return db.Data.DeploymentID -} - -// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state -// header. It is set during deploy, after CreateDeployment returns the -// server-generated ID, and persisted to the state file by Finalize. Storing it -// on db.Data (not the WAL header, which is written before the ID is known) -// means the subsequent state write carries it forward. -// -// The header is marked dirty so Finalize persists it even when the deploy wrote -// no resource entries; otherwise a bundle with no resources would mint a fresh -// deployment record on every deploy, leaking one orphan per run. -func (db *DeploymentState) SetDeploymentID(id string) { - db.mu.Lock() - defer db.mu.Unlock() - if db.Data.DeploymentID == id { - return - } - db.Data.DeploymentID = id - db.headerDirty = true -} - type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -259,19 +220,32 @@ type ( WithWrite bool ) +// DMSSource tells Open to read resource state from the deployment metadata +// service instead of the state file. A nil *DMSSource keeps Open file-only. +type DMSSource struct { + // Client is the DMS client used to list the deployment's resources. + Client bundledeployments.BundleDeploymentsInterface + + // Config accompanies Client (both come from the same workspace client) and is + // used only for a temporary raw read of last_successful_version_id; see the + // TODO in deploymentHasSuccessfulVersion. + Config *sdkconfig.Config + + // DeploymentID identifies the deployment in DMS, resolved from the + // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a + // bundle that has not recorded a deployment yet. + DeploymentID string +} + // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// withRecovery is set). When dmsSource is non-nil, the deployment metadata // service is the source of truth for resource state: if DMS holds a // successfully completed version for this deployment, the resources read from // the file are replaced with the ones recorded in DMS. The local identity -// (lineage, serial, and deployment ID) always comes from the file, since that -// is what the write path increments and carries forward. A nil dmsClient keeps -// the behavior file-only. -// -// dmsCfg accompanies dmsClient (both come from the same workspace client) and -// is used only for a temporary raw read of last_successful_version_id; see the -// TODO in deploymentHasSuccessfulVersion. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { +// (lineage and serial) always comes from the file, since that is what the write +// path increments and carries forward. A nil dmsSource keeps the behavior +// file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -319,7 +293,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil { + if dmsSource != nil { // Only deployments that start out empty are recorded in DMS. Resources // tracked in a state file that DMS does not know about are not in DMS and // never will be: the first recorded deploy would create a deployment whose @@ -327,9 +301,9 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // authoritative for everything (see overlayDMSState). Resources this bundle // already owns would look absent and be created a second time. // - // A state file that DMS already owns (it carries a deployment ID) is fine — - // that is a bundle that opted in while it was still empty. So is a state file - // with no resources, e.g. one left behind by a destroy. + // A deployment DMS already owns (deploymentID is non-empty) is fine — that is + // a bundle that opted in while it was still empty. So is a state file with no + // resources, e.g. one left behind by a destroy. // // TODO(DMS): lift this restriction by upgrading an existing state in place. // That means writing the state at featureStateVersion (3) with a feature flag @@ -339,11 +313,11 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // exists (see featureStateVersion and Header.Features); once it is written, // this check goes away and record_deployment_history becomes usable on // existing bundles. - if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - if db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + if dmsSource.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsSource); err != nil { return err } } @@ -488,12 +462,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - hasEntries := lineNumber > 1 - - // A header-only WAL still has to be persisted when a header field changed in - // memory during this deployment (the DMS deployment ID): dropping the write - // would lose the ID and make the next deploy create a second deployment. - persist := hasEntries || db.headerDirty + persist := lineNumber > 1 // Only advance the serial when the state file is actually written, because // the caller (replayWAL) persists it only in that case. A header-only WAL diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 6530ca049d9..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,91 +20,30 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } -func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - assert.Empty(t, db.GetDeploymentID()) - - // The deployment ID is set during deploy (after CreateDeployment) and - // persisted by Finalize even though it is not part of the WAL header. - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - mustFinalize(t, &reopened) -} - func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) assert.ErrorIs(t, err, os.ErrNotExist) } -func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - // A bundle with no resources writes no WAL entries, but the deployment ID - // still has to be persisted: otherwise the next deploy sees no ID and creates - // a second deployment record, leaking one per deploy. - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - assert.Equal(t, 1, reopened.Data.Serial) - mustFinalize(t, &reopened) -} - -func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - before, err := os.ReadFile(path) - require.NoError(t, err) - - // Re-setting the same ID is not a header change, so a deploy that commits - // nothing must not bump the serial (see mergeWalIntoState). - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) - reopened.SetDeploymentID("server-assigned-id") - mustFinalize(t, &reopened) - - after, err := os.ReadFile(path) - require.NoError(t, err) - assert.Equal(t, string(before), string(after)) -} - func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ @@ -154,10 +93,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -168,12 +107,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -189,7 +128,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -232,17 +171,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -254,7 +193,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -271,7 +210,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 792c016f963..3d70a218b15 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -169,7 +169,11 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. // CompleteVersion is deferred before lock.Release so it runs while the lock // is still held (defers run last-in-first-out). - recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) @@ -276,11 +280,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if recorder != nil { - // On a first deploy the server assigned the deployment ID; persist it in - // state (Finalize writes it to disk) so later deploys reuse the record. // Record operations under the version just created so DMS holds the - // deployed resource state. - b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + // deployed resource state. On a first deploy the deployment ID was only + // assigned by CreateVersion above, so this must come after it. b.DeploymentBundle.OpRec = direct.NewOperationRecorder( b.WorkspaceClient(ctx).BundleDeployments, recorder.DeploymentID(), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 244f593476f..2925e80bca8 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -137,7 +137,11 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // created until the destroy is approved (below), so a cancelled destroy // records nothing; the deferred CompleteVersion is a no-op until then. It is // deferred before lock.Release so it runs while the lock is still held. - recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 667ef8627aa..02254237595 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -14,24 +14,31 @@ import ( // // Recording is enabled only when experimental.record_deployment_history is set // AND the engine is direct: DMS resource state is tracked per direct-engine -// deployment, and only the direct engine opens the state DB where the -// deployment ID is stored. Returning nil for terraform leaves those deployments -// untouched. +// deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID passed to the recorder is the one persisted in state from a -// previous deploy; it is empty on a bundle's first recorded deploy, in which -// case the recorder creates the deployment and the server assigns the ID. -func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { +// The deployment ID is resolved from the workspace rather than from local state +// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment +// lock has been acquired, so it observes any deployment a concurrent deploy +// created. It is empty on a bundle's first recorded deploy, in which case the +// recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil + return nil, nil } if !eng.IsDirect() { - return nil + return nil, nil + } + + statePath := b.Config.Workspace.StatePath + deploymentID, err := dms.ResolveDeploymentID(ctx, b.WorkspaceClient(ctx), statePath) + if err != nil { + return nil, err } return dms.NewRecorder( b.WorkspaceClient(ctx).BundleDeployments, - b.DeploymentBundle.StateDB.GetDeploymentID(), + deploymentID, + statePath, b.Config.Bundle.Target, versionType, - ) + ), nil } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 4866f27c5b3..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index b5dbeed6c56..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f556c2b3450..e9840188db4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -20,13 +20,12 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -215,18 +214,24 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open its client to overlay DMS - // state on top of the local identity (lineage/serial/deployment ID). - // Reads open the state write-disabled, so no lineage is minted here. - // dmsCfg accompanies the client for a temporary raw read (see the TODO - // in dstate.deploymentHasSuccessfulVersion). - var dmsClient bundledeployments.BundleDeploymentsInterface - var dmsCfg *sdkconfig.Config + // service owns resource state, so hand Open a DMS source to overlay that + // state on top of the local identity (lineage/serial). Reads open the + // state write-disabled, so no lineage is minted here. + var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { - dmsClient = b.WorkspaceClient(ctx).BundleDeployments - dmsCfg = b.WorkspaceClient(ctx).Config + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + dmsSource = &dstate.DMSSource{ + Client: w.BundleDeployments, + Config: w.Config, + DeploymentID: deploymentID, + } } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 1a113491cf9..6fa9a1a24be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -30,13 +30,15 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // // The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID persisted in state (empty on a bundle's first-ever recorded -// deploy), and CreateVersion creates the deployment record when that ID is -// empty and exposes the server-assigned ID via DeploymentID so the caller can -// persist it. Later deploys pass the stored ID back in and reuse the record. +// is given the ID resolved from the workspace (empty on a bundle's first-ever +// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the +// deployment record when that ID is empty. Later deploys resolve the same ID +// from the deployment's workspace node and reuse the record; a destroy deletes +// the record and its node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string + statePath string targetName string versionType VersionType @@ -46,12 +48,15 @@ type Recorder struct { } // NewRecorder returns a Recorder for the given deployment. deploymentID is the -// DMS deployment ID persisted in state, or empty if this bundle has not yet -// recorded a deployment (the server assigns one during CreateVersion). -func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { +// ID resolved from the deployment's workspace node, or empty if this bundle has +// not yet recorded a deployment (the server assigns one during CreateVersion). +// statePath is the bundle's remote state directory, under which DMS registers +// the deployment node. +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, statePath, targetName string, versionType VersionType) *Recorder { return &Recorder{ svc: svc, deploymentID: deploymentID, + statePath: statePath, targetName: targetName, versionType: versionType, } @@ -59,8 +64,7 @@ func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, // DeploymentID returns the DMS deployment ID this recorder is bound to. It is // empty until CreateVersion has created the deployment record (on a first -// deploy) and non-empty afterwards, so callers persist it once CreateVersion -// succeeds. +// deploy) and non-empty afterwards, so callers can parent operations under it. func (r *Recorder) DeploymentID() string { if r == nil { return "" @@ -140,41 +144,40 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. When no deployment ID is stored, or the stored one no -// longer exists in DMS, it creates the deployment and lets the server assign the -// ID; otherwise it reads the existing deployment to compute the next version -// number. +// new version under it. With no deployment ID it creates the deployment and lets +// the server assign the ID; otherwise it reads the existing deployment to +// compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. + // Existing deployment: read it to compute the next version number. A 404 is + // not recovered from by creating a second deployment: the ID was just + // resolved from the deployment's workspace node, which the service trashes + // when it deletes the record, so a missing record here means the two are out + // of sync and creating another one would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - switch { - case getErr == nil: - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) - case errors.Is(getErr, apierr.ErrNotFound): - // The record the state points at is gone: a successful destroy deletes - // it (leaving the ID behind in the local state file), and it can also be - // deleted out of band. Recording must not dead-end on it, so fall back to - // creating a new deployment; the caller persists the new ID. - log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) - r.deploymentID = "" - default: + if getErr != nil { return "", fmt.Errorf("failed to get deployment: %w", getErr) } - } - - if r.deploymentID == "" { - // First deploy: create the deployment with an empty ID so the server - // assigns one, then start at version 1. + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } else { + // First deploy: create the deployment so the server assigns an ID, then + // start at version 1. + // + // initial_parent_path is required: the service creates the deployment's + // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the + // deployment ID that ResolveDeploymentID reads back on later deploys. The + // folder must already exist, which it does by this point - the deployment + // lock lives in the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ - TargetName: r.targetName, + InitialParentPath: r.statePath, + TargetName: r.targetName, }, }) if createErr != nil { diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 9b60078635f..6c2f334c946 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -12,6 +12,10 @@ import ( "github.com/stretchr/testify/require" ) +// testStatePath is the bundle state directory the recorder registers the +// deployment node under; several tests assert it round-trips to the service. +const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" + // fakeDMS records the calls the recorder makes and lets a test script the // server-side responses. It embeds the SDK interface so it satisfies it while // only overriding the methods the recorder uses. @@ -33,11 +37,9 @@ type fakeDMS struct { func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) - id := req.DeploymentId - if id == "" { - id = f.assignedID - } - return &bundledeployments.Deployment{Name: "deployments/" + id}, nil + // The server always assigns the ID; it is the ID of the workspace node it + // creates under initial_parent_path. + return &bundledeployments.Deployment{Name: "deployments/" + f.assignedID}, nil } func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { @@ -66,16 +68,18 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} - // A first deploy has no stored deployment ID. - r := NewRecorder(f, "", "dev", VersionTypeDeploy) + // A first deploy resolves no deployment ID from the workspace. + r := NewRecorder(f, "", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) - // The deployment was created with an empty ID so the server assigns one, and - // the recorder exposes the assigned ID for the caller to persist. + // The server assigned the ID, and the recorder exposes it for the rest of the + // deploy (it parents the operations recorded under this version). require.Len(t, f.created, 1) - assert.Empty(t, f.created[0].DeploymentId) assert.Equal(t, "server-generated-id", r.DeploymentID()) + // initial_parent_path is required: the service creates the deployment node + // under it, and that node is what ResolveDeploymentID looks up later. + assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) // The first version is 1, parented under the assigned deployment. require.Len(t, f.versions, 1) @@ -96,7 +100,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) @@ -107,39 +111,28 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } -func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { - f := &fakeDMS{ - assignedID: "fresh-id", - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) - }, - } - // A deploy after a destroy still has the destroyed deployment's ID in state, - // but the record is gone. Recording must recover rather than fail the deploy. - r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) - - require.NoError(t, r.CreateVersion(t.Context())) - - require.Len(t, f.created, 1) - assert.Equal(t, "fresh-id", r.DeploymentID()) - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) -} - func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, errors.New("boom") - }, + cases := map[string]error{ + // A resolved ID whose record is missing means the record and the workspace + // node it was resolved from are out of sync. Creating a second deployment + // would collide on the same node path, so fail instead. + "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), + "other": errors.New("boom"), + } + for name, getErr := range cases { + t.Run(name, func(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, getErr + }, + } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) + }) } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) - - // Only a missing deployment is recovered from; any other read failure is fatal - // rather than silently forking a second deployment record. - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -148,7 +141,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) @@ -164,7 +157,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -184,7 +177,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go new file mode 100644 index 00000000000..0d26448e558 --- /dev/null +++ b/libs/dms/resolve.go @@ -0,0 +1,45 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// DeploymentNodeName is the workspace node DMS creates for a deployment. The +// name is fixed for every deployment: the node *is* the bundle's state file. +// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// ResolveDeploymentID returns the DMS deployment ID for the bundle whose state +// lives under statePath, or an empty string when the bundle has no deployment +// recorded yet. +// +// The ID is not stored anywhere by the CLI. DMS registers each deployment as a +// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the +// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: +// deploymentId = Long.toString(createdNode.getId())). So a get-status on that +// path is the lookup, which keeps the workspace the single source of truth: a +// deployment that was destroyed or deleted out of band reports absent here +// rather than leaving a dangling ID behind in the local state file. +func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil + } + return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + + if obj.ObjectId == 0 { + return "", fmt.Errorf("deployment at %s has no object ID", nodePath) + } + return strconv.FormatInt(obj.ObjectId, 10), nil +} diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go new file mode 100644 index 00000000000..24d5303983d --- /dev/null +++ b/libs/dms/resolve_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestClient returns a workspace client pointed at a server that serves a +// single get-status response. +func newTestClient(t *testing.T, statusCode int, body string) *databricks.WorkspaceClient { + t.Helper() + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Query().Get("path") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { + assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: srv.URL, + Token: "token", + Credentials: config.PatCredentials{}, + }) + require.NoError(t, err) + return w +} + +func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { + w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) + + // The workspace node ID is the deployment ID, so no local state is consulted. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Equal(t, "123456789", id) +} + +func TestResolveDeploymentIDAbsentWhenNodeMissing(t *testing.T) { + w := newTestClient(t, http.StatusNotFound, `{"error_code":"RESOURCE_DOES_NOT_EXIST","message":"Path (/Workspace/state/`+DeploymentNodeName+`) doesn't exist."}`) + + // A bundle that never recorded a deployment, or whose deployment was + // destroyed (the service trashes the node), has no ID rather than an error. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Empty(t, id) +} + +func TestResolveDeploymentIDPropagatesOtherErrors(t *testing.T) { + w := newTestClient(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"nope"}`) + + // Anything other than a missing node is fatal: silently treating it as absent + // would create a second deployment for a bundle that already has one. + _, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.Error(t, err) + assert.ErrorContains(t, err, "looking up deployment at /Workspace/state/"+DeploymentNodeName) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index a1b0cba24a9..6dffbe34a75 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -3,15 +3,23 @@ package testserver import ( "bytes" "encoding/json" + "path" "slices" "strconv" "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" ) // Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. // State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. +// dmsDeploymentNodeName is the name of the workspace node the service creates +// for every deployment. It must match DEPLOYMENT_NODE_NAME on the service side +// (DeploymentWhsClient); the literal is repeated here rather than shared with +// the CLI so a test would catch the CLI drifting from the service. +const dmsDeploymentNodeName = "resources.deployment.json" + // dmsDeployment holds a deployment record together with the versions and // resources recorded under it, so the read APIs (ListVersions/ListResources) // can serve back what deploys wrote. @@ -27,29 +35,49 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string + // nodePath is the workspace node whose object ID is this deployment's ID. + // Kept so DeleteDeployment can trash the node, the way the service does. + nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { - // The client either supplies the deployment ID or, in the server-generated - // flow, leaves it empty for the server to mint one. - deploymentID := req.URL.Query().Get("deployment_id") - if deploymentID == "" { - deploymentID = nextUUID() - } - var dep bundledeployments.Deployment if err := json.Unmarshal(req.Body, &dep); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } defer s.LockUnlock()() + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: nodePath, + ObjectId: objectID, + }, + } + + deploymentID := strconv.FormatInt(objectID, 10) dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive s.dmsDeployments[deploymentID] = &dmsDeployment{ deployment: dep, versions: map[string]*bundledeployments.Version{}, resources: map[string]bundledeployments.Resource{}, + nodePath: nodePath, } return Response{Body: dep} } @@ -99,6 +127,11 @@ func deploymentBody(d *dmsDeployment) (map[string]any, error) { func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { defer s.LockUnlock()() + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if d, ok := s.dmsDeployments[deploymentID]; ok { + delete(s.files, d.nodePath) + } delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } From 4009eb4ae58bd899bfe6d8210995823e73a42f7a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 00:30:33 +0000 Subject: [PATCH 12/56] bundle: address review of the DMS state PR - Limit recorded state to 64 KB, checked when the payload is built so an oversized resource fails itself rather than the drain at close. - Collapse the operation queue's pending/inflight pair into one `owned` set. The two maps encoded a single question ("is this key already claimed?") and had to be read together; `take` now only releases ownership. - Only log "Coalescing" when an operation was actually merged. The old code logged it for in-flight keys too, where nothing was coalesced. - Restore mergeWalIntoState's `hasEntries` naming and comment from main. The `persist` rename existed for the headerDirty case, which is gone. - Trim the comments added by this PR. Also note in fetchDeploymentResources that DMS has no field for dependency edges and they cannot be recovered from the recorded state (references are resolved to literals before serialization), so depends_on is carried over from the local state file. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 41 ++++++++----------- bundle/direct/dstate/state.go | 74 +++++++++++++---------------------- bundle/direct/opqueue.go | 64 +++++++++++++++--------------- bundle/direct/opqueue_test.go | 15 ++++++- bundle/direct/oprecorder.go | 23 +++++++---- bundle/phases/dms.go | 9 ++--- libs/dms/recorder.go | 30 ++++++-------- libs/dms/resolve.go | 19 ++++----- 8 files changed, 129 insertions(+), 146 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 0941d87d35c..8709ceec57f 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -15,12 +15,9 @@ import ( ) // overlayDMSState replaces the file-derived resource state with the state -// recorded in the deployment metadata service (DMS), when DMS owns this -// deployment. Once DMS is authoritative its resource set is trusted even when -// empty (a successful deploy with no resources); the file's resources are only -// used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu, has already populated -// db.Data from the file, and has resolved src.DeploymentID. +// recorded in DMS, when DMS owns this deployment. An authoritative DMS is +// trusted even when its resource set is empty (a successful deploy of nothing). +// The caller holds db.mu and has already populated db.Data from the file. func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { @@ -45,21 +42,13 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS holds a successfully -// completed version for the deployment. It is the signal that DMS owns the -// state: if the deployment was never recorded to DMS, or its initial DMS deploy -// did not complete successfully, DMS state is absent or partial and Open keeps -// the local file's resources instead. +// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server +// advances last_successful_version_id only when a version completes (unlike +// last_version_id, which also advances on failure), so a non-empty value means +// DMS holds a complete resource set. Otherwise Open keeps the file's resources. // -// The deployment carries last_successful_version_id, which the server advances -// only when a version completes successfully (unlike last_version_id, which -// also advances on failure). So a non-empty value is exactly the "DMS owns the -// state" signal, readable in a single GetDeployment. -// -// TODO(DMS): this reads the deployment via a raw GET into a local struct -// because last_successful_version_id is still stage:DEVELOPMENT in the proto -// and therefore stripped from the generated SDK. Once the field is promoted to -// PRIVATE_PREVIEW and regenerated, replace the raw call with +// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and +// stripped from the generated SDK. Once it ships, use // client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) @@ -88,10 +77,14 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. // -// DMS does not record dependency edges, so depends_on is carried over from the -// local state entry for the same key. It is derived from the local config on -// every deploy and is only consumed for delete ordering, so falling back to an -// empty list when the local state has no entry is safe. +// DMS has no field for dependency edges, and they cannot be recovered from the +// recorded state either: references are resolved to literals before it is +// serialized. So depends_on is carried over from the local state file. +// +// TODO(DMS): resources present in DMS but not in the local file therefore get no +// depends_on. Plan recomputes it from config for everything it still declares, +// so this only affects deletes of resources dropped from config, which are +// ordered arbitrarily among themselves. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index c27de3ca44d..0359feba89a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -223,28 +223,22 @@ type ( // DMSSource tells Open to read resource state from the deployment metadata // service instead of the state file. A nil *DMSSource keeps Open file-only. type DMSSource struct { - // Client is the DMS client used to list the deployment's resources. Client bundledeployments.BundleDeploymentsInterface - // Config accompanies Client (both come from the same workspace client) and is - // used only for a temporary raw read of last_successful_version_id; see the - // TODO in deploymentHasSuccessfulVersion. + // Config is only for a temporary raw read of last_successful_version_id; see + // the TODO in deploymentHasSuccessfulVersion. Config *sdkconfig.Config - // DeploymentID identifies the deployment in DMS, resolved from the - // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a - // bundle that has not recorded a deployment yet. + // DeploymentID is resolved from the deployment's workspace node (see + // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsSource is non-nil, the deployment metadata -// service is the source of truth for resource state: if DMS holds a -// successfully completed version for this deployment, the resources read from -// the file are replaced with the ones recorded in DMS. The local identity -// (lineage and serial) always comes from the file, since that is what the write -// path increments and carries forward. A nil dmsSource keeps the behavior -// file-only. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS +// instead of the file whenever DMS holds a successful version. Lineage and +// serial always come from the file, since that is what the write path +// increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -294,25 +288,15 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only deployments that start out empty are recorded in DMS. Resources - // tracked in a state file that DMS does not know about are not in DMS and - // never will be: the first recorded deploy would create a deployment whose - // resource set covers only what that deploy touched, and DMS would then be - // authoritative for everything (see overlayDMSState). Resources this bundle - // already owns would look absent and be created a second time. + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // overlayDMSState), so pre-existing resources it never saw would look absent + // and get created a second time. // - // A deployment DMS already owns (deploymentID is non-empty) is fine — that is - // a bundle that opted in while it was still empty. So is a state file with no - // resources, e.g. one left behind by a destroy. - // - // TODO(DMS): lift this restriction by upgrading an existing state in place. - // That means writing the state at featureStateVersion (3) with a feature flag - // recording that DMS owns it, plus a tombstone entry per resource so a CLI - // that predates DMS refuses the state instead of silently deploying against a - // resource set it cannot see. The feature-flag scaffolding for this already - // exists (see featureStateVersion and Header.Features); once it is written, - // this check goes away and record_deployment_history becomes usable on - // existing bundles. + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } @@ -364,7 +348,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - persist, err := db.mergeWalIntoState(ctx) + hasEntries, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -373,7 +357,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if persist { + if hasEntries { if err := db.unlockedSave(); err != nil { return err } @@ -384,9 +368,6 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } -// mergeWalIntoState replays the WAL into db.Data and reports whether the caller -// must persist the state file: either the WAL carried resource entries, or a -// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -462,20 +443,19 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - persist := lineNumber > 1 + hasEntries := lineNumber > 1 - // Only advance the serial when the state file is actually written, because - // the caller (replayWAL) persists it only in that case. A header-only WAL - // that changed nothing is a deploy that started but committed nothing; - // advancing the serial for it leaves the in-memory serial ahead of the - // persisted one, so the next deploy writes its WAL header at serial+2 and - // recovery rejects it as "ahead of expected". - // See acceptance/bundle/deploy/wal/header-only-wal. - if persist { + // Only advance the serial when the WAL carried entries, because the caller + // (replayWAL) persists the new state file only in that case. A header-only + // WAL is a deploy that started but committed nothing; advancing the serial + // for it leaves the in-memory serial ahead of the persisted one, so the + // next deploy writes its WAL header at serial+2 and recovery rejects it as + // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. + if hasEntries { db.Data.Serial = newSerial } - return persist, nil + return hasEntries, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 72a2fa9c39e..68e8e2e9f96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -22,19 +22,15 @@ const ( ) // operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip before moving on to the -// next resource. +// worker does not wait for the CreateOperation round trip. // -// It guarantees at most one upload in flight per resource key: within a key the -// worker that owns it uploads sequentially, so the last operation recorded for a -// resource is also the last one the service sees. +// At most one upload is in flight per resource key, so the last operation +// recorded for a resource is also the last one the service sees. // -// Uploads are not fire-and-forget: close drains the queue and returns the first -// failure, which fails the deploy. That matters because a successfully completed -// version makes DMS the source of truth for resource state (see -// dstate.overlayDMSState); silently dropping an operation would leave DMS with an -// incomplete resource set, and the next deploy would plan to create resources -// that already exist. +// Uploads are not fire-and-forget: close returns the first failure and fails the +// deploy. A dropped operation would leave DMS with an incomplete resource set, +// and since DMS then becomes the source of truth (see dstate.overlayDMSState), +// the next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader @@ -51,10 +47,11 @@ type operationQueue struct { // picked up yet. pending map[string]recordedOperation - // inflight holds the resource keys a worker currently owns. A key that is - // in flight is not queued again: the owning worker re-checks pending after its - // upload and picks up anything recorded in the meantime. - inflight map[string]bool + // owned holds the resource keys that are already queued or being uploaded. + // Such a key is never queued a second time; recording writes to pending + // instead, which the owning worker re-checks after each upload. This is what + // keeps uploads for one resource sequential. + owned map[string]bool err error closed bool @@ -74,7 +71,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati uploader: uploader, queue: make(chan string, operationQueueSize), pending: make(map[string]recordedOperation), - inflight: make(map[string]bool), + owned: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -85,16 +82,14 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It performs no API -// call, so upload failures surface from close rather than here; the error -// returned is only about turning the applied resource into a payload. +// record serializes an operation and queues it for upload. It makes no API call, +// so upload failures surface from close; an error here only means the applied +// resource could not be turned into a payload. // -// When an operation for the same resource is already waiting it is replaced -// instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. The -// merged operation keeps the action of a queued create (see mergeAction), so -// collapsing a create and a later update still records a create. This is best -// effort - only operations that have not been picked up yet are collapsed. +// An operation for a resource that is still waiting replaces it rather than +// queueing again: DMS keeps one state per key, so one upload records both. The +// merge keeps a queued create's action (see mergeAction). Best effort — only +// operations no worker has picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -107,15 +102,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action q.mu.Lock() queued, waiting := q.pending[resourceKey] - owned := waiting || q.inflight[resourceKey] if waiting { op.action = mergeAction(queued.action, op.action) } q.pending[resourceKey] = op + owned := q.owned[resourceKey] + q.owned[resourceKey] = true q.mu.Unlock() - if owned { + if waiting { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + } + + // Someone already owns this key, so pending is enough: a worker will pick the + // operation up. Queueing again would upload the resource twice in parallel. + if owned { return nil } @@ -167,21 +168,20 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey, marking the key in flight so -// record does not queue it a second time. It reports false, and releases the key, -// when nothing is waiting. +// take claims the operation waiting for resourceKey. It reports false and gives +// up ownership when nothing is waiting, which is what lets the next record +// queue the key again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.inflight, resourceKey) + delete(q.owned, resourceKey) return recordedOperation{}, false } delete(q.pending, resourceKey) - q.inflight[resourceKey] = true return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a8e05141fc7..cdd1083a06d 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strconv" + "strings" "sync" "testing" @@ -153,6 +154,18 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { assert.Empty(t, f.recorded()) } +func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + require.ErrorContains(t, err, "exceeds the 65536 byte limit") + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + func TestOperationQueueCloseIsIdempotent(t *testing.T) { f := &fakeUploader{err: errors.New("boom")} q := newOperationQueue(t.Context(), f) @@ -226,7 +239,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.inflight) + assert.Empty(t, q.owned) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 63d0c84a621..3b3d631f4a9 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,6 +12,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// maxOperationStateSize is the largest serialized state DMS accepts per +// operation. Uploading more is rejected server-side, so fail early with a +// message that names the resource. +const maxOperationStateSize = 64 * 1024 + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -28,7 +33,8 @@ type recordedOperation struct { } // newRecordedOperation serializes an applied operation for upload. state is the -// local config after the operation and must be nil for delete operations. +// local config after the operation and must be nil for delete operations. It +// errors when the serialized state exceeds maxOperationStateSize. func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { @@ -37,19 +43,20 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. + // Operation.State carries the serialized config, which DMS serves back as + // resource state. Unset for delete: the resource is gone. // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. + // Redact secrets, like dstate.SaveState does for the local state file: + // otherwise we leak them to the service and the read path writes them back + // into a local state file in plaintext. if state != nil { raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } + if len(raw) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(raw), maxOperationStateSize) + } op.state = raw } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 02254237595..3d2f4f54009 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -16,11 +16,10 @@ import ( // AND the engine is direct: DMS resource state is tracked per direct-engine // deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID is resolved from the workspace rather than from local state -// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment -// lock has been acquired, so it observes any deployment a concurrent deploy -// created. It is empty on a bundle's first recorded deploy, in which case the -// recorder creates the deployment and the server assigns the ID. +// The deployment ID is resolved from the workspace, not local state (see +// dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is +// held, so it sees any deployment a concurrent deploy created. It is empty on the +// first recorded deploy, where the recorder creates the deployment instead. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil, nil diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 6fa9a1a24be..83f180ba3be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,12 +29,10 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // -// The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID resolved from the workspace (empty on a bundle's first-ever -// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the -// deployment record when that ID is empty. Later deploys resolve the same ID -// from the deployment's workspace node and reuse the record; a destroy deletes -// the record and its node, so the next deploy starts over from empty. +// The server assigns the deployment ID on the first deploy, i.e. when the ID +// resolved from the workspace is empty (see ResolveDeploymentID). Later deploys +// resolve the same ID and reuse the record; a destroy deletes the record and its +// node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string @@ -150,10 +148,10 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment: the ID was just - // resolved from the deployment's workspace node, which the service trashes - // when it deletes the record, so a missing record here means the two are out - // of sync and creating another one would collide on the same node path. + // not recovered from by creating a second deployment. The service trashes the + // workspace node when it deletes the record, so a node that resolved but has + // no record means the two are out of sync, and creating another deployment + // would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) @@ -166,14 +164,12 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } versionID = strconv.FormatInt(lastVersion+1, 10) } else { - // First deploy: create the deployment so the server assigns an ID, then - // start at version 1. + // First deploy: create the deployment so the server assigns an ID. // - // initial_parent_path is required: the service creates the deployment's - // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the - // deployment ID that ResolveDeploymentID reads back on later deploys. The - // folder must already exist, which it does by this point - the deployment - // lock lives in the same directory. + // initial_parent_path is required. The service creates the deployment node + // under it, and that node's ID is the deployment ID ResolveDeploymentID reads + // back later. The folder already exists by now: the deployment lock lives in + // the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go index 0d26448e558..9bbf5a84b56 100644 --- a/libs/dms/resolve.go +++ b/libs/dms/resolve.go @@ -11,22 +11,17 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" ) -// DeploymentNodeName is the workspace node DMS creates for a deployment. The -// name is fixed for every deployment: the node *is* the bundle's state file. -// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. const DeploymentNodeName = "resources.deployment.json" // ResolveDeploymentID returns the DMS deployment ID for the bundle whose state -// lives under statePath, or an empty string when the bundle has no deployment -// recorded yet. +// lives under statePath, or empty if it has never recorded a deployment. // -// The ID is not stored anywhere by the CLI. DMS registers each deployment as a -// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the -// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: -// deploymentId = Long.toString(createdNode.getId())). So a get-status on that -// path is the lookup, which keeps the workspace the single source of truth: a -// deployment that was destroyed or deleted out of band reports absent here -// rather than leaving a dangling ID behind in the local state file. +// The CLI stores the ID nowhere: DMS registers the deployment as a workspace +// node and that node's ID *is* the deployment ID, so a get-status is the lookup. +// This keeps the workspace the single source of truth — a destroyed deployment +// reports absent instead of leaving a dangling ID in the local state file. func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { nodePath := path.Join(statePath, DeploymentNodeName) From 70741aeceb38fd9155b952e9f2d8785d08555338 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 10:38:59 +0000 Subject: [PATCH 13/56] bundle: record depends_on in the state uploaded to DMS The read path carried depends_on over from the local state file, which is empty exactly when it matters: a fresh checkout reconstructs state from DMS and got no dependency edges. Deletes are the one case that cannot recompute them, because the resource is gone from config, so two dropped resources with a real dependency could be deleted in the wrong order. DMS has no field for dependency edges, and they cannot be recovered from the recorded config either: references are resolved to literals before it is serialized. So Operation.State now carries an envelope, dstate.RecordedState, holding the config plus depends_on. Nesting depends_on inside the config would have collided with resource fields of the same name (jobs.Task.depends_on). The envelope mirrors the local ResourceEntry, so both sides of the round trip have the same shape. acceptance/bundle/dms/depends-on covers it: a job referencing another records its edge, and after wiping the local state a destroy still deletes the referencing job first. Co-authored-by: Isaac --- .../bundle/dms/depends-on/databricks.yml | 13 ++++++ .../bundle/dms/depends-on/out.test.toml | 3 ++ acceptance/bundle/dms/depends-on/output.txt | 26 ++++++++++++ acceptance/bundle/dms/depends-on/script | 8 ++++ .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 22 +++++----- .../dms/redeploy-after-destroy/output.txt | 22 +++++----- bundle/direct/bundle_apply.go | 7 ++-- bundle/direct/dstate/dms.go | 40 ++++++++++++------- bundle/direct/dstate/dms_test.go | 38 ++++++++---------- bundle/direct/opqueue.go | 4 +- bundle/direct/opqueue_test.go | 20 +++++----- bundle/direct/oprecorder.go | 11 +++-- bundle/direct/oprecorder_test.go | 21 ++++++++-- 14 files changed, 157 insertions(+), 80 deletions(-) create mode 100644 acceptance/bundle/dms/depends-on/databricks.yml create mode 100644 acceptance/bundle/dms/depends-on/out.test.toml create mode 100644 acceptance/bundle/dms/depends-on/output.txt create mode 100644 acceptance/bundle/dms/depends-on/script diff --git a/acceptance/bundle/dms/depends-on/databricks.yml b/acceptance/bundle/dms/depends-on/databricks.yml new file mode 100644 index 00000000000..f97e3a38809 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: dms-depends-on + +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..2bb8d06b9dc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,26 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs --oneline +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..905d022619c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --oneline + +title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" +rm -rf .databricks +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs --oneline diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6aeff83c04b..116584d10c1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -42,4 +42,4 @@ Deployment complete! {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index ed10f06e699..e792d5d8d99 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -45,16 +45,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ed7b6ede989..ff8694b5424 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -70,16 +70,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index b26861128b7..f29aa18a186 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -94,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil, nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -129,8 +129,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads - // the ID assigned by Deploy. - if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + // the ID assigned by Deploy. depends_on is recorded alongside the config + // because it cannot be recomputed from it (see dstate.RecordedState). + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value, d.DependsOn); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 8709ceec57f..e6ae63f6095 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" @@ -14,6 +15,22 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// RecordedState is what the CLI serializes into the DMS Operation.State field. +// +// It is an envelope rather than the bare resource config, because depends_on has +// to survive the round trip: DMS has no field for dependency edges, and they +// cannot be recomputed from the config once it is recorded (references are +// resolved to literals before serialization). Nesting depends_on inside the +// config instead would collide with resource fields of the same name, e.g. +// jobs.Task.depends_on. +// +// The shape deliberately matches the local ResourceEntry so both sides of the +// state round trip look the same. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + // overlayDMSState replaces the file-derived resource state with the state // recorded in DMS, when DMS owns this deployment. An authoritative DMS is // trusted even when its resource set is empty (a successful deploy of nothing). @@ -29,7 +46,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } - resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err } @@ -76,16 +93,7 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -// -// DMS has no field for dependency edges, and they cannot be recovered from the -// recorded state either: references are resolved to literals before it is -// serialized. So depends_on is carried over from the local state file. -// -// TODO(DMS): resources present in DMS but not in the local file therefore get no -// depends_on. Plan recomputes it from config for everything it still declares, -// so this only affects deletes of resources dropped from config, which are -// ordered arbitrarily among themselves. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -102,15 +110,17 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund // ("resources.jobs.foo"), so prepend it here. key := "resources." + res.ResourceKey - var state json.RawMessage + var recorded RecordedState if res.State != nil { - state = *res.State + if err := json.Unmarshal(*res.State, &recorded); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } } out[key] = ResourceEntry{ ID: res.ResourceId, - State: state, - DependsOn: local[key].DependsOn, + State: recorded.State, + DependsOn: recorded.DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index df1084b9de7..35fe7acbb0c 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -35,40 +35,34 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy ) } -func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { - state := json.RawMessage(`{"name":"foo"}`) +func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { + recorded := json.RawMessage(`{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, {ResourceKey: "pipelines.bar", ResourceId: "456"}, }} - dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} - local := map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, - } - - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") require.NoError(t, err) - // DMS owns the ID and state, but it does not record dependency edges, so - // depends_on must survive from the local entry. Losing it breaks delete - // ordering and --select expansion. + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, "resources.pipelines.bar": {ID: "456"}, }, got) } -func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { +func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { + recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123"}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, }} - // A bundle whose local state was wiped has no entry to carry depends_on from; - // the resource is still recovered from DMS. - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) - require.NoError(t, err) - assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123"}, - }, got) + _, err := fetchDeploymentResources(t.Context(), f, "dep-1") + assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 68e8e2e9f96..1f38c2b7dd6 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -90,12 +90,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // queueing again: DMS keeps one state per key, so one upload records both. The // merge keeps a queued create's action (see mergeAction). Best effort — only // operations no worker has picked up yet are collapsed. -func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil } - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index cdd1083a06d..6bf0da8107b 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -59,7 +59,7 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -94,8 +94,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { // Two uploads, not three: v2 was superseded by v3 while both were queued, and // the last recorded state is the one the service ends up with. assert.Equal(t, []string{ - `resources.jobs.foo={"name":"v1"}`, - `resources.jobs.foo={"name":"v3"}`, + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } @@ -114,15 +114,15 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) // The state is the later one, but the action stays CREATE: recording an update // would tell DMS the resource already existed before this deploy. - assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -147,7 +147,7 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { // Serialization failures surface at record time, on the resource that caused // them, rather than from the drain at the end of apply. - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil, nil) require.Error(t, err) require.NoError(t, q.close()) @@ -159,7 +159,7 @@ func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { q := newOperationQueue(t.Context(), f) big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big, nil) require.ErrorContains(t, err, "exceeds the 65536 byte limit") require.NoError(t, q.close()) @@ -224,7 +224,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) } }) } @@ -247,6 +247,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, nil)) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 3b3d631f4a9..91bcb65a4f6 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -35,7 +36,7 @@ type recordedOperation struct { // newRecordedOperation serializes an applied operation for upload. state is the // local config after the operation and must be nil for delete operations. It // errors when the serialized state exceeds maxOperationStateSize. -func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err @@ -43,14 +44,18 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // Operation.State carries the serialized config, which DMS serves back as + // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // // Redact secrets, like dstate.SaveState does for the local state file: // otherwise we leak them to the service and the read path writes them back // into a local state file in plaintext. if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + raw, err := json.Marshal(dstate.RecordedState{State: config, DependsOn: dependsOn}) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 70afb788cd3..556f7f59f90 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -30,7 +30,7 @@ func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployment // an operationQueue worker does. func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { t.Helper() - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, nil) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } @@ -71,18 +71,31 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - op, err := newRecordedOperation(deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, - `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + string(op.state)) +} + +func TestNewRecordedOperationRecordsDependsOn(t *testing.T) { + // depends_on rides in an envelope alongside the config: it cannot be + // recomputed from the config, whose references are already resolved. + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}} + + op, err := newRecordedOperation(deployplan.Create, "job-123", map[string]string{"name": "foo"}, dependsOn) + require.NoError(t, err) + + assert.JSONEq(t, + `{"state":{"name":"foo"},"depends_on":[{"node":"resources.jobs.bar","label":"${resources.jobs.bar.id}"}]}`, string(op.state)) } func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil, nil) assert.Error(t, err) } From 386a0b616f6a409ab1c31fb34383b74bcd098547 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 11:03:26 +0000 Subject: [PATCH 14/56] bundle: drop the DMS state overlay check Migrating an existing deployment is not supported: Open rejects a bundle that already has resources in state, so a deployment that exists in DMS was created by an opted-in CLI and DMS owns its resource set outright. The last_successful_version_id probe that decided whether to trust DMS was therefore always true by the time it ran. Removing it takes with it the raw GET that read the field (it is stage:DEVELOPMENT and stripped from the generated SDK) and DMSSource.Config, which existed only to make that call. overlayDMSState is now readDMSState, since it no longer overlays anything conditionally: it just reads. Recording stays opt-in via experimental.record_deployment_history; that flag is what makes the caller pass a DMSSource at all. acceptance/bundle/dms/existing-state also now covers wiping the local cache: deploy pulls the state file back from the workspace, so the resources stay tracked and opting in is still rejected. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 7 +++ acceptance/bundle/dms/existing-state/script | 5 ++ acceptance/bundle/dms/no-resources/output.txt | 4 -- bundle/direct/dstate/dms.go | 59 ++----------------- bundle/direct/dstate/state.go | 21 +++---- cmd/bundle/utils/process.go | 8 +-- libs/dms/resolve_test.go | 4 +- 7 files changed, 33 insertions(+), 75 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 116584d10c1..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -18,6 +18,13 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS >>> print_requests.py //api/2.0/bundle --sort --oneline +=== Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +>>> print_requests.py //api/2.0/bundle --sort --oneline + === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 7bc296465d5..ae9be95f701 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -9,6 +9,11 @@ trace musterr $CLI bundle deploy title "No deployment was created in DMS" trace print_requests.py //api/2.0/bundle --sort --oneline +title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" +rm -rf .databricks +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 73bd3adfa11..b400fb9a67e 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -51,10 +51,6 @@ Deployment complete! "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e6ae63f6095..ec51e2c4479 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,15 +3,9 @@ package dstate import ( "context" "encoding/json" - "errors" "fmt" - "net/http" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/auth" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/client" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -31,21 +25,12 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// overlayDMSState replaces the file-derived resource state with the state -// recorded in DMS, when DMS owns this deployment. An authoritative DMS is -// trusted even when its resource set is empty (a successful deploy of nothing). -// The caller holds db.mu and has already populated db.Data from the file. -func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) - if err != nil { - return err - } - if !authoritative { - // DMS has no completed version for this deployment: a prior direct deploy - // that has not yet successfully recorded to DMS. Keep the file state. - return nil - } - +// readDMSState replaces the file-derived resource state with the state recorded +// in DMS. Recording is only enabled for net-new deployments, so once a +// deployment exists DMS owns its resource set outright - including when that set +// is empty, which is a successful deploy of nothing rather than missing data. +// The caller holds db.mu. +func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err @@ -59,38 +44,6 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server -// advances last_successful_version_id only when a version completes (unlike -// last_version_id, which also advances on failure), so a non-empty value means -// DMS holds a complete resource set. Otherwise Open keeps the file's resources. -// -// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and -// stripped from the generated SDK. Once it ships, use -// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. -func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { - apiClient, err := client.New(cfg) - if err != nil { - return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) - } - - // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with - // name=deployments/{id}); we unmarshal into a local struct so we can read - // last_successful_version_id, which the typed SDK response drops. - var dep struct { - LastSuccessfulVersionID string `json:"last_successful_version_id"` - } - err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) - } - return dep.LastSuccessfulVersionID != "", nil -} - // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 0359feba89a..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -221,24 +220,20 @@ type ( ) // DMSSource tells Open to read resource state from the deployment metadata -// service instead of the state file. A nil *DMSSource keeps Open file-only. +// service instead of the state file. Callers pass it only when the bundle set +// experimental.record_deployment_history; a nil *DMSSource keeps Open file-only. type DMSSource struct { Client bundledeployments.BundleDeploymentsInterface - // Config is only for a temporary raw read of last_successful_version_id; see - // the TODO in deploymentHasSuccessfulVersion. - Config *sdkconfig.Config - // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). With a non-nil dmsSource, resources come from DMS -// instead of the file whenever DMS holds a successful version. Lineage and -// serial always come from the file, since that is what the write path -// increments. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS rather +// than the file. Lineage and serial always come from the file, since that is +// what the write path increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -290,8 +285,8 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W if dmsSource != nil { // Only bundles that start out empty can be recorded. Once DMS owns a // deployment it is authoritative for the whole resource set (see - // overlayDMSState), so pre-existing resources it never saw would look absent - // and get created a second time. + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. // // TODO(DMS): allow this by upgrading the state in place, writing it at // featureStateVersion with a feature flag plus a tombstone per resource so an @@ -301,7 +296,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } if dmsSource.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsSource); err != nil { + if err := db.readDMSState(ctx, dmsSource); err != nil { return err } } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e9840188db4..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -214,9 +214,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open a DMS source to overlay that - // state on top of the local identity (lineage/serial). Reads open the - // state write-disabled, so no lineage is minted here. + // service owns resource state, so hand Open a DMS source to read it from + // there instead of the file. The local identity (lineage/serial) still + // comes from the file. Reads open the state write-disabled, so no lineage + // is minted here. var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { w := b.WorkspaceClient(ctx) @@ -227,7 +228,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, - Config: w.Config, DeploymentID: deploymentID, } } diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go index 24d5303983d..6838f4dd15c 100644 --- a/libs/dms/resolve_test.go +++ b/libs/dms/resolve_test.go @@ -25,7 +25,7 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp })) t.Cleanup(srv.Close) t.Cleanup(func() { - assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + assert.Equal(t, nodePath, gotPath) }) w, err := databricks.NewWorkspaceClient(&databricks.Config{ @@ -37,6 +37,8 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp return w } +const nodePath = "/Workspace/state/" + DeploymentNodeName + func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) From efe7320cbeeaacecde8dc76ef68eed74b9e6d658 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 23:44:57 +0000 Subject: [PATCH 15/56] bundle: gate DMS state on a state feature flag Reading resource state from DMS now requires the state file to record a "deployment_history" feature flag, rather than inferring eligibility from the resource count. The old check refused a state that had resources and no DMS deployment ID. That happened to be right, but it inferred intent from a side effect: a state with resources could equally be one DMS already owns. The flag says so directly. Header.Features was already scaffolded for exactly this, so this fills it in: - Open writes the flag when recording is enabled, and refuses a state that has resources without it. Migrating such a target is not supported, so the error names the target and the three ways out: use a new target, destroy this one and redeploy, or unset the feature. - A state recording any feature is written at featureStateVersion, so a CLI that predates the flag refuses it (see migrateState) instead of deploying against a resource set that lives in DMS and looks empty on disk. - migrateState accepts features it implements and refuses only unknown ones, naming just those in the error. - WAL replay carries the features forward, so recovering a WAL from a recording deploy still produces a state marked as DMS-owned. The flag is per target, which matches how experimental.record_deployment_history is set. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 4 +- acceptance/bundle/dms/record/output.txt | 9 ++ acceptance/bundle/dms/record/script | 3 + bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++++++++------ bundle/direct/dstate/state_test.go | 21 +++- cmd/bundle/utils/process.go | 1 + 7 files changed, 132 insertions(+), 51 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..229b3bea03f 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: cannot record deployment history for a bundle that already has deployed r === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index e792d5d8d99..3266e0d3af8 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,6 +73,15 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false +=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 63eaa323b1b..3298c8fb131 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -6,6 +6,9 @@ title "The deployment ID is the ID of the workspace node the service registered trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json +title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..288fdcf863f 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,25 +12,34 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +// knownFeatures lists the state feature flags this CLI implements. A state that +// records anything outside this set is refused by migrateState. +var knownFeatures = map[string]bool{ + FeatureDeploymentHistory: true, +} + // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. + // featureStateVersion states carry a feature list (see the featureStateVersion + // doc comment). A featureStateVersion state with no features is equivalent to + // currentStateVersion, so accept it and return without running the migrations + // below, leaving the on-disk version at featureStateVersion rather than flipping + // it down. Same for a state whose features this CLI implements. One that records + // a feature this CLI does not know depends on capabilities it lacks, so refuse it + // and tell the user to upgrade. if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) + unknown := make([]string, 0, len(db.Features)) for name := range db.Features { - features = append(features, name) + if !knownFeatures[name] { + unknown = append(unknown, name) + } + } + if len(unknown) == 0 { + return nil } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + slices.Sort(unknown) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bad87a6a293..b367e97001b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,22 +31,28 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade + // featureStateVersion is the schema version written once a state records + // deployment state "feature flags" (see Header.Features). Reading such a state + // (see migrateState): + // - featureStateVersion with no features -> accept, leave the version as-is + // - featureStateVersion with known features -> accept + // - featureStateVersion with unknown features -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. + // featureStateVersion. That way a release can start writing + // featureStateVersion + features without older CLIs either mishandling a feature + // they lack or rejecting a featureless state outright. featureStateVersion is + // always 3. featureStateVersion = 3 + // FeatureDeploymentHistory marks a state whose resources live in the deployment + // metadata service rather than in the state file. A CLI that does not know this + // feature refuses the state instead of deploying against a resource set it + // cannot see - the file's resources are not authoritative. + FeatureDeploymentHistory = "deployment_history" + // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -82,13 +88,27 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. A CLI that does not implement one of them refuses the state rather than + // deploying against it (see migrateState). It is a map so a future CLI can attach + // per-feature data without reshaping the state. Empty/omitted for states that use + // no features. Features map[string]struct{} `json:"features,omitempty"` } +// hasFeature reports whether the state records the given feature flag. +func (h *Header) hasFeature(name string) bool { + _, ok := h.Features[name] + return ok +} + +// setFeature records a feature flag in the state. +func (h *Header) setFeature(name string) { + if h.Features == nil { + h.Features = make(map[string]struct{}) + } + h.Features[name] = struct{}{} +} + type Database struct { Header @@ -228,6 +248,10 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string + + // TargetName names the bundle target in the error Open returns for a state that + // predates the opt-in, since the feature is enabled per target. + TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -283,18 +307,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only bundles that start out empty can be recorded. Once DMS owns a - // deployment it is authoritative for the whole resource set (see - // readDMSState), so pre-existing resources it never saw would look absent and - // get created a second time. - // - // TODO(DMS): allow this by upgrading the state in place, writing it at - // featureStateVersion with a feature flag plus a tombstone per resource so an - // older CLI refuses the state instead of deploying against resources it - // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { - return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + // An existing state file has to be marked as DMS-owned before its resources + // can be read from DMS. Recording only starts on an empty state, so a state + // with resources and no feature flag predates the opt-in: DMS never saw those + // resources and is authoritative for the whole set once enabled (see + // readDMSState), so they would look absent and be created a second time. + // Migrating such a target is not supported yet. + if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { + return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) } + // Mark the state as DMS-owned so a CLI without this feature refuses it rather + // than deploying against a resource set it cannot see. + db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -311,13 +335,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - walHead := Header{ - Lineage: db.GetOrInitLineage(), - Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, - CLIVersion: build.GetInfo().Version, - } - return appendJSONLine(db.walFile, walHead) + return appendJSONLine(db.walFile, db.newWalHeader()) } return nil @@ -404,6 +422,16 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial + + // Carry the WAL's features (and the version that goes with them) into the + // state being written, so recovering a WAL written by a DMS-recording deploy + // still produces a state marked as DMS-owned. + for name := range header.Features { + db.Data.setFeature(name) + } + if len(db.Data.Features) > 0 { + db.Data.StateVersion = featureStateVersion + } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -507,13 +535,25 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - walHead := Header{ + return appendJSONLine(db.walFile, db.newWalHeader()) +} + +// newWalHeader builds the header for a fresh WAL. Features carry over from the +// state being written, and a state that records any feature is written at +// featureStateVersion so a CLI that lacks the feature refuses it instead of +// deploying against it. The caller holds db.mu. +func (db *DeploymentState) newWalHeader() Header { + version := currentStateVersion + if len(db.Data.Features) > 0 { + version = featureStateVersion + } + return Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, + StateVersion: version, CLIVersion: build.GetInfo().Version, + Features: db.Data.Features, } - return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a9c90530514..35050575121 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,7 +154,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 that records a feature is refused: this CLI does not understand features. + // v3 recording a feature this CLI implements is accepted. + known := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}}, + }} + require.NoError(t, migrateState(known)) + assert.Equal(t, featureStateVersion, known.StateVersion) + + // v3 recording a feature this CLI does not know is refused: its resources may + // live somewhere this CLI cannot see. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -165,6 +174,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) + + // Only the unknown feature is named, so the message tells the user what to do. + mixed := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, + }} + err = migrateState(mixed) + require.Error(t, err) + assert.Contains(t, err.Error(), "future_feature") + assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 209dc874403..b000da60264 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,6 +229,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, + TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From 92eac95b722c6148fedcc03f79db081257c5e152 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 09:49:01 +0000 Subject: [PATCH 16/56] bundle: combine queued operations into a single Create action Coalescing now keeps the newest operation outright instead of merging fields. Each operation carries the resource's full state rather than a delta, so a newer one entirely supersedes an older one - including its resource_id, which is the field that actually changes between two records (a create learns the ID only after the API call returns it). The previous code merged the action and overwrote the ID, which is backwards: the action cannot differ between records of the same resource, while the ID can. mergeAction is gone. Also renames `owned` to `queuedOrUploading`. Nothing is owned by a particular worker: a key can be handled by one worker, released, and picked up later by another. The mark only means "some worker will get to this", which is all record needs to know in order to not queue the key twice. The doc comments now lead with the two rules that shape the design (no overlapping uploads per resource; only the newest operation matters) so the mechanism reads as a consequence of them rather than as bookkeeping. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 94 ++++++++++++++++++-------------- bundle/direct/opqueue_test.go | 33 +++++++---- bundle/direct/oprecorder.go | 18 ------ bundle/direct/oprecorder_test.go | 30 ---------- 4 files changed, 74 insertions(+), 101 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 1f38c2b7dd6..9a1a1cd918e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -21,37 +21,48 @@ const ( operationUploadWorkers = 4 ) -// operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip. +// operationQueue hands recorded operations to background workers, so an apply +// worker does not wait for the CreateOperation round trip before deploying the +// next resource. // -// At most one upload is in flight per resource key, so the last operation -// recorded for a resource is also the last one the service sees. +// Two rules shape the design: +// +// - Uploads for one resource never overlap. DMS stores one state per resource +// key, so concurrent uploads could land out of order and leave stale state. +// - Only the newest operation for a resource matters. Each operation carries the +// resource's full state, not a delta, so a newer one entirely supersedes an +// older one. When both are still waiting, the older is dropped ("coalesced") +// and one upload records the result. // // Uploads are not fire-and-forget: close returns the first failure and fails the // deploy. A dropped operation would leave DMS with an incomplete resource set, -// and since DMS then becomes the source of truth (see dstate.overlayDMSState), -// the next deploy would recreate resources that already exist. +// and since DMS then becomes the source of truth (see dstate.readDMSState), the +// next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader - // queue carries resource keys, not the operations themselves: a worker looks - // the operation up in pending when it picks the key up, which is what lets - // record collapse repeated writes to the same resource. + // queue carries resource keys, not operations. A worker looks the operation up + // when it picks the key up, so recording again before then just overwrites the + // entry in pending - that is what makes coalescing work. queue chan string wg sync.WaitGroup // mu guards the fields below. mu sync.Mutex - // pending is the latest operation recorded per resource key that no worker has - // picked up yet. + // pending holds the newest operation per resource key that no worker has taken + // yet. Empty for a key means everything recorded for it has been uploaded. pending map[string]recordedOperation - // owned holds the resource keys that are already queued or being uploaded. - // Such a key is never queued a second time; recording writes to pending - // instead, which the owning worker re-checks after each upload. This is what - // keeps uploads for one resource sequential. - owned map[string]bool + // queuedOrUploading marks keys that are already in the queue channel or being + // uploaded right now. Such a key must not be queued again, or two workers could + // upload the same resource at once; recording writes to pending instead, and + // the worker handling the key picks it up when its current upload finishes. + // + // No single worker "owns" a key for the whole time it is marked: a key can be + // handled by one worker, released, and later picked up by another. The mark only + // means "some worker will get to this", which is all record needs to know. + queuedOrUploading map[string]bool err error closed bool @@ -68,10 +79,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati } q := &operationQueue{ - uploader: uploader, - queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), - owned: make(map[string]bool), + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + queuedOrUploading: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -82,14 +93,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It makes no API call, -// so upload failures surface from close; an error here only means the applied -// resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. It makes no +// API call, so upload failures surface from close; an error here only means the +// applied resource could not be turned into a payload. // -// An operation for a resource that is still waiting replaces it rather than -// queueing again: DMS keeps one state per key, so one upload records both. The -// merge keeps a queued create's action (see mergeAction). Best effort — only -// operations no worker has picked up yet are collapsed. +// Recording a resource that is still waiting replaces the waiting operation +// outright, since the newer one carries the resource's full state. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil @@ -101,22 +110,20 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - queued, waiting := q.pending[resourceKey] - if waiting { - op.action = mergeAction(queued.action, op.action) - } + _, replaced := q.pending[resourceKey] q.pending[resourceKey] = op - owned := q.owned[resourceKey] - q.owned[resourceKey] = true + alreadyHandled := q.queuedOrUploading[resourceKey] + q.queuedOrUploading[resourceKey] = true q.mu.Unlock() - if waiting { + if replaced { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) } - // Someone already owns this key, so pending is enough: a worker will pick the - // operation up. Queueing again would upload the resource twice in parallel. - if owned { + // A worker is already going to handle this key, and it re-reads pending before + // finishing, so it will see the operation written above. Queueing the key again + // would let a second worker upload the same resource concurrently. + if alreadyHandled { return nil } @@ -152,7 +159,7 @@ func (q *operationQueue) work(ctx context.Context) { defer q.wg.Done() for resourceKey := range q.queue { - // Keep uploading this key until nothing new was recorded for it, instead of + // Keep uploading this key until nothing new was recorded for it, rather than // putting it back on the queue: a worker sending to the channel it consumes // from can deadlock once the queue is full. for { @@ -168,16 +175,19 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey. It reports false and gives -// up ownership when nothing is waiting, which is what lets the next record -// queue the key again. +// take claims the operation waiting for resourceKey. It reports false and clears +// the queuedOrUploading mark when nothing is waiting, which is what lets the next +// record queue the key again. +// +// Clearing the mark and observing pending empty happen under one lock, so record +// can never skip queueing a key that no worker is going to look at again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.owned, resourceKey) + delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 6bf0da8107b..356a7cfb319 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -22,9 +22,10 @@ type fakeUploader struct { started chan string err error - mu sync.Mutex - uploads []string - actions map[string]bundledeployments.OperationActionType + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -40,8 +41,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} + f.resourceIDs = map[string]string{} } f.actions[resourceKey] = op.action + f.resourceIDs[resourceKey] = op.resourceID return f.err } @@ -57,6 +60,12 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation return f.actions[resourceKey] } +func (f *fakeUploader) resourceIDFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.resourceIDs[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) @@ -99,9 +108,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } -func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { - // Hold the first upload so the create below stays queued and the update - // coalesces into it. +func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { + // Hold the first upload so the operations below stay queued and coalesce. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) @@ -114,15 +122,18 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) + // A resource whose ID is only known after it was created: the first operation + // has no ID, the second fills it in. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) - // The state is the later one, but the action stays CREATE: recording an update - // would tell DMS the resource already existed before this deploy. + // Everything comes from the newest operation: it carries the resource's full + // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -239,7 +250,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.owned) + assert.Empty(t, q.queuedOrUploading) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 91bcb65a4f6..2401e34fcf8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -68,24 +68,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } -// mergeAction returns the action to record when a later operation coalesces into -// one still queued for the same resource (see operationQueue.record). The state -// uploaded is the later one, but the action must not be downgraded: Create and -// Recreate tell DMS the resource ID is new, and a subsequent Update only refines -// the state of that same new resource. Recording the pair as an Update would -// claim the resource already existed. A Delete is the exception - the resource is -// gone, so nothing earlier is worth reporting. -func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { - if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { - return next - } - if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || - queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { - return queued - } - return next -} - // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 556f7f59f90..f7262c04632 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -99,36 +99,6 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } -func TestMergeAction(t *testing.T) { - const ( - create = bundledeployments.OperationActionTypeOperationActionTypeCreate - recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate - update = bundledeployments.OperationActionTypeOperationActionTypeUpdate - resize = bundledeployments.OperationActionTypeOperationActionTypeResize - del = bundledeployments.OperationActionTypeOperationActionTypeDelete - ) - - cases := []struct { - queued, next, want bundledeployments.OperationActionType - }{ - // A queued create is not downgraded: the resource is still new. - {create, update, create}, - {create, resize, create}, - {recreate, update, recreate}, - {create, create, create}, - // A delete wins: the resource is gone, so the earlier action is moot. - {create, del, del}, - {update, del, del}, - // Neither side is a create, so the later action stands. - {update, resize, resize}, - {resize, update, update}, - {del, create, create}, - } - for _, c := range cases { - assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) - } -} - func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 96c0826849483f30565a609174637834bc909bc8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:13:16 +0000 Subject: [PATCH 17/56] acceptance: set MSYS_NO_PATHCONV for the DMS tests The dms tests pass workspace paths to $CLI (`workspace get-status /Workspace/...`). On Windows, Git Bash rewrites a leading-'/' argument into a Windows path before the CLI sees it, so the lookup goes to C:/Program Files/Git/Workspace/... and 404s, failing record, no-resources and redeploy-after-destroy on that platform only. Same fix and reason as acceptance/cmd/workspace/export-dir-*/test.toml. Co-authored-by: Isaac --- acceptance/bundle/dms/test.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..40cc45a44a5 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,11 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a +# leading-'/' argument into a Windows path before the CLI sees it, so +# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... +# and 404s. Quoting the argument does not help - the conversion happens in the +# Windows binary's argument processing. +[Env] +MSYS_NO_PATHCONV = "1" From b2ef0e83a76c48527379f2b5f39edb82a87169e4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:39:34 +0000 Subject: [PATCH 18/56] acceptance: scope MSYS_NO_PATHCONV to the get-status commands Setting it in test.toml applied it to the whole test, which stopped Git Bash converting the PATH too - so python3 could not find print_requests.py ("can't open file 'C:\\c\\a\\cli\\cli\\acceptance\\bin\\print_requests.py'") and every dms test failed on Windows, including the three that were passing. trace exports leading KEY=value pairs in a subshell, so setting it there fixes the CLI's leading-'/' argument without reaching the helpers. The precedent this was copied from (acceptance/cmd/workspace/export-dir-*/test.toml) uses no python helpers, which is why the test.toml form is safe there but not here. Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/script | 2 +- acceptance/bundle/dms/record/output.txt | 2 +- acceptance/bundle/dms/record/script | 6 +++++- acceptance/bundle/dms/redeploy-after-destroy/output.txt | 4 ++-- acceptance/bundle/dms/redeploy-after-destroy/script | 4 ++-- acceptance/bundle/dms/test.toml | 8 -------- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index b400fb9a67e..86009c71b94 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -34,7 +34,7 @@ Deployment complete! } } ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 847935af5d1..9b14355bd27 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,7 +1,7 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 3266e0d3af8..c2b3333e587 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -64,7 +64,7 @@ Deployment complete! } === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 3298c8fb131..0ad4dd96d98 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -3,7 +3,11 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path +# into C:/Program Files/Git/Workspace/... before the CLI sees it. Set per command +# rather than in test.toml: trace exports it in a subshell, so it cannot reach the +# python helpers, whose PATH does need converting. +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ff8694b5424..b8b6f3c630d 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -15,7 +15,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. === Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node @@ -25,7 +25,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 04c639019c9..a39edf3c0d8 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -3,9 +3,9 @@ trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 40cc45a44a5..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,11 +10,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a -# leading-'/' argument into a Windows path before the CLI sees it, so -# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... -# and 404s. Quoting the argument does not help - the conversion happens in the -# Windows binary's argument processing. -[Env] -MSYS_NO_PATHCONV = "1" From 30adb997c90de6c5831773909387190372b0f663 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:25:27 +0000 Subject: [PATCH 19/56] bundle: drop the locks in the operation queue's close close runs on one goroutine after every apply worker has returned, so nothing else touches the queue by then and the closed flag needs no protection. The wg.Wait orders the upload workers' writes to err before it is read, so that read needs none either. Also tightens the coalescing test to count uploads for the resource instead of only checking the payload of the last one. It asserted the merged content but would have passed if the operations had been uploaded twice, which is the thing coalescing exists to prevent. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 14 ++++++-------- bundle/direct/opqueue_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 9a1a1cd918e..8d1bea2ff72 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -135,23 +135,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // record must have returned first: record on a closed queue panics. Calling close // more than once is safe, so callers can defer it and still check the error at a // specific point. +// +// Unlike the other methods this one takes no lock. It runs on one goroutine after +// every apply worker has returned, so nothing else touches the queue by then, and +// the wg.Wait below orders the workers' writes to err before it is read. func (q *operationQueue) close() error { if q == nil { return nil } - q.mu.Lock() - closed := q.closed - q.closed = true - q.mu.Unlock() - - if !closed { + if !q.closed { + q.closed = true close(q.queue) q.wg.Wait() } - q.mu.Lock() - defer q.mu.Unlock() return q.err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 356a7cfb319..69492c2e63e 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -130,6 +130,16 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { close(f.block) require.NoError(t, q.close()) + // One upload, not two: the second operation replaced the first while every + // worker was busy, so the extra CreateOperation round trip never happens. + var uploadsForFoo int + for _, u := range f.recorded() { + if strings.HasPrefix(u, "resources.jobs.foo=") { + uploadsForFoo++ + } + } + assert.Equal(t, 1, uploadsForFoo, "the two operations should coalesce into one upload") + // Everything comes from the newest operation: it carries the resource's full // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) From 8f56ef6878728e88ffbc6da8d58f61130a0aea54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:33:25 +0000 Subject: [PATCH 20/56] bundle: record operation state without redacting it Drops the RedactSensitiveFields call, leaving a TODO: fields marked bundle:"sensitive" now reach DMS in plaintext, and the read path writes them back into the local state file unredacted. This has to be restored before the feature ships to users. Also documents why take leaves the key in queuedOrUploading when it hands an operation to a worker, and covers the case the comment describes: recording while that key's upload is in flight. The operation cannot join the in-flight request, so it is uploaded next by the same worker rather than being dropped or picked up concurrently by a second one. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 5 +++++ bundle/direct/opqueue_test.go | 27 +++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 10 ++++------ bundle/direct/oprecorder_test.go | 9 ++++----- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 8d1bea2ff72..81163915b96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -189,6 +189,11 @@ func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { return recordedOperation{}, false } + // The key stays in queuedOrUploading: the worker keeps coming back here until + // nothing is pending for it, so anything recorded while this operation uploads + // is still picked up. The mark is only cleared above, once there is nothing + // left - which is also what stops a second worker from taking the key and + // uploading the same resource concurrently. delete(q.pending, resourceKey) return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 69492c2e63e..b28d75dbf43 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -149,6 +149,33 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { f.actionFor("resources.jobs.foo")) } +func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { + // Record while the key's own upload is in flight: the key is off the queue but + // still marked, so record does not queue it again. The worker that holds the key + // has to come back for it, or the operation would be silently dropped. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.started) + + // The worker has taken the key off the queue and is uploading v1 right now. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v2"}, nil)) + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // after v1 rather than replacing it. The service ends up with the newest state. + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + }, f.recorded()) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 2401e34fcf8..4f932f09c3a 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -8,8 +8,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -47,11 +45,11 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // - // Redact secrets, like dstate.SaveState does for the local state file: - // otherwise we leak them to the service and the read path writes them back - // into a local state file in plaintext. + // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, + // unlike dstate.SaveState which redacts them before writing the local state + // file. Redact them before this ships to users. if state != nil { - config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := json.Marshal(state) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index f7262c04632..a948b609441 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,7 +64,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { +func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -74,10 +73,10 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Sensitive fields are redacted before leaving the CLI, matching what - // dstate.SaveState writes to the local state file. + // Recorded as-is for now, unlike dstate.SaveState which redacts before writing + // the local state file. See the TODO in newRecordedOperation. assert.JSONEq(t, - `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) } From 329886f2b2e62a821cead6ad54aaaebefee2381f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:38:48 +0000 Subject: [PATCH 21/56] bundle: drop the redaction TODO from the operation recorder Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 4 ---- bundle/direct/oprecorder_test.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 4f932f09c3a..d0d2694d358 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -44,10 +44,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. - // - // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, - // unlike dstate.SaveState which redacts them before writing the local state - // file. Redact them before this ships to users. if state != nil { config, err := json.Marshal(state) if err != nil { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index a948b609441..1b68c3a626b 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -73,8 +73,8 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is for now, unlike dstate.SaveState which redacts before writing - // the local state file. See the TODO in newRecordedOperation. + // Recorded as-is, unlike dstate.SaveState which redacts before writing the + // local state file. assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) From 2818d11bcab9cf6b958f67ee736c91e50698771d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:00:00 +0000 Subject: [PATCH 22/56] bundle: revert the schema annotation and tighten CompleteVersion's guard Drops the record_deployment_history annotation change (and the generated schema that followed from it), leaving both files as they are on main. CompleteVersion now keys its no-op on versionNum rather than on the heartbeat handle. Both are set together by CreateVersion, so the behaviour is the same - a deploy that was cancelled, or whose CreateVersion failed, does not complete a version that was never created - but the check now names the thing it is actually guarding. Callers defer CompleteVersion unconditionally, so this is the only thing standing between a failed CreateVersion and a CompleteVersion call against a nonexistent version. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 2 -- bundle/schema/jsonschema.json | 2 +- libs/dms/recorder.go | 7 +++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 100d33356fd..10832fe04a6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,8 +178,6 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - - Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c52e96d5dc9..4c78bd7c384 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", "$ref": "#/$defs/bool" }, "scripts": { diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 83f180ba3be..e285a80e4a2 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -102,9 +102,12 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { } // CompleteVersion finalizes the version created by CreateVersion. A nil -// Recorder, or one whose CreateVersion never ran, is a no-op. +// Recorder, or one whose CreateVersion never ran or failed, is a no-op: there is +// no version on the server to complete. Callers defer it unconditionally, so this +// is the check that keeps a cancelled or failed deploy from completing a version +// that was never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { - if r == nil || r.stopHeartbeat == nil { + if r == nil || r.versionNum == 0 { return nil } From b84ae7c631453e3cba92d83f729b44fab0588900 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:24:31 +0000 Subject: [PATCH 23/56] bundle: gate record_deployment_history off again Restores validate.ValidateRecordDeploymentHistory and the hidden DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, so setting experimental.record_deployment_history is an error unless that variable is set. The service side is not ready for users: DMS is only deployed to dev and staging, and reading state back needs the workspace APIs to expose the deployment's tree node, which is still behind a flag. With the flag unreachable, the state feature flag added earlier is not needed yet, so dstate is back to the resource-count check in Open. The deployment_history feature, hasFeature/setFeature, the version bump on write and the WAL carry-over all come back with the state upgrade in a follow-up. The dms acceptance tests force allow the flag, and bundle/dms/not-supported covers the error users see. Operation requests in bundle/dms/depends-on print multi-line now: the state envelope nests two levels, which --oneline made unreadable. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 67 ++++++++++- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/existing-state/output.txt | 4 +- .../bundle/dms/not-supported/databricks.yml | 10 ++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++ acceptance/bundle/dms/not-supported/script | 5 + acceptance/bundle/dms/not-supported/test.toml | 6 + acceptance/bundle/dms/record/output.txt | 9 -- acceptance/bundle/dms/record/script | 3 - acceptance/bundle/dms/test.toml | 6 + .../validate_record_deployment_history.go | 47 ++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++ bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++------------ bundle/direct/dstate/state_test.go | 21 +--- .../force_allow_record_deployment_history.go | 19 +++ bundle/phases/initialize.go | 5 + cmd/bundle/utils/process.go | 1 - 19 files changed, 296 insertions(+), 136 deletions(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 2bb8d06b9dc..67fd0165669 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -6,9 +6,70 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //versions/1/operations --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +>>> print_requests.py //versions/1/operations --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.child" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.child", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.parent" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.parent", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} === Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 905d022619c..be1b9d622f8 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy -trace print_requests.py //versions/1/operations --sort --oneline +trace print_requests.py //versions/1/operations --sort title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" rm -rf .databricks diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 229b3bea03f..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: target "default" was deployed without experimental.record_deployment_hist === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..b665d545d12 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: the service side is not ready for users yet +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..10a0d995d87 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: the service side is not ready for users yet" +trace musterr $CLI bundle validate + +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..4617ff88f20 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,6 @@ +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index c2b3333e587..33cf719ddfd 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,15 +73,6 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false -=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 0ad4dd96d98..32ee3ec972f 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -10,9 +10,6 @@ title "The deployment ID is the ID of the workspace node the service registered trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" -trace jq '{state_version, features}' .databricks/bundle/default/resources.json - title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..7c21473f724 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected outright (see +# validate.ValidateRecordDeploymentHistory). These tests exercise the feature itself, +# so they force allow it the same way DMS development does. bundle/dms/not-supported +# covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..4f0137fae0f --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but the service side is not +// ready for users: the deployment metadata service is only deployed to dev and staging, +// and reading state back needs the workspace APIs to expose the deployment's tree node, +// which is still behind a flag. Enabling this today also makes DMS the source of truth +// for resource state, so a bundle that turns it on cannot be turned back. +// +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.ForceAllowRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..1bb172766f2 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + forceAllow string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 288fdcf863f..e4d21a7054a 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,34 +12,25 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) -// knownFeatures lists the state feature flags this CLI implements. A state that -// records anything outside this set is refused by migrateState. -var knownFeatures = map[string]bool{ - FeatureDeploymentHistory: true, -} - // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list (see the featureStateVersion - // doc comment). A featureStateVersion state with no features is equivalent to - // currentStateVersion, so accept it and return without running the migrations - // below, leaving the on-disk version at featureStateVersion rather than flipping - // it down. Same for a state whose features this CLI implements. One that records - // a feature this CLI does not know depends on capabilities it lacks, so refuse it - // and tell the user to upgrade. + // featureStateVersion states carry a feature list this CLI does not yet write or + // understand (see the featureStateVersion doc comment). A featureStateVersion + // state with no features is equivalent to currentStateVersion, so accept it and + // return without running the migrations below, leaving the on-disk version at + // featureStateVersion rather than flipping it down. One that records any feature + // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. if db.StateVersion == featureStateVersion { - unknown := make([]string, 0, len(db.Features)) - for name := range db.Features { - if !knownFeatures[name] { - unknown = append(unknown, name) - } - } - if len(unknown) == 0 { + if len(db.Features) == 0 { return nil } - slices.Sort(unknown) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) + features := make([]string, 0, len(db.Features)) + for name := range db.Features { + features = append(features, name) + } + slices.Sort(features) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index b367e97001b..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,28 +31,22 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version written once a state records - // deployment state "feature flags" (see Header.Features). Reading such a state - // (see migrateState): - // - featureStateVersion with no features -> accept, leave the version as-is - // - featureStateVersion with known features -> accept - // - featureStateVersion with unknown features -> refuse, tell the user to upgrade + // featureStateVersion is the schema version a future CLI will write once it + // records deployment state "feature flags" (see Header.Features). This CLI does + // not write it and records no features; it exists now only so this CLI reads + // such states correctly (see migrateState): + // - featureStateVersion with no features -> accept and leave the version as-is + // - featureStateVersion with any feature -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. That way a release can start writing - // featureStateVersion + features without older CLIs either mishandling a feature - // they lack or rejecting a featureless state outright. featureStateVersion is - // always 3. + // featureStateVersion. This is forward-compat scaffolding so that a later release + // can start writing featureStateVersion + features without older CLIs (with this + // change) either mishandling a feature they lack or rejecting a featureless state + // outright. featureStateVersion is always 3. featureStateVersion = 3 - // FeatureDeploymentHistory marks a state whose resources live in the deployment - // metadata service rather than in the state file. A CLI that does not know this - // feature refuses the state instead of deploying against a resource set it - // cannot see - the file's resources are not authoritative. - FeatureDeploymentHistory = "deployment_history" - // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -88,27 +82,13 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. A CLI that does not implement one of them refuses the state rather than - // deploying against it (see migrateState). It is a map so a future CLI can attach - // per-feature data without reshaping the state. Empty/omitted for states that use - // no features. + // value. This CLI writes no features; it only reads the field to detect a state + // that depends on features it lacks and refuse it (see migrateState). It is a + // map so a future CLI can attach per-feature data without reshaping the state. + // Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } -// hasFeature reports whether the state records the given feature flag. -func (h *Header) hasFeature(name string) bool { - _, ok := h.Features[name] - return ok -} - -// setFeature records a feature flag in the state. -func (h *Header) setFeature(name string) { - if h.Features == nil { - h.Features = make(map[string]struct{}) - } - h.Features[name] = struct{}{} -} - type Database struct { Header @@ -248,10 +228,6 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string - - // TargetName names the bundle target in the error Open returns for a state that - // predates the opt-in, since the feature is enabled per target. - TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -307,18 +283,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // An existing state file has to be marked as DMS-owned before its resources - // can be read from DMS. Recording only starts on an empty state, so a state - // with resources and no feature flag predates the opt-in: DMS never saw those - // resources and is authoritative for the whole set once enabled (see - // readDMSState), so they would look absent and be created a second time. - // Migrating such a target is not supported yet. - if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { - return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. + // + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - // Mark the state as DMS-owned so a CLI without this feature refuses it rather - // than deploying against a resource set it cannot see. - db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -335,7 +311,13 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) + walHead := Header{ + Lineage: db.GetOrInitLineage(), + Serial: db.Data.Serial + 1, + StateVersion: currentStateVersion, + CLIVersion: build.GetInfo().Version, + } + return appendJSONLine(db.walFile, walHead) } return nil @@ -422,16 +404,6 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial - - // Carry the WAL's features (and the version that goes with them) into the - // state being written, so recovering a WAL written by a DMS-recording deploy - // still produces a state marked as DMS-owned. - for name := range header.Features { - db.Data.setFeature(name) - } - if len(db.Data.Features) > 0 { - db.Data.StateVersion = featureStateVersion - } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -535,25 +507,13 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) -} - -// newWalHeader builds the header for a fresh WAL. Features carry over from the -// state being written, and a state that records any feature is written at -// featureStateVersion so a CLI that lacks the feature refuses it instead of -// deploying against it. The caller holds db.mu. -func (db *DeploymentState) newWalHeader() Header { - version := currentStateVersion - if len(db.Data.Features) > 0 { - version = featureStateVersion - } - return Header{ + walHead := Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: version, + StateVersion: currentStateVersion, CLIVersion: build.GetInfo().Version, - Features: db.Data.Features, } + return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 35050575121..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,16 +154,7 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 recording a feature this CLI implements is accepted. - known := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}}, - }} - require.NoError(t, migrateState(known)) - assert.Equal(t, featureStateVersion, known.StateVersion) - - // v3 recording a feature this CLI does not know is refused: its resources may - // live somewhere this CLI cannot see. + // v3 that records a feature is refused: this CLI does not understand features. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -174,16 +165,6 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) - - // Only the unknown feature is named, so the message tells the user what to do. - mixed := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, - }} - err = migrateState(mixed) - require.Error(t, err) - assert.Contains(t, err.Error(), "future_feature") - assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..70ea74fa182 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) + // Rejects experimental.record_deployment_history: the feature is not usable yet. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index b000da60264..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,7 +229,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, - TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From daa69e28df3826732cde4e90edd787620b625749 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:47:33 +0000 Subject: [PATCH 24/56] bundle: stop the deploy when an operation upload fails An upload failure was only reported at close, so a DMS outage let apply deploy every remaining resource and fail at the end. That leaves resources in the workspace that DMS has no record of, and since a completed version makes DMS the source of truth for resource state, the next deploy would create them again. record now returns the first upload error, which the apply worker turns into a failed node, so the deploy stops shortly after the failure instead of running to completion. It refuses new work only: operations already recorded still upload, because close drains them, so the records DMS ends up with match the resources that were actually applied. Resources already mid-apply also finish. Also repeats the one test whose bug depends on a scheduler interleaving rather than on a forced handshake, so a single run gets many chances to hit the bad ordering. The other tests pin their interleaving with the started/block channels, so repetition would not add coverage; the new error tests use a `done` channel to wait for an upload to have finished rather than merely started. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 31 ++++++++- bundle/direct/opqueue_test.go | 121 ++++++++++++++++++++++++++-------- 2 files changed, 120 insertions(+), 32 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 81163915b96..ba189c13d2e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -93,9 +93,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers. It makes no -// API call, so upload failures surface from close; an error here only means the -// applied resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. The upload +// itself happens on a worker, so an error returned here is either a failure to +// turn the applied resource into a payload, or an earlier upload's error +// resurfaced (see below). // // Recording a resource that is still waiting replaces the waiting operation // outright, since the newer one carries the resource's full state. @@ -104,6 +105,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return nil } + // Report an earlier upload failure to the apply worker that is about to record + // the next resource, so the deploy stops instead of running to completion and + // only failing at close. That matters because a successfully completed version + // makes DMS the source of truth for resource state (see dstate.readDMSState): + // deploying everything while its records are missing leaves resources the next + // deploy would create a second time. + // + // This refuses new work only. Operations already recorded still upload - close + // drains them - so the records DMS does end up with match the resources that + // were actually applied. Resources already mid-apply also finish, so the deploy + // stops shortly after the first failure rather than exactly at it. + if err := q.firstErr(); err != nil { + return err + } + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err @@ -208,3 +224,12 @@ func (q *operationQueue) setErr(err error) { q.err = err } } + +// firstErr returns the first upload error, or nil if every upload so far +// succeeded. +func (q *operationQueue) firstErr() error { + q.mu.Lock() + defer q.mu.Unlock() + + return q.err +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index b28d75dbf43..a68560493ce 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -20,7 +20,10 @@ import ( type fakeUploader struct { block chan struct{} started chan string - err error + // done receives the resource key after the upload returns, for tests that need + // an upload to have completed rather than merely started. + done chan string + err error mu sync.Mutex uploads []string @@ -37,7 +40,6 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.mu.Lock() - defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} @@ -45,6 +47,13 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.actions[resourceKey] = op.action f.resourceIDs[resourceKey] = op.resourceID + f.mu.Unlock() + + // Sent outside the lock: a test that stops reading this channel would otherwise + // hold f.mu and deadlock every other worker. + if f.done != nil { + f.done <- resourceKey + } return f.err } @@ -189,6 +198,54 @@ func TestOperationQueueReturnsUploadError(t *testing.T) { assert.Contains(t, err.Error(), "resources.jobs.foo") } +func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { + // An upload failure stops the deploy at the next resource instead of surfacing + // only at close, so the apply workers do not keep creating resources that DMS + // has no record of. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Wait for the failing upload to finish, so the error is stored before the next + // record rather than racing it. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.done) + + // The next resource an apply worker tries to record is refused, with the upload + // error that caused it. + err := q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil) + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + + // The refused resource was not queued, and close still reports the failure. + require.ErrorIs(t, q.close(), uploadErr) + assert.Equal(t, []string{`resources.jobs.foo={"state":{"name":"v1"}}`}, f.recorded()) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + +func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { + // A failure refuses new work but does not discard work already recorded: the + // records DMS ends up with have to match the resources that were applied. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Every worker is parked mid-upload, so these stay queued. + for i := range operationUploadWorkers { + require.NoError(t, q.record(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + require.NoError(t, q.record(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + + close(f.block) + require.ErrorIs(t, q.close(), uploadErr) + + // The queued operation was uploaded rather than dropped on the way out. + assert.Contains(t, f.recorded(), `resources.jobs.queued={"state":{"name":"v1"}}`) + assert.Len(t, f.recorded(), operationUploadWorkers+1) +} + func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) @@ -254,40 +311,46 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // case where a coalesced key can be handed to a second worker while the first // is still uploading it. The service keeps one state per key, so overlapping // uploads for a key could land out of order and leave a stale state behind. + // + // The interleaving that breaks this is scheduler-dependent, so one pass proves + // little: repeat it so a single run has many chances to hit the bad ordering. const ( + iterations = 200 workers = 10 perWorker = 5 distinctKeyMod = 12 ) - ctx := t.Context() - u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(ctx, u) - - // Collect record errors instead of asserting inside the goroutines: testify - // assertions may only run on the goroutine running the test function. - errs := make(chan error, workers*perWorker) - var wg sync.WaitGroup - for w := range workers { - wg.Go(func() { - for i := range perWorker { - key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) - } - }) - } - wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } - require.NoError(t, q.close()) + for range iterations { + ctx := t.Context() + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(ctx, u) + + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) + } + }) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.NoError(t, q.close()) - assert.False(t, u.uneven, "two uploads overlapped for the same resource key") - // Every distinct key was recorded, and close drained all of them. - assert.Len(t, u.last, distinctKeyMod) - assert.Empty(t, q.pending) - assert.Empty(t, q.queuedOrUploading) + require.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + require.Len(t, u.last, distinctKeyMod) + require.Empty(t, q.pending) + require.Empty(t, q.queuedOrUploading) + } } func TestNilOperationQueueIsNoOp(t *testing.T) { From 2c2e209f3abc8cccdc37d91f354cf277bf642a28 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:14:02 +0000 Subject: [PATCH 25/56] bundle: stop applying resources once an operation upload fails The previous commit made record return the upload error, but record runs after the resource has already been created or updated, so every node that started before the failure was noticed still modified the workspace. Apply now checks for a recorded failure before it touches anything, right after the dependency check, so a node that has not started yet is refused rather than applied. Resources already mid-apply still finish - the check cannot unwind those - but the deploy no longer runs to completion against a service that is rejecting its records. acceptance/bundle/dms/operation-upload-fails covers it. Which resources get refused depends on how far apply got before a background upload failed, so the per-resource errors go to a LOG file and requests are not recorded; the test asserts the deploy fails and reports the upload error. Also drops --sort from the dms tests that deploy zero or one resource: their request order is already deterministic, and the unsorted output reads in chronological order. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 10 ++--- acceptance/bundle/dms/existing-state/script | 8 ++-- acceptance/bundle/dms/no-resources/output.txt | 8 ++-- acceptance/bundle/dms/no-resources/script | 4 +- .../dms/operation-upload-fails/databricks.yml | 24 +++++++++++ .../dms/operation-upload-fails/out.test.toml | 3 ++ .../dms/operation-upload-fails/output.txt | 4 ++ .../bundle/dms/operation-upload-fails/script | 6 +++ .../dms/operation-upload-fails/test.toml | 12 ++++++ acceptance/bundle/dms/record/output.txt | 42 +++++++++---------- acceptance/bundle/dms/record/script | 6 +-- .../dms/redeploy-after-destroy/output.txt | 16 +++---- .../bundle/dms/redeploy-after-destroy/script | 4 +- bundle/direct/bundle_apply.go | 11 +++++ bundle/direct/opqueue.go | 6 ++- 15 files changed, 114 insertions(+), 50 deletions(-) create mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml create mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails/script create mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..755981c414d 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true @@ -16,14 +16,14 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false @@ -45,8 +45,8 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index ae9be95f701..9e448161259 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,22 +1,22 @@ title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace musterr $CLI bundle deploy title "No deployment was created in DMS" -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" rm -rf .databricks trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 86009c71b94..61e520c20a2 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -46,14 +46,14 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 9b14355bd27..f1b9ad5fa60 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml b/acceptance/bundle/dms/operation-upload-fails/databricks.yml new file mode 100644 index 00000000000..1edbd7add88 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: dms-operation-upload-fails + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..26c748d9e79 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,6 @@ +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..ce222ac9e87 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,12 @@ +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# The service rejects every recorded operation. Deploy must stop rather than +# create every remaining resource: a completed version makes DMS the source of +# truth for resource state, so resources it has no record of would be created a +# second time by the next deploy. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 33cf719ddfd..dcb6b3efa22 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -27,13 +27,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -62,6 +55,13 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json @@ -80,7 +80,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -111,11 +111,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //api/2.0/bundle --sort -{ - "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -128,13 +124,6 @@ Destroy complete! "version_type": "VERSION_TYPE_DESTROY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", @@ -147,3 +136,14 @@ Destroy complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 32ee3ec972f..895a9033a39 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -1,6 +1,6 @@ title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" # MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path @@ -13,8 +13,8 @@ trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" trace $CLI bundle destroy --auto-approve -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index b8b6f3c630d..783082c1f83 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -31,7 +31,7 @@ Deployment complete! "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -52,13 +52,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -87,3 +80,10 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index a39edf3c0d8..8dabf189f49 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,11 +1,11 @@ title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve -print_requests.py //api/2.0/bundle --sort --get > /dev/null +print_requests.py //api/2.0/bundle --get > /dev/null trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index f29aa18a186..46d70c7b135 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -69,6 +69,17 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop before touching the workspace once recording an operation has failed. + // A completed version makes DMS the source of truth for resource state (see + // dstate.readDMSState), so continuing would create resources it has no record + // of and the next deploy would create them a second time. Checked here rather + // than only where operations are recorded, which is after the resource has + // already been modified. + if err := opQueue.firstErr(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba189c13d2e..48d2887ec7c 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -226,8 +226,12 @@ func (q *operationQueue) setErr(err error) { } // firstErr returns the first upload error, or nil if every upload so far -// succeeded. +// succeeded. A nil queue (recording disabled) never errors. func (q *operationQueue) firstErr() error { + if q == nil { + return nil + } + q.mu.Lock() defer q.mu.Unlock() From 9329441d2f4391271c886fc5305addd76c9680a2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:37:48 +0000 Subject: [PATCH 26/56] testserver: create the DMS deployment record on the first version CreateDeployment now only registers the workspace node whose ID names the deployment; the record itself is created by the first CreateVersion. A client that registers a deployment and then fails before recording a version leaves just the node behind, not an empty deployment. That state is reachable, so both sides of the CLI handle it: - the recorder starts at version 1 under the ID the node already names, instead of failing on the missing record or creating a second deployment that would collide on the same node path - the read path keeps the local (empty) state instead of surfacing the 404 from ListResources acceptance/bundle/dms/version-never-created covers it end to end: the first version fails, and the next deploy reuses the same deployment ID. Also drops libs/testserver/bundle_test.go. The fake is exercised by every dms acceptance test, so unit tests for it only duplicate that coverage. Co-authored-by: Isaac --- .../dms/version-never-created/databricks.yml | 10 ++++ .../dms/version-never-created/out.test.toml | 3 ++ .../dms/version-never-created/output.txt | 34 +++++++++++++ .../bundle/dms/version-never-created/script | 7 +++ .../dms/version-never-created/test.toml | 7 +++ bundle/direct/dstate/dms.go | 12 +++++ libs/dms/recorder.go | 26 ++++++---- libs/dms/recorder_test.go | 46 ++++++++++------- libs/testserver/bundle.go | 37 +++++++++----- libs/testserver/bundle_test.go | 51 ------------------- libs/testserver/fake_workspace.go | 7 +++ 11 files changed, 147 insertions(+), 93 deletions(-) create mode 100644 acceptance/bundle/dms/version-never-created/databricks.yml create mode 100644 acceptance/bundle/dms/version-never-created/out.test.toml create mode 100644 acceptance/bundle/dms/version-never-created/output.txt create mode 100644 acceptance/bundle/dms/version-never-created/script create mode 100644 acceptance/bundle/dms/version-never-created/test.toml delete mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..f6fae9e6f56 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,34 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "FILE" +} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> print_requests.py //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..39566343e98 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,7 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..f0a8407e21d --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,7 @@ +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index ec51e2c4479..094f114a0a6 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,9 +3,12 @@ package dstate import ( "context" "encoding/json" + "errors" "fmt" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -33,6 +36,15 @@ type RecordedState struct { func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { + // The deployment's record is created by its first version, so a node can + // resolve to an ID that has none yet: a deploy that registered the deployment + // and then failed before recording a version. There is nothing to read, and + // the file's resources are still empty, so carry on and let this deploy record + // the first version. + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + log.Debugf(ctx, "No deployment record for %s yet; keeping local state", src.DeploymentID) + return nil + } return err } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index e285a80e4a2..6973a09b4b5 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -150,22 +150,26 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment. The service trashes the - // workspace node when it deletes the record, so a node that resolved but has - // no record means the two are out of sync, and creating another deployment - // would collide on the same node path. + // A resolved node names the deployment, but its record is created by the + // first version, so there may be none yet: a deploy that registered the + // deployment and then failed before recording a version. Start at version 1 + // under the ID the node already names, rather than creating a second + // deployment, which would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - if getErr != nil { + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + versionID = "1" + default: return "", fmt.Errorf("failed to get deployment: %w", getErr) } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } else { // First deploy: create the deployment so the server assigns an ID. // diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 6c2f334c946..d4c7efaca03 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -112,27 +112,35 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing } func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - cases := map[string]error{ - // A resolved ID whose record is missing means the record and the workspace - // node it was resolved from are out of sync. Creating a second deployment - // would collide on the same node path, so fail instead. - "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), - "other": errors.New("boom"), + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, } - for name, getErr := range cases { - t.Run(name, func(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, getErr - }, - } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) - - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) - }) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + +func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { + // The record is created by the first version, so a node can name a deployment + // that has none yet - an earlier deploy registered it and then failed. Record + // version 1 under that same ID instead of creating a second deployment, which + // would collide on the node path. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + }, } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/stored-id", f.versions[0].Parent) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 6dffbe34a75..4e04c65ba81 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -35,9 +35,6 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string - // nodePath is the workspace node whose object ID is this deployment's ID. - // Kept so DeleteDeployment can trash the node, the way the service does. - nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -70,15 +67,15 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } + // Only the node is created here. The deployment record itself is created by the + // first CreateVersion, so a client that creates a deployment and then fails + // before recording a version leaves no record behind - just the node, which + // names the ID that first version will be created under. deploymentID := strconv.FormatInt(objectID, 10) + s.dmsDeploymentNodes[deploymentID] = nodePath + dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive - s.dmsDeployments[deploymentID] = &dmsDeployment{ - deployment: dep, - versions: map[string]*bundledeployments.Version{}, - resources: map[string]bundledeployments.Resource{}, - nodePath: nodePath, - } return Response{Body: dep} } @@ -129,9 +126,10 @@ func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { // The service trashes the deployment's workspace node, so a later get-status // on the node path reports the deployment as absent. - if d, ok := s.dmsDeployments[deploymentID]; ok { - delete(s.files, d.nodePath) + if nodePath, ok := s.dmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) } + delete(s.dmsDeploymentNodes, deploymentID) delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } @@ -148,7 +146,22 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d, ok := s.dmsDeployments[deploymentID] if !ok { - return dmsNotFound("deployment " + deploymentID) + // The deployment record is created by its first version, not by + // CreateDeployment. That call only registered the workspace node, so the node + // existing is what makes this ID valid. + if _, known := s.dmsDeploymentNodes[deploymentID]; !known { + return dmsNotFound("deployment " + deploymentID) + } + d = &dmsDeployment{ + deployment: bundledeployments.Deployment{ + Name: "deployments/" + deploymentID, + Status: bundledeployments.DeploymentStatusDeploymentStatusActive, + TargetName: version.TargetName, + }, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + s.dmsDeployments[deploymentID] = d } // Mirror the server-side optimistic concurrency check: the new version must diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go deleted file mode 100644 index 4d28624ba3e..00000000000 --- a/libs/testserver/bundle_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package testserver - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against -// serializing the deployment through a struct that embeds -// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is -// promoted to the embedding struct and silently drops last_successful_version_id. -// The CLI read path treats a missing value as "DMS does not own the state", so -// losing the field here makes the whole overlay path untestable. -func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{lastSuccessfulVersionID: "2"} - d.deployment.Name = "deployments/abc" - d.deployment.LastVersionId = "3" - d.deployment.TargetName = "default" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.Equal(t, "deployments/abc", body["name"]) - assert.Equal(t, "3", body["last_version_id"]) - assert.Equal(t, "default", body["target_name"]) - assert.Equal(t, "2", body["last_successful_version_id"]) - - // The response must round-trip as JSON the same way, since that is what the - // client actually reads. - raw, err := json.Marshal(body) - require.NoError(t, err) - assert.JSONEq(t, - `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, - string(raw)) -} - -// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment -// with no successful version does not advertise one: the read path must keep -// using the local state file in that case. -func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{} - d.deployment.Name = "deployments/abc" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.NotContains(t, body, "last_successful_version_id") -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index a3c4519ccc4..13d9f76e00a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -231,6 +231,12 @@ type FakeWorkspace struct { // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by // deployment ID. Each record carries its versions and latest resource state. dmsDeployments map[string]*dmsDeployment + + // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment + // registered for it. A deployment appears here before it has a record in + // dmsDeployments: the record is created by its first version, so the node is + // what makes an ID valid in between. + dmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -383,6 +389,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, dmsDeployments: map[string]*dmsDeployment{}, + dmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, From 38b6f54081ae33008edd866a64ffd696c6dc90af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 14:05:39 +0000 Subject: [PATCH 27/56] bundle: simplify the concurrency test comment Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a68560493ce..910b58d470f 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -307,13 +307,16 @@ func (s *serialUploader) upload(ctx context.Context, resourceKey string, op reco } func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { - // Concurrent apply workers repeatedly record overlapping resource keys, the - // case where a coalesced key can be handed to a second worker while the first - // is still uploading it. The service keeps one state per key, so overlapping - // uploads for a key could land out of order and leave a stale state behind. + // Two workers must never upload the same resource at the same time. DMS stores + // one state per resource, so concurrent uploads can finish out of order and + // leave the older state as the final one. // - // The interleaving that breaks this is scheduler-dependent, so one pass proves - // little: repeat it so a single run has many chances to hit the bad ordering. + // Lots of goroutines record a small set of keys, so the same key is recorded + // repeatedly while its earlier upload may still be running. serialUploader flags + // any overlap it sees. + // + // Whether a bug shows up depends on how the scheduler interleaves things, so one + // pass proves little - repeat it to get many chances at a bad ordering. const ( iterations = 200 workers = 10 From 4f45539f06f8071c8441433dbe69f2ba9b7669da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 15:16:42 +0000 Subject: [PATCH 28/56] bundle: trim the operation recorder tests Drops TestOperationRecorderDeleteHasNoState: the dms acceptance goldens already show a delete operation recorded without a state field. Renames the redaction test to say what it checks - the state is recorded as-is - rather than contrasting it with dstate.SaveState. Co-authored-by: Isaac --- bundle/direct/oprecorder_test.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 1b68c3a626b..674c78abf77 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -52,19 +52,7 @@ func TestOperationRecorderStripsResourcePrefix(t *testing.T) { require.NotNil(t, req.Operation.State) } -func TestOperationRecorderDeleteHasNoState(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 3) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) - - require.Len(t, f.requests, 1) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) - // Delete operations carry no serialized state. - assert.Nil(t, f.requests[0].Operation.State) -} - -func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { +func TestNewRecordedOperationRecordsStateAsIs(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -73,8 +61,7 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is, unlike dstate.SaveState which redacts before writing the - // local state file. + // The state is serialized as-is, including fields tagged bundle:"sensitive". assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) From 2bf49fb490b1af8305b996e777da075385c698bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Aug 2026 16:47:17 +0000 Subject: [PATCH 29/56] bundle: send recorded state as a JSON string DMS types Operation.state as a string, so the JSON has to go on the wire quoted. The CLI was assigning the raw object to the SDK's json.RawMessage field, which the service rejected: Invalid value: {"state":{...}} for expected type: STRING Every CreateOperation failed while the deploy still reported success, so DMS ended up owning an empty resource set - and because a completed version makes it authoritative, the next deploy would have recreated everything. Verified on dogfood: before this change ListResources returned {} after a successful deploy; after it, the resource is recorded and a deploy following `rm -rf .databricks` plans "0 to add, 1 unchanged" from DMS state alone. The read path unquotes symmetrically, and the test server now stores and returns state the way the service does, so the acceptance goldens show the real wire format rather than a shape only the fake produces. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 39 +------------------ .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 16 +------- .../dms/redeploy-after-destroy/output.txt | 16 +------- bundle/direct/dstate/dms.go | 9 ++++- bundle/direct/dstate/dms_test.go | 6 ++- bundle/direct/oprecorder.go | 11 +++++- 7 files changed, 28 insertions(+), 71 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 67fd0165669..db3624d9fc1 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -17,28 +17,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ] - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -52,21 +31,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 755981c414d..e8067f822f5 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -48,5 +48,5 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index dcb6b3efa22..3f9b8a8eadc 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -37,21 +37,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 783082c1f83..c06c1d9b7cd 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -62,21 +62,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 094f114a0a6..3d7131580e2 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -77,7 +77,14 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund var recorded RecordedState if res.State != nil { - if err := json.Unmarshal(*res.State, &recorded); err != nil { + // State is a string field, so it arrives as a quoted JSON string (see the + // write side in direct.operationRecorder.upload). Unquote it, then parse + // the envelope it holds. + var envelope string + if err := json.Unmarshal(*res.State, &envelope); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } + if err := json.Unmarshal([]byte(envelope), &recorded); err != nil { return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) } } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 35fe7acbb0c..8145424ca93 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -36,7 +36,11 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy } func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { - recorded := json.RawMessage(`{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}`) + // DMS types state as a string, so the envelope arrives as a quoted JSON string. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + quoted, err := json.Marshal(envelope) + require.NoError(t, err) + recorded := json.RawMessage(quoted) f := &fakeResourceLister{resources: []bundledeployments.Resource{ {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, {ResourceKey: "pipelines.bar", ResourceId: "456"}, diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index d0d2694d358..a1a5cf641bc 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -99,7 +99,16 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r Status: bundledeployments.OperationStatusOperationStatusSucceeded, } if op.state != nil { - operation.State = &op.state + // DMS types state as a string, so the JSON goes on the wire as a quoted + // string rather than an embedded object. The SDK field is a json.RawMessage, + // so quote the payload here; sending the object directly is rejected with + // "Invalid value: {...} for expected type: STRING". + quoted, err := json.Marshal(string(op.state)) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + raw := json.RawMessage(quoted) + operation.State = &raw } _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ From 93e95a4a6f41773a612e4071739ad9dffd1e8a4b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 10:24:19 +0000 Subject: [PATCH 30/56] bundle: send previous_version_id and display_name when recording a version Two fields the service needs were never sent, because the generated bundledeployments.Version struct does not have them both: previous_version_id is the service's concurrency check. Without it every deploy after the first was rejected with "previous_version_id is outdated; the deployment's most recent version is N", so recording only ever worked once per bundle. The struct has no such field, so the CLI now builds the CreateVersion body itself via a small local type rather than the generated client. display_name is what names the deployment in the UI. The service copies it from the version onto the deployment's workspace node, which is where GetDeployment reads it back from, and it only does so when the version carries one - so every deployment showed up unnamed. It comes from bundle.name. The test server enforced "version_id == last_version_id + 1", a rule the real service does not have (it requires numerically greater, plus a matching previous_version_id). Corrected, so the fake rejects a stale previous_version_id the way the service does instead of accepting a contract only it implements. Verified on dogfood: two consecutive deploys both succeed (the second used to fail), GetDeployment and ListDeployments both return display_name "isaac-fix2-check" - the only named deployment among 38 - and a plan after rm -rf .databricks still reports "0 to add, 1 unchanged" from DMS state. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/output.txt | 7 +- acceptance/bundle/dms/record/output.txt | 11 +- .../dms/redeploy-after-destroy/output.txt | 3 +- .../dms/version-never-created/output.txt | 4 +- bundle/phases/dms.go | 21 ++-- libs/dms/recorder.go | 113 ++++++++++++++---- libs/dms/recorder_test.go | 74 +++++++++--- libs/testserver/bundle.go | 37 ++++-- 10 files changed, 210 insertions(+), 64 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index e8067f822f5..e7bb54440b5 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -47,6 +47,6 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 75086ceb5f7..1d208e1e85e 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 61e520c20a2..06e346f3029 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -22,8 +22,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-no-resources" } } { @@ -63,8 +64,10 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-no-resources", + "previous_version_id": "1" } } { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 3f9b8a8eadc..f65d67d0991 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -23,8 +23,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-record" } } { @@ -75,8 +76,10 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-record", + "previous_version_id": "1" } } { @@ -106,8 +109,10 @@ Destroy complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", "target_name": "default", - "version_type": "VERSION_TYPE_DESTROY" + "display_name": "dms-record", + "previous_version_id": "2" } } { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index c06c1d9b7cd..6e160478e7b 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -48,8 +48,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-redeploy-after-destroy" } } { diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index f6fae9e6f56..23f84279395 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -28,7 +28,7 @@ API message: Internal error >>> print_requests.py //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 3d2f4f54009..346d6ca70c5 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/client" ) // newDeploymentRecorder returns a dms.Recorder for the current deployment, or @@ -33,11 +34,17 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng if err != nil { return nil, err } - return dms.NewRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - deploymentID, - statePath, - b.Config.Bundle.Target, - versionType, - ), nil + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + return nil, err + } + return dms.NewRecorder(dms.RecorderOptions{ + Service: b.WorkspaceClient(ctx).BundleDeployments, + Versions: dms.NewAPIVersionCreator(apiClient), + DeploymentID: deploymentID, + StatePath: statePath, + TargetName: b.Config.Bundle.Target, + DisplayName: b.Config.Bundle.Name, + VersionType: versionType, + }), nil } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 6973a09b4b5..2e8798fb665 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -10,8 +10,10 @@ import ( "time" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -27,6 +29,54 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) +// createVersionRequest is the CreateVersion request body. +// +// The CLI builds the body itself instead of using bundledeployments.Version +// because the generated struct has no previous_version_id field, which the +// service requires as its concurrency check. Without it every deploy after the +// first is rejected. +type createVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + TargetName string `json:"target_name,omitempty"` + // DisplayName names the deployment in the UI. The service copies it onto the + // deployment's workspace node, which is where GetDeployment reads it from, so + // a version that omits it leaves the deployment unnamed. + DisplayName string `json:"display_name,omitempty"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` +} + +// versionCreator creates a version under a deployment. It exists because the +// generated client cannot express the request body (see createVersionRequest). +type versionCreator interface { + CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) +} + +// apiVersionCreator creates versions through the workspace API client. +type apiVersionCreator struct { + client *client.DatabricksClient +} + +// NewAPIVersionCreator returns a versionCreator that posts to the DMS API. +func NewAPIVersionCreator(c *client.DatabricksClient) versionCreator { + return &apiVersionCreator{client: c} +} + +func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + var version bundledeployments.Version + path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + // Recorder records a single deploy/destroy as a version with DMS. // // The server assigns the deployment ID on the first deploy, i.e. when the ID @@ -35,9 +85,11 @@ const ( // node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface + versions versionCreator deploymentID string statePath string targetName string + displayName string versionType VersionType // populated by CreateVersion @@ -45,18 +97,34 @@ type Recorder struct { stopHeartbeat context.CancelFunc } -// NewRecorder returns a Recorder for the given deployment. deploymentID is the -// ID resolved from the deployment's workspace node, or empty if this bundle has -// not yet recorded a deployment (the server assigns one during CreateVersion). -// statePath is the bundle's remote state directory, under which DMS registers -// the deployment node. -func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, statePath, targetName string, versionType VersionType) *Recorder { +// RecorderOptions are the dependencies and deployment identity a Recorder needs. +type RecorderOptions struct { + // Service handles every DMS call except CreateVersion. + Service bundledeployments.BundleDeploymentsInterface + // Versions handles CreateVersion; see versionCreator. + Versions versionCreator + // DeploymentID is the ID resolved from the deployment's workspace node, or + // empty if this bundle has not recorded a deployment yet (the server assigns + // one during CreateVersion). + DeploymentID string + // StatePath is the bundle's remote state directory, under which DMS registers + // the deployment node. + StatePath string + TargetName string + DisplayName string + VersionType VersionType +} + +// NewRecorder returns a Recorder for the deployment described by opts. +func NewRecorder(opts RecorderOptions) *Recorder { return &Recorder{ - svc: svc, - deploymentID: deploymentID, - statePath: statePath, - targetName: targetName, - versionType: versionType, + svc: opts.Service, + versions: opts.Versions, + deploymentID: opts.DeploymentID, + statePath: opts.StatePath, + targetName: opts.TargetName, + displayName: opts.DisplayName, + versionType: opts.VersionType, } } @@ -149,6 +217,9 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // the server assign the ID; otherwise it reads the existing deployment to // compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + // The version this one supersedes, sent as the concurrency check. Empty for a + // deployment's first version. + var previousVersionID string if r.deploymentID != "" { // A resolved node names the deployment, but its record is created by the // first version, so there may be none yet: a deploy that registered the @@ -165,6 +236,7 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) } versionID = strconv.FormatInt(lastVersion+1, 10) + previousVersionID = dep.LastVersionId case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): versionID = "1" default: @@ -194,16 +266,15 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin versionID = "1" } - // The server validates that versionID equals last_version_id + 1 and returns - // ABORTED otherwise (e.g. a concurrent deploy already created this version). - version, versionErr := r.svc.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ - Parent: "deployments/" + r.deploymentID, - VersionId: versionID, - Version: bundledeployments.Version{ - CliVersion: build.GetInfo().Version, - VersionType: r.versionType, - TargetName: r.targetName, - }, + // The server rejects the call unless versionID is numerically greater than + // last_version_id and previous_version_id matches it, so a deploy racing + // another is rejected rather than overwriting it. + version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.targetName, + DisplayName: r.displayName, + PreviousVersionId: previousVersionID, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index d4c7efaca03..f7695d62eb0 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -16,6 +16,10 @@ import ( // deployment node under; several tests assert it round-trips to the service. const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" +// testDisplayName is the bundle name the recorder sends as the version's display +// name; the service copies it onto the deployment's workspace node. +const testDisplayName = "proj" + // fakeDMS records the calls the recorder makes and lets a test script the // server-side responses. It embeds the SDK interface so it satisfies it while // only overriding the methods the recorder uses. @@ -30,11 +34,30 @@ type fakeDMS struct { // captured requests created []bundledeployments.CreateDeploymentRequest - versions []bundledeployments.CreateVersionRequest + versions []fakeVersionRequest completed []bundledeployments.CompleteVersionRequest deleted []string } +// fakeVersionRequest is a CreateVersion call captured by fakeVersions. +type fakeVersionRequest struct { + deploymentID string + versionID string + body createVersionRequest +} + +// fakeVersions captures CreateVersion calls. It is separate from fakeDMS because +// the CLI does not create versions through the generated client (see +// createVersionRequest), so the two use different signatures. +type fakeVersions struct { + requests *[]fakeVersionRequest +} + +func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) + return &bundledeployments.Version{VersionId: versionID}, nil +} + func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) // The server always assigns the ID; it is the ID of the workspace node it @@ -47,11 +70,6 @@ func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDe return f.getDeployment(id) } -func (f *fakeDMS) CreateVersion(ctx context.Context, req bundledeployments.CreateVersionRequest) (*bundledeployments.Version, error) { - f.versions = append(f.versions, req) - return &bundledeployments.Version{VersionId: req.VersionId}, nil -} - func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { f.completed = append(f.completed, req) return &bundledeployments.Version{}, nil @@ -69,7 +87,7 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} // A first deploy resolves no deployment ID from the workspace. - r := NewRecorder(f, "", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -83,8 +101,8 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) // The first version is 1, parented under the assigned deployment. require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/server-generated-id", f.versions[0].Parent) + assert.Equal(t, "1", f.versions[0].versionID) + assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) assert.Equal(t, int64(1), r.Version()) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -100,15 +118,33 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) // No new deployment is created; the version increments to last_version_id + 1. assert.Empty(t, f.created) require.Len(t, f.versions, 1) - assert.Equal(t, "5", f.versions[0].VersionId) + assert.Equal(t, "5", f.versions[0].versionID) assert.Equal(t, "stored-id", r.DeploymentID()) + // The version it supersedes is the concurrency check; without it the service + // rejects every deploy after the first. + assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) +} + +func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.CreateVersion(t.Context())) + + require.Len(t, f.versions, 1) + // The service copies display_name onto the deployment's workspace node, which + // is where GetDeployment reads it from; a version that omits it leaves the + // deployment unnamed in the UI. + assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + // A first version supersedes nothing, so previous_version_id is unset. + assert.Empty(t, f.versions[0].body.PreviousVersionId) } func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { @@ -117,7 +153,7 @@ func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { return nil, errors.New("boom") }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) err := r.CreateVersion(t.Context()) assert.ErrorContains(t, err, "failed to get deployment") @@ -134,13 +170,13 @@ func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) assert.Empty(t, f.created) require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/stored-id", f.versions[0].Parent) + assert.Equal(t, "1", f.versions[0].versionID) + assert.Equal(t, "stored-id", f.versions[0].deploymentID) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -149,10 +185,10 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) - assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) require.NoError(t, r.CompleteVersion(t.Context(), true)) // A successful destroy deletes the deployment record. @@ -165,7 +201,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -185,7 +221,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 4e04c65ba81..5d91f47b693 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -142,6 +142,14 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + // previous_version_id is absent from the generated struct, so read it separately. + var concurrency struct { + PreviousVersionId string `json:"previous_version_id"` + } + if err := json.Unmarshal(req.Body, &concurrency); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + defer s.LockUnlock()() d, ok := s.dmsDeployments[deploymentID] @@ -164,15 +172,22 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response s.dmsDeployments[deploymentID] = d } - // Mirror the server-side optimistic concurrency check: the new version must - // be exactly last_version_id + 1. - want := "1" + // Mirror the server-side checks: version_id must be numerically greater than + // the most recent version (not exactly one more), and previous_version_id must + // name that version, which is what detects a concurrent deploy. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 if d.deployment.LastVersionId != "" { - last, _ := strconv.ParseInt(d.deployment.LastVersionId, 10, 64) - want = strconv.FormatInt(last+1, 10) + last, _ = strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.deployment.LastVersionId) } - if versionID != want { - return dmsAborted("expected version " + want + ", got " + versionID) + if concurrency.PreviousVersionId != d.deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } d.deployment.LastVersionId = versionID @@ -284,6 +299,14 @@ func dmsNotFound(what string) Response { // dmsAborted returns the 409 ABORTED error the server uses for the version // optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + func dmsAborted(message string) Response { return Response{ StatusCode: 409, From 9acbeccd53d3b15ce5254409280c69711307167c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 10:35:52 +0000 Subject: [PATCH 31/56] bundle: fix recording a destroy Two problems, both only reachable on a real destroy: The delete operation sent no resource_id, so the service rejected it with "resource_id is required for OPERATION_ACTION_TYPE_DELETE operations" and the destroy failed. A delete carries no state, so resource_id is the only thing identifying the resource. It has to be read before the delete, which removes it from the local state. CompleteVersion then ran after files.Delete(). The deployment is a node under the state directory, so deleting the files deletes the deployment, and completing the version afterwards failed with 404. The destroy now completes the version before deleting the files; CompleteVersion is idempotent so Destroy can still defer it unconditionally. The test server accepted a delete without resource_id, which is why the acceptance tests passed while the real destroy failed. It now requires one. Verified on dogfood: deploy, redeploy, then destroy all succeed. Co-authored-by: Isaac --- acceptance/bundle/dms/record/output.txt | 1 + bundle/direct/bundle_apply.go | 5 ++++- bundle/phases/destroy.go | 13 +++++++++++-- libs/dms/recorder.go | 7 ++++++- libs/dms/recorder_test.go | 20 ++++++++++++++++++++ libs/testserver/bundle.go | 6 ++++++ 6 files changed, 48 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index f65d67d0991..b2cb6f240ed 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -123,6 +123,7 @@ Destroy complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "[NUMID]", "resource_key": "jobs.foo", "status": "OPERATION_STATUS_SUCCEEDED" } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 46d70c7b135..4f7eb770bb2 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -93,6 +93,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { + // Read the ID before the delete removes it from state; DMS requires it to + // identify which resource the delete operation refers to. + deletedID := b.StateDB.GetResourceID(resourceKey) if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. @@ -105,7 +108,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, "", nil, nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, deletedID, nil, nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2925e80bca8..c33b95110e2 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -82,7 +82,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, recorder *dms.Recorder) { if engine.IsDirect() { b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { @@ -106,6 +106,15 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete the version before deleting the remote files. The deployment is a + // node under the state directory, so files.Delete removes it and any later call + // fails with 404. CompleteVersion is idempotent, so the deferred call in Destroy + // is a no-op after this. + if err := recorder.CompleteVersion(ctx, true); err != nil { + logdiag.LogError(ctx, err) + return + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) { @@ -215,7 +224,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { recorder.Version(), ) } - destroyCore(ctx, b, plan, engine) + destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 2e8798fb665..cc9fdd0d87f 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -95,6 +95,10 @@ type Recorder struct { // populated by CreateVersion versionNum int64 stopHeartbeat context.CancelFunc + + // completed makes CompleteVersion idempotent, so a caller that completes the + // version early can still defer it unconditionally. + completed bool } // RecorderOptions are the dependencies and deployment identity a Recorder needs. @@ -175,9 +179,10 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { // is the check that keeps a cancelled or failed deploy from completing a version // that was never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { - if r == nil || r.versionNum == 0 { + if r == nil || r.versionNum == 0 || r.completed { return nil } + r.completed = true r.stopHeartbeat() diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index f7695d62eb0..694fab0699a 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -211,6 +211,26 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { assert.Empty(t, f.deleted) } +func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { + // Destroy completes the version before deleting the remote files, because that + // deletes the deployment's node, and still defers CompleteVersion. The second + // call must not reach the server, which would fail with 404. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + + assert.Len(t, f.completed, 1) + // The destroy deletes the deployment record once, not once per call. + assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + func TestNilRecorderIsNoOp(t *testing.T) { var r *Recorder assert.NoError(t, r.CreateVersion(t.Context())) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 5d91f47b693..eecf4153e3b 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -242,6 +242,12 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsNotFound("deployment " + deploymentID) } + // A delete carries no state, so resource_id is the only thing identifying which + // resource it refers to; the service rejects a delete without one. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") + } + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey op.ResourceKey = resourceKey From b34ed0a08b115103e9781712ca5a34cf1c26dd4d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 14:39:44 +0000 Subject: [PATCH 32/56] bundle: record failed operations, and upload at most 2 at a time Recording only covered resources that applied. A resource that failed was left out of the deployment history entirely, so the history said nothing about why a deploy failed - the one thing you want it for. DMS has error_message and a FAILED status for exactly this, and the CLI never used either. A failed operation carries no state: the resource was not written, so there is nothing to serve back as its state. The service does list the resource, but with no resource_id and no state, so a later deploy plans to create it rather than treating it as deployed. Verified against the service, and the test server now matches that shape rather than the shape I assumed. A message over the 16 KiB limit is truncated rather than rejected, since failing to record would hide the error being reported. Uploads drop from 4 workers to 2. Concurrent CreateOperation calls under one version contend on the version's operation_count, and the resulting transaction conflict is reported as a 500, which fails the deploy. Measured against the service: 3 concurrent writes succeeded, 4 did not, and 8 resources reliably failed. 2 keeps some overlap without reaching the conflict. The real fix is a batch upload API that commits every operation in one transaction; this is a stopgap until that exists. Verified on dogfood: a bundle with one good and one bad job records the good one as SUCCEEDED and the bad one as FAILED carrying the API's error, the version completes with VERSION_COMPLETE_FAILURE, and a plan from DMS state alone reports "1 to add, 1 unchanged". Co-authored-by: Isaac --- .../bundle/dms/record-failure/databricks.yml | 10 +++ .../bundle/dms/record-failure/out.test.toml | 3 + .../bundle/dms/record-failure/output.txt | 75 +++++++++++++++++++ acceptance/bundle/dms/record-failure/script | 15 ++++ .../bundle/dms/record-failure/test.toml | 6 ++ bundle/direct/bundle_apply.go | 4 + bundle/direct/opqueue.go | 38 +++++++++- bundle/direct/oprecorder.go | 54 +++++++++++-- bundle/direct/oprecorder_test.go | 21 ++++++ libs/testserver/bundle.go | 18 ++++- 10 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 acceptance/bundle/dms/record-failure/databricks.yml create mode 100644 acceptance/bundle/dms/record-failure/out.test.toml create mode 100644 acceptance/bundle/dms/record-failure/output.txt create mode 100644 acceptance/bundle/dms/record-failure/script create mode 100644 acceptance/bundle/dms/record-failure/test.toml diff --git a/acceptance/bundle/dms/record-failure/databricks.yml b/acceptance/bundle/dms/record-failure/databricks.yml new file mode 100644 index 00000000000..8e8573fa70f --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record-failure + +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..07b0c043f26 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,75 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files... +Deploying resources... +Error: cannot create resources.jobs.doomed: cluster spec is invalid (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: cluster spec is invalid + + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record-failure" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.doomed" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "error_message": "cluster spec is invalid", + "resource_key": "jobs.doomed", + "status": "OPERATION_STATUS_FAILED" + } +} + +=== The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "name": "deployments/[NUMID]/resources/jobs.doomed", + "resource_key": "jobs.doomed", + "resource_type": "" + } + ] +} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..6a1890973a6 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,15 @@ +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | jq -r .object_id) +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..bf8710439b1 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,6 @@ +# The job cannot be created, so applying it fails and the operation is recorded as +# failed rather than omitted. +[[Server]] +Pattern = "POST /api/2.2/jobs/create" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "cluster spec is invalid"}''' diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 4f7eb770bb2..bdd0c243e7f 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,6 +104,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { + opQueue.recordFailure(ctx, resourceKey, action, deletedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -137,6 +138,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // GetResourceID is empty for a create that never got an ID, which is + // what the service expects for a failed create. + opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 48d2887ec7c..c3f4439ee4e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -18,7 +18,13 @@ const ( // operationUploadWorkers is how many uploads run at a time. It is below // operationQueueSize so a burst of operations is absorbed by the queue rather // than by one request per resource. - operationUploadWorkers = 4 + // + // Capped at 2 because concurrent CreateOperation calls under the same version + // contend on shared state server-side and the transaction conflict surfaces as + // a 500, which fails the deploy. Measured against the service: 3 concurrent + // writes still succeeded, 4 did not. The real fix is a batch upload API that + // commits every operation in one transaction; until then, keep this at 2. + operationUploadWorkers = 2 ) // operationQueue hands recorded operations to background workers, so an apply @@ -125,6 +131,33 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return err } + q.enqueue(ctx, resourceKey, op) + return nil +} + +// recordFailure records that applying a resource failed, so the deployment +// history explains the failure instead of omitting the resource. +// +// Unlike record, this does not resurface an earlier upload error: the deploy is +// already failing, and returning a different error here would replace the one the +// user needs to see. A failure to upload this record is reported at close. +func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { + if q == nil { + return + } + + op, err := newFailedOperation(action, resourceID, cause) + if err != nil { + log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) + return + } + + q.enqueue(ctx, resourceKey, op) +} + +// enqueue publishes op as the pending operation for resourceKey and makes sure a +// worker will pick it up. +func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() _, replaced := q.pending[resourceKey] q.pending[resourceKey] = op @@ -140,11 +173,10 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // finishing, so it will see the operation written above. Queueing the key again // would let a second worker upload the same resource concurrently. if alreadyHandled { - return nil + return } q.queue <- resourceKey - return nil } // close drains the queue and returns the first upload error. All callers of diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index a1a5cf641bc..a4595a89c08 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -16,6 +16,12 @@ import ( // message that names the resource. const maxOperationStateSize = 64 * 1024 +// maxOperationErrorMessageSize is the largest error message DMS accepts per +// operation. A longer message is truncated rather than rejected, so a failing +// resource is still recorded with its error instead of the recording itself +// failing and masking the error we are trying to report. +const maxOperationErrorMessageSize = 16 * 1024 + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -25,9 +31,15 @@ const maxOperationStateSize = 64 * 1024 type recordedOperation struct { action bundledeployments.OperationActionType resourceID string + status bundledeployments.OperationStatus + + // errorMessage is why the operation failed. It is set only when status is + // failed, which the service enforces. + errorMessage string // state is the serialized local config after the operation. It is nil for a - // delete, where the resource no longer exists. + // delete, where the resource no longer exists, and for a failure, where the + // resource was not written. state json.RawMessage } @@ -40,7 +52,11 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return recordedOperation{}, err } - op := recordedOperation{action: actionType, resourceID: resourceID} + op := recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusSucceeded, + } // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. @@ -62,6 +78,31 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } +// newFailedOperation records an operation that did not apply, so the deployment +// history says why a resource failed rather than just omitting it. +// +// No state is recorded: the resource was not written, so there is nothing to +// serve back as its state. CREATE and RECREATE may have no resourceID yet, which +// the service allows for exactly those two actions. +func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + message := cause.Error() + if len(message) > maxOperationErrorMessageSize { + message = message[:maxOperationErrorMessageSize] + } + + return recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusFailed, + errorMessage: message, + }, nil +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { @@ -93,10 +134,11 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r dmsKey := strings.TrimPrefix(resourceKey, "resources.") operation := bundledeployments.Operation{ - ActionType: op.action, - ResourceId: op.resourceID, - ResourceKey: dmsKey, - Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ActionType: op.action, + ResourceId: op.resourceID, + ResourceKey: dmsKey, + Status: op.status, + ErrorMessage: op.errorMessage, } if op.state != nil { // DMS types state as a string, so the JSON goes on the wire as a quoted diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 674c78abf77..a76a91edcdb 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,8 @@ package direct import ( "context" + "errors" + "strings" "sync" "testing" @@ -85,6 +87,25 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } +func TestNewFailedOperationRecordsError(t *testing.T) { + op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "cluster spec is invalid", op.errorMessage) + // The resource was never written, so there is no state to serve back for it. + assert.Nil(t, op.state) +} + +func TestNewFailedOperationTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + require.NoError(t, err) + + assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index eecf4153e3b..72a3ae257e0 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -243,19 +243,31 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str } // A delete carries no state, so resource_id is the only thing identifying which - // resource it refers to; the service rejects a delete without one. + // resource it refers to; the service rejects a delete without one. A failed + // delete is exempt only for create-flavored actions, which may not have an ID. if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") } + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey op.ResourceKey = resourceKey // Reflect the operation onto the deployment-level resource set the way the // backend does: a delete removes the resource, anything else upserts it. - if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete { + // + // A failed operation is upserted too, matching the service, but it carries + // neither a resource_id nor state, so the read path treats the resource as not + // yet created rather than as existing state (verified against the service: a + // failed create is listed with an empty resource_id and no state). + switch { + case op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed: delete(d.resources, resourceKey) - } else { + default: d.resources[resourceKey] = bundledeployments.Resource{ Name: "deployments/" + deploymentID + "/resources/" + resourceKey, ResourceKey: resourceKey, From b9ca9a3ae287d519f2f843e58bed084ce64aa56a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 09:38:20 +0000 Subject: [PATCH 33/56] bundle: report the recorded deployment in bundle summary The deployment metadata service assigns a deployment an ID and a version per deploy, but nothing surfaced either, so a caller had no way to go from a bundle to the deployment it recorded. Summary now reports both under bundle.deployment.history. The ID comes from the deployment's workspace node, which is where the CLI already resolves it from; the version comes from GetDeployment. Both are output only, so the field is annotated readonly and stays out of the user-facing JSON schema. Like InitializeURLs, the mutator makes extra API calls and only runs when the fields are needed, and it is a no-op unless the bundle records deployment history, so bundles that do not record pay nothing. A deployment whose record does not exist yet (a deploy that registered it and then failed before recording a version) reports the ID without a version rather than failing summary. The test server was returning its 404 without a JSON content-type, so the SDK could not parse it into a typed error and callers matching apierr.ErrResourceDoesNotExist saw a generic failure instead. Also fixes bundle/dms/record-failure, which read a 19-digit object ID through jq: 1.6 rounds it, so the test only passed against a newer jq than CI runs. Verified on dogfood: summary reports the deployment ID and version 1, then version 2 after a redeploy, matching GetDeployment; a bundle without recording has no history field at all. Co-authored-by: Isaac --- acceptance/bundle/dms/record-failure/script | 3 +- acceptance/bundle/dms/summary/databricks.yml | 10 +++ acceptance/bundle/dms/summary/out.test.toml | 3 + acceptance/bundle/dms/summary/output.txt | 39 +++++++++++ acceptance/bundle/dms/summary/script | 15 ++++ bundle/config/deployment.go | 16 +++++ .../mutator/initialize_deployment_history.go | 68 +++++++++++++++++++ .../initialize_deployment_history_test.go | 36 ++++++++++ cmd/bundle/utils/process.go | 3 +- libs/testserver/bundle.go | 3 + 10 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/dms/summary/databricks.yml create mode 100644 acceptance/bundle/dms/summary/out.test.toml create mode 100644 acceptance/bundle/dms/summary/output.txt create mode 100644 acceptance/bundle/dms/summary/script create mode 100644 bundle/config/mutator/initialize_deployment_history.go create mode 100644 bundle/config/mutator/initialize_deployment_history_test.go diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 6a1890973a6..66af9601402 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -4,7 +4,8 @@ trace print_requests.py //api/2.0/bundle --sort title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" # The deployment ID is the workspace node's ID; read it back the way the CLI does. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | jq -r .object_id) +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" diff --git a/acceptance/bundle/dms/summary/databricks.yml b/acceptance/bundle/dms/summary/databricks.yml new file mode 100644 index 00000000000..c0698376223 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-summary + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..c792a60fcfa --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,39 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary/default + +Deleting files... +Destroy complete! + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..6f44e7c2880 --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,15 @@ +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go new file mode 100644 index 00000000000..91a3f1352c9 --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -0,0 +1,68 @@ +package mutator + +import ( + "context" + "errors" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +type initializeDeploymentHistory struct{} + +// InitializeDeploymentHistory populates bundle.deployment.history with the +// deployment recorded by the deployment metadata service, for the output of the +// 'bundle summary' command. +// +// NOTE: this makes extra API calls, so like InitializeURLs it should only be used +// when the fields are needed. It is a no-op unless the bundle records deployment +// history. +func InitializeDeploymentHistory() bundle.Mutator { + return &initializeDeploymentHistory{} +} + +func (m *initializeDeploymentHistory) Name() string { + return "InitializeDeploymentHistory" +} + +func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + return diag.FromErr(err) + } + if deploymentID == "" { + // Nothing recorded yet: the bundle has not been deployed, or its deployment + // was destroyed. + return nil + } + + history := &config.DeploymentHistory{DeploymentID: deploymentID} + + // The deployment's record is created by its first version, so a resolved ID can + // name a deployment that has none yet (a deploy that registered the deployment + // and then failed). Report the ID without a version rather than failing summary. + dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + deploymentID, + }) + switch { + case err == nil: + history.LatestVersionID = dep.LastVersionId + case errors.Is(err, apierr.ErrNotFound), errors.Is(err, apierr.ErrResourceDoesNotExist): + log.Debugf(ctx, "No deployment record for %s yet; reporting the ID without a version", deploymentID) + default: + return diag.FromErr(err) + } + + b.Config.Bundle.Deployment.History = history + return nil +} diff --git a/bundle/config/mutator/initialize_deployment_history_test.go b/bundle/config/mutator/initialize_deployment_history_test.go new file mode 100644 index 00000000000..a0478819bfd --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history_test.go @@ -0,0 +1,36 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitializeDeploymentHistoryIsNoOpWithoutRecording(t *testing.T) { + // Without recording there is no deployment to report, and the mutator must not + // make the API calls that would find one. + cases := []struct { + name string + experimental *config.Experimental + }{ + {"experimental unset", nil}, + {"recording disabled", &config.Experimental{RecordDeploymentHistory: false}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: tc.experimental, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, InitializeDeploymentHistory()) + require.NoError(t, diags.Error()) + assert.Nil(t, b.Config.Bundle.Deployment.History) + }) + } +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 209dc874403..30eefcac586 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -258,8 +258,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle statemgmt.Load(state, modes...), } // InitializeURLs makes an extra API call; only run it when URLs are needed. + // InitializeDeploymentHistory likewise, and only for bundles that record it. if opts.InitIDs { - mutators = append(mutators, mutator.InitializeURLs()) + mutators = append(mutators, mutator.InitializeURLs(), mutator.InitializeDeploymentHistory()) } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 72a3ae257e0..698e0bab79f 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -308,6 +308,9 @@ func (s *FakeWorkspace) ListResources(deploymentID string) Response { func dmsNotFound(what string) Response { return Response{ StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, Body: map[string]string{ "error_code": "RESOURCE_DOES_NOT_EXIST", "message": what + " does not exist", From 1a05f7a6290b8fdf58b49f099b805abb7ec3fbc8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 11:14:43 +0000 Subject: [PATCH 34/56] bundle: record git, workspace and mode provenance with a version A version reported only the CLI version, target and display name, so a deployment carried nothing about the source it came from. The service already has fields for this and denormalizes them onto the deployment, so all three were silently empty for every recorded deploy. The mapping mirrors what bundle/deploy/metadata computes for the metadata file, including its two conditionals: a source-linked deployment reports the sync root as file_path, and git_folder_path is set only for a deploy from a Databricks Git folder. bundle_root_path is relative to git_folder_path, so it is sent with it or not at all - the service rejects one without the other, which a local git deploy hit. Fixes a second bug found the same way: a deployment whose first version was rejected still leaves the record behind, with an empty last_version_id. createDeploymentVersion parsed that unconditionally and failed with "failed to parse last_version_id" on every later deploy, leaving the bundle permanently unable to record. It now retries at version 1, the same way it handles a record that does not exist yet. The test server now denormalizes provenance onto the deployment and enforces the git_folder_path/bundle_root_path pairing, so acceptance tests observe the real contract rather than a shape only the fake accepts. Verified on dogfood from a real git repo (branch master, origin bundle-examples): GetDeployment reports deployment_mode DEPLOYMENT_MODE_DEVELOPMENT, git_info with branch/commit/origin_url, and workspace_info with root_path and file_path. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/output.txt | 12 +++- .../bundle/dms/provenance/databricks.yml | 15 ++++ .../bundle/dms/provenance/out.test.toml | 3 + acceptance/bundle/dms/provenance/output.txt | 69 +++++++++++++++++++ acceptance/bundle/dms/provenance/script | 15 ++++ acceptance/bundle/dms/provenance/test.toml | 4 ++ .../bundle/dms/record-failure/output.txt | 6 +- acceptance/bundle/dms/record/output.txt | 18 ++++- .../dms/redeploy-after-destroy/output.txt | 6 +- .../dms/version-never-created/output.txt | 4 +- bundle/phases/dms.go | 52 ++++++++++++++ libs/dms/recorder.go | 27 ++++++++ libs/testserver/bundle.go | 15 ++++ 15 files changed, 239 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/dms/provenance/databricks.yml create mode 100644 acceptance/bundle/dms/provenance/out.test.toml create mode 100644 acceptance/bundle/dms/provenance/output.txt create mode 100644 acceptance/bundle/dms/provenance/script create mode 100644 acceptance/bundle/dms/provenance/test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index e7bb54440b5..a8923c7f36d 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -47,6 +47,6 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 1d208e1e85e..b593998dbd1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 06e346f3029..8801d4179b1 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -24,7 +24,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-no-resources" + "display_name": "dms-no-resources", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } } } { @@ -67,7 +71,11 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-no-resources", - "previous_version_id": "1" + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } } } { diff --git a/acceptance/bundle/dms/provenance/databricks.yml b/acceptance/bundle/dms/provenance/databricks.yml new file mode 100644 index 00000000000..df3b2d033c4 --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: dms-provenance + +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..ed6035fb6eb --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,69 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "dev", + "display_name": "dms-provenance", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..eb36baf6067 --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,15 @@ +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py //versions --sort + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 07b0c043f26..cb4073087d7 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -30,7 +30,11 @@ API message: cluster spec is invalid "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-record-failure" + "display_name": "dms-record-failure", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default" + } } } { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index b2cb6f240ed..f436470e073 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -25,7 +25,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-record" + "display_name": "dms-record", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { @@ -79,7 +83,11 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-record", - "previous_version_id": "1" + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { @@ -112,7 +120,11 @@ Destroy complete! "version_type": "VERSION_TYPE_DESTROY", "target_name": "default", "display_name": "dms-record", - "previous_version_id": "2" + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 6e160478e7b..33aa1938db1 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -50,7 +50,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-redeploy-after-destroy" + "display_name": "dms-redeploy-after-destroy", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default" + } } } { diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 23f84279395..e87ed786023 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -28,7 +28,7 @@ API message: Internal error >>> print_requests.py //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 346d6ca70c5..8e204208813 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,11 +2,14 @@ package phases import ( "context" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) // newDeploymentRecorder returns a dms.Recorder for the current deployment, or @@ -46,5 +49,54 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng TargetName: b.Config.Bundle.Target, DisplayName: b.Config.Bundle.Name, VersionType: versionType, + Provenance: deploymentProvenance(b), }), nil } + +// deploymentProvenance describes the source this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentProvenance(b *bundle.Bundle) dms.Provenance { + p := dms.Provenance{Mode: deploymentModeToSDK(b.Config.Bundle.Mode)} + + git := b.Config.Bundle.Git + if git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + p.Git = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = git.BundleRootPath + } + p.Workspace = ws + + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index cc9fdd0d87f..0306507913a 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -46,6 +46,12 @@ type createVersionRequest struct { // PreviousVersionId is the deployment's most recent version, unset for a // deployment's first version. PreviousVersionId string `json:"previous_version_id,omitempty"` + // DeploymentMode is the bundle target's mode, unset when the target sets none. + DeploymentMode bundledeployments.DeploymentMode `json:"deployment_mode,omitempty"` + // GitInfo and WorkspaceInfo record where the deployed source came from and + // where it landed. The service denormalizes both onto the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` } // versionCreator creates a version under a deployment. It exists because the @@ -91,6 +97,7 @@ type Recorder struct { targetName string displayName string versionType VersionType + provenance Provenance // populated by CreateVersion versionNum int64 @@ -117,6 +124,18 @@ type RecorderOptions struct { TargetName string DisplayName string VersionType VersionType + // Provenance records where the deployed source came from; see Provenance. + Provenance Provenance +} + +// Provenance is what a version records about the source it deployed and where it +// landed. The service denormalizes these onto the deployment, so they describe the +// deployment as of its most recent version. +type Provenance struct { + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Git *bundledeployments.GitInfo + Workspace *bundledeployments.WorkspaceInfo } // NewRecorder returns a Recorder for the deployment described by opts. @@ -129,6 +148,7 @@ func NewRecorder(opts RecorderOptions) *Recorder { targetName: opts.TargetName, displayName: opts.DisplayName, versionType: opts.VersionType, + provenance: opts.Provenance, } } @@ -235,6 +255,10 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin Name: "deployments/" + r.deploymentID, }) switch { + case getErr == nil && dep.LastVersionId == "": + // The record exists but carries no version: a deploy whose first version was + // rejected still leaves the record behind. Retry at version 1. + versionID = "1" case getErr == nil: lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) if parseErr != nil { @@ -280,6 +304,9 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin TargetName: r.targetName, DisplayName: r.displayName, PreviousVersionId: previousVersionID, + DeploymentMode: r.provenance.Mode, + GitInfo: r.provenance.Git, + WorkspaceInfo: r.provenance.Workspace, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 698e0bab79f..4f2571691a2 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -190,11 +190,26 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } + // bundle_root_path is relative to git_folder_path, so the service rejects a + // workspace_info that carries one without the other. + if ws := version.WorkspaceInfo; ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together") + } + d.deployment.LastVersionId = versionID version.Name = "deployments/" + deploymentID + "/versions/" + versionID version.VersionId = versionID version.Status = bundledeployments.VersionStatusVersionStatusInProgress d.versions[versionID] = &version + + // The service denormalizes the version's provenance onto the deployment, which + // is where the read APIs serve it from. display_name is excluded: the service + // keeps that on the deployment's workspace node instead. + d.deployment.TargetName = version.TargetName + d.deployment.DeploymentMode = version.DeploymentMode + d.deployment.GitInfo = version.GitInfo + d.deployment.WorkspaceInfo = version.WorkspaceInfo + return Response{Body: version} } From c08abbdd51784f945bb27f5936a6a91e284a02b1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 22:31:58 +0000 Subject: [PATCH 35/56] bundle: stamp the deployment and version onto jobs and pipelines AnnotateJobs/AnnotatePipelines set deployment.kind and metadata_file_path, but not deployment_id or version_id, so a job or pipeline in the workspace had no way back to the deployment that produced it. That link is what lineage resolves to attribute a job to its bundle. The version has to exist before the resources are planned: the plan snapshots the resource config, and apply deploys from that snapshot, so stamping after the plan never reaches the API. Verified: with the stamp applied post-plan the deployed job still had only kind and metadata_file_path. CreateVersion therefore moves ahead of planning, which means a cancelled deploy now leaves a version behind - completed as a failure by the deferred CompleteVersion, the same as any other failed deploy. This drops the incidental stale-plan guard the old ordering gave us: CreateVersion ran after approval, so a concurrent deploy that advanced the deployment made the ABORTED check reject a stale plan. previous_version_id still detects the race, just at claim time rather than apply time. Verified on dogfood via the raw API (the SDK hides these preview fields): the job reports deployment_id 3908151894982320 / version_id 2 and the pipeline the same deployment with version_id 5, and that ID resolves to the real DMS deployment. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 4 +- .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 2 +- .../dms/redeploy-after-destroy/output.txt | 2 +- .../metadata/annotate_deployment_version.go | 47 +++++++++++++++++ .../annotate_deployment_version_test.go | 50 +++++++++++++++++++ bundle/phases/deploy.go | 29 +++++++---- 8 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 bundle/deploy/metadata/annotate_deployment_version.go create mode 100644 bundle/deploy/metadata/annotate_deployment_version_test.go diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index db3624d9fc1..952dcb00a86 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -17,7 +17,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -31,7 +31,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a8923c7f36d..1897f6c4e86 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -48,5 +48,5 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index ed6035fb6eb..c567a12d66c 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -47,7 +47,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index f436470e073..a707e5e46d7 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -42,7 +42,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 33aa1938db1..12ce1a298c4 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -67,7 +67,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..1b8139168ce --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,47 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeploymentVersion struct { + deploymentID string + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS deployment and version onto every job +// and pipeline, so a resource in the workspace points back at the deployment that +// produced it (which is how lineage resolves a job to its bundle). +// +// AnnotateJobs/AnnotatePipelines set the rest of the deployment metadata during +// initialize, but the version - and, on a first deploy, the deployment ID - only +// exist once CreateVersion has run, so these two fields are stamped separately +// from the deploy phase. +func AnnotateDeploymentVersion(deploymentID string, version int64) bundle.Mutator { + return &annotateDeploymentVersion{deploymentID: deploymentID, version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..aca51358ace --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeploymentVersion("dep-123", 7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 3d70a218b15..808ca6c012c 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -228,6 +228,22 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // Create the version before planning: the plan snapshots the resource config, + // so the deployment and version have to be stamped onto the resources before it + // is computed or the applied resources would not carry them. A cancelled deploy + // therefore leaves a version behind, completed as a failure by the deferred + // CompleteVersion. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(recorder.DeploymentID(), recorder.Version())) + if logdiag.HasError(ctx) { + return + } + } + planFromFile := plan != nil if plan == nil { // State is already open for read by process.go (for direct engine) @@ -271,18 +287,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { - // Record the DMS version now that the plan is approved and the state WAL - // has been opened. CreateVersion requests version_id == last_version_id + 1; - // the server returns ABORTED if a concurrent deploy advanced the deployment - // since the plan was computed, so a stale plan is not applied. - if err := recorder.CreateVersion(ctx); err != nil { - logdiag.LogError(ctx, err) - return - } if recorder != nil { - // Record operations under the version just created so DMS holds the - // deployed resource state. On a first deploy the deployment ID was only - // assigned by CreateVersion above, so this must come after it. + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. b.DeploymentBundle.OpRec = direct.NewOperationRecorder( b.WorkspaceClient(ctx).BundleDeployments, recorder.DeploymentID(), From 41c41eee510e546dd7719ee895f949d70b34337b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 08:31:53 +0000 Subject: [PATCH 36/56] bundle: raise operation upload workers back to 8 The cap of 2 was a stopgap for a service-side limit: concurrent CreateOperation calls under one version contended on the version row, and the transaction conflict surfaced as a 500 that failed the deploy. That is fixed, so the cap can go. Verified against the service rather than assumed. The raw-API probe that originally pinned the threshold now reports 8/8 and 16/16 concurrent writes succeeding, where 4 used to fail and 8 gave 3/8. End to end at 8 workers: the 12-job bundle that previously failed on jobs.j04 deploys cleanly, 35 jobs deploy and destroy with zero 500s, and a plan from DMS state alone after wiping .databricks reports "0 to add, 12 to change" - every operation recorded, nothing duplicated. Left at 8 rather than higher: the probe shows headroom at 16, but 8 matches the apply-side parallelism, and a batch upload API remains the better way to cut the request count. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index c3f4439ee4e..551c91b2874 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -19,12 +19,12 @@ const ( // operationQueueSize so a burst of operations is absorbed by the queue rather // than by one request per resource. // - // Capped at 2 because concurrent CreateOperation calls under the same version - // contend on shared state server-side and the transaction conflict surfaces as - // a 500, which fails the deploy. Measured against the service: 3 concurrent - // writes still succeeded, 4 did not. The real fix is a batch upload API that - // commits every operation in one transaction; until then, keep this at 2. - operationUploadWorkers = 2 + // This was temporarily capped at 2 while concurrent CreateOperation calls under + // the same version contended on shared state server-side, surfacing the + // transaction conflict as a 500 that failed the deploy. The service now handles + // them: measured against it, 8 and 16 concurrent writes both succeed where 4 + // used to fail. + operationUploadWorkers = 8 ) // operationQueue hands recorded operations to background workers, so an apply From af8c851d81410fc06f8c695850d85025a09c273f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 10:05:39 +0000 Subject: [PATCH 37/56] bundle: spell out how to recover from the record-deployment-history error The old remedy read "Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", which is misleading in two ways. It reads as two alternatives when the second one is really three ordered steps, and following it as written does not work: this error also fires on destroy, so a user who tries to "destroy the bundle and deploy it again" hits the same error and cannot get out. The setting has to come off first, which the old wording never said. It also did not say what the first option costs. Removing the setting is not a fix for recording, it is the choice to keep the existing resources and not record them. The message now numbers the three steps in the order they have to happen, and states the keep-the-resources option separately so the two outcomes are not confused. Verified on dogfood by reproducing the report: a bundle deployed without recording, then with the setting added, fails destroy with the new message; following the three steps destroys the bundle and then deploys with recording enabled. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 18 ++++++++++++++++-- bundle/direct/dstate/state.go | 11 ++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 1897f6c4e86..6233f158974 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,14 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out === No deployment was created in DMS @@ -20,7 +27,14 @@ Error: cannot record deployment history for a bundle that already has deployed r === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out >>> print_requests.py //api/2.0/bundle --oneline diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bad87a6a293..c88e6e0dedb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -293,7 +293,16 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // older CLI refuses the state instead of deploying against resources it // cannot see. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { - return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + // The remedy is ordered deliberately: this error also blocks destroy, so the + // setting has to come out first or there is no way to tear the bundle down. + return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out`, path) } if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { From 610b71f2df754a292879829d38db83ef4ee46fc3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 10:20:57 +0000 Subject: [PATCH 38/56] bundle: add a bugbash guide for deployment history recording The repo already has a bugbash mechanism (internal/bugbash/exec.sh drops you into a shell with a branch's CLI on PATH), but nothing explaining how to exercise this feature once you are there. Recording needs three separate things set before it does anything, and several of its behaviours look like bugs until you know they are not, so the guide covers both. Also notes in the README that the branch needs a successful release-build run for exec.sh to have something to download, and that the workflow only triggers on main, demo-* and bugbash-* - which is not obvious from the script and is the first thing to go wrong when pointing it at a feature branch. Co-authored-by: Isaac --- internal/bugbash/README.md | 8 + internal/bugbash/record-deployment-history.md | 151 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 internal/bugbash/record-deployment-history.md diff --git a/internal/bugbash/README.md b/internal/bugbash/README.md index 941ab6227cc..1c5995a8188 100644 --- a/internal/bugbash/README.md +++ b/internal/bugbash/README.md @@ -11,3 +11,11 @@ but works without command completion with earlier versions. ```shell bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch ``` + +The branch needs a successful `release-build` run to download a snapshot from. That +workflow runs on `main` and on any branch named `demo-*` or `bugbash-*`, so push the +branch under one of those names. + +## Feature guides + +- [Deployment history recording](./record-deployment-history.md) diff --git a/internal/bugbash/record-deployment-history.md b/internal/bugbash/record-deployment-history.md new file mode 100644 index 00000000000..db0a7172653 --- /dev/null +++ b/internal/bugbash/record-deployment-history.md @@ -0,0 +1,151 @@ +# Bugbash: deployment history recording + +Records every `bundle deploy` and `bundle destroy` with the Deployment Metadata +Service (DMS), so a deployment has a server-side history and its resource state +lives in the workspace rather than only in the local cache. + +## Get a CLI with the feature + +```shell +bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) bugbash-record-deployment-history +``` + +That drops you into a shell with `databricks` on `$PATH`. Check you have the right +build with `databricks --version`. + +## Turn the feature on + +Three things are needed. Missing any one of them means nothing is recorded. + +```shell +export DATABRICKS_BUNDLE_ENGINE=direct +export DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=true +``` + +and in `databricks.yml`: + +```yaml +experimental: + record_deployment_history: true +``` + +The env var only unlocks the gate; the YAML flag is what enables recording. Without +the env var the CLI refuses: + +``` +Error: experimental.record_deployment_history is not supported yet +``` + +Recording is direct-engine only. On terraform the flag is rejected, and no +`/api/2.0/bundle/*` calls are made. + +The feature must be enabled from the bundle's **first** deploy. Turning it on for a +bundle that already has deployed resources is refused, because DMS would then own a +resource set it never saw and the next deploy would create everything a second time. +The error spells out the three steps to start over. + +## A bundle to start from + +```yaml +bundle: + name: my-dms-test + +experimental: + record_deployment_history: true + +resources: + jobs: + hello: + name: my-dms-test-job + tasks: + - task_key: main + notebook_task: + notebook_path: ./noop.py +``` + +with `noop.py` beside it: + +``` +# Databricks notebook source +print(1) +``` + +## Find the deployment + +The CLI stores the deployment ID nowhere. DMS registers the deployment as a workspace +node, and that node's object ID *is* the deployment ID: + +```shell +databricks workspace get-status \ + "/Workspace/Users/$(databricks current-user me | jq -r .userName)/.bundle/my-dms-test/default/state/resources.deployment.json" \ + -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])' +``` + +Use `python3`, not `jq`, for that ID. It exceeds 2^53 and jq below 1.7 silently +rounds it, which looks like "deployment does not exist". + +Or read it straight off the summary: + +```shell +databricks bundle summary -o json | jq .bundle.deployment.history +``` + +## What to look at + +```shell +databricks api get "/api/2.0/bundle/deployments/$DID" # the deployment +databricks api get "/api/2.0/bundle/deployments/$DID/versions" # one version per deploy +databricks api get "/api/2.0/bundle/deployments/$DID/versions/$V/operations" +databricks api get "/api/2.0/bundle/deployments/$DID/resources" # current resource state +databricks api get "/api/2.0/bundle/deployments" # all deployments +``` + +`resources` and `operations` paginate at 20 with a `next_page_token`. A bundle with +more than 20 resources is not truncated; page through it. + +Jobs and pipelines carry a back-reference to the deployment, but the SDK hides those +fields, so read them raw: + +```shell +databricks api get "/api/2.0/jobs/get?job_id=$JID" | jq .settings.deployment +databricks api get "/api/2.0/pipelines/$PID" | jq .spec.deployment +``` + +Both should show `deployment_id` and `version_id` next to `kind: BUNDLE`. + +## Worth exercising + +- **Iterate.** Deploy, change a field, deploy again. Each deploy claims a version; + only changed resources get an operation. +- **Wipe the local cache.** `rm -rf .databricks`, then `bundle plan`. It should report + your resources as unchanged, reconstructed from DMS. It must never plan to create + something that already exists. +- **Break a resource.** Give a job an invalid cron expression. The failed resource is + recorded with `status: OPERATION_STATUS_FAILED` and an `error_message`, the version + completes with `VERSION_COMPLETE_FAILURE`, and a later plan wants to create it. +- **Destroy.** A destroy records its own version with a DELETE per resource, then + deletes the deployment record. +- **Non-job resources.** Pipelines, schemas, volumes, experiments, registered models, + secret scopes and dashboards are all recorded. Each has a differently-shaped + resource id (numeric, UUID, `catalog.schema.name`, a scope name). +- **Targets.** Each target has its own state path, so `-t dev` and `-t prod` are + separate deployments with separate version chains. +- **Provenance.** Deploy from a git repo and check `git_info` on the version; + `deployment_mode` reflects the target's `mode`. + +## Not bugs + +- A redeploy with no changes still creates a version, with no operations under it. +- After `destroy`, `GetDeployment` still returns the record with + `status: DEPLOYMENT_STATUS_DELETED`. That is a soft delete. +- `state` on an operation or resource is a **quoted JSON string**, not an embedded + object. Parse it once to get `{"state": {...}, "depends_on": [...]}`. +- DMS resource keys have no `resources.` prefix (`jobs.foo`), unlike local state keys. +- Sub-resources get their own operation, e.g. `secret_scopes.mine.permissions`. +- Permissions are not set on the deployment node. It inherits from the state folder, + which the bundle's `permissions:` section already governs. + +## Reporting + +Include the deployment ID, the version, and the request/response for anything that +looks wrong. `databricks bundle deploy --log-level debug` logs the DMS calls. From d3bda0c4c3eaea4be48656cd7d10b6dc1c1601a1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 11:03:27 +0000 Subject: [PATCH 39/56] bundle: report the recorded deployment after a deploy A deploy said nothing about the deployment it had just recorded, so finding it meant knowing that the ID is the object ID of a workspace node and running get-status by hand. Deploy now ends with: Recorded deployment 996980114755597 version 2 at /Workspace/Users/.../state/resources.deployment.json The path is printed rather than a workspace URL. A deployment is a BUNDLE_DEPLOYMENT tree node with no page of its own: there is no entry for it in workspaceurls, and the file browser is fed by GraphQL tree messages that have no component handling that node type yet, so any URL would 404. The path is what exists today and is enough to look the node up. The line is skipped when recording is off, so a deploy that does not record prints nothing new - checked both ways on dogfood. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 1 + .../bundle/dms/existing-state/output.txt | 1 + .../bundle/dms/multiple-resources/output.txt | 2 ++ acceptance/bundle/dms/no-resources/output.txt | 2 ++ acceptance/bundle/dms/provenance/output.txt | 1 + acceptance/bundle/dms/record/output.txt | 2 ++ .../dms/redeploy-after-destroy/output.txt | 2 ++ acceptance/bundle/dms/summary/output.txt | 2 ++ bundle/phases/deploy.go | 5 +++-- bundle/phases/dms.go | 21 +++++++++++++++++++ 10 files changed, 37 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 952dcb00a86..695b24cd6c6 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/resources.deployment.json >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6233f158974..2e34713c0c2 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,6 +58,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index b593998dbd1..8bc9358ad44 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -19,6 +20,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 8801d4179b1..0766ad59906 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,6 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --get { @@ -50,6 +51,7 @@ Deployment complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index c567a12d66c..86f8caf3a10 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/resources.deployment.json >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index a707e5e46d7..8602c252eff 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle { @@ -70,6 +71,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 12ce1a298c4..ad83e46b575 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -24,6 +25,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index c792a60fcfa..e378456b9c3 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json >>> [CLI] bundle summary -o json { @@ -18,6 +19,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json >>> [CLI] bundle summary -o json { diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 808ca6c012c..ba923638ab5 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -77,7 +77,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting) { +func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, recorder *dms.Recorder) { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. cmdio.LogString(ctx, "Deploying resources...") @@ -120,6 +120,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") + logDeploymentHistory(ctx, b, recorder) } // Once the deploy is complete, dry-run the migration to the direct engine @@ -296,7 +297,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand recorder.Version(), ) } - deployCore(ctx, b, plan, stateEngine, requestedEngine) + deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) } else { cmdio.LogString(ctx, "Deployment cancelled!") return diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 8e204208813..5ebf652c639 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,11 +2,14 @@ package phases import ( "context" + "fmt" + "path" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -53,6 +56,24 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// logDeploymentHistory reports the deployment this deploy was recorded under, so +// the user can look its history up without hunting for the ID. A nil recorder means +// recording is off, and a zero version means the version was never created. +// +// It prints the deployment's workspace path rather than a UI link: the deployment is +// a BUNDLE_DEPLOYMENT tree node with no page of its own yet, so a URL would 404. +func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil || recorder.Version() == 0 { + return + } + + cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d at %s", + recorder.DeploymentID(), + recorder.Version(), + path.Join(b.Config.Workspace.StatePath, dms.DeploymentNodeName), + )) +} + // deploymentProvenance describes the source this deploy came from and where it // landed, mirroring what bundle/deploy/metadata computes for the metadata file. func deploymentProvenance(b *bundle.Bundle) dms.Provenance { From 45c4ed221fcbd76f7735195a245fd63ead382754 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 11:12:06 +0000 Subject: [PATCH 40/56] bundle: link to the recorded deployment after a deploy Deploy printed the deployment node's workspace path, which was only useful for an API lookup. There is a page for a deployment, so link to it instead: Deployment history: https:///deployments/996980114757453?version=2 The version pins the page to the deploy that just ran, and advances with each deploy. The workspace ID is omitted - the page redirects correctly without it, and leaving it out keeps the line short enough to stay clickable in a terminal. DeploymentURL lives in libs/workspaceurls next to the resource URL patterns, but is a separate function rather than another entry in resourceURLPatterns: a deployment is not a bundle resource type, and it takes a query parameter that none of those patterns do. It preserves an existing query so a base URL carrying ?w= for a vanity or legacy host still addresses the right workspace. An unparseable host degrades to reporting the id and version without a link, rather than failing a deploy that already succeeded. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 2 +- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 4 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 4 +- .../dms/redeploy-after-destroy/output.txt | 4 +- acceptance/bundle/dms/summary/output.txt | 4 +- bundle/phases/dms.go | 27 ++++++---- libs/workspaceurls/urls.go | 23 +++++++++ libs/workspaceurls/urls_test.go | 49 +++++++++++++++++++ 11 files changed, 102 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 695b24cd6c6..5b9ca490227 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 2e34713c0c2..3b5da470cfa 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,7 +58,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 8bc9358ad44..7e81865e11a 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -20,7 +20,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 0766ad59906..e4c0341934b 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --get { @@ -51,7 +51,7 @@ Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 86f8caf3a10..a4084f82728 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 8602c252eff..9a51c82e60e 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -71,7 +71,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ad83e46b575..64c4e6b8c4f 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -25,7 +25,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index e378456b9c3..13a86eb3a21 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle summary -o json { @@ -19,7 +19,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> [CLI] bundle summary -o json { diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 5ebf652c639..aa6b1d8e810 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -3,7 +3,7 @@ package phases import ( "context" "fmt" - "path" + "net/url" "strings" "github.com/databricks/cli/bundle" @@ -11,6 +11,8 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -56,22 +58,27 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } -// logDeploymentHistory reports the deployment this deploy was recorded under, so -// the user can look its history up without hunting for the ID. A nil recorder means +// logDeploymentHistory links to the deployment this deploy was recorded under, so +// the user can open its history without hunting for the ID. A nil recorder means // recording is off, and a zero version means the version was never created. // -// It prints the deployment's workspace path rather than a UI link: the deployment is -// a BUNDLE_DEPLOYMENT tree node with no page of its own yet, so a URL would 404. +// The workspace ID is left out of the URL: the page redirects correctly without it, +// and omitting it keeps the line short enough to stay clickable in a terminal. func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { if recorder == nil || recorder.Version() == 0 { return } - cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d at %s", - recorder.DeploymentID(), - recorder.Version(), - path.Join(b.Config.Workspace.StatePath, dms.DeploymentNodeName), - )) + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the deployment without it rather than + // failing a deploy that already succeeded. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d", recorder.DeploymentID(), recorder.Version())) + return + } + + cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) } // deploymentProvenance describes the source this deploy came from and where it diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index a1bf973801f..7efc2156ef3 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -68,6 +69,28 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment recorded with +// the deployment metadata service, of the form +// +// /deployments/?version= +// +// The version pins the page to the deploy that produced it. It is separate from +// ResourceURL because a deployment is not a bundle resource type: it has no entry +// in resourceURLPatterns and takes a query parameter none of those do. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index e39d28d9aaf..ba8299efb05 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string From a8a5148f9fdf28ae34ee164ef6c7117092b04b90 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 12:55:58 +0000 Subject: [PATCH 41/56] bundle: send the prior state when recording a failed operation Recording a failed operation sent no state, which the service rejects for any action that acts on an existing resource: Error: recording operation for resources.jobs.test_job with the deployment metadata service: state is required for a OPERATION_ACTION_TYPE_UPDATE operation, because it acts on a resource that already exists and cannot destroy it, even when it fails (400 INVALID_PARAMETER_VALUE) So a failed update replaced the real error with this one, and the failure itself was never recorded. The rule makes sense: dropping the state would leave DMS unable to describe a resource it still owns. A failed operation now carries the resource's state from before the deploy, unchanged - the resource is whatever it was before the attempt. It stays nil for a create, where there is no prior state and no resource to describe, which is also why resource_id may be empty for CREATE and RECREATE. Verified on dogfood: a job deployed, then given an invalid cron so its update fails. The 400 is gone, the deploy reports only the real quartz error, and version 2 records the operation as FAILED with the error message and the pre-deploy state (no schedule). Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 4 ++-- bundle/direct/opqueue.go | 5 +++-- bundle/direct/oprecorder.go | 29 +++++++++++++++++++++++++---- bundle/direct/oprecorder_test.go | 4 ++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index bdd0c243e7f..c3f7f02cbad 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,7 +104,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { - opQueue.recordFailure(ctx, resourceKey, action, deletedID, err) + opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState(&b.StateDB, resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -140,7 +140,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if err != nil { // GetResourceID is empty for a create that never got an ID, which is // what the service expects for a failed create. - opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), err) + opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), priorState(&b.StateDB, resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 551c91b2874..ba64d52ebd9 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "fmt" "sync" @@ -141,12 +142,12 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // Unlike record, this does not resurface an earlier upload error: the deploy is // already failing, and returning a different error here would replace the one the // user needs to see. A failure to upload this record is reported at close. -func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { +func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if q == nil { return } - op, err := newFailedOperation(action, resourceID, cause) + op, err := newFailedOperation(action, resourceID, priorState, cause) if err != nil { log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) return diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index a4595a89c08..2055384ffa8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -81,10 +81,13 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // newFailedOperation records an operation that did not apply, so the deployment // history says why a resource failed rather than just omitting it. // -// No state is recorded: the resource was not written, so there is nothing to -// serve back as its state. CREATE and RECREATE may have no resourceID yet, which -// the service allows for exactly those two actions. -func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { +// priorState is the resource's state from before the deploy, carried through +// unchanged: an action other than a create acts on a resource that still exists, and +// the service rejects such an operation without state because dropping it would +// leave DMS unable to describe a resource it still owns. It is nil for a create, +// where there is no prior state and no resource to describe - which is also why the +// resourceID may be empty for CREATE and RECREATE. +func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err @@ -100,9 +103,27 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, cause e resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusFailed, errorMessage: message, + state: priorState, }, nil } +// priorState returns the resource's recorded state from before this deploy, in the +// same envelope form the success path uploads, or nil when the resource has none +// (a create). A failed operation reports this unchanged: the resource is whatever it +// was before the attempt. +func priorState(db *dstate.DeploymentState, resourceKey string) json.RawMessage { + entry, ok := db.GetResourceEntry(resourceKey) + if !ok || len(entry.State) == 0 { + return nil + } + + raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return nil + } + return raw +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index a76a91edcdb..ec5068563e2 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -88,7 +88,7 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { } func TestNewFailedOperationRecordsError(t *testing.T) { - op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) + op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) require.NoError(t, err) assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) @@ -100,7 +100,7 @@ func TestNewFailedOperationRecordsError(t *testing.T) { func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. - op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) require.NoError(t, err) assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) From cf2d9cd957771fb914507214c499606d4533fb98 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 13:18:12 +0000 Subject: [PATCH 42/56] bundle: take the id and state of a failed operation from the same record Recording a failed recreate sent state without an id, which the service rejects: Error: recording operation for resources.schemas.foo with the deployment metadata service: resource_id is required for an operation that records state, because state records a resource that exists (400 INVALID_PARAMETER_VALUE) So a failed recreate replaced the real error with this one. The id came from live state while the state came from the pre-deploy record, and a recreate deletes before it creates, so by the time the create failed the id was gone and the state was not. Both now come from the same pre-deploy entry, which is the only pairing the service accepts: state describes a resource that exists, so it needs the id to say which one. They stay empty together for a create, which never had either. Verified on dogfood: a schema deployed, then pointed at a nonexistent catalog so its recreate fails. The 400 is gone, the deploy reports only the real catalog error, and the operation records as FAILED with the prior catalog_name and the id. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 10 ++++++---- bundle/direct/oprecorder.go | 21 +++++++++++++-------- bundle/direct/oprecorder_test.go | 13 +++++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c3f7f02cbad..2047f65a498 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,7 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { - opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState(&b.StateDB, resourceKey), err) + _, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -138,9 +139,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { - // GetResourceID is empty for a create that never got an ID, which is - // what the service expects for a failed create. - opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), priorState(&b.StateDB, resourceKey), err) + // Both are empty for a create that never got an ID, which is what the + // service expects for a failed create. + priorID, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, priorID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 2055384ffa8..32494c07325 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -107,21 +107,26 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt }, nil } -// priorState returns the resource's recorded state from before this deploy, in the -// same envelope form the success path uploads, or nil when the resource has none -// (a create). A failed operation reports this unchanged: the resource is whatever it -// was before the attempt. -func priorState(db *dstate.DeploymentState, resourceKey string) json.RawMessage { +// priorRecord returns the resource's id and state from before this deploy, in the +// same envelope form the success path uploads, or empty values when the resource has +// no prior record (a create). A failed operation reports these unchanged: the resource +// is whatever it was before the attempt. +// +// Both come from the same pre-deploy entry because the service requires an id +// alongside state: state describes a resource that exists, so it needs the id to say +// which one. Reading the id from live state instead would return "" for a failed +// recreate, whose delete step already dropped it, and the mismatch is rejected. +func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.RawMessage) { entry, ok := db.GetResourceEntry(resourceKey) if !ok || len(entry.State) == 0 { - return nil + return "", nil } raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { - return nil + return "", nil } - return raw + return entry.ID, raw } // operationUploader records an applied resource operation with DMS. Uploads run diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index ec5068563e2..b793ab65a08 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "errors" "strings" "sync" @@ -97,6 +98,18 @@ func TestNewFailedOperationRecordsError(t *testing.T) { assert.Nil(t, op.state) } +func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { + // A failed recreate has already deleted the resource, so the id must come from + // the pre-deploy record alongside the state: the service rejects state without + // an id, since state describes a resource that exists. + op, err := newFailedOperation(deployplan.Recreate, "main.some_schema", json.RawMessage(`{"state":{"catalog_name":"main"}}`), errors.New("Catalog 'mainx' does not exist")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "main.some_schema", op.resourceID) + assert.JSONEq(t, `{"state":{"catalog_name":"main"}}`, string(op.state)) +} + func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. From 015af745e828edae2e370fa1fb6292aef2e705da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 13:30:48 +0000 Subject: [PATCH 43/56] bundle: stop the deployment stamp from showing as drift After a deploy, planning an untouched bundle reported an update on every job and pipeline: "deployment.deployment_id": { "action": "update", "old": "261237257843077", "remote": "261237257843077" } old and remote agree and new is absent, which is the tell: only the deploy phase stamps deployment_id, because it is not known until the version is claimed. A plain `bundle plan` never runs that mutator, so the field is set in the state and in the workspace but empty in the local config, and the absence reads as a change. version_id was already ignored as a local change for a different reason (it changes on every deploy). deployment_id was left out deliberately, on the grounds that it is stable so a change to it is worth showing - but the value it is compared against is never populated at plan time, so the rule only ever fired on this phantom. It is now ignored as a local change too, for jobs and pipelines. The stamp itself is unaffected: verified on dogfood that the deployed job still carries deployment_id, and the drift is gone (0 to change, 1 unchanged). The new acceptance test plans an untouched bundle after deploying it, which no existing DMS test did - that gap is what let this through. Co-authored-by: Isaac --- acceptance/bundle/dms/no-drift/databricks.yml | 16 +++++ acceptance/bundle/dms/no-drift/out.test.toml | 3 + acceptance/bundle/dms/no-drift/output.txt | 58 +++++++++++++++++++ acceptance/bundle/dms/no-drift/script | 12 ++++ bundle/direct/dresources/resources.yml | 16 +++-- 5 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/dms/no-drift/databricks.yml create mode 100644 acceptance/bundle/dms/no-drift/out.test.toml create mode 100644 acceptance/bundle/dms/no-drift/output.txt create mode 100644 acceptance/bundle/dms/no-drift/script diff --git a/acceptance/bundle/dms/no-drift/databricks.yml b/acceptance/bundle/dms/no-drift/databricks.yml new file mode 100644 index 00000000000..ca2ed6bd96d --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: dms-no-drift + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + name: bar + catalog: main + schema: default + serverless: true diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..0307d57910b --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,58 @@ + +=== Deploy, then plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== A second deploy is a no-op too: no update request for either resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 + +>>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "name": "bar", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..c67c2c28a74 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,12 @@ +title "Deploy, then plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle deploy + +# Only the deploy phase stamps deployment.deployment_id (it is not known until the +# version is claimed), so plan sees it in the state and in the workspace but not in the +# local config. Without an ignore_local_changes rule that absence plans an update on a +# job and a pipeline nobody edited. +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "A second deploy is a no-op too: no update request for either resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c9df091551f..7345f19c506 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -26,11 +26,17 @@ resources: # it changes constantly. Ignoring it as a local and remote change keeps that # churn from driving an update or showing as drift on its own; when the job is # updated for any other reason, DoUpdate sends the full config via Reset, so - # the current version_id is still recorded. deployment_id is intentionally - # left out: it is stable across versions, so a change to it is worth showing. + # the current version_id is still recorded. + # + # deployment_id is ignored as a local change for a different reason: only the + # deploy phase stamps it (it is not known until the version is claimed), so + # during a plain `bundle plan` the local config has none while the state and the + # workspace both do, and the absence would show as drift on an untouched job. ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service + - field: deployment.deployment_id + reason: managed by the deployment metadata service ignore_remote_changes: - field: deployment.version_id @@ -168,8 +174,8 @@ resources: - field: ingestion_definition.ingest_from_uc_foreign_catalog reason: immutable - # See jobs above: version_id is set on every deploy, so it is ignored as a - # local/remote change. deployment_id is left out so a change to it still shows. + # See jobs above: version_id is set on every deploy, and deployment_id is only + # stamped by the deploy phase, so both are ignored as local changes. ignore_remote_changes: - field: deployment.version_id reason: managed by the deployment metadata service @@ -184,6 +190,8 @@ resources: ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service + - field: deployment.deployment_id + reason: managed by the deployment metadata service # "id" is output-only, providing it in config would be a mistake - field: id reason: "!drop" From a49f16f5227c1a622b660cfbc024f7f039f1d1df Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:07:57 +0000 Subject: [PATCH 44/56] bundle: group what a version records into dms.Metadata Provenance held the deployment mode, git info and workspace info, while the display name and target name travelled as separate RecorderOptions fields even though they are the same kind of thing: what a version records about the deploy, which the service denormalizes onto the deployment. They are now one Metadata struct, renamed from Provenance since it no longer describes only the source. Two not-found branches go away with it. A deployment ID is always the object ID of a BUNDLE_DEPLOYMENT node that get-status just returned, and the service has a deployment for every such node, so GetDeployment cannot report not-found for it. The recorder now says so instead of quietly retrying at version 1: internal error: no deployment found for the file with object id 2612372578 The fake server modelled the record as created by the first version rather than by CreateDeployment, which is what made those branches look reachable. It now creates both together, with last_version_id empty until the first version - so the "registered then failed" case still reaches the retry-at-1 path, via an empty last_version_id rather than a 404. InitializeDeploymentHistory was wired to InitIDs, so bundle open and three pipelines commands paid for its API calls while only bundle summary reports the result. It has its own option now. Also drops the bugbash guide this PR had added. Co-authored-by: Isaac --- .../mutator/initialize_deployment_history.go | 23 +-- bundle/phases/dms.go | 14 +- cmd/bundle/summary.go | 9 +- cmd/bundle/utils/process.go | 18 ++- internal/bugbash/README.md | 8 - internal/bugbash/record-deployment-history.md | 151 ------------------ libs/dms/recorder.go | 58 ++++--- libs/dms/recorder_test.go | 34 ++-- libs/testserver/bundle.go | 30 ++-- 9 files changed, 90 insertions(+), 255 deletions(-) delete mode 100644 internal/bugbash/record-deployment-history.md diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 91a3f1352c9..99aad70c7cc 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -2,14 +2,11 @@ package mutator import ( "context" - "errors" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -46,23 +43,19 @@ func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundl return nil } - history := &config.DeploymentHistory{DeploymentID: deploymentID} - - // The deployment's record is created by its first version, so a resolved ID can - // name a deployment that has none yet (a deploy that registered the deployment - // and then failed). Report the ID without a version rather than failing summary. + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so this get does not + // have a not-found case. last_version_id is empty until the first version. dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + deploymentID, }) - switch { - case err == nil: - history.LatestVersionID = dep.LastVersionId - case errors.Is(err, apierr.ErrNotFound), errors.Is(err, apierr.ErrResourceDoesNotExist): - log.Debugf(ctx, "No deployment record for %s yet; reporting the ID without a version", deploymentID) - default: + if err != nil { return diag.FromErr(err) } - b.Config.Bundle.Deployment.History = history + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: dep.LastVersionId, + } return nil } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index aa6b1d8e810..cb8f6ad2db4 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -51,10 +51,8 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng Versions: dms.NewAPIVersionCreator(apiClient), DeploymentID: deploymentID, StatePath: statePath, - TargetName: b.Config.Bundle.Target, - DisplayName: b.Config.Bundle.Name, VersionType: versionType, - Provenance: deploymentProvenance(b), + Metadata: deploymentMetadata(b), }), nil } @@ -81,10 +79,14 @@ func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.R cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) } -// deploymentProvenance describes the source this deploy came from and where it +// deploymentMetadata describes the bundle this deploy came from and where it // landed, mirroring what bundle/deploy/metadata computes for the metadata file. -func deploymentProvenance(b *bundle.Bundle) dms.Provenance { - p := dms.Provenance{Mode: deploymentModeToSDK(b.Config.Bundle.Mode)} +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } git := b.Config.Bundle.Git if git.Branch != "" || git.Commit != "" || git.OriginURL != "" { diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index b3a55a607cc..d533517f998 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -27,10 +27,11 @@ Useful after deployment to see what was created and where to find it.`, cmd.RunE = func(cmd *cobra.Command, args []string) error { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ - ReadState: true, - AlwaysPull: forcePull, - IncludeLocations: includeLocations, - InitIDs: true, + ReadState: true, + AlwaysPull: forcePull, + IncludeLocations: includeLocations, + InitIDs: true, + InitDeploymentHistory: true, }) if err != nil { return err diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 30eefcac586..1cf67b52132 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -56,6 +56,12 @@ type ProcessOptions struct { // Implies ReadState InitIDs bool + // If true, calls InitializeDeploymentHistory() to look up the bundle's recorded + // deployment. Separate from InitIDs because it costs its own API calls and only + // 'bundle summary' reports the result. + // Implies InitIDs + InitDeploymentHistory bool + // if true, pass ErrorOnEmptyState to statemgmt.Load // Implies ReadState ErrorOnEmptyState bool @@ -89,6 +95,11 @@ func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, err func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle, stateDesc *statemgmt.StateDesc, retErr error) { var err error + // The deployment history is looked up alongside the resource IDs, so asking for + // it implies them. Normalized here so the options below only test InitIDs. + if opts.InitDeploymentHistory { + opts.InitIDs = true + } ctx := cmd.Context() if opts.SkipInitContext { if !logdiag.IsSetup(ctx) { @@ -258,9 +269,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle statemgmt.Load(state, modes...), } // InitializeURLs makes an extra API call; only run it when URLs are needed. - // InitializeDeploymentHistory likewise, and only for bundles that record it. if opts.InitIDs { - mutators = append(mutators, mutator.InitializeURLs(), mutator.InitializeDeploymentHistory()) + mutators = append(mutators, mutator.InitializeURLs()) + } + // Same for InitializeDeploymentHistory, which only 'bundle summary' reports. + if opts.InitDeploymentHistory { + mutators = append(mutators, mutator.InitializeDeploymentHistory()) } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { diff --git a/internal/bugbash/README.md b/internal/bugbash/README.md index 1c5995a8188..941ab6227cc 100644 --- a/internal/bugbash/README.md +++ b/internal/bugbash/README.md @@ -11,11 +11,3 @@ but works without command completion with earlier versions. ```shell bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch ``` - -The branch needs a successful `release-build` run to download a snapshot from. That -workflow runs on `main` and on any branch named `demo-*` or `bugbash-*`, so push the -branch under one of those names. - -## Feature guides - -- [Deployment history recording](./record-deployment-history.md) diff --git a/internal/bugbash/record-deployment-history.md b/internal/bugbash/record-deployment-history.md deleted file mode 100644 index db0a7172653..00000000000 --- a/internal/bugbash/record-deployment-history.md +++ /dev/null @@ -1,151 +0,0 @@ -# Bugbash: deployment history recording - -Records every `bundle deploy` and `bundle destroy` with the Deployment Metadata -Service (DMS), so a deployment has a server-side history and its resource state -lives in the workspace rather than only in the local cache. - -## Get a CLI with the feature - -```shell -bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) bugbash-record-deployment-history -``` - -That drops you into a shell with `databricks` on `$PATH`. Check you have the right -build with `databricks --version`. - -## Turn the feature on - -Three things are needed. Missing any one of them means nothing is recorded. - -```shell -export DATABRICKS_BUNDLE_ENGINE=direct -export DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=true -``` - -and in `databricks.yml`: - -```yaml -experimental: - record_deployment_history: true -``` - -The env var only unlocks the gate; the YAML flag is what enables recording. Without -the env var the CLI refuses: - -``` -Error: experimental.record_deployment_history is not supported yet -``` - -Recording is direct-engine only. On terraform the flag is rejected, and no -`/api/2.0/bundle/*` calls are made. - -The feature must be enabled from the bundle's **first** deploy. Turning it on for a -bundle that already has deployed resources is refused, because DMS would then own a -resource set it never saw and the next deploy would create everything a second time. -The error spells out the three steps to start over. - -## A bundle to start from - -```yaml -bundle: - name: my-dms-test - -experimental: - record_deployment_history: true - -resources: - jobs: - hello: - name: my-dms-test-job - tasks: - - task_key: main - notebook_task: - notebook_path: ./noop.py -``` - -with `noop.py` beside it: - -``` -# Databricks notebook source -print(1) -``` - -## Find the deployment - -The CLI stores the deployment ID nowhere. DMS registers the deployment as a workspace -node, and that node's object ID *is* the deployment ID: - -```shell -databricks workspace get-status \ - "/Workspace/Users/$(databricks current-user me | jq -r .userName)/.bundle/my-dms-test/default/state/resources.deployment.json" \ - -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])' -``` - -Use `python3`, not `jq`, for that ID. It exceeds 2^53 and jq below 1.7 silently -rounds it, which looks like "deployment does not exist". - -Or read it straight off the summary: - -```shell -databricks bundle summary -o json | jq .bundle.deployment.history -``` - -## What to look at - -```shell -databricks api get "/api/2.0/bundle/deployments/$DID" # the deployment -databricks api get "/api/2.0/bundle/deployments/$DID/versions" # one version per deploy -databricks api get "/api/2.0/bundle/deployments/$DID/versions/$V/operations" -databricks api get "/api/2.0/bundle/deployments/$DID/resources" # current resource state -databricks api get "/api/2.0/bundle/deployments" # all deployments -``` - -`resources` and `operations` paginate at 20 with a `next_page_token`. A bundle with -more than 20 resources is not truncated; page through it. - -Jobs and pipelines carry a back-reference to the deployment, but the SDK hides those -fields, so read them raw: - -```shell -databricks api get "/api/2.0/jobs/get?job_id=$JID" | jq .settings.deployment -databricks api get "/api/2.0/pipelines/$PID" | jq .spec.deployment -``` - -Both should show `deployment_id` and `version_id` next to `kind: BUNDLE`. - -## Worth exercising - -- **Iterate.** Deploy, change a field, deploy again. Each deploy claims a version; - only changed resources get an operation. -- **Wipe the local cache.** `rm -rf .databricks`, then `bundle plan`. It should report - your resources as unchanged, reconstructed from DMS. It must never plan to create - something that already exists. -- **Break a resource.** Give a job an invalid cron expression. The failed resource is - recorded with `status: OPERATION_STATUS_FAILED` and an `error_message`, the version - completes with `VERSION_COMPLETE_FAILURE`, and a later plan wants to create it. -- **Destroy.** A destroy records its own version with a DELETE per resource, then - deletes the deployment record. -- **Non-job resources.** Pipelines, schemas, volumes, experiments, registered models, - secret scopes and dashboards are all recorded. Each has a differently-shaped - resource id (numeric, UUID, `catalog.schema.name`, a scope name). -- **Targets.** Each target has its own state path, so `-t dev` and `-t prod` are - separate deployments with separate version chains. -- **Provenance.** Deploy from a git repo and check `git_info` on the version; - `deployment_mode` reflects the target's `mode`. - -## Not bugs - -- A redeploy with no changes still creates a version, with no operations under it. -- After `destroy`, `GetDeployment` still returns the record with - `status: DEPLOYMENT_STATUS_DELETED`. That is a soft delete. -- `state` on an operation or resource is a **quoted JSON string**, not an embedded - object. Parse it once to get `{"state": {...}, "depends_on": [...]}`. -- DMS resource keys have no `resources.` prefix (`jobs.foo`), unlike local state keys. -- Sub-resources get their own operation, e.g. `secret_scopes.mine.permissions`. -- Permissions are not set on the deployment node. It inherits from the state folder, - which the bundle's `permissions:` section already governs. - -## Reporting - -Include the deployment ID, the version, and the request/response for anything that -looks wrong. `databricks bundle deploy --log-level debug` logs the DMS calls. diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 0306507913a..eda0233931b 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -94,10 +94,8 @@ type Recorder struct { versions versionCreator deploymentID string statePath string - targetName string - displayName string versionType VersionType - provenance Provenance + metadata Metadata // populated by CreateVersion versionNum int64 @@ -121,17 +119,19 @@ type RecorderOptions struct { // StatePath is the bundle's remote state directory, under which DMS registers // the deployment node. StatePath string - TargetName string - DisplayName string VersionType VersionType - // Provenance records where the deployed source came from; see Provenance. - Provenance Provenance + // Metadata is what the version records about the deploy; see Metadata. + Metadata Metadata } -// Provenance is what a version records about the source it deployed and where it -// landed. The service denormalizes these onto the deployment, so they describe the -// deployment as of its most recent version. -type Provenance struct { +// Metadata is what a version records about the bundle it deployed, the source it +// came from, and where it landed. The service denormalizes these onto the +// deployment, so they describe the deployment as of its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string // Mode is the bundle target's mode, empty when the target sets none. Mode bundledeployments.DeploymentMode Git *bundledeployments.GitInfo @@ -145,10 +145,8 @@ func NewRecorder(opts RecorderOptions) *Recorder { versions: opts.Versions, deploymentID: opts.DeploymentID, statePath: opts.StatePath, - targetName: opts.TargetName, - displayName: opts.DisplayName, versionType: opts.VersionType, - provenance: opts.Provenance, + metadata: opts.Metadata, } } @@ -246,30 +244,28 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin // deployment's first version. var previousVersionID string if r.deploymentID != "" { - // A resolved node names the deployment, but its record is created by the - // first version, so there may be none yet: a deploy that registered the - // deployment and then failed before recording a version. Start at version 1 - // under the ID the node already names, rather than creating a second - // deployment, which would collide on the same node path. + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so a not-found + // here means that invariant is broken rather than anything the user did. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) switch { - case getErr == nil && dep.LastVersionId == "": + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + return "", fmt.Errorf("internal error: no deployment found for the file with object id %s: %w", r.deploymentID, getErr) + case getErr != nil: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + case dep.LastVersionId == "": // The record exists but carries no version: a deploy whose first version was // rejected still leaves the record behind. Retry at version 1. versionID = "1" - case getErr == nil: + default: lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) if parseErr != nil { return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) } versionID = strconv.FormatInt(lastVersion+1, 10) previousVersionID = dep.LastVersionId - case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): - versionID = "1" - default: - return "", fmt.Errorf("failed to get deployment: %w", getErr) } } else { // First deploy: create the deployment so the server assigns an ID. @@ -281,7 +277,7 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, - TargetName: r.targetName, + TargetName: r.metadata.TargetName, }, }) if createErr != nil { @@ -301,12 +297,12 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: r.versionType, - TargetName: r.targetName, - DisplayName: r.displayName, + TargetName: r.metadata.TargetName, + DisplayName: r.metadata.DisplayName, PreviousVersionId: previousVersionID, - DeploymentMode: r.provenance.Mode, - GitInfo: r.provenance.Git, - WorkspaceInfo: r.provenance.Workspace, + DeploymentMode: r.metadata.Mode, + GitInfo: r.metadata.Git, + WorkspaceInfo: r.metadata.Workspace, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 694fab0699a..56d5d41631d 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -87,7 +87,7 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} // A first deploy resolves no deployment ID from the workspace. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -118,7 +118,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -134,7 +134,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -153,30 +153,28 @@ func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { return nil, errors.New("boom") }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) err := r.CreateVersion(t.Context()) assert.ErrorContains(t, err, "failed to get deployment") assert.Empty(t, f.created) } -func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { - // The record is created by the first version, so a node can name a deployment - // that has none yet - an earlier deploy registered it and then failed. Record - // version 1 under that same ID instead of creating a second deployment, which - // would collide on the node path. +func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { + // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found + // for a node get-status just returned is a broken invariant, not a state the + // deploy can recover from. f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context())) + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") assert.Empty(t, f.created) - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].versionID) - assert.Equal(t, "stored-id", f.versions[0].deploymentID) + assert.Empty(t, f.versions) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -185,7 +183,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) @@ -201,7 +199,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -220,7 +218,7 @@ func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -241,7 +239,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 4f2571691a2..74a8d2156b3 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -67,15 +67,20 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } - // Only the node is created here. The deployment record itself is created by the - // first CreateVersion, so a client that creates a deployment and then fails - // before recording a version leaves no record behind - just the node, which - // names the ID that first version will be created under. + // The record is created together with the node, so a get on it always resolves + // for a node that exists. It carries no version yet: last_version_id stays empty + // until the first CreateVersion, which is how a client that registers a + // deployment and then fails leaves a record with no versions. deploymentID := strconv.FormatInt(objectID, 10) s.dmsDeploymentNodes[deploymentID] = nodePath dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } return Response{Body: dep} } @@ -154,22 +159,7 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d, ok := s.dmsDeployments[deploymentID] if !ok { - // The deployment record is created by its first version, not by - // CreateDeployment. That call only registered the workspace node, so the node - // existing is what makes this ID valid. - if _, known := s.dmsDeploymentNodes[deploymentID]; !known { - return dmsNotFound("deployment " + deploymentID) - } - d = &dmsDeployment{ - deployment: bundledeployments.Deployment{ - Name: "deployments/" + deploymentID, - Status: bundledeployments.DeploymentStatusDeploymentStatusActive, - TargetName: version.TargetName, - }, - versions: map[string]*bundledeployments.Version{}, - resources: map[string]bundledeployments.Resource{}, - } - s.dmsDeployments[deploymentID] = d + return dmsNotFound("deployment " + deploymentID) } // Mirror the server-side checks: version_id must be numerically greater than From 37a8b942fed90c15d761931f734622617ae6be35 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:13:09 +0000 Subject: [PATCH 45/56] bundle: shorten the comment on the DMS upload-failure check Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 2047f65a498..9fa882867fc 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -69,12 +69,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - // Stop before touching the workspace once recording an operation has failed. - // A completed version makes DMS the source of truth for resource state (see - // dstate.readDMSState), so continuing would create resources it has no record - // of and the next deploy would create them a second time. Checked here rather - // than only where operations are recorded, which is after the resource has - // already been modified. + // Stop resource CRUD once uploading DMS state has failed. if err := opQueue.firstErr(); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false From 636c9cfbae9db628e90a3a4fbcbad5beb3e8c38d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:49:33 +0000 Subject: [PATCH 46/56] bundle: stamp the deployment before planning, and keep recording after a failure Three changes. The phantom drift on deployment.deployment_id is fixed by setting the field instead of ignoring it. AnnotateDeploymentVersion stamped the id and the version together from the deploy phase, so a plain `bundle plan` left the id empty while the state and the workspace both had it, and the absence read as a change on an untouched job. It splits into AnnotateDeployment (the id, applied where the state is opened, which is before anything diffs) and AnnotateDeploymentVersion (the version, which does not exist until CreateVersion claims one). The ignore_local_changes entries added for deployment_id are dropped: resources.yml is back to what main has, and the local config now matches the workspace rather than hiding a mismatch. A failed upload no longer stops the ones behind it. record refused new work once any upload had failed, so a resource that was applied went unrecorded and DMS drifted from reality in the other direction. The workers already continued past a failure; now record does too, and close still reports the error, so the deploy fails either way. Stopping resource CRUD is unchanged - bundle_apply still checks firstErr before touching the workspace, which is the check that matters. Comments through the DMS files are cut to one or two lines each. opqueue.go keeps its structure, since the concurrency rules there are not obvious from the code. Verified on dogfood: `bundle plan` on an untouched bundle reports 0 to change with no ignore rule, and the deployed job still carries deployment_id. Co-authored-by: Isaac --- .../metadata/annotate_deployment_version.go | 50 +++++-- .../annotate_deployment_version_test.go | 2 +- bundle/direct/dresources/resources.yml | 26 ++-- bundle/direct/dstate/dms.go | 37 ++--- bundle/direct/opqueue.go | 135 ++++++------------ bundle/direct/opqueue_test.go | 25 ++-- bundle/phases/deploy.go | 16 ++- cmd/bundle/utils/process.go | 10 ++ libs/dms/recorder.go | 57 +++----- 9 files changed, 155 insertions(+), 203 deletions(-) diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go index 1b8139168ce..46e0ab7ed41 100644 --- a/bundle/deploy/metadata/annotate_deployment_version.go +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -8,21 +8,46 @@ import ( "github.com/databricks/cli/libs/diag" ) -type annotateDeploymentVersion struct { +type annotateDeployment struct { deploymentID string - version int64 } -// AnnotateDeploymentVersion stamps the DMS deployment and version onto every job -// and pipeline, so a resource in the workspace points back at the deployment that -// produced it (which is how lineage resolves a job to its bundle). +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a +// resource in the workspace points back at the deployment that produced it (which is +// how lineage resolves a job to its bundle). // -// AnnotateJobs/AnnotatePipelines set the rest of the deployment metadata during -// initialize, but the version - and, on a first deploy, the deployment ID - only -// exist once CreateVersion has run, so these two fields are stamped separately -// from the deploy phase. -func AnnotateDeploymentVersion(deploymentID string, version int64) bundle.Mutator { - return &annotateDeploymentVersion{deploymentID: deploymentID, version: version} +// It runs before the plan is computed, since a resource whose deployment is unset +// locally but set in the workspace would otherwise show as drift. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. It +// is separate from AnnotateDeployment because the version only exists once +// CreateVersion has claimed one, which happens during deploy. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} } func (m *annotateDeploymentVersion) Name() string { @@ -33,13 +58,10 @@ func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) d versionID := strconv.FormatInt(m.version, 10) for _, job := range b.Config.Resources.Jobs { - // Deployment is set by AnnotateJobs, which runs during initialize. - job.Deployment.DeploymentId = m.deploymentID job.Deployment.VersionId = versionID } for _, pipeline := range b.Config.Resources.Pipelines { - pipeline.Deployment.DeploymentId = m.deploymentID pipeline.Deployment.VersionId = versionID } diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go index aca51358ace..297ba95f8ee 100644 --- a/bundle/deploy/metadata/annotate_deployment_version_test.go +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -34,7 +34,7 @@ func TestAnnotateDeploymentVersion(t *testing.T) { }, } - diags := bundle.ApplySeq(t.Context(), b, AnnotateDeploymentVersion("dep-123", 7)) + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) require.NoError(t, diags.Error()) job := b.Config.Resources.Jobs["my-job"].Deployment diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 7345f19c506..6061db0fb04 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -26,17 +26,11 @@ resources: # it changes constantly. Ignoring it as a local and remote change keeps that # churn from driving an update or showing as drift on its own; when the job is # updated for any other reason, DoUpdate sends the full config via Reset, so - # the current version_id is still recorded. - # - # deployment_id is ignored as a local change for a different reason: only the - # deploy phase stamps it (it is not known until the version is claimed), so - # during a plain `bundle plan` the local config has none while the state and the - # workspace both do, and the absence would show as drift on an untouched job. + # the current version_id is still recorded. deployment_id is intentionally + # left out: it is stable across versions, so a change to it is worth showing. ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service - - field: deployment.deployment_id - reason: managed by the deployment metadata service ignore_remote_changes: - field: deployment.version_id @@ -174,8 +168,8 @@ resources: - field: ingestion_definition.ingest_from_uc_foreign_catalog reason: immutable - # See jobs above: version_id is set on every deploy, and deployment_id is only - # stamped by the deploy phase, so both are ignored as local changes. + # See jobs above: version_id is set on every deploy, so it is ignored as a + # local/remote change. deployment_id is left out so a change to it still shows. ignore_remote_changes: - field: deployment.version_id reason: managed by the deployment metadata service @@ -184,14 +178,22 @@ resources: # Thus it shows up as a remote change since we don't set on the object. - field: id reason: "!drop" + # QQQ should this be here? When run_as is explicitly set, the GET response echoes it back + # as a structured run_as.user_name (verified on e2-dogfood with a real user), so it may not + # be truly input-only. The explicit-set case could not be confirmed on aws-cli, azure-cli, + # or gcp-cli: those envs authenticate as a service principal that lacks servicePrincipal.user + # on itself, so it can't self-bind run_as. In the default (unset) case on all three clouds, + # GET returns only the flat run_as_user_name and no structured run_as. - field: run_as reason: input_only + # Carried by CreatePipeline/EditPipeline but never returned by GET, so remote + # always reads back false and a config value of true never converges. + - field: allow_duplicate_names + reason: input_only ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service - - field: deployment.deployment_id - reason: managed by the deployment metadata service # "id" is output-only, providing it in config would be a mistake - field: id reason: "!drop" diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 3d7131580e2..4ec68dea82d 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,48 +3,29 @@ package dstate import ( "context" "encoding/json" - "errors" "fmt" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// RecordedState is what the CLI serializes into the DMS Operation.State field. -// -// It is an envelope rather than the bare resource config, because depends_on has -// to survive the round trip: DMS has no field for dependency edges, and they -// cannot be recomputed from the config once it is recorded (references are -// resolved to literals before serialization). Nesting depends_on inside the -// config instead would collide with resource fields of the same name, e.g. -// jobs.Task.depends_on. -// -// The shape deliberately matches the local ResourceEntry so both sides of the -// state round trip look the same. +// RecordedState is what the CLI serializes into the DMS Operation.State field. It +// wraps the config rather than being it, so depends_on survives the round trip: DMS +// has no field for dependency edges, and they cannot be recomputed once references +// are resolved to literals. Nesting them in the config would collide with resource +// fields of the same name (e.g. jobs.Task.depends_on). type RecordedState struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// readDMSState replaces the file-derived resource state with the state recorded -// in DMS. Recording is only enabled for net-new deployments, so once a -// deployment exists DMS owns its resource set outright - including when that set -// is empty, which is a successful deploy of nothing rather than missing data. -// The caller holds db.mu. +// readDMSState replaces the file-derived resource state with the state recorded in +// DMS. Recording is only enabled for net-new deployments, so once a deployment +// exists DMS owns its resource set outright - an empty set means a successful deploy +// of nothing, not missing data. The caller holds db.mu. func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { - // The deployment's record is created by its first version, so a node can - // resolve to an ID that has none yet: a deploy that registered the deployment - // and then failed before recording a version. There is nothing to read, and - // the file's resources are still empty, so carry on and let this deploy record - // the first version. - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - log.Debugf(ctx, "No deployment record for %s yet; keeping local state", src.DeploymentID) - return nil - } return err } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba64d52ebd9..ec5b97ac37d 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -11,46 +11,30 @@ import ( ) const ( - // operationQueueSize bounds how many recorded operations wait for upload. - // Apply deploys at most defaultParallelism resources at a time, so a queue - // this deep means an apply worker practically never blocks on a free slot. + // operationQueueSize bounds how many recorded operations wait for upload. Deep + // enough that an apply worker practically never blocks on a free slot. operationQueueSize = 10 - // operationUploadWorkers is how many uploads run at a time. It is below - // operationQueueSize so a burst of operations is absorbed by the queue rather - // than by one request per resource. - // - // This was temporarily capped at 2 while concurrent CreateOperation calls under - // the same version contended on shared state server-side, surfacing the - // transaction conflict as a 500 that failed the deploy. The service now handles - // them: measured against it, 8 and 16 concurrent writes both succeed where 4 - // used to fail. + // operationUploadWorkers is how many uploads run at a time. operationUploadWorkers = 8 ) -// operationQueue hands recorded operations to background workers, so an apply -// worker does not wait for the CreateOperation round trip before deploying the -// next resource. +// operationQueue uploads recorded operations from background workers, so a deploy +// never waits on the CreateOperation round trip. Two rules shape it: // -// Two rules shape the design: +// - One resource, one upload at a time. DMS keeps a single state per resource, so +// overlapping uploads could land out of order and leave the older state. +// - Newest operation wins. Each carries the resource's full state, so a queued +// operation superseded by a newer one is dropped ("coalesced"). // -// - Uploads for one resource never overlap. DMS stores one state per resource -// key, so concurrent uploads could land out of order and leave stale state. -// - Only the newest operation for a resource matters. Each operation carries the -// resource's full state, not a delta, so a newer one entirely supersedes an -// older one. When both are still waiting, the older is dropped ("coalesced") -// and one upload records the result. -// -// Uploads are not fire-and-forget: close returns the first failure and fails the -// deploy. A dropped operation would leave DMS with an incomplete resource set, -// and since DMS then becomes the source of truth (see dstate.readDMSState), the -// next deploy would recreate resources that already exist. +// close reports the first upload failure, which fails the deploy: DMS becomes the +// source of truth (see dstate.readDMSState), so a missing record would make the +// next deploy create a resource that already exists. type operationQueue struct { uploader operationUploader - // queue carries resource keys, not operations. A worker looks the operation up - // when it picks the key up, so recording again before then just overwrites the - // entry in pending - that is what makes coalescing work. + // queue carries resource keys, not operations: a worker looks the operation up + // when it picks the key up, which is what makes coalescing work. queue chan string wg sync.WaitGroup @@ -61,25 +45,16 @@ type operationQueue struct { // yet. Empty for a key means everything recorded for it has been uploaded. pending map[string]recordedOperation - // queuedOrUploading marks keys that are already in the queue channel or being - // uploaded right now. Such a key must not be queued again, or two workers could - // upload the same resource at once; recording writes to pending instead, and - // the worker handling the key picks it up when its current upload finishes. - // - // No single worker "owns" a key for the whole time it is marked: a key can be - // handled by one worker, released, and later picked up by another. The mark only - // means "some worker will get to this", which is all record needs to know. + // queuedOrUploading means "some worker will get to this key". Recording such a + // key writes to pending only, so two workers never upload one resource at once. queuedOrUploading map[string]bool err error closed bool } -// newOperationQueue starts the upload workers. It returns nil when uploader is -// nil (recording disabled), and every method is a no-op on a nil queue so callers -// do not have to branch. -// -// ctx is used for the uploads, so it must stay valid until close returns. +// newOperationQueue starts the upload workers, returning nil when uploader is nil +// (recording off; every method is a no-op on a nil queue). ctx must outlive close. func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { if uploader == nil { return nil @@ -100,33 +75,16 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers. The upload -// itself happens on a worker, so an error returned here is either a failure to -// turn the applied resource into a payload, or an earlier upload's error -// resurfaced (see below). +// record serializes an operation and hands it to the upload workers, so an error +// here means the payload could not be built; upload errors surface at close. // -// Recording a resource that is still waiting replaces the waiting operation -// outright, since the newer one carries the resource's full state. +// An earlier upload failure does not stop this: every applied resource is still +// recorded, best effort, so DMS ends up as close to reality as it can get. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil } - // Report an earlier upload failure to the apply worker that is about to record - // the next resource, so the deploy stops instead of running to completion and - // only failing at close. That matters because a successfully completed version - // makes DMS the source of truth for resource state (see dstate.readDMSState): - // deploying everything while its records are missing leaves resources the next - // deploy would create a second time. - // - // This refuses new work only. Operations already recorded still upload - close - // drains them - so the records DMS does end up with match the resources that - // were actually applied. Resources already mid-apply also finish, so the deploy - // stops shortly after the first failure rather than exactly at it. - if err := q.firstErr(); err != nil { - return err - } - op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err @@ -136,12 +94,9 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return nil } -// recordFailure records that applying a resource failed, so the deployment -// history explains the failure instead of omitting the resource. -// -// Unlike record, this does not resurface an earlier upload error: the deploy is -// already failing, and returning a different error here would replace the one the -// user needs to see. A failure to upload this record is reported at close. +// recordFailure records that applying a resource failed, so the deployment history +// explains the failure instead of omitting the resource. It returns nothing: the +// deploy is already failing, and a second error would mask the one the user needs. func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if q == nil { return @@ -170,9 +125,8 @@ func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op rec log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) } - // A worker is already going to handle this key, and it re-reads pending before - // finishing, so it will see the operation written above. Queueing the key again - // would let a second worker upload the same resource concurrently. + // A worker will re-read pending before it finishes, so it picks up the operation + // written above. Queueing again would let a second worker upload the same key. if alreadyHandled { return } @@ -180,14 +134,12 @@ func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op rec q.queue <- resourceKey } -// close drains the queue and returns the first upload error. All callers of -// record must have returned first: record on a closed queue panics. Calling close -// more than once is safe, so callers can defer it and still check the error at a -// specific point. +// close drains the queue and returns the first upload error. Every record caller +// must have returned first (record on a closed queue panics); calling close twice +// is safe, so it can be deferred and still checked at a specific point. // -// Unlike the other methods this one takes no lock. It runs on one goroutine after -// every apply worker has returned, so nothing else touches the queue by then, and -// the wg.Wait below orders the workers' writes to err before it is read. +// It takes no lock: it runs after every apply worker returned, and wg.Wait orders +// the workers' writes to err before it is read here. func (q *operationQueue) close() error { if q == nil { return nil @@ -206,15 +158,16 @@ func (q *operationQueue) work(ctx context.Context) { defer q.wg.Done() for resourceKey := range q.queue { - // Keep uploading this key until nothing new was recorded for it, rather than - // putting it back on the queue: a worker sending to the channel it consumes - // from can deadlock once the queue is full. + // Drain this key here instead of re-queueing it: a worker sending to the + // channel it consumes from deadlocks once the queue is full. for { op, ok := q.take(resourceKey) if !ok { break } + // Keep going after a failure, so one bad upload does not drop the records + // for every resource behind it. if err := q.uploader.upload(ctx, resourceKey, op); err != nil { q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) } @@ -222,12 +175,9 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey. It reports false and clears -// the queuedOrUploading mark when nothing is waiting, which is what lets the next -// record queue the key again. -// -// Clearing the mark and observing pending empty happen under one lock, so record -// can never skip queueing a key that no worker is going to look at again. +// take claims the operation waiting for resourceKey, reporting false and clearing +// the queuedOrUploading mark when nothing is left, which lets record queue it again. +// Both happen under one lock, so a key can never be left for no worker to pick up. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() @@ -238,11 +188,8 @@ func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { return recordedOperation{}, false } - // The key stays in queuedOrUploading: the worker keeps coming back here until - // nothing is pending for it, so anything recorded while this operation uploads - // is still picked up. The mark is only cleared above, once there is nothing - // left - which is also what stops a second worker from taking the key and - // uploading the same resource concurrently. + // The mark stays until the branch above clears it, so anything recorded during + // this upload is still picked up and no second worker takes the key meanwhile. delete(q.pending, resourceKey) return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 910b58d470f..7a98dc43774 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -198,10 +198,9 @@ func TestOperationQueueReturnsUploadError(t *testing.T) { assert.Contains(t, err.Error(), "resources.jobs.foo") } -func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { - // An upload failure stops the deploy at the next resource instead of surfacing - // only at close, so the apply workers do not keep creating resources that DMS - // has no record of. +func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { + // A failed upload must not stop the ones behind it: every applied resource is + // recorded best effort, so DMS ends up as close to reality as it can get. uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} q := newOperationQueue(t.Context(), f) @@ -211,22 +210,22 @@ func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) assert.Equal(t, "resources.jobs.foo", <-f.done) - // The next resource an apply worker tries to record is refused, with the upload - // error that caused it. - err := q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil) - require.Error(t, err) - assert.ErrorIs(t, err, uploadErr) + // The next resource is still accepted, even though the first upload failed. + require.NoError(t, q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) - // The refused resource was not queued, and close still reports the failure. + // Both were attempted, and close still reports the failure so the deploy fails. require.ErrorIs(t, q.close(), uploadErr) - assert.Equal(t, []string{`resources.jobs.foo={"state":{"name":"v1"}}`}, f.recorded()) + assert.ElementsMatch(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.bar={"state":{"name":"v1"}}`, + }, f.recorded()) assert.Empty(t, q.pending) assert.Empty(t, q.queuedOrUploading) } func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { - // A failure refuses new work but does not discard work already recorded: the - // records DMS ends up with have to match the resources that were applied. + // A failure does not discard work already recorded: the records DMS ends up with + // have to match the resources that were applied. uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index ba923638ab5..c24efc51f11 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -229,17 +229,21 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Create the version before planning: the plan snapshots the resource config, - // so the deployment and version have to be stamped onto the resources before it - // is computed or the applied resources would not carry them. A cancelled deploy - // therefore leaves a version behind, completed as a failure by the deferred - // CompleteVersion. + // Create the version before planning: the plan snapshots the resource config, so + // the version has to be stamped on before it is computed or the applied resources + // would not carry it. A cancelled deploy therefore leaves a version behind, + // completed as a failure by the deferred CompleteVersion. if err := recorder.CreateVersion(ctx); err != nil { logdiag.LogError(ctx, err) return } if recorder != nil { - bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(recorder.DeploymentID(), recorder.Version())) + // The deployment ID is stamped earlier, when the state is opened; only the + // version is new here. A first deploy has no ID until now, so stamp both. + bundle.ApplySeqContext(ctx, b, + metadata.AnnotateDeployment(recorder.DeploymentID()), + metadata.AnnotateDeploymentVersion(recorder.Version()), + ) if logdiag.HasError(ctx) { return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 1cf67b52132..f7fdf35e255 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -241,6 +242,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle Client: w.BundleDeployments, DeploymentID: deploymentID, } + + // Stamp the deployment onto the resources before anything diffs them. + // The workspace has it, so a plan that left it unset would report drift + // on a resource nobody touched. The version is stamped by the deploy + // phase instead, once it claims one. + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(deploymentID)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index eda0233931b..001126f4fb6 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,19 +29,15 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) -// createVersionRequest is the CreateVersion request body. -// -// The CLI builds the body itself instead of using bundledeployments.Version -// because the generated struct has no previous_version_id field, which the -// service requires as its concurrency check. Without it every deploy after the -// first is rejected. +// createVersionRequest is the CreateVersion request body. Hand-written because the +// generated struct has no previous_version_id, which the service needs as its +// concurrency check - without it every deploy after the first is rejected. type createVersionRequest struct { CliVersion string `json:"cli_version"` VersionType VersionType `json:"version_type"` TargetName string `json:"target_name,omitempty"` - // DisplayName names the deployment in the UI. The service copies it onto the - // deployment's workspace node, which is where GetDeployment reads it from, so - // a version that omits it leaves the deployment unnamed. + // DisplayName names the deployment in the UI. The service keeps it on the + // deployment's node, so a version that omits it leaves the deployment unnamed. DisplayName string `json:"display_name,omitempty"` // PreviousVersionId is the deployment's most recent version, unset for a // deployment's first version. @@ -83,12 +79,10 @@ func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, ver return &version, nil } -// Recorder records a single deploy/destroy as a version with DMS. -// -// The server assigns the deployment ID on the first deploy, i.e. when the ID -// resolved from the workspace is empty (see ResolveDeploymentID). Later deploys -// resolve the same ID and reuse the record; a destroy deletes the record and its -// node, so the next deploy starts over from empty. +// Recorder records a single deploy/destroy as a version with DMS. The server +// assigns the deployment ID on the first deploy and later deploys reuse it; a +// destroy deletes the record, so the next deploy starts over (see +// ResolveDeploymentID). type Recorder struct { svc bundledeployments.BundleDeploymentsInterface versions versionCreator @@ -112,9 +106,8 @@ type RecorderOptions struct { Service bundledeployments.BundleDeploymentsInterface // Versions handles CreateVersion; see versionCreator. Versions versionCreator - // DeploymentID is the ID resolved from the deployment's workspace node, or - // empty if this bundle has not recorded a deployment yet (the server assigns - // one during CreateVersion). + // DeploymentID is resolved from the deployment's workspace node, empty until the + // first recorded deploy (CreateVersion assigns one then). DeploymentID string // StatePath is the bundle's remote state directory, under which DMS registers // the deployment node. @@ -124,9 +117,9 @@ type RecorderOptions struct { Metadata Metadata } -// Metadata is what a version records about the bundle it deployed, the source it -// came from, and where it landed. The service denormalizes these onto the -// deployment, so they describe the deployment as of its most recent version. +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. type Metadata struct { // DisplayName is the bundle's name, which the deployment is listed under. DisplayName string @@ -191,11 +184,9 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { return nil } -// CompleteVersion finalizes the version created by CreateVersion. A nil -// Recorder, or one whose CreateVersion never ran or failed, is a no-op: there is -// no version on the server to complete. Callers defer it unconditionally, so this -// is the check that keeps a cancelled or failed deploy from completing a version -// that was never created. +// CompleteVersion finalizes the version created by CreateVersion. It is a no-op +// when CreateVersion never ran, which is what lets callers defer it and still not +// complete a version a cancelled deploy never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { if r == nil || r.versionNum == 0 || r.completed { return nil @@ -235,10 +226,9 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { return nil } -// createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. With no deployment ID it creates the deployment and lets -// the server assign the ID; otherwise it reads the existing deployment to -// compute the next version number. +// createDeploymentVersion ensures the deployment record exists, then creates a new +// version under it: with no ID it creates the deployment, otherwise it reads the +// existing one for the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { // The version this one supersedes, sent as the concurrency check. Empty for a // deployment's first version. @@ -269,11 +259,8 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } } else { // First deploy: create the deployment so the server assigns an ID. - // - // initial_parent_path is required. The service creates the deployment node - // under it, and that node's ID is the deployment ID ResolveDeploymentID reads - // back later. The folder already exists by now: the deployment lock lives in - // the same directory. + // initial_parent_path is required - the node the service creates under it is + // what ResolveDeploymentID reads back later. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, From ff96ef01de7c012c0bf177a90dc20032d49cf41e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 01:24:24 +0000 Subject: [PATCH 47/56] bundle: decouple InitDeploymentHistory from InitIDs The two are unrelated: InitIDs loads resource IDs out of the state, while the deployment history is read from the service. They were coupled only because the mutator sat inside the block gated on InitIDs, so reaching it meant forcing that option on - which made 'bundle summary' look like it needed resource IDs to report a deployment ID. It runs in its own block now, and implies ReadState instead. bundle summary asks for both because it happens to want both. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f7fdf35e255..3b90aef5c6a 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -58,9 +58,8 @@ type ProcessOptions struct { InitIDs bool // If true, calls InitializeDeploymentHistory() to look up the bundle's recorded - // deployment. Separate from InitIDs because it costs its own API calls and only - // 'bundle summary' reports the result. - // Implies InitIDs + // deployment. Independent of InitIDs, and costs its own API calls. + // Implies ReadState InitDeploymentHistory bool // if true, pass ErrorOnEmptyState to statemgmt.Load @@ -96,11 +95,6 @@ func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, err func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle, stateDesc *statemgmt.StateDesc, retErr error) { var err error - // The deployment history is looked up alongside the resource IDs, so asking for - // it implies them. Normalized here so the options below only test InitIDs. - if opts.InitDeploymentHistory { - opts.InitIDs = true - } ctx := cmd.Context() if opts.SkipInitContext { if !logdiag.IsSetup(ctx) { @@ -192,7 +186,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } - shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.InitDeploymentHistory || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { // PullResourcesState depends on stateFiler which needs b.Config.Workspace.StatePath which is set in phases.Initialize @@ -282,15 +276,20 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.InitIDs { mutators = append(mutators, mutator.InitializeURLs()) } - // Same for InitializeDeploymentHistory, which only 'bundle summary' reports. - if opts.InitDeploymentHistory { - mutators = append(mutators, mutator.InitializeDeploymentHistory()) - } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } } + + // Independent of the resource IDs above: this reads the deployment record, not + // the state. It makes its own API calls, so only 'bundle summary' asks for it. + if opts.InitDeploymentHistory { + bundle.ApplyContext(ctx, b, mutator.InitializeDeploymentHistory()) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } } var plan *deployplan.Plan From 89369d5966563be4725dbc6b6cc1b6b8062ce558 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 01:36:40 +0000 Subject: [PATCH 48/56] bundle: fold the display-name assertions into the first-deploy test Both tests set up the same recorder and called CreateVersion; the second only added two assertions about the request body, so they move into the first. Co-authored-by: Isaac --- libs/dms/recorder_test.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 56d5d41631d..51ce63e5971 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -105,6 +105,13 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) assert.Equal(t, int64(1), r.Version()) + // The service copies display_name onto the deployment's workspace node, which is + // where GetDeployment reads it from; a version that omits it leaves the deployment + // unnamed in the UI. + assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + // A first version supersedes nothing, so previous_version_id is unset. + assert.Empty(t, f.versions[0].body.PreviousVersionId) + require.NoError(t, r.CompleteVersion(t.Context(), true)) require.Len(t, f.completed, 1) assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) @@ -132,21 +139,6 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) } -func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { - f := &fakeDMS{assignedID: "server-generated-id"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - require.NoError(t, r.CreateVersion(t.Context())) - - require.Len(t, f.versions, 1) - // The service copies display_name onto the deployment's workspace node, which - // is where GetDeployment reads it from; a version that omits it leaves the - // deployment unnamed in the UI. - assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) - // A first version supersedes nothing, so previous_version_id is unset. - assert.Empty(t, f.versions[0].body.PreviousVersionId) -} - func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { From 02c35fd1dc67595af1ace3237cde2493088f743c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 10:56:05 +0000 Subject: [PATCH 49/56] bundle: record DMS operations from the state writes Operations were recorded from bundle_apply, once per resource after Deploy returned, so a resource that writes state more than once in a deploy reported only the end result. Recreate is the case that matters: it drops the state entry, then saves the new resource, and the intermediate step was invisible. SaveState and DeleteState now record, so what DMS holds follows the WAL. Both take the action to report and a context; the sink is installed on the state DB during apply and is nil everywhere else (migration and bind/unbind write state without a DMS version, so they record nothing). Three things fell out of it: The upload queue no longer coalesces. It kept only the newest operation per resource, which is exactly the intermediate write we want to keep, so pending holds a per- resource FIFO and every write is uploaded oldest-first. The operations API is called directly instead of through the generated client. The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints that way) while the SDK types it int64, so reading a CreateOperation response fails with "invalid character '1' after top-level value" - the write succeeds, only the parse does not. The testserver now emits the same string form, so tests exercise the real wire format. A separate fix for the spec/SDK is in flight. A recreate's intermediate delete is not recorded. The service keeps one operation per resource per version, with action_type fixed at creation, and it rejects a succeeded recreate that carries no state ("it leaves a resource that exists"). So the drop cannot be its own event; the save that follows reports the recreate, and the failure path reports it if that save never happens. Verified on dogfood across create, update, recreate and destroy: no recording errors, and the recreate records the new resource id and name. Co-authored-by: Isaac --- .../bundle/dms/partial-update/databricks.yml | 12 ++ .../bundle/dms/partial-update/out.test.toml | 3 + .../bundle/dms/partial-update/output.txt | 167 ++++++++++++++++++ acceptance/bundle/dms/partial-update/script | 15 ++ bundle/direct/apply.go | 18 +- bundle/direct/bind.go | 8 +- bundle/direct/bundle_apply.go | 31 ++-- bundle/direct/dstate/dms.go | 10 ++ bundle/direct/dstate/state.go | 62 ++++++- bundle/direct/dstate/state_test.go | 75 +++++++- bundle/direct/opclient.go | 80 +++++++++ bundle/direct/opqueue.go | 63 +++---- bundle/direct/opqueue_test.go | 105 ++++++----- bundle/direct/oprecorder.go | 111 ++++++++---- bundle/direct/oprecorder_test.go | 120 ++++++++----- bundle/migrate/build_state.go | 4 +- bundle/phases/deploy.go | 13 +- bundle/phases/destroy.go | 9 +- bundle/phases/dms.go | 19 ++ libs/testserver/bundle.go | 123 ++++++++++++- libs/testserver/handlers.go | 3 + 21 files changed, 834 insertions(+), 217 deletions(-) create mode 100644 acceptance/bundle/dms/partial-update/databricks.yml create mode 100644 acceptance/bundle/dms/partial-update/out.test.toml create mode 100644 acceptance/bundle/dms/partial-update/output.txt create mode 100644 acceptance/bundle/dms/partial-update/script create mode 100644 bundle/direct/opclient.go diff --git a/acceptance/bundle/dms/partial-update/databricks.yml b/acceptance/bundle/dms/partial-update/databricks.yml new file mode 100644 index 00000000000..fc61c680fb1 --- /dev/null +++ b/acceptance/bundle/dms/partial-update/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-partial-update + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_partial_update_schema + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt new file mode 100644 index 00000000000..8c36883a9db --- /dev/null +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -0,0 +1,167 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "main.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: other + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script new file mode 100644 index 00000000000..9310ded83aa --- /dev/null +++ b/acceptance/bundle/dms/partial-update/script @@ -0,0 +1,15 @@ +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# Only the save is recorded. The service keeps one operation per resource per version +# and rejects a succeeded recreate that carries no state, so the intermediate drop +# cannot be its own event; the save that follows reports the recreate instead. +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" +trace $CLI bundle deploy --auto-approve +trace print_requests.py //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index cbb0a2d45ff..e8f95daa48a 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.Create) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -116,7 +116,11 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate, not a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate rather + // than deliberately removed. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Recreate) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -158,12 +162,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Update) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Update) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -208,7 +212,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.UpdateWithID) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -250,7 +254,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -291,7 +295,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Resize) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ec910b2734e..4de9c8d736a 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -93,7 +93,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -151,7 +151,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -221,7 +221,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, deployplan.Delete) if err != nil { return err } @@ -235,7 +235,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, deployplan.Delete) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 9fa882867fc..76710211c05 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -37,7 +37,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Operations are recorded with DMS from background workers so a resource's // deploy is not held up by the CreateOperation round trip. The queue is // drained below, once every apply worker has finished recording. + // + // The state DB records through it, so every state write becomes an operation and + // DMS mirrors the WAL. opQueue := newOperationQueue(ctx, b.OpRec) + if opQueue != nil { + // Assigned only when non-nil: a nil *operationQueue in an interface is not a + // nil interface, so the state DB's nil check would not see it. + b.StateDB.SetOperationSink(opQueue) + } g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) @@ -88,13 +96,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - // Read the ID before the delete removes it from state; DMS requires it to - // identify which resource the delete operation refers to. + // Read the ID before the delete removes it from state; recording a failure + // below needs it to say which resource the operation refers to. deletedID := b.StateDB.GetResourceID(resourceKey) if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, action) } else { err = d.Destroy(ctx, &b.StateDB) } @@ -104,11 +112,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } - // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, deletedID, nil, nil); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } return true } @@ -132,6 +135,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a resource that + // writes state more than once (a recreate) reports each step. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { // Both are empty for a create that never got an ID, which is what the @@ -141,15 +147,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } - - // Record the operation with DMS. The resource ID and applied config - // (sv.Value) come from the write just performed; GetResourceID reads - // the ID assigned by Deploy. depends_on is recorded alongside the config - // because it cannot be recomputed from it (see dstate.RecordedState). - if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value, d.DependsOn); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 4ec68dea82d..56647fe471c 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -19,6 +19,16 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// OperationSink records one resource operation with the deployment metadata service. +// SaveState and DeleteState call it for every state write, so what DMS holds mirrors +// the WAL - including the intermediate writes of a recreate. +// +// It does not return an error: the upload happens on a background worker, and the +// deploy learns about a failure when the queue is drained. +type OperationSink interface { + RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) +} + // readDMSState replaces the file-derived resource state with the state recorded in // DMS. Recording is only enabled for net-new deployments, so once a deployment // exists DMS owns its resource set outright - an empty set means a successful deploy diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 06d0bb3dd25..ff1d437300b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -71,6 +71,18 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // sink records each state write with DMS. Nil unless the bundle records + // deployment history, in which case SetOperationSink installs it. + sink OperationSink +} + +// SetOperationSink makes every subsequent state write also record an operation with +// DMS. It is set after the version is created, which is why it is not an Open option. +func (db *DeploymentState) SetOperationSink(sink OperationSink) { + db.mu.Lock() + defer db.mu.Unlock() + db.sink = sink } type Header struct { @@ -119,7 +131,10 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after action was applied to it. action is +// what the deployment metadata service reports for the write; it is ignored when the +// bundle does not record deployment history. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -140,13 +155,27 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) - if err == nil { - db.stateIDs[key] = newID + if err != nil { + return err } - return err + db.stateIDs[key] = newID + + // Recorded after the WAL write, so DMS never reports a state the deploy failed to + // persist locally. + if db.sink != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + db.sink.RecordOperation(ctx, key, action, newID, recorded) + } + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry. action distinguishes a real delete +// from the intermediate drop a recreate performs, both of which are recorded. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -155,11 +184,28 @@ func (db *DeploymentState) DeleteState(key string) error { return nil } + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] + err := appendJSONLine(db.walFile, WALEntry{Key: key}) - if err == nil { - delete(db.stateIDs, key) + if err != nil { + return err } - return err + delete(db.stateIDs, key) + + // State is nil: the resource no longer exists. + // + // A recreate is the exception. It drops the entry and then saves the new + // resource, but the service keeps one operation per resource per version whose + // action_type is fixed at creation, and it rejects a succeeded recreate that + // carries no state ("it leaves a resource that exists"). So the intermediate drop + // cannot be recorded as its own event; the save that follows reports the recreate, + // and if that save never happens the failure path reports it instead. + if db.sink != nil && action != deployplan.Recreate { + db.sink.RecordOperation(ctx, key, action, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a9c90530514..9f0374c0ec4 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -1,11 +1,14 @@ package dstate import ( + "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" + "github.com/databricks/cli/bundle/deployplan" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,13 +19,75 @@ func mustFinalize(t *testing.T, db *DeploymentState) { require.NoError(t, err) } +// fakeSink captures what the state writes report to DMS. +type fakeSink struct { + ops []string +} + +func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { + f.ops = append(f.ops, fmt.Sprintf("%s %s id=%s state=%s", action, resourceKey, resourceID, string(state))) +} + +func TestStateWritesRecordOperations(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + // A recreate: the old entry is dropped, then the new resource is saved. + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) + mustFinalize(t, &db) + + // The recreate's intermediate drop is not reported: the service keeps one + // operation per resource per version and rejects a succeeded recreate carrying no + // state, so the save that follows is what reports it. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{"key":"old"}}`, + `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, + }, sink.ops) +} + +func TestDeleteStateRecordsRealDelete(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) + + // A real delete reports the id it had and no state: the resource is gone. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{}}`, + `delete jobs.my_job id=123 state=`, + }, sink.ops) +} + +func TestStateWritesRecordNothingWithoutSink(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // No sink: recording is off, and the writes still succeed. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) +} + func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open and verify persisted data. @@ -108,7 +173,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var committed DeploymentState @@ -172,12 +237,12 @@ func TestDeleteState(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) mustFinalize(t, &db2) var db3 DeploymentState @@ -205,7 +270,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go new file mode 100644 index 00000000000..009089cdb85 --- /dev/null +++ b/bundle/direct/opclient.go @@ -0,0 +1,80 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The CLI calls the operations API directly rather than through the generated +// client because the SDK cannot read the response: it types sequence_id as an +// int64, while the service sends it as a JSON string (proto3 encodes 64-bit ints +// that way), so unmarshalling a CreateOperation response fails with +// "invalid character '1' after top-level value". The write itself succeeds - the +// status is 200 - so only the response parse is affected. + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update of this operation. + // Typed as a string because that is what the service sends; see above. + SequenceId string `json:"sequence_id,omitempty"` +} + +// updateOperationRequest carries the fields a later write for the same resource +// changes. action_type and resource_key are omitted: the service fixes them when +// the operation is created and ignores them here. +type updateOperationRequest struct { + State *json.RawMessage `json:"state,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ResourceId string `json:"resource_id,omitempty"` + Status bundledeployments.OperationStatus `json:"status,omitempty"` + SequenceId string `json:"sequence_id,omitempty"` +} + +// operationClient records operations under a deployment version. +type operationClient interface { + CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) + UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) +} + +// apiOperationClient talks to the operations API through the workspace client. +type apiOperationClient struct { + client *client.DatabricksClient +} + +// newAPIOperationClient returns an operationClient that posts to the DMS API. +func newAPIOperationClient(c *client.DatabricksClient) operationClient { + return &apiOperationClient{client: c} +} + +func (a *apiOperationClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations", parent) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"resource_key": resourceKey}, + op, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} + +func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) + err := a.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"update_mask": strings.Join(updatableFields, ",")}, + body, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ec5b97ac37d..ea796abf84d 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -41,9 +41,11 @@ type operationQueue struct { // mu guards the fields below. mu sync.Mutex - // pending holds the newest operation per resource key that no worker has taken - // yet. Empty for a key means everything recorded for it has been uploaded. - pending map[string]recordedOperation + // pending holds the operations waiting per resource key, oldest first. Every one + // is uploaded: a resource can write state more than once in a deploy (a recreate + // drops it, then saves the new resource), and each write is its own event, so + // dropping the older one would hide a step. No key means nothing is waiting. + pending map[string][]recordedOperation // queuedOrUploading means "some worker will get to this key". Recording such a // key writes to pending only, so two workers never upload one resource at once. @@ -63,7 +65,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati q := &operationQueue{ uploader: uploader, queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), + pending: make(map[string][]recordedOperation), queuedOrUploading: make(map[string]bool), } @@ -75,23 +77,26 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers, so an error -// here means the payload could not be built; upload errors surface at close. +// RecordOperation implements dstate.OperationSink: every state write becomes an +// operation, so DMS mirrors the WAL. state is already the serialized envelope, and +// nil for a delete. // -// An earlier upload failure does not stop this: every applied resource is still -// recorded, best effort, so DMS ends up as close to reality as it can get. -func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// An earlier upload failure does not stop this: every write is still recorded, best +// effort, so DMS ends up as close to reality as it can get. +func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { if q == nil { - return nil + return } - op, err := newRecordedOperation(action, resourceID, state, dependsOn) + op, err := newStateOperation(action, resourceID, state) if err != nil { - return err + // The deploy already persisted this write locally, so failing it here would + // report an error about history for a resource that deployed fine. + log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) + return } q.enqueue(ctx, resourceKey, op) - return nil } // recordFailure records that applying a resource failed, so the deployment history @@ -111,22 +116,17 @@ func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, q.enqueue(ctx, resourceKey, op) } -// enqueue publishes op as the pending operation for resourceKey and makes sure a +// enqueue appends op to the operations waiting for resourceKey and makes sure a // worker will pick it up. func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() - _, replaced := q.pending[resourceKey] - q.pending[resourceKey] = op + q.pending[resourceKey] = append(q.pending[resourceKey], op) alreadyHandled := q.queuedOrUploading[resourceKey] q.queuedOrUploading[resourceKey] = true q.mu.Unlock() - if replaced { - log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) - } - // A worker will re-read pending before it finishes, so it picks up the operation - // written above. Queueing again would let a second worker upload the same key. + // appended above. Queueing again would let a second worker upload the same key. if alreadyHandled { return } @@ -175,23 +175,26 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey, reporting false and clearing -// the queuedOrUploading mark when nothing is left, which lets record queue it again. -// Both happen under one lock, so a key can never be left for no worker to pick up. +// take claims the oldest operation waiting for resourceKey, reporting false and +// clearing the queuedOrUploading mark when nothing is left, which lets record queue +// it again. Both happen under one lock, so a key can never be left for no worker to +// pick up. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() - op, ok := q.pending[resourceKey] - if !ok { + ops := q.pending[resourceKey] + if len(ops) == 0 { + delete(q.pending, resourceKey) delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } - // The mark stays until the branch above clears it, so anything recorded during - // this upload is still picked up and no second worker takes the key meanwhile. - delete(q.pending, resourceKey) - return op, true + // Oldest first, so the service sees the writes in the order they happened. The + // mark stays until the branch above clears it, so anything recorded during this + // upload is still picked up and no second worker takes the key meanwhile. + q.pending[resourceKey] = ops[1:] + return ops[0], true } // setErr keeps the first upload error; later ones are dropped because one failure diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 7a98dc43774..1048db35d9c 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "errors" "strconv" "strings" @@ -9,6 +10,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -75,9 +77,17 @@ func (f *fakeUploader) resourceIDFor(resourceKey string) string { return f.resourceIDs[resourceKey] } +// envelope builds the serialized RecordedState the state DB hands the queue. +func envelope(t *testing.T, name string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(dstate.RecordedState{State: json.RawMessage(`{"name":"` + name + `"}`)}) + require.NoError(t, err) + return raw +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) + q.RecordOperation(t.Context(), resourceKey, deployplan.Update, "id-1", envelope(t, name)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -92,10 +102,14 @@ func TestOperationQueueUploadsEachOperation(t *testing.T) { assert.Len(t, f.recorded(), 20) } -func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { - // Hold the first upload so later operations for the same resource pile up in - // the queue and are collapsed into one. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} +func TestOperationQueueUploadsEveryWriteForSameResource(t *testing.T) { + // Hold the first upload so the writes behind it queue up. Each one is its own + // event, so all three are uploaded, oldest first - a resource can legitimately + // write state several times in one deploy (see Recreate). + // + // started is buffered for all three: every write now uploads, and a worker + // blocking on an unread send would deadlock the drain below. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 3)} q := newOperationQueue(t.Context(), f) recordState(t, q, "resources.jobs.foo", "v1") @@ -109,17 +123,20 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { close(f.block) require.NoError(t, q.close()) - // Two uploads, not three: v2 was superseded by v3 while both were queued, and - // the last recorded state is the one the service ends up with. assert.Equal(t, []string{ `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } -func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { - // Hold the first upload so the operations below stay queued and coalesce. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} +func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { + // Every worker is parked mid-upload, so the writes below sit in pending rather + // than being picked up. Both still go out, in order, once a worker frees up. + // + // started is buffered for the two foo writes as well: nothing reads it after the + // loop below, and a worker blocking on the send would deadlock the drain. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} q := newOperationQueue(t.Context(), f) recordState(t, q, "resources.jobs.hold", "v1") @@ -131,27 +148,26 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - // A resource whose ID is only known after it was created: the first operation - // has no ID, the second fills it in. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "created"}, nil)) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "updated"}, nil)) + // A resource whose ID is only known after it was created: the first write has no + // ID, the second fills it in. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "created")) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "updated")) close(f.block) require.NoError(t, q.close()) - // One upload, not two: the second operation replaced the first while every - // worker was busy, so the extra CreateOperation round trip never happens. - var uploadsForFoo int + var uploadsForFoo []string for _, u := range f.recorded() { if strings.HasPrefix(u, "resources.jobs.foo=") { - uploadsForFoo++ + uploadsForFoo = append(uploadsForFoo, u) } } - assert.Equal(t, 1, uploadsForFoo, "the two operations should coalesce into one upload") + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"created"}}`, + `resources.jobs.foo={"state":{"name":"updated"}}`, + }, uploadsForFoo) - // Everything comes from the newest operation: it carries the resource's full - // state, and the ID it learned after the create. - assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) + // The ID recorded last is the one the create learned. assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, @@ -165,11 +181,11 @@ func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.started) // The worker has taken the key off the queue and is uploading v1 right now. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v2"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v2")) close(f.block) require.NoError(t, q.close()) @@ -207,11 +223,11 @@ func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { // Wait for the failing upload to finish, so the error is stored before the next // record rather than racing it. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.done) // The next resource is still accepted, even though the first upload failed. - require.NoError(t, q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "v1")) // Both were attempted, and close still reports the failure so the deploy fails. require.ErrorIs(t, q.close(), uploadErr) @@ -232,10 +248,10 @@ func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { // Every worker is parked mid-upload, so these stay queued. for i := range operationUploadWorkers { - require.NoError(t, q.record(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", envelope(t, "v1")) close(f.block) require.ErrorIs(t, q.close(), uploadErr) @@ -245,26 +261,23 @@ func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { assert.Len(t, f.recorded(), operationUploadWorkers+1) } -func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { +func TestOperationQueueRecordDropsUnsupportedAction(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) - // Serialization failures surface at record time, on the resource that caused - // them, rather than from the drain at the end of apply. - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil, nil) - require.Error(t, err) + // The state write already succeeded, so an operation that cannot be described is + // dropped with a warning rather than failing the deploy. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) } -func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { +func TestOperationQueueRecordDropsOversizedState(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) - big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big, nil) - require.ErrorContains(t, err, "exceeds the 65536 byte limit") + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) @@ -328,23 +341,23 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} q := newOperationQueue(ctx, u) - // Collect record errors instead of asserting inside the goroutines: testify - // assertions may only run on the goroutine running the test function. - errs := make(chan error, workers*perWorker) + // The envelopes are built up front: json.Marshal is fine on many goroutines, + // but the helper takes *testing.T, which is not. + states := make([]json.RawMessage, workers) + for w := range workers { + states[w] = envelope(t, strconv.Itoa(w)) + } + var wg sync.WaitGroup for w := range workers { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) + q.RecordOperation(ctx, key, deployplan.Update, "id-1", states[w]) } }) } wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } require.NoError(t, q.close()) require.False(t, u.uneven, "two uploads overlapped for the same resource key") @@ -360,6 +373,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 32494c07325..26afccd8f2f 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -5,9 +5,11 @@ import ( "encoding/json" "fmt" "strings" + "sync" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -43,39 +45,25 @@ type recordedOperation struct { state json.RawMessage } -// newRecordedOperation serializes an applied operation for upload. state is the -// local config after the operation and must be nil for delete operations. It -// errors when the serialized state exceeds maxOperationStateSize. -func newRecordedOperation(action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) (recordedOperation, error) { +// newStateOperation describes a state write for upload. state is the serialized +// RecordedState envelope the state DB just persisted, and nil for a delete, where +// the resource is gone. It errors when the state exceeds maxOperationStateSize. +func newStateOperation(action deployplan.ActionType, resourceID string, state json.RawMessage) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err } - op := recordedOperation{ + if len(state) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) + } + + return recordedOperation{ action: actionType, resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusSucceeded, - } - - // Operation.State carries the serialized state, which DMS serves back as - // resource state. Unset for delete: the resource is gone. - if state != nil { - config, err := json.Marshal(state) - if err != nil { - return recordedOperation{}, fmt.Errorf("serializing state: %w", err) - } - raw, err := json.Marshal(dstate.RecordedState{State: config, DependsOn: dependsOn}) - if err != nil { - return recordedOperation{}, fmt.Errorf("serializing state: %w", err) - } - if len(raw) > maxOperationStateSize { - return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(raw), maxOperationStateSize) - } - op.state = raw - } - - return op, nil + state: state, + }, nil } // newFailedOperation records an operation that did not apply, so the deployment @@ -135,24 +123,45 @@ type operationUploader interface { upload(ctx context.Context, resourceKey string, op recordedOperation) error } -// operationRecorder uploads operations via the DMS CreateOperation API. +// operationRecorder uploads operations via the DMS operations API. type operationRecorder struct { - client bundledeployments.BundleDeploymentsInterface + ops operationClient // parent is the version the operations are recorded under, formatted as // "deployments/{deployment_id}/versions/{version_id}". parent string + + // mu guards sequenceIDs. + mu sync.Mutex + + // sequenceIDs holds the last sequence_id the service returned per resource key, + // which is how a resource already recorded in this version is recognised. The + // service names operations "operations/{resource_key}", so it keeps one per + // resource per version: the second write for a resource has to update that + // operation, and echo this value as the concurrency precondition. + sequenceIDs map[string]string } -// NewOperationRecorder returns an operationUploader backed by the DMS -// CreateOperation API. deploymentID and version identify the deployment version -// assigned by DMS that the operations are recorded under. -func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) operationUploader { +// NewOperationRecorder returns an operationUploader backed by the DMS operations +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID string, version int64) operationUploader { + return newOperationRecorder(newAPIOperationClient(apiClient), deploymentID, version) +} + +// newOperationRecorder is the internal constructor, so tests can supply their own +// operationClient. +func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { return &operationRecorder{ - client: client, - parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + ops: ops, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + sequenceIDs: make(map[string]string), } } +// updatableFields are the operation fields a later write for the same resource can +// change. resource_id is included because a recreate learns a new one. +var updatableFields = []string{"state", "error_message", "resource_id", "status"} + func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on @@ -179,12 +188,36 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r operation.State = &raw } - _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ - Parent: r.parent, - ResourceKey: dmsKey, - Operation: operation, - }) - return err + r.mu.Lock() + sequenceID, recorded := r.sequenceIDs[dmsKey] + r.mu.Unlock() + + var result operationResponse + var err error + if recorded { + // Only the masked fields and sequence_id are read on an update; action_type + // stays as the operation was created, so sending it would just be misleading. + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, updateOperationRequest{ + State: operation.State, + ErrorMessage: operation.ErrorMessage, + ResourceId: operation.ResourceId, + Status: operation.Status, + SequenceId: sequenceID, + }) + } else { + result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) + } + if err != nil { + return err + } + + // Remember the sequence the service assigned, so the next write for this + // resource updates rather than re-creates. + r.mu.Lock() + r.sequenceIDs[dmsKey] = result.SequenceId + r.mu.Unlock() + + return nil } // deployActionToSDK maps a deployplan action to its DMS operation action type. diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index b793ab65a08..d80caa09341 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -14,80 +14,120 @@ import ( "github.com/stretchr/testify/require" ) +// fakeOpCall is one recorded call to the operations API. +type fakeOpCall struct { + method string + parent string + resourceKey string + op bundledeployments.Operation + update updateOperationRequest +} + type fakeOpClient struct { - bundledeployments.BundleDeploymentsInterface + mu sync.Mutex + calls []fakeOpCall + // sequence is what the service reports back; a string, as the service sends it. + sequence string +} - mu sync.Mutex - requests []bundledeployments.CreateOperationRequest +func (f *fakeOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) + return operationResponse{SequenceId: f.sequence}, nil } -func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { +func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { f.mu.Lock() defer f.mu.Unlock() - f.requests = append(f.requests, req) - return &bundledeployments.Operation{}, nil + f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body}) + return operationResponse{SequenceId: f.sequence}, nil } // uploadOne records a single operation through the given uploader, mirroring what // an operationQueue worker does. -func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { t.Helper() - op, err := newRecordedOperation(action, resourceID, state, nil) + op, err := newStateOperation(action, resourceID, state) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } func TestOperationRecorderStripsResourcePrefix(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 2) + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) - uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", envelope(t, "foo")) - require.Len(t, f.requests, 1) - req := f.requests[0] + require.Len(t, f.calls, 1) + c := f.calls[0] // The wire key drops the CLI-internal "resources." prefix, both in the query // param and the operation body. - assert.Equal(t, "jobs.foo", req.ResourceKey) - assert.Equal(t, "jobs.foo", req.Operation.ResourceKey) - assert.Equal(t, "deployments/dep-1/versions/2", req.Parent) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, req.Operation.ActionType) - assert.Equal(t, "job-123", req.Operation.ResourceId) - require.NotNil(t, req.Operation.State) + assert.Equal(t, "create", c.method) + assert.Equal(t, "jobs.foo", c.resourceKey) + assert.Equal(t, "jobs.foo", c.op.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", c.parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, c.op.ActionType) + assert.Equal(t, "job-123", c.op.ResourceId) + require.NotNil(t, c.op.State) } -func TestNewRecordedOperationRecordsStateAsIs(t *testing.T) { - state := struct { - Name string `json:"name"` - Token string `json:"token" bundle:"sensitive"` - }{Name: "foo", Token: "super-secret"} +func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { + // One operation per resource per version: the second write has to update the + // first, echoing the sequence_id the service returned as its precondition. + f := &fakeOpClient{sequence: "7"} + r := newOperationRecorder(f, "dep-1", 2) - op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "", nil) + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "job-456", envelope(t, "new")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) - // The state is serialized as-is, including fields tagged bundle:"sensitive". - assert.JSONEq(t, - `{"state":{"name":"foo","token":"super-secret"}}`, - string(op.state)) + assert.Equal(t, "update", f.calls[1].method) + assert.Equal(t, "jobs.foo", f.calls[1].resourceKey) + assert.Equal(t, "7", f.calls[1].update.SequenceId) + assert.Equal(t, "job-456", f.calls[1].update.ResourceId) + require.NotNil(t, f.calls[1].update.State) } -func TestNewRecordedOperationRecordsDependsOn(t *testing.T) { - // depends_on rides in an envelope alongside the config: it cannot be - // recomputed from the config, whose references are already resolved. - dependsOn := []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}} +func TestOperationRecorderTracksSequencePerResource(t *testing.T) { + // A different resource has its own operation, so its first write creates. + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) - op, err := newRecordedOperation(deployplan.Create, "job-123", map[string]string{"name": "foo"}, dependsOn) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "foo")) + uploadOne(t, r, "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "bar")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, "create", f.calls[1].method) +} + +func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { + // The state DB serializes the envelope (see dstate.SaveState); the operation + // carries it through untouched, sensitive fields and all. + state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) + + op, err := newStateOperation(deployplan.Create, "job-123", state) require.NoError(t, err) - assert.JSONEq(t, - `{"state":{"name":"foo"},"depends_on":[{"node":"resources.jobs.bar","label":"${resources.jobs.bar.id}"}]}`, - string(op.state)) + assert.JSONEq(t, string(state), string(op.state)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, op.status) } -func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newRecordedOperation(deployplan.Skip, "job-123", nil, nil) +func TestNewStateOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newStateOperation(deployplan.Skip, "job-123", nil) assert.Error(t, err) } +func TestNewStateOperationRejectsOversizedState(t *testing.T) { + big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) + + _, err := newStateOperation(deployplan.Create, "job-123", big) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") +} + func TestNewFailedOperationRecordsError(t *testing.T) { op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) require.NoError(t, err) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index c08cf01c31f..459c14a0b6d 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -233,7 +233,9 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + // Migration rebuilds local state from terraform's; nothing is deployed, and + // the DMS sink is never set on this state, so the action is not reported. + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, deployplan.Create); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 5513944cf7a..d0faca36bca 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -17,7 +17,6 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -295,15 +294,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { - if recorder != nil { - // Record operations under the version created before planning, so DMS holds - // the deployed resource state. - b.DeploymentBundle.OpRec = direct.NewOperationRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - recorder.DeploymentID(), - recorder.Version(), - ) - } + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. + setOperationRecorder(ctx, b, recorder) deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) } else { cmdio.LogString(ctx, "Deployment cancelled!") diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index d2e072d23ce..e09d5716f5f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -13,7 +13,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -217,13 +216,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { logdiag.LogError(ctx, err) return } - if recorder != nil { - b.DeploymentBundle.OpRec = direct.NewOperationRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - recorder.DeploymentID(), - recorder.Version(), - ) - } + setOperationRecorder(ctx, b, recorder) destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index cb8f6ad2db4..23da595d4c8 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -9,9 +9,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -56,6 +58,23 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// setOperationRecorder points the deployment at the version the recorder claimed, so +// the state writes during apply are recorded under it. A nil recorder means recording +// is off and leaves the deployment's uploader unset. +func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil { + return + } + + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) +} + // logDeploymentHistory links to the deployment this deploy was recorded under, so // the user can open its history without hunting for the ID. A nil recorder means // recording is off, and a zero version means the version was never created. diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 74a8d2156b3..574570ce194 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -29,6 +29,10 @@ type dmsDeployment struct { // resources is the latest resource state per resource key, updated as // operations are recorded. resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + operations map[string]*bundledeployments.Operation // lastSuccessfulVersionID is the highest version that completed // successfully. The server advances last_successful_version_id only on // success (unlike last_version_id), and the read path treats a non-empty @@ -80,6 +84,7 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { deployment: dep, versions: map[string]*bundledeployments.Version{}, resources: map[string]bundledeployments.Resource{}, + operations: map[string]*bundledeployments.Operation{}, } return Response{Body: dep} } @@ -259,8 +264,29 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } - op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + // The service names operations after the resource key, so it keeps one per + // resource per version: creating a second one for the same resource conflicts, + // and the caller has to use UpdateOperation instead. + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + if _, exists := d.operations[opName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": "operation for " + resourceKey + " already exists in this version"}, + } + } + + op.Name = opName op.ResourceKey = resourceKey + op.SequenceId = 1 + d.operations[opName] = &op + + // The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints + // that way) while the SDK struct types it as an int64, so the response is built + // by hand to match the wire format the CLI actually parses. + body, err := operationBody(&op) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } // Reflect the operation onto the deployment-level resource set the way the // backend does: a delete removes the resource, anything else upserts it. @@ -283,7 +309,100 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str State: op.State, } } - return Response{Body: op} + return Response{Body: body} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields and once for the + // precondition. + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // Only the mutable fields change; action_type and resource_key stay as created. + existing.State = op.State + existing.ErrorMessage = op.ErrorMessage + existing.ResourceId = op.ResourceId + existing.Status = op.Status + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Mirror onto the resource set the same way CreateOperation does, so the read + // path reflects the newest write. + if existing.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + + return Response{Body: body} } func (s *FakeWorkspace) ListResources(deploymentID string) Response { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a05f79986ec..1c9f7b50bb8 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -305,6 +305,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { return req.Workspace.ListResources(req.Vars["deployment_id"]) }) From 3f1c296be9c64b637d36cb48fba460bc820de01d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 11:31:48 +0000 Subject: [PATCH 50/56] bundle: add DATABRICKS_BUNDLE_DMS to enable deployment history recording Recording could only be turned on by setting experimental.record_deployment_history in the bundle, which is impractical for running the acceptance suite with DMS enabled: that would mean editing 650-odd databricks.yml files. DATABRICKS_BUNDLE_DMS turns it on for a whole run instead. The three places that branch on recording now go through env.RecordsDeploymentHistory, so the setting and the variable cannot drift apart. The validation that rejects the feature for users is deliberately left alone: it gates the yaml field, which the variable does not set. Verified on dogfood: a bundle with no experimental block records its deployment with DATABRICKS_BUNDLE_DMS=true alone. This is the groundwork for running the bundle suite with recording on. That run is not enabled yet - see the report on the state-file limitation. Co-authored-by: Isaac --- .../mutator/initialize_deployment_history.go | 4 +- bundle/env/dms.go | 25 ++++++++++++ bundle/env/dms_test.go | 38 +++++++++++++++++++ bundle/phases/dms.go | 16 ++++++-- cmd/bundle/utils/process.go | 3 +- 5 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 bundle/env/dms.go create mode 100644 bundle/env/dms_test.go diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 99aad70c7cc..b90800e4e70 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -28,7 +29,8 @@ func (m *initializeDeploymentHistory) Name() string { } func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + if !env.RecordsDeploymentHistory(ctx, configured) { return nil } diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..53aeaccc011 --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,25 @@ +package env + +import "context" + +// DMSVariable names the environment variable that turns on deployment history +// recording without setting experimental.record_deployment_history in the bundle. +// It exists for the CLI's own acceptance tests, which run the whole bundle suite +// with DMS enabled: setting it here beats adding the field to every databricks.yml. +// +// Like ForceAllowRecordDeploymentHistoryVariable it is deliberately undocumented; see +// validate.ValidateRecordDeploymentHistory for why the feature is still gated off. +const DMSVariable = "DATABRICKS_BUNDLE_DMS" + +// DMS reports whether the environment turns on deployment history recording. +func DMS(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSVariable}) + return ok && value != "" && value != "0" && value != "false" +} + +// RecordsDeploymentHistory reports whether this deploy records deployment history, +// from either the bundle setting or DMSVariable. It is the single predicate the +// recording code paths branch on, so the env var and the config field cannot drift. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || DMS(ctx) +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..f449d5ec6b9 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,38 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestDMS(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"1", true}, + {"", false}, + {"0", false}, + {"false", false}, + } { + ctx := env.Set(t.Context(), DMSVariable, tc.value) + assert.Equal(t, tc.want, DMS(ctx), "value %q", tc.value) + } +} + +func TestDMSUnset(t *testing.T) { + assert.False(t, DMS(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), DMSVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 23da595d4c8..8ca73fac94e 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" @@ -23,16 +24,16 @@ import ( // nil when DMS recording does not apply. A nil recorder is a no-op, so callers // do not need to branch on it. // -// Recording is enabled only when experimental.record_deployment_history is set -// AND the engine is direct: DMS resource state is tracked per direct-engine -// deployment. Returning nil for terraform leaves those deployments untouched. +// Recording is enabled only when the bundle asks for it (see +// recordsDeploymentHistory) AND the engine is direct: DMS resource state is tracked +// per direct-engine deployment. Returning nil for terraform leaves those untouched. // // The deployment ID is resolved from the workspace, not local state (see // dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is // held, so it sees any deployment a concurrent deploy created. It is empty on the // first recorded deploy, where the recorder creates the deployment instead. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + if !recordsDeploymentHistory(ctx, b) { return nil, nil } if !eng.IsDirect() { @@ -58,6 +59,13 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// recordsDeploymentHistory reports whether this bundle records deployment history, +// from experimental.record_deployment_history or DATABRICKS_BUNDLE_DMS. +func recordsDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // setOperationRecorder points the deployment at the version the recorder claimed, so // the state writes during apply are recorded under it. A nil recorder means recording // is off and leaves the deployment's uploader unset. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 3b90aef5c6a..36205181115 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" @@ -225,7 +226,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // comes from the file. Reads open the state write-disabled, so no lineage // is minted here. var dmsSource *dstate.DMSSource - if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { + if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { w := b.WorkspaceClient(ctx) deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) if err != nil { From 0ddb8053da708f8ac73fc1cc84826328faa8dd13 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 13:14:15 +0000 Subject: [PATCH 51/56] bundle: skip a recorded resource that was never created, and prepare the DMS test run Two fixes and the groundwork for running the bundle suite with recording on. A failed create is recorded with its error and nothing else, so DMS reports the resource with no id. The read path turned that into a state entry with an empty id, which looks tracked but refers to nothing: a later destroy then failed with "cannot plan resources.jobs.foo: internal error, missing in state" and the resource could not be removed at all. Such a resource is now left out of the state, so the next deploy creates it and a destroy skips it. DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES records a bundle whose state file already tracks resources, which is otherwise refused. It exists for the acceptance suite: most tests there seed a state fixture, and they assert the output of a deploy rather than reading state back, so the duplication the refusal prevents cannot bite them. bundle/dms/existing-state turns it back off, since that test asserts the refusal. The tests that assert a job or pipeline request now drop deployment_id and version_id from it, so they pass with recording on and off from one set of golden files. Two of them switched to print_requests.py --del-body, which grew dotted-path support for the purpose - dropping the whole deployment block would also stop asserting kind and metadata_file_path. acceptance/bundle/resources normalizes the same two fields out of plan JSON and state dumps, where they appear too deep for a per-request filter. Measured on bundle/resources/jobs: 9 tests failed with recording on before this, 0 do now, and the two that still fail (big_id, update) fail the same way on main. The matrix entry that turns the second run on is not added yet: the conversion covers one subtree of about thirty. Co-authored-by: Isaac --- acceptance/bin/print_requests.py | 13 +++++++- .../bundle/dms/depends-on/out.test.toml | 1 + .../bundle/dms/existing-state/out.test.toml | 1 + .../dms/multiple-resources/out.test.toml | 1 + acceptance/bundle/dms/no-drift/out.test.toml | 1 + .../bundle/dms/no-resources/out.test.toml | 1 + .../bundle/dms/not-supported/out.test.toml | 1 + .../dms/operation-upload-fails/out.test.toml | 1 + .../bundle/dms/partial-update/out.test.toml | 1 + .../bundle/dms/provenance/out.test.toml | 1 + .../bundle/dms/record-failure/out.test.toml | 1 + acceptance/bundle/dms/record/out.test.toml | 1 + .../dms/redeploy-after-destroy/out.test.toml | 1 + acceptance/bundle/dms/summary/out.test.toml | 1 + acceptance/bundle/dms/test.toml | 9 +++++ .../dms/version-never-created/out.test.toml | 1 + .../bundle/resources/jobs/delete_job/script | 2 +- .../resources/jobs/num_workers/output.txt | 2 +- .../bundle/resources/jobs/num_workers/script | 2 +- .../jobs/remote_matches_config/output.txt | 2 +- .../jobs/remote_matches_config/script | 2 +- .../resources/jobs/task-source/output.txt | 8 ++--- .../bundle/resources/jobs/task-source/script | 8 ++--- .../jobs/update_single_node/output.txt | 8 ++--- .../resources/jobs/update_single_node/script | 6 ++-- .../jobs/webhook-reorder-remote/output.txt | 2 +- .../jobs/webhook-reorder-remote/script | 2 +- acceptance/bundle/resources/test.toml | 33 +++++++++++++++++++ acceptance/bundle/test.toml | 4 +-- bundle/direct/dstate/dms.go | 9 +++++ bundle/direct/dstate/dms_test.go | 17 ++++++++++ bundle/direct/dstate/state.go | 13 +++++++- bundle/env/dms.go | 12 +++++++ cmd/bundle/utils/process.go | 5 +-- 34 files changed, 145 insertions(+), 28 deletions(-) diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index 0e8a74c0759..dbc7d5a6d90 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -178,6 +178,17 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal return filtered_requests +def del_path(body, field): + """Delete field from body. A dotted field descends into nested objects, e.g. + deployment.version_id removes only that key from the deployment block.""" + *parents, leaf = field.split(".") + for name in parents: + body = body.get(name) + if not isinstance(body, dict): + return + body.pop(leaf, None) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("path_filters", nargs="*", help="Path substring filters") @@ -238,7 +249,7 @@ def main(): body = req.get("body") if isinstance(body, dict): for field in del_body_fields: - body.pop(field, None) + del_path(body, field) for field in del_fields: req.pop(field, None) if args.verbose: diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/depends-on/out.test.toml +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/existing-state/out.test.toml +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/multiple-resources/out.test.toml +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/no-drift/out.test.toml +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/no-resources/out.test.toml +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/not-supported/out.test.toml +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/operation-upload-fails/out.test.toml +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/partial-update/out.test.toml +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/provenance/out.test.toml +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/record-failure/out.test.toml +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/record/out.test.toml +++ b/acceptance/bundle/dms/record/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/summary/out.test.toml +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 7c21473f724..e491f968d76 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -5,6 +5,10 @@ Cloud = false # engine; it is a no-op on terraform. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# These tests enable recording through experimental.record_deployment_history, so the +# DATABRICKS_BUNDLE_DMS variant the rest of the suite adds would just duplicate them. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + RecordRequests = true Ignore = [ @@ -16,3 +20,8 @@ Ignore = [ # so they force allow it the same way DMS development does. bundle/dms/not-supported # covers the rejection. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" + +# The parent lets the rest of the suite record a bundle whose state already tracks +# resources, since most of those tests seed a state fixture. bundle/dms/existing-state +# asserts that refusal, so it has to stay on here. +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "" diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/version-never-created/out.test.toml +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 022d90a82b7..c9242b9e209 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -2,4 +2,4 @@ trace $CLI bundle deploy cp empty.yml databricks.yml $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 16e4d600cd2..f61ab034296 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 8e430e43063..83d9321bcc4 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index a067a141773..9ce9dba13c1 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 44a0eb849fe..972181ba497 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -18,4 +18,4 @@ $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/task-source/output.txt b/acceptance/bundle/resources/jobs/task-source/output.txt index 1815edbc0eb..2409cb86a2a 100644 --- a/acceptance/bundle/resources/jobs/task-source/output.txt +++ b/acceptance/bundle/resources/jobs/task-source/output.txt @@ -9,9 +9,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt >>> [CLI] bundle plan update jobs.git_job @@ -24,6 +24,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt diff --git a/acceptance/bundle/resources/jobs/task-source/script b/acceptance/bundle/resources/jobs/task-source/script index 838aac092c3..e8908986c50 100644 --- a/acceptance/bundle/resources/jobs/task-source/script +++ b/acceptance/bundle/resources/jobs/task-source/script @@ -2,8 +2,8 @@ trace $CLI bundle deploy # For terraform we expect the task source to be explicitly set always # For direct we do not expect it to be set unless explicitly specified in the bundle config -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt # Removing the git_source block and deploying again @@ -13,7 +13,7 @@ trace $CLI bundle deploy # In direct mode update should not contain source unless explicitly specified in the bundle config # In terraform mode update should always contain source field -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index aca783207f3..aba6e239b86 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [FOO_ID], + "job_id": [NUMID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 9e84f677e73..55ce937b978 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -4,14 +4,14 @@ $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt +trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id trace $CLI bundle plan @@ -24,7 +24,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index e8c0f8a022a..31a759d0c51 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index ebbcb3409b3..fda553c3cea 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -16,4 +16,4 @@ EOF trace $CLI bundle plan $CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 159efe02696..a8d852b9821 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1 +1,34 @@ RecordRequests = true + +# Recording adds two things to a deploy's output, and both are normalized away so the +# DMS run asserts the same goldens as the engine runs. That is the point of the run: +# every test then checks that recording does not change what a deploy does, rather than +# needing a second copy of 600-odd output files. They live here rather than in the parent so +# bundle/dms, which asserts the recording itself, does not inherit them. +# +# The link printed after a deploy: +[[Repls]] +Old = '(?m)^Deployment history: .*\n' +New = '' + +# And the stamp on jobs and pipelines, which the plan reports as a change of its own. +# Matched with the trailing comma and without, since it can be the only entry - in which +# case the whole "changes" object exists only because of recording, and goes too. +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\}\n *\},?\n' +New = '' + +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\},\n' +New = '' + +# The stamp itself, where it appears inside a serialized deployment block (plan JSON, +# state dumps). Both orderings are covered: the pair can sit before or after the fields +# that stay, so the comma may be on this line or the one before. +[[Repls]] +Old = '(?m)^( *)"(deployment_id|version_id)": "[^"]*",\n' +New = '' + +[[Repls]] +Old = ',(\n *"(deployment_id|version_id)": "[^"]*")+' +New = '' diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index fca8259a3b5..c9acc1e0635 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -1,11 +1,11 @@ # This allows recording per-deployment output files, e.g. $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" +Ignore = ["databricks.yml"] + # The lowest Python version we support. Alternative to "uv run --python 3.10" Env.UV_PYTHON = "3.10" -Ignore = ["databricks.yml"] - # User-agent: [[Repls]] Old = 'os/darwin' diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 56647fe471c..a96e864c761 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -80,6 +80,15 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } } + // A resource with no id was never created: the deploy that recorded it failed + // before the API assigned one (a failed create is recorded with the error and + // nothing else). Leaving it out keeps it untracked, so the next deploy creates + // it and a destroy skips it - an entry with an empty id would instead look + // tracked and fail the delete with "missing in state". + if res.ResourceId == "" { + continue + } + out[key] = ResourceEntry{ ID: res.ResourceId, State: recorded.State, diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 8145424ca93..c475942f829 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -61,6 +61,23 @@ func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { }, got) } +func TestFetchDeploymentResourcesSkipsResourceWithoutID(t *testing.T) { + // A failed create is recorded with its error and nothing else, so the resource has + // no id. Keeping it would make the resource look tracked while referring to nothing, + // and a later destroy fails with "missing in state" instead of skipping it. + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.created", ResourceId: "123"}, + {ResourceKey: "jobs.failed"}, + }} + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") + require.NoError(t, err) + + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.created": {ID: "123"}, + }, got) +} + func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ff1d437300b..91d0aae453b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -269,6 +269,17 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string + + // AllowExistingResources records a bundle whose state file already tracks + // resources, instead of refusing it. Those resources are not handed over to DMS: + // the first recorded deploy reports only what it touches, so the ones it does not + // touch are absent from DMS and a later deploy plans them as creates. + // + // It exists for the CLI's own acceptance tests, which run the whole bundle suite + // with recording on. Most of those tests seed a state fixture, and they assert the + // output of a single deploy rather than reading state back, so the duplication the + // refusal prevents cannot bite them. + AllowExistingResources bool } // Open reads the deployment state from disk (and recovers the WAL when @@ -333,7 +344,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // featureStateVersion with a feature flag plus a tombstone per resource so an // older CLI refuses the state instead of deploying against resources it // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 && !dmsSource.AllowExistingResources { // The remedy is ordered deliberately: this error also blocks destroy, so the // setting has to come out first or there is no way to tear the bundle down. return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 53aeaccc011..812d492d4bc 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -23,3 +23,15 @@ func DMS(ctx context.Context) bool { func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { return configured || DMS(ctx) } + +// DMSAllowExistingResourcesVariable names the environment variable that lets a bundle +// with resources already in its state file be recorded, which is otherwise refused +// (see dstate.DMSSource.AllowExistingResources for what that costs). +const DMSAllowExistingResourcesVariable = "DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES" + +// DMSAllowExistingResources reports whether the environment allows recording a bundle +// that already tracks resources. +func DMSAllowExistingResources(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSAllowExistingResourcesVariable}) + return ok && value != "" && value != "0" && value != "false" +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 36205181115..f2b1750205e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -234,8 +234,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } dmsSource = &dstate.DMSSource{ - Client: w.BundleDeployments, - DeploymentID: deploymentID, + Client: w.BundleDeployments, + DeploymentID: deploymentID, + AllowExistingResources: env.DMSAllowExistingResources(ctx), } // Stamp the deployment onto the resources before anything diffs them. From 42d3cb40fa5a41f9a6c13beb2efae1168efdb5ce Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 17:00:07 +0000 Subject: [PATCH 52/56] bundle: run the acceptance suite with deployment history recording on DATABRICKS_BUNDLE_DMS adds a second run of the bundle suite with recording enabled, so the deployment metadata service is exercised by every test rather than only the handful under bundle/dms. It is direct-engine and local only: DMS is not deployed to the cloud test environments, and terraform does not record at all. Recording changes a deploy's observable output in three places, and each is dropped at the assertion rather than by keeping a second copy of the golden files - that way every test also checks that recording does not change what a deploy does: - the deployment link printed after a deploy, dropped in bundle/test.toml. The URL is covered by workspaceurls.TestDeploymentURL and the calls behind it by bundle/dms. - deployment_id and version_id on a job or pipeline request, dropped with print_requests.py --del-body. The stamp also appears under new_settings for a jobs/reset, so both paths are listed. - the same two fields in a plan dump or a jobs/get response, dropped with jq in the test that writes them. print_requests.py --del-body now takes a dotted path, so a test drops only those two fields instead of the whole deployment block, which would also stop asserting kind and metadata_file_path. Two things are skipped for now, each with the reason in its test.toml: - bundle/dms pins recording off, since those tests turn it on through experimental.record_deployment_history and would otherwise run twice. - the four saved-plan tests (big_id, update, delete_task, remote_delete/deploy). `deploy --plan` applies the state the plan was saved with, and the plan is written before the deployment version exists, so the stamp never reaches the applied resource and the next plan reports it as a change. That needs the stamp written into the saved plan; the exclusion notes it. bundle/resources/jobs passes both runs. Note the suite needs jq 1.7: jq 1.6 rounds a 19-digit job id to 16 significant digits, which silently weakens an id assertion in update_single_node (acceptance/acceptance_test.go already requires 1.7). Co-authored-by: Isaac --- .../empty_code_source/out.test.toml | 1 + .../local_code_source/out.test.toml | 1 + acceptance/bundle/apps/app_yaml/out.test.toml | 1 + .../artifact_and_app_same_path/out.test.toml | 1 + .../bundle/apps/compute_size/out.test.toml | 1 + .../bundle/apps/delete_deleting/out.test.toml | 1 + .../bundle/apps/git_source/out.test.toml | 1 + .../bundle/apps/job_permissions/out.test.toml | 1 + .../job_permissions_warning/out.test.toml | 1 + .../apps/value_from_warning/out.test.toml | 1 + .../ai_runtime_code_source/out.test.toml | 1 + .../volume_doesnot_exist/out.test.toml | 1 + .../volume_not_deployed/out.test.toml | 1 + .../artifact_upload_for_volumes/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../artifacts_dynamic_version/out.test.toml | 1 + .../artifacts/build_and_files/out.test.toml | 1 + .../build_and_files_whl/out.test.toml | 1 + .../artifacts/glob_exact_whl/out.test.toml | 1 + .../artifacts/globs_in_files/out.test.toml | 1 + .../globs_in_files_in_include/out.test.toml | 1 + .../artifacts/globs_invalid/out.test.toml | 1 + .../bundle/artifacts/issue_3109/out.test.toml | 1 + .../artifacts/nil_artifacts/out.test.toml | 1 + .../same_name_libraries/out.test.toml | 1 + .../bundle/artifacts/shell/bash/out.test.toml | 1 + .../artifacts/shell/basic/out.test.toml | 1 + .../bundle/artifacts/shell/cmd/out.test.toml | 1 + .../artifacts/shell/default/out.test.toml | 1 + .../artifacts/shell/err-bash/out.test.toml | 1 + .../artifacts/shell/err-sh/out.test.toml | 1 + .../artifacts/shell/invalid/out.test.toml | 1 + .../bundle/artifacts/shell/sh/out.test.toml | 1 + .../unique_name_libraries/out.test.toml | 1 + .../upload_multiple_libraries/out.test.toml | 1 + .../whl_change_version/out.test.toml | 1 + .../bundle/artifacts/whl_dbfs/out.test.toml | 1 + .../artifacts/whl_dynamic/out.test.toml | 1 + .../artifacts/whl_explicit/out.test.toml | 1 + .../artifacts/whl_implicit/out.test.toml | 1 + .../whl_implicit_custom_path/out.test.toml | 1 + .../whl_implicit_notebook/out.test.toml | 1 + .../artifacts/whl_multiple/out.test.toml | 1 + .../artifacts/whl_no_cleanup/out.test.toml | 1 + .../whl_prebuilt_multiple/out.test.toml | 1 + .../whl_prebuilt_outside/out.test.toml | 1 + .../out.test.toml | 1 + .../whl_via_environment_key/out.test.toml | 1 + .../bundle/benchmarks/deploy/out.test.toml | 1 + .../bundle/benchmarks/plan/out.test.toml | 1 + .../bundle/benchmarks/validate/out.test.toml | 1 + acceptance/bundle/bundle_tag/id/out.test.toml | 1 + .../bundle/bundle_tag/url/out.test.toml | 1 + .../bundle/bundle_tag/url_ref/out.test.toml | 1 + .../cli_defaults/out.test.toml | 1 + .../config_edits/out.test.toml | 1 + .../dashboard_etag/out.test.toml | 1 + .../flushed_cache/out.test.toml | 1 + .../formatting_preserved/out.test.toml | 1 + .../job_fields/out.test.toml | 1 + .../job_multiple_tasks/out.test.toml | 1 + .../job_params_variables/out.test.toml | 1 + .../job_pipeline_task/out.test.toml | 1 + .../multiple_files/out.test.toml | 1 + .../multiple_resources/out.test.toml | 1 + .../output_json/out.test.toml | 1 + .../output_no_changes/out.test.toml | 1 + .../pipeline_fields/out.test.toml | 1 + .../out.test.toml | 1 + .../resolve_variables/out.test.toml | 1 + .../select_basic/out.test.toml | 1 + .../select_multiple/out.test.toml | 1 + .../skip_permissions/out.test.toml | 1 + .../cli_default_split_element/out.test.toml | 1 + .../split/dotted_target/out.test.toml | 1 + .../split/isolation/out.test.toml | 1 + .../split/keyed_edit/out.test.toml | 1 + .../split/keyed_remove/out.test.toml | 1 + .../split/keyed_rename/out.test.toml | 1 + .../split/keyed_twoblock/out.test.toml | 1 + .../split/multifile/out.test.toml | 1 + .../nested_add_split_parent/out.test.toml | 1 + .../split/nested_sequence/out.test.toml | 1 + .../split/positional/out.test.toml | 1 + .../remove_field_both_blocks/out.test.toml | 1 + .../remove_with_unrelated_add/out.test.toml | 1 + .../rename_ambiguous_pairing/out.test.toml | 1 + .../out.test.toml | 1 + .../rename_two_removes_one_add/out.test.toml | 1 + .../split/target_variable/out.test.toml | 1 + .../split/variable_file_order/out.test.toml | 1 + .../target_override/out.test.toml | 1 + .../task_rename_revert/out.test.toml | 1 + .../validation_errors/out.test.toml | 1 + .../bundle/debug/list-targets/out.test.toml | 1 + acceptance/bundle/debug/out.test.toml | 1 + .../bundle/deploy/empty-bundle/out.test.toml | 1 + .../deploy/experimental-python/out.test.toml | 1 + .../deploy/fail-on-active-runs/out.test.toml | 1 + .../files/no-snapshot-sync/out.test.toml | 1 + .../files/out-of-band-delete/out.test.toml | 1 + .../deploy/force-lock-config/out.test.toml | 1 + .../immutable-no-artifacts/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/deploy/immutable/out.test.toml | 1 + .../bundle/deploy/mlops-stacks/out.test.toml | 1 + .../deploy/pipeline-config-dots/out.test.toml | 1 + .../deploy/python-notebook/out.test.toml | 1 + .../deploy/readplan/basic/out.test.toml | 1 + .../cli-version-mismatch/out.test.toml | 1 + .../grants-remove-principal/out.test.toml | 1 + .../readplan/invalid-plan/out.test.toml | 1 + .../readplan/lineage-mismatch/out.test.toml | 1 + .../readplan/plan-not-found/out.test.toml | 1 + .../plan-version-mismatch/out.test.toml | 1 + .../readplan/postgres_role/out.test.toml | 1 + .../readplan/serial-mismatch/out.test.toml | 1 + .../readplan/terraform-error/out.test.toml | 1 + .../readplan/unknown-field/out.test.toml | 1 + .../deploy/snapshot-comparison/out.test.toml | 1 + .../deploy/spark-jar-task/out.test.toml | 1 + .../deploy/wal/chain-3-jobs/out.test.toml | 1 + .../wal/corrupted-wal-entry/out.test.toml | 1 + .../wal/crash-after-create/out.test.toml | 1 + .../bundle/deploy/wal/empty-wal/out.test.toml | 1 + .../wal/failed-plan-no-wal/out.test.toml | 1 + .../wal/future-serial-wal/out.test.toml | 1 + .../deploy/wal/header-only-wal/out.test.toml | 1 + .../deploy/wal/lineage-mismatch/out.test.toml | 1 + .../bundle/deploy/wal/stale-wal/out.test.toml | 1 + .../deploy/wal/wal-with-delete/out.test.toml | 1 + .../yaml-sync-empty-grants/out.test.toml | 1 + .../deployment/bind/alert/out.test.toml | 1 + .../deployment/bind/catalog/out.test.toml | 1 + .../deployment/bind/cluster/out.test.toml | 1 + .../deployment/bind/dashboard/out.test.toml | 1 + .../bind/dashboard/recreation/out.test.toml | 1 + .../bind/database_instance/out.test.toml | 1 + .../deployment/bind/experiment/out.test.toml | 1 + .../bind/external_location/out.test.toml | 1 + .../deployment/bind/genie_space/out.test.toml | 1 + .../already-managed-different/out.test.toml | 1 + .../job/already-managed-same/out.test.toml | 1 + .../bind/job/engine-from-config/out.test.toml | 1 + .../bind/job/generate-and-bind/out.test.toml | 1 + .../bind/job/job-abort-bind/out.test.toml | 1 + .../job/job-spark-python-task/out.test.toml | 1 + .../bind/job/noop-job/out.test.toml | 1 + .../bind/job/python-job/out.test.toml | 1 + .../bind/job/stale-state/out.test.toml | 1 + .../bind/model-serving-endpoint/out.test.toml | 1 + .../bind/pipelines/recreate/out.test.toml | 1 + .../bind/pipelines/update/out.test.toml | 1 + .../bind/postgres_database/out.test.toml | 1 + .../bind/postgres_role/out.test.toml | 1 + .../bind/quality-monitor/out.test.toml | 1 + .../bind/registered-model/out.test.toml | 1 + .../deployment/bind/schema/out.test.toml | 1 + .../bind/secret-scope/out.test.toml | 1 + .../bind/sql_warehouse/out.test.toml | 1 + .../bind/vector_search_endpoint/out.test.toml | 1 + .../bind/vector_search_index/out.test.toml | 1 + .../deployment/bind/volume/out.test.toml | 1 + .../unbind/engine-from-config/out.test.toml | 1 + .../deployment/unbind/grants/out.test.toml | 1 + .../deployment/unbind/job/out.test.toml | 1 + .../unbind/permissions/out.test.toml | 1 + .../unbind/python-job/out.test.toml | 1 + .../destroy/all-resources/out.test.toml | 1 + .../force-lock-node-limit/out.test.toml | 1 + .../destroy/jobs-and-pipeline/out.test.toml | 1 + acceptance/bundle/dms/depends-on/output.txt | 1 - .../bundle/dms/existing-state/output.txt | 1 - .../bundle/dms/multiple-resources/output.txt | 2 - acceptance/bundle/dms/no-drift/output.txt | 2 - acceptance/bundle/dms/no-resources/output.txt | 2 - .../bundle/dms/partial-update/output.txt | 2 - acceptance/bundle/dms/provenance/output.txt | 1 - acceptance/bundle/dms/record/output.txt | 2 - .../dms/redeploy-after-destroy/output.txt | 2 - acceptance/bundle/dms/summary/output.txt | 2 - .../bundle/empty_string_dropped/out.test.toml | 1 + .../empty_string_variable/out.test.toml | 1 + .../environments/dependencies/out.test.toml | 1 + .../skip_name_prefix_for_schema/out.test.toml | 1 + .../bundle/generate/alert/out.test.toml | 1 + .../alert_existing_id_not_found/out.test.toml | 1 + .../app_not_yet_deployed/out.test.toml | 1 + .../generate/app_subfolders/out.test.toml | 1 + .../bundle/generate/auto-bind/out.test.toml | 1 + .../generate/dashboard-inplace/out.test.toml | 1 + .../bundle/generate/dashboard/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../generate/designer_job/out.test.toml | 1 + .../bundle/generate/genie_space/out.test.toml | 1 + .../out.test.toml | 1 + .../genie_space_inplace/out.test.toml | 1 + .../bundle/generate/git_job/out.test.toml | 1 + .../generate/include_warning/out.test.toml | 1 + .../bundle/generate/ipynb_job/out.test.toml | 1 + .../job_nested_notebooks/out.test.toml | 1 + .../generate/lakeflow_pipelines/out.test.toml | 1 + .../bundle/generate/pipeline/out.test.toml | 1 + .../pipeline_and_deploy/out.test.toml | 1 + .../generate/pipeline_with_glob/out.test.toml | 1 + .../generate/pipeline_with_sql/out.test.toml | 1 + .../bundle/generate/python_job/out.test.toml | 1 + .../python_job_and_deploy/out.test.toml | 1 + .../spark_python_task_job/out.test.toml | 1 + acceptance/bundle/git-permerror/out.test.toml | 1 + .../bundle/help/bundle-deploy/out.test.toml | 1 + .../bundle-deployment-migrate/out.test.toml | 1 + .../help/bundle-deployment/out.test.toml | 1 + .../bundle/help/bundle-destroy/out.test.toml | 1 + .../bundle-generate-dashboard/out.test.toml | 1 + .../help/bundle-generate-job/out.test.toml | 1 + .../bundle-generate-pipeline/out.test.toml | 1 + .../bundle/help/bundle-generate/out.test.toml | 1 + .../bundle/help/bundle-init/out.test.toml | 1 + .../bundle/help/bundle-open/out.test.toml | 1 + .../bundle/help/bundle-run/out.test.toml | 1 + .../bundle/help/bundle-schema/out.test.toml | 1 + .../bundle/help/bundle-summary/out.test.toml | 1 + .../bundle/help/bundle-sync/out.test.toml | 1 + .../bundle/help/bundle-validate/out.test.toml | 1 + acceptance/bundle/help/bundle/out.test.toml | 1 + .../includes/glob_in_root_path/out.test.toml | 1 + .../include_outside_root/out.test.toml | 1 + .../non_yaml_in_include/out.test.toml | 1 + .../includes/yml_outside_root/out.test.toml | 1 + .../bundle/integration_whl/base/out.test.toml | 1 + .../custom_params/out.test.toml | 1 + .../interactive_cluster/out.test.toml | 1 + .../out.test.toml | 1 + .../interactive_single_user/out.test.toml | 1 + .../integration_whl/serverless/out.test.toml | 1 + .../serverless_custom_params/out.test.toml | 1 + .../serverless_dynamic_version/out.test.toml | 1 + .../integration_whl/wrapper/out.test.toml | 1 + .../wrapper_custom_params/out.test.toml | 1 + .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + .../bundle/libraries/maven/out.test.toml | 1 + .../outside_of_bundle_root/out.test.toml | 1 + .../bundle/libraries/pypi/out.test.toml | 1 + .../lifecycle/prevent-destroy/out.test.toml | 1 + .../started-validation/out.test.toml | 1 + .../bundle/lifecycle/started/out.test.toml | 1 + .../local_state_staleness/out.test.toml | 1 + acceptance/bundle/migrate/added/out.test.toml | 1 + .../migrate/auto-migrate-clean/out.test.toml | 1 + .../auto-migrate-empty-tfstate/out.test.toml | 1 + .../migrate/auto-migrate-envvar/out.test.toml | 1 + .../auto-migrate-push-failure/out.test.toml | 1 + .../out.test.toml | 1 + acceptance/bundle/migrate/basic/out.test.toml | 1 + .../bundle/migrate/dashboards/out.test.toml | 1 + .../migrate/default-python/out.test.toml | 1 + .../engine-config-direct/out.test.toml | 1 + .../engine-config-terraform/out.test.toml | 1 + .../bundle/migrate/grants/out.test.toml | 1 + .../bundle/migrate/permissions/out.test.toml | 1 + .../bundle/migrate/profile_arg/out.test.toml | 1 + .../bundle/migrate/removed/out.test.toml | 1 + acceptance/bundle/migrate/runas/out.test.toml | 1 + .../bundle/migrate/var_arg/out.test.toml | 1 + .../multi_profile/auto_select/out.test.toml | 1 + .../multi_profile/env_auth_skip/out.test.toml | 1 + .../no_workspace_profiles/out.test.toml | 1 + .../non_interactive_error/out.test.toml | 1 + acceptance/bundle/open/out.test.toml | 1 + .../bundle/override/clusters/out.test.toml | 1 + .../bundle/override/job_cluster/out.test.toml | 1 + .../override/job_cluster_var/out.test.toml | 1 + .../bundle/override/job_tasks/out.test.toml | 1 + .../override/merge-string-map/out.test.toml | 1 + .../override/pipeline_cluster/out.test.toml | 1 + .../paths/designer_notebook/out.test.toml | 1 + .../bundle/paths/fallback/out.test.toml | 1 + .../paths/git_source_jobs/out.test.toml | 1 + .../invalid_pipeline_globs/out.test.toml | 1 + acceptance/bundle/paths/nominal/out.test.toml | 1 + .../paths/outside_root_no_sync/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/paths/pipeline_globs/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../relative_path_outside_root/out.test.toml | 1 + .../relative_path_translation/out.test.toml | 1 + .../bundle/plan/no_upload/out.test.toml | 1 + .../presets/preset_vs_dev_mode/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../experimental-compatibility/out.test.toml | 1 + .../python/grants-aliases/out.test.toml | 1 + .../python/mutator-ordering/out.test.toml | 1 + .../out.test.toml | 1 + .../python/pipelines-support/out.test.toml | 1 + .../python/propagates-auth-env/out.test.toml | 1 + .../python/resolve-variable/out.test.toml | 1 + .../python/resource-loading/out.test.toml | 1 + .../python/restricted-execution/out.test.toml | 1 + .../python/schemas-support/out.test.toml | 1 + .../python/unicode-support/out.test.toml | 1 + .../python/volumes-support/out.test.toml | 1 + .../bundle/quality_monitor/out.test.toml | 1 + acceptance/bundle/refschema/out.test.toml | 1 + .../bad_ref_string_to_int/out.test.toml | 1 + .../resource_deps/bad_syntax/out.test.toml | 1 + .../computed_volume_path/out.test.toml | 1 + .../resource_deps/create_error/out.test.toml | 1 + .../resource_deps/duplicate_ref/out.test.toml | 1 + .../resource_deps/grant_ref/out.test.toml | 1 + .../resource_deps/id_chain/out.test.toml | 1 + .../resource_deps/id_star/out.test.toml | 1 + .../immutable_field_ref/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../implicit_deps_volume/out.test.toml | 1 + .../bundle/resource_deps/job_id/out.test.toml | 1 + .../job_id_big_graph/delete_all/out.test.toml | 1 + .../job_id_big_graph/destroy/out.test.toml | 1 + .../job_id_delete_bar/out.test.toml | 1 + .../job_id_delete_foo/out.test.toml | 1 + .../resource_deps/job_tasks/out.test.toml | 1 + .../resource_deps/jobs_update/out.test.toml | 1 + .../jobs_update_remote/out.test.toml | 1 + .../resource_deps/loop_jobs/out.test.toml | 1 + .../resource_deps/loop_self/out.test.toml | 1 + .../out.test.toml | 1 + .../missing_map_key/out.test.toml | 1 + .../missing_string_field/out.test.toml | 1 + .../resource_deps/model_id_ref/out.test.toml | 1 + .../non_existent_field/out.test.toml | 1 + .../permission_ref/out.test.toml | 1 + .../pipelines_recreate/out.test.toml | 1 + .../out.test.toml | 1 + .../remote_app_url/out.test.toml | 1 + .../out.test.toml | 1 + .../remote_pipeline/out.test.toml | 1 + .../resource_deps/resources_var/out.test.toml | 1 + .../resources_var_presets/out.test.toml | 1 + .../out.test.toml | 1 + .../tf_path_only_error/out.test.toml | 1 + .../tf_path_renames/out.test.toml | 1 + .../unicode_reference/out.test.toml | 1 + .../volume_path_contains_id/out.test.toml | 1 + .../volume_path_job_ref/out.test.toml | 1 + .../resources/alerts/basic/out.test.toml | 1 + .../resources/alerts/with_file/out.test.toml | 1 + .../out.test.toml | 1 + .../with_file_run_from_subdir/out.test.toml | 1 + .../out.test.toml | 1 + .../apps/config-drift-stopped/out.test.toml | 1 + .../resources/apps/config-drift/out.test.toml | 1 + .../apps/config-no-deployment/out.test.toml | 1 + .../apps/create_already_exists/out.test.toml | 1 + .../apps/default_description/out.test.toml | 1 + .../git-source-no-deployment/out.test.toml | 1 + .../resources/apps/immutable/out.test.toml | 1 + .../apps/inline_config/out.test.toml | 1 + .../lifecycle-started-omitted/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../apps/lifecycle-started/out.test.toml | 1 + .../apps/readplan-lifecycle/out.test.toml | 1 + .../apps/resource-refs/out.test.toml | 1 + .../resources/apps/update/out.test.toml | 1 + .../catalogs/auto-approve/out.test.toml | 1 + .../resources/catalogs/basic/out.test.toml | 1 + .../drift/managed_properties/out.test.toml | 1 + .../catalogs/empty-name/out.test.toml | 1 + .../catalogs/with-schemas/out.test.toml | 1 + .../deploy/data_security_mode/out.test.toml | 1 + .../deploy/instance_pool/out.test.toml | 1 + .../instance_pool_and_node_type/out.test.toml | 1 + .../deploy/local_ssd_count/out.test.toml | 1 + .../deploy/num_workers_absent/out.test.toml | 1 + .../clusters/deploy/simple/out.test.toml | 1 + .../deploy/update-after-create/out.test.toml | 1 + .../update-and-resize-autoscale/out.test.toml | 1 + .../deploy/update-and-resize/out.test.toml | 1 + .../deploy/workload_type/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../clusters/lifecycle-started/out.test.toml | 1 + .../clusters/readplan-lifecycle/out.test.toml | 1 + .../resize-terminated-fallback/out.test.toml | 1 + .../run/spark_python_task/out.test.toml | 1 + .../change-embed-credentials/out.test.toml | 1 + .../dashboards/change-name/out.test.toml | 1 + .../change-parent-path/out.test.toml | 1 + .../change-serialized-dashboard/out.test.toml | 1 + .../dataset-catalog-schema/out.test.toml | 1 + .../delete-trashed-out-of-band/out.test.toml | 1 + .../dashboards/destroy/out.test.toml | 1 + .../dashboards/detect-change/out.test.toml | 1 + .../dashboards/generate_inplace/out.test.toml | 1 + .../dashboards/nested-folders/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../resources/dashboards/simple/out.test.toml | 1 + .../simple_outside_bundle_root/out.test.toml | 1 + .../dashboards/simple_syncroot/out.test.toml | 1 + .../unpublish-out-of-band/out.test.toml | 1 + .../database_catalogs/basic/out.test.toml | 1 + .../database_catalogs/recreate/out.test.toml | 1 + .../database_instances/recreate/out.test.toml | 1 + .../single-instance/out.test.toml | 1 + .../resources/experiments/basic/out.test.toml | 1 + .../external_locations/out.test.toml | 1 + .../genie_spaces/delete_warning/out.test.toml | 1 + .../genie_spaces/inline/out.test.toml | 1 + .../parent_path_update/out.test.toml | 1 + .../recreate_when_gone/out.test.toml | 1 + .../serialized_space/out.test.toml | 1 + .../genie_spaces/simple/out.test.toml | 1 + .../version_migration/out.test.toml | 1 + .../resources/grants/catalogs/out.test.toml | 1 + .../grants/registered_models/out.test.toml | 1 + .../schemas/all_privileges/out.test.toml | 1 + .../all_privileges_coexist/out.test.toml | 1 + .../schemas/change_privilege/out.test.toml | 1 + .../duplicate_principals/out.test.toml | 1 + .../duplicate_privileges/out.test.toml | 1 + .../grants/schemas/empty_array/out.test.toml | 1 + .../out_of_band_principal/out.test.toml | 1 + .../grants/schemas/remove_all/out.test.toml | 1 + .../schemas/remove_principal/out.test.toml | 1 + .../resources/grants/volumes/out.test.toml | 1 + .../resources/independent/out.test.toml | 1 + .../resources/instance_pools/out.test.toml | 1 + .../resources/job_runs/basic/out.test.toml | 1 + .../job_runs/job_parameters/out.test.toml | 1 + .../resources/job_runs/redeploy/out.test.toml | 1 + .../resources/jobs/alert-task/out.test.toml | 1 + .../resources/jobs/big_id/out.test.toml | 1 + .../bundle/resources/jobs/big_id/output.txt | 4 +- .../bundle/resources/jobs/big_id/script | 4 +- .../bundle/resources/jobs/big_id/test.toml | 6 +++ .../jobs/check-metadata/out.test.toml | 1 + .../resources/jobs/create-error/out.test.toml | 1 + .../resources/jobs/delete_job/out.test.toml | 1 + .../bundle/resources/jobs/delete_job/script | 4 +- .../resources/jobs/delete_task/out.test.toml | 1 + .../resources/jobs/delete_task/test.toml | 6 +++ .../jobs/double-underscore-keys/out.test.toml | 1 + .../jobs/fail-on-active-runs/out.test.toml | 1 + .../instance_pool_and_node_type/out.test.toml | 1 + .../jobs/no-git-provider/out.test.toml | 1 + .../resources/jobs/num_workers/out.test.toml | 1 + .../resources/jobs/num_workers/output.txt | 2 +- .../bundle/resources/jobs/num_workers/script | 2 +- .../jobs/on_failure_empty_slice/out.test.toml | 1 + .../jobs/remote_add_tag/out.test.toml | 1 + .../resources/jobs/remote_add_tag/script | 2 +- .../jobs/remote_delete/deploy/out.test.toml | 1 + .../jobs/remote_delete/deploy/test.toml | 6 +++ .../jobs/remote_delete/destroy/out.test.toml | 1 + .../removed_from_config/out.test.toml | 1 + .../removed_from_config/output.txt | 2 +- .../remote_delete/removed_from_config/script | 2 +- .../jobs/remote_matches_config/out.test.toml | 1 + .../jobs/remote_matches_config/output.txt | 2 +- .../jobs/remote_matches_config/script | 4 +- .../jobs/shared-root-path/out.test.toml | 1 + .../jobs/tags_empty_map/out.test.toml | 1 + .../resources/jobs/task-source/out.test.toml | 1 + .../jobs/tasks-reorder-locally/out.test.toml | 1 + .../unknown-terraform-field/out.test.toml | 1 + .../resources/jobs/update/out.test.toml | 1 + .../bundle/resources/jobs/update/output.txt | 8 ++-- .../bundle/resources/jobs/update/script | 8 ++-- .../bundle/resources/jobs/update/test.toml | 6 +++ .../jobs/update_single_node/out.test.toml | 1 + .../jobs/update_single_node/output.txt | 8 ++-- .../resources/jobs/update_single_node/script | 14 +++---- .../jobs/webhook-reorder-remote/out.test.toml | 1 + .../jobs/webhook-reorder-remote/output.txt | 2 +- .../jobs/webhook-reorder-remote/script | 4 +- .../basic/out.test.toml | 1 + .../drift/write_only/out.test.toml | 1 + .../recreate/catalog-name/out.test.toml | 1 + .../recreate/name-change/out.test.toml | 1 + .../recreate/route-optimized/out.test.toml | 1 + .../recreate/schema-name/out.test.toml | 1 + .../recreate/table-prefix/out.test.toml | 1 + .../running-endpoint/out.test.toml | 1 + .../update/ai-gateway/out.test.toml | 1 + .../both_gateway_and_tags/out.test.toml | 1 + .../update/config/out.test.toml | 1 + .../update/email-notifications/out.test.toml | 1 + .../update/tags/out.test.toml | 1 + .../resources/models/basic/out.test.toml | 1 + .../resources/models/empty-name/out.test.toml | 1 + .../models/readplan-permissions/out.test.toml | 1 + .../apps/current_can_manage/out.test.toml | 1 + .../apps/other_can_manage/out.test.toml | 1 + .../clusters/current_can_manage/out.test.toml | 1 + .../permissions/clusters/target/out.test.toml | 1 + .../dashboards/create/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../permissions/factcheck/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../out_of_band_deletion/out.test.toml | 1 + .../jobs/added_remotely/out.test.toml | 1 + .../jobs/current_can_manage/out.test.toml | 1 + .../jobs/current_can_manage_run/out.test.toml | 1 + .../jobs/current_is_owner/out.test.toml | 1 + .../permissions/jobs/delete_one/out.test.toml | 1 + .../jobs/deleted_remotely/out.test.toml | 1 + .../with_permissions/out.test.toml | 1 + .../without_permissions/out.test.toml | 1 + .../permissions/jobs/empty_list/out.test.toml | 1 + .../jobs/other_can_manage/out.test.toml | 1 + .../jobs/other_can_manage_run/out.test.toml | 1 + .../jobs/other_is_owner/out.test.toml | 1 + .../jobs/reorder_locally/out.test.toml | 1 + .../jobs/reorder_remotely/out.test.toml | 1 + .../permissions/jobs/update/out.test.toml | 1 + .../permissions/jobs/viewers/out.test.toml | 1 + .../models/current_can_manage/out.test.toml | 1 + .../resources/permissions/out.test.toml | 1 + .../pipelines/504/create/out.test.toml | 1 + .../pipelines/504/plan/out.test.toml | 1 + .../pipelines/504/update/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../pipelines/current_is_owner/out.test.toml | 1 + .../pipelines/empty_list/out.test.toml | 1 + .../pipelines/other_can_manage/out.test.toml | 1 + .../pipelines/other_is_owner/out.test.toml | 1 + .../pipelines/update/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../target_permissions/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../allow-duplicate-names/out.test.toml | 1 + .../pipelines/auto-approve/out.test.toml | 1 + .../pipelines/drift/parameters/out.test.toml | 1 + .../pipelines/lakeflow-pipeline/out.test.toml | 1 + .../pipelines/num-workers-zero/out.test.toml | 1 + .../pipelines/photon-true/out.test.toml | 1 + .../change-ingestion-definition/out.test.toml | 1 + .../change-storage/out.test.toml | 1 + .../pipelines/recreate/out.test.toml | 1 + .../remote_matches_config/out.test.toml | 1 + .../resources/pipelines/update/out.test.toml | 1 + .../pipelines/zero-value-fields/out.test.toml | 1 + .../postgres_branches/basic/out.test.toml | 1 + .../purge_on_delete/out.test.toml | 1 + .../purge_on_delete_transitions/out.test.toml | 1 + .../postgres_branches/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../update_protected/out.test.toml | 1 + .../without_branch_id/out.test.toml | 1 + .../postgres_catalogs/basic/out.test.toml | 1 + .../postgres_catalogs/recreate/out.test.toml | 1 + .../postgres_databases/basic/out.test.toml | 1 + .../live_errors/bad_database_id/out.test.toml | 1 + .../live_errors/bad_role_ref/out.test.toml | 1 + .../postgres_databases/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../postgres_databases/update/out.test.toml | 1 + .../postgres_endpoints/basic/out.test.toml | 1 + .../postgres_endpoints/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../update_autoscaling/out.test.toml | 1 + .../without_endpoint_id/out.test.toml | 1 + .../postgres_projects/basic/out.test.toml | 1 + .../purge_on_delete/out.test.toml | 1 + .../purge_on_delete_transitions/out.test.toml | 1 + .../postgres_projects/recreate/out.test.toml | 1 + .../update_display_name/out.test.toml | 1 + .../without_project_id/out.test.toml | 1 + .../postgres_roles/basic/out.test.toml | 1 + .../inherited-role-bind/out.test.toml | 1 + .../inherited-role-conflict/out.test.toml | 1 + .../recreate-postgres-role/out.test.toml | 1 + .../postgres_roles/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../postgres_roles/update/out.test.toml | 1 + .../basic/out.test.toml | 1 + .../recreate/out.test.toml | 1 + .../change_assets_dir/out.test.toml | 1 + .../change_output_schema_name/out.test.toml | 1 + .../change_table_name/out.test.toml | 1 + .../quality_monitors/create/out.test.toml | 1 + .../aliases_converge/out.test.toml | 1 + .../registered_models/basic/out.test.toml | 1 + .../drift/browse_only/out.test.toml | 1 + .../schemas/auto-approve/out.test.toml | 1 + .../drift/managed_properties/out.test.toml | 1 + .../resources/schemas/recreate/out.test.toml | 1 + .../resources/schemas/update/out.test.toml | 1 + .../secret_scopes/backend-type/out.test.toml | 1 + .../secret_scopes/basic/out.test.toml | 1 + .../secret_scopes/delete_scope/out.test.toml | 1 + .../permissions-collapse/out.test.toml | 1 + .../secret_scopes/permissions/out.test.toml | 1 + .../resources/secrets/basic/out.test.toml | 1 + .../secrets/direct-only/out.test.toml | 1 + .../secrets/update-value/out.test.toml | 1 + .../out.test.toml | 1 + .../validate-no-plain-text/out.test.toml | 1 + .../lifecycle-started-edit/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../lifecycle-started/out.test.toml | 1 + .../resources/sql_warehouses/out.test.toml | 1 + .../basic/out.test.toml | 1 + .../recreate/out.test.toml | 1 + acceptance/bundle/resources/test.toml | 41 ++++++++----------- .../basic/out.test.toml | 1 + .../drift/budget_policy/out.test.toml | 1 + .../drift/recreated_same_name/out.test.toml | 1 + .../drift/target_qps/out.test.toml | 1 + .../recreate/create-fails/out.test.toml | 1 + .../recreate/endpoint_type/out.test.toml | 1 + .../update/budget_policy/out.test.toml | 1 + .../update/target_qps/out.test.toml | 1 + .../vector_search_indexes/basic/out.test.toml | 1 + .../drift/deleted_remotely/out.test.toml | 1 + .../drift/orphaned_endpoint/out.test.toml | 1 + .../grants/select/out.test.toml | 1 + .../embedding_dimension/out.test.toml | 1 + .../recreate/pending_deletion/out.test.toml | 1 + .../recreate/with_endpoint/out.test.toml | 1 + .../schema_normalization/out.test.toml | 1 + .../volumes/catalog-var-ref/out.test.toml | 1 + .../volumes/change-comment/out.test.toml | 1 + .../volumes/change-name/out.test.toml | 1 + .../volumes/change-schema-name/out.test.toml | 1 + .../resources/volumes/recreate/out.test.toml | 1 + .../volumes/remote-change-name/out.test.toml | 1 + .../volumes/remote-delete/out.test.toml | 1 + .../set-storage-location/out.test.toml | 1 + .../volumes/set-volume-path/out.test.toml | 1 + .../volumes/uppercase-name/out.test.toml | 1 + .../root/env-not-a-directory/out.test.toml | 1 + .../bundle/root/env-not-found/out.test.toml | 1 + .../bundle/root/not-found/out.test.toml | 1 + .../bundle/root/real-empty-dir/out.test.toml | 1 + .../bundle/run/app-with-job/out.test.toml | 1 + acceptance/bundle/run/basic/out.test.toml | 1 + .../bundle/run/diagnostics/out.test.toml | 1 + .../run/inline-script/basic/out.test.toml | 1 + .../run/inline-script/cwd/out.test.toml | 1 + .../profile-is-passed/from_flag/out.test.toml | 1 + .../target-is-passed/default/out.test.toml | 1 + .../target-is-passed/from_flag/out.test.toml | 1 + .../run/inline-script/no-auth/out.test.toml | 1 + .../run/inline-script/no-bundle/out.test.toml | 1 + .../inline-script/no-separator/out.test.toml | 1 + .../bundle/run/jobs/partial_run/out.test.toml | 1 + acceptance/bundle/run/no-state/out.test.toml | 1 + .../bundle/run/refresh-flags/out.test.toml | 1 + .../bundle/run/scripts/basic/out.test.toml | 1 + .../bundle/run/scripts/cwd/out.test.toml | 1 + .../profile-is-passed/from_flag/out.test.toml | 1 + .../target-is-passed/default/out.test.toml | 1 + .../target-is-passed/from_flag/out.test.toml | 1 + .../run/scripts/env-bad-prefix/out.test.toml | 1 + .../run/scripts/env-precedence/out.test.toml | 1 + .../run/scripts/env-section/out.test.toml | 1 + .../run/scripts/exit_code/out.test.toml | 1 + .../bundle/run/scripts/io/out.test.toml | 1 + .../bundle/run/scripts/no-auth/out.test.toml | 1 + .../scripts/no-interpolation/out.test.toml | 1 + .../run/scripts/no_content/out.test.toml | 1 + .../run/scripts/shell/envvar/out.test.toml | 1 + .../run/scripts/shell/math/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/run/state-wiped/out.test.toml | 1 + .../run_as/allowed/regular_user/out.test.toml | 1 + .../allowed/service_principal/out.test.toml | 1 + .../run_as/dashboard_embed/out.test.toml | 1 + .../run_as/empty_override/out.test.toml | 1 + .../bundle/run_as/empty_run_as/out.test.toml | 1 + .../run_as/empty_run_as_dict/out.test.toml | 1 + .../bundle/run_as/empty_sp/out.test.toml | 1 + .../bundle/run_as/empty_user/out.test.toml | 1 + .../run_as/empty_user_and_sp/out.test.toml | 1 + .../invalid_both_sp_and_user/out.test.toml | 1 + .../bundle/run_as/job_default/out.test.toml | 1 + .../model_serving_different/out.test.toml | 1 + .../model_serving_matching/out.test.toml | 1 + acceptance/bundle/run_as/out.test.toml | 1 + .../pipelines/regular_user/out.test.toml | 1 + .../pipelines/service_principal/out.test.toml | 1 + .../run_as/pipelines_legacy/out.test.toml | 1 + .../scripts/no-trailing-newline/out.test.toml | 1 + acceptance/bundle/scripts/out.test.toml | 1 + .../restricted-execution/out.test.toml | 1 + .../bundle/select/ambiguous/out.test.toml | 1 + acceptance/bundle/select/basic/out.test.toml | 1 + .../select/grants_permissions/out.test.toml | 1 + .../bundle/select/missing/out.test.toml | 1 + .../bundle/select/rejected/out.test.toml | 1 + acceptance/bundle/state/bad_env/out.test.toml | 1 + .../bundle/state/bad_json_local/out.test.toml | 1 + acceptance/bundle/state/basic/out.test.toml | 1 + .../bundle/state/engine_default/out.test.toml | 1 + .../state/engine_mismatch/out.test.toml | 1 + .../bundle/state/feature_flags/out.test.toml | 1 + .../state/force_pull_commands/out.test.toml | 1 + .../bundle/state/future_version/out.test.toml | 1 + .../state/lineage_different/out.test.toml | 1 + .../permission_level_migration/out.test.toml | 1 + .../bundle/state/same_serial/out.test.toml | 1 + .../bundle/state/state_present/out.test.toml | 1 + .../missing-libraries-file-path/out.test.toml | 1 + .../summary/modified_status/out.test.toml | 1 + acceptance/bundle/sync/dryrun/out.test.toml | 1 + acceptance/bundle/sync/out.test.toml | 1 + .../bundle/syncroot/dotdot-git/out.test.toml | 1 + .../syncroot/dotdot-nogit/out.test.toml | 1 + .../config-remote-sync-error/out.test.toml | 1 + .../config-remote-sync-recreate/out.test.toml | 1 + .../config-remote-sync-save/out.test.toml | 1 + .../config-remote-sync/out.test.toml | 1 + .../out.test.toml | 1 + .../deploy-artifact-path-type/out.test.toml | 1 + .../deploy-artifacts-variables/out.test.toml | 1 + .../deploy-compute-type/out.test.toml | 1 + .../deploy-config-file-count/out.test.toml | 1 + .../deploy-error-message/out.test.toml | 1 + .../telemetry/deploy-error/out.test.toml | 1 + .../deploy-experimental/out.test.toml | 1 + .../telemetry/deploy-mode/out.test.toml | 1 + .../deploy-name-prefix/custom/out.test.toml | 1 + .../mode-development/out.test.toml | 1 + .../telemetry/deploy-no-uuid/out.test.toml | 1 + .../telemetry/deploy-run-as/out.test.toml | 1 + .../deploy-target-count/out.test.toml | 1 + .../deploy-variable-count/out.test.toml | 1 + .../deploy-whl-artifacts/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/telemetry/deploy/out.test.toml | 1 + .../helper_upper_lower/out.test.toml | 1 + .../helper_username/out.test.toml | 1 + .../helpers-error/out.test.toml | 1 + .../number-precision/out.test.toml | 1 + .../supported-url/out.test.toml | 1 + .../unsupported-url/out.test.toml | 1 + .../wrong-path/out.test.toml | 1 + .../wrong-url/out.test.toml | 1 + .../bundle/templates/dbt-sql/out.test.toml | 1 + .../default-minimal/python/out.test.toml | 1 + .../default-minimal/skip/out.test.toml | 1 + .../default-minimal/sql/out.test.toml | 1 + .../azure-government/out.test.toml | 1 + .../default-python/classic/out.test.toml | 1 + .../combinations/classic/out.test.toml | 1 + .../combinations/serverless/out.test.toml | 1 + .../fail-missing-uv/out.test.toml | 1 + .../integration_classic/out.test.toml | 1 + .../default-python/no-uc/out.test.toml | 1 + .../serverless-customcatalog/out.test.toml | 1 + .../default-python/serverless/out.test.toml | 1 + .../templates/default-scala/out.test.toml | 1 + .../templates/default-sql/out.test.toml | 1 + .../lakeflow-integrations/out.test.toml | 1 + .../lakeflow-pipelines/python/out.test.toml | 1 + .../lakeflow-pipelines/sql/out.test.toml | 1 + .../templates/nested-output/out.test.toml | 1 + .../pydabs/check-consistency/out.test.toml | 1 + .../pydabs/check-formatting/out.test.toml | 1 + .../pydabs/deploy-classic/out.test.toml | 1 + .../pydabs/init-classic/out.test.toml | 1 + .../telemetry/custom-template/out.test.toml | 1 + .../templates/telemetry/dbt-sql/out.test.toml | 1 + .../telemetry/default-python/out.test.toml | 1 + .../telemetry/default-sql/out.test.toml | 1 + acceptance/bundle/test.toml | 27 ++++++++++++ .../trampoline/warning_message/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/undefined_resources/out.test.toml | 1 + .../internal_server_error/out.test.toml | 1 + .../bundle/upload/timeout/out.test.toml | 1 + acceptance/bundle/user_agent/out.test.toml | 1 + .../bundle/user_agent/simple/out.test.toml | 1 + .../validate/anchor_containers/out.test.toml | 1 + .../out.test.toml | 1 + .../validate/dashboard_defaults/out.test.toml | 1 + .../dashboard_required_name/out.test.toml | 1 + .../out.test.toml | 1 + .../definitions_yaml_anchors/out.test.toml | 1 + .../duplicate_yaml_merge_key/out.test.toml | 1 + .../empty_resources/empty_def/out.test.toml | 1 + .../empty_resources/empty_dict/out.test.toml | 1 + .../empty_resources/null/out.test.toml | 1 + .../empty_resources/with_grants/out.test.toml | 1 + .../with_permissions/out.test.toml | 1 + .../bundle/validate/empty_tasks/out.test.toml | 1 + .../engine-config-valid/out.test.toml | 1 + acceptance/bundle/validate/enum/out.test.toml | 1 + .../validate/enum_resource_refs/out.test.toml | 1 + .../genie_space_complex/out.test.toml | 1 + .../genie_space_defaults/out.test.toml | 1 + .../out.test.toml | 1 + .../grants_required_principal/out.test.toml | 1 + .../immutable_workspace_paths/out.test.toml | 1 + .../validate/include_locations/out.test.toml | 1 + .../invalid-engine-bundle/out.test.toml | 1 + .../invalid-engine-target/out.test.toml | 1 + .../validate/job-references/out.test.toml | 1 + .../out.test.toml | 1 + .../model_serving_conversion/out.test.toml | 1 + .../models/missing_name/out.test.toml | 1 + .../validate/models/user_id/out.test.toml | 1 + .../validate/no_dashboard_etag/out.test.toml | 1 + .../no_genie_space_etag/out.test.toml | 1 + .../bundle/validate/permissions/out.test.toml | 1 + .../permissions_overlap/out.test.toml | 1 + .../presets_max_concurrent_runs/out.test.toml | 1 + .../presets_name_prefix/out.test.toml | 1 + .../presets_name_prefix_dev/out.test.toml | 1 + .../validate/presets_tags/out.test.toml | 1 + .../bundle/validate/required/out.test.toml | 1 + .../reserved_deployment_fields/out.test.toml | 1 + .../sql_warehouse_required_name/out.test.toml | 1 + .../bundle/validate/strict/out.test.toml | 1 + .../validate/sync_patterns/out.test.toml | 1 + .../validate/var_in_bundle_name/out.test.toml | 1 + .../validate/volume_defaults/out.test.toml | 1 + .../bundle/variables/arg-repeat/out.test.toml | 1 + .../variables/complex-cross-ref/out.test.toml | 1 + .../complex-cycle-self/out.test.toml | 1 + .../variables/complex-cycle/out.test.toml | 1 + .../variables/complex-simple/out.test.toml | 1 + .../complex-transitive-deep/out.test.toml | 1 + .../complex-transitive-deeper/out.test.toml | 1 + .../complex-transitive/out.test.toml | 1 + .../complex-with-var-reference/out.test.toml | 1 + .../complex-within-complex/out.test.toml | 1 + .../bundle/variables/complex/out.test.toml | 1 + .../complex_multiple_files/out.test.toml | 1 + .../bundle/variables/cycle/out.test.toml | 1 + .../variables/double_underscore/out.test.toml | 1 + .../bundle/variables/empty/out.test.toml | 1 + .../variables/env_overrides/out.test.toml | 1 + .../variables/file-defaults/out.test.toml | 1 + .../bundle/variables/git-branch/out.test.toml | 1 + .../bundle/variables/host/out.test.toml | 1 + acceptance/bundle/variables/int/out.test.toml | 1 + .../bundle/variables/issue_2436/out.test.toml | 1 + .../issue_3039_lookup_with_ref/out.test.toml | 1 + .../bundle/variables/lookup/out.test.toml | 1 + .../prepend-workspace-var/out.test.toml | 1 + .../variables/resolve-builtin/out.test.toml | 1 + .../variables/resolve-empty/out.test.toml | 1 + .../out.test.toml | 1 + .../resolve-nonstrings/out.test.toml | 1 + .../resolve-resources-fields/out.test.toml | 1 + .../resolve-vars-in-root-path/out.test.toml | 1 + .../variables/unicode_reference/out.test.toml | 1 + .../bundle/variables/vanilla/out.test.toml | 1 + .../bundle/variables/var_in_var/out.test.toml | 1 + .../variable_in_resource_key/out.test.toml | 1 + .../out.test.toml | 1 + .../without_definition/out.test.toml | 1 + .../volume_path/invalid_file/out.test.toml | 1 + .../invalid_resource/out.test.toml | 1 + .../volume_path/invalid_root/out.test.toml | 1 + .../volume_path/invalid_state/out.test.toml | 1 + .../bundle/volume_path/valid/out.test.toml | 1 + 879 files changed, 951 insertions(+), 77 deletions(-) diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/app_yaml/out.test.toml b/acceptance/bundle/apps/app_yaml/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/app_yaml/out.test.toml +++ b/acceptance/bundle/apps/app_yaml/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml +++ b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/compute_size/out.test.toml b/acceptance/bundle/apps/compute_size/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/apps/compute_size/out.test.toml +++ b/acceptance/bundle/apps/compute_size/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/apps/git_source/out.test.toml b/acceptance/bundle/apps/git_source/out.test.toml index 8f6c4a03c57..dfb2766ed88 100644 --- a/acceptance/bundle/apps/git_source/out.test.toml +++ b/acceptance/bundle/apps/git_source/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions/out.test.toml b/acceptance/bundle/apps/job_permissions/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/apps/job_permissions/out.test.toml +++ b/acceptance/bundle/apps/job_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions_warning/out.test.toml b/acceptance/bundle/apps/job_permissions_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/job_permissions_warning/out.test.toml +++ b/acceptance/bundle/apps/job_permissions_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/value_from_warning/out.test.toml b/acceptance/bundle/apps/value_from_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/value_from_warning/out.test.toml +++ b/acceptance/bundle/apps/value_from_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml +++ b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files/out.test.toml b/acceptance/bundle/artifacts/build_and_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml +++ b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files/out.test.toml b/acceptance/bundle/artifacts/globs_in_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_invalid/out.test.toml b/acceptance/bundle/artifacts/globs_invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_invalid/out.test.toml +++ b/acceptance/bundle/artifacts/globs_invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/issue_3109/out.test.toml b/acceptance/bundle/artifacts/issue_3109/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/issue_3109/out.test.toml +++ b/acceptance/bundle/artifacts/issue_3109/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml +++ b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/bash/out.test.toml b/acceptance/bundle/artifacts/shell/bash/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/bash/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/basic/out.test.toml b/acceptance/bundle/artifacts/shell/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/basic/out.test.toml +++ b/acceptance/bundle/artifacts/shell/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/cmd/out.test.toml b/acceptance/bundle/artifacts/shell/cmd/out.test.toml index 8471d88c7f3..af56bea47fb 100644 --- a/acceptance/bundle/artifacts/shell/cmd/out.test.toml +++ b/acceptance/bundle/artifacts/shell/cmd/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false GOOS.darwin = false GOOS.linux = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/default/out.test.toml b/acceptance/bundle/artifacts/shell/default/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/default/out.test.toml +++ b/acceptance/bundle/artifacts/shell/default/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/invalid/out.test.toml b/acceptance/bundle/artifacts/shell/invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/invalid/out.test.toml +++ b/acceptance/bundle/artifacts/shell/invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/sh/out.test.toml b/acceptance/bundle/artifacts/shell/sh/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/sh/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_change_version/out.test.toml b/acceptance/bundle/artifacts/whl_change_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_change_version/out.test.toml +++ b/acceptance/bundle/artifacts/whl_change_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_explicit/out.test.toml b/acceptance/bundle/artifacts/whl_explicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_explicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_explicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit/out.test.toml b/acceptance/bundle/artifacts/whl_implicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml +++ b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/deploy/out.test.toml b/acceptance/bundle/benchmarks/deploy/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/deploy/out.test.toml +++ b/acceptance/bundle/benchmarks/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/plan/out.test.toml b/acceptance/bundle/benchmarks/plan/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/plan/out.test.toml +++ b/acceptance/bundle/benchmarks/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/validate/out.test.toml b/acceptance/bundle/benchmarks/validate/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/validate/out.test.toml +++ b/acceptance/bundle/benchmarks/validate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/id/out.test.toml b/acceptance/bundle/bundle_tag/id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/id/out.test.toml +++ b/acceptance/bundle/bundle_tag/id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url/out.test.toml b/acceptance/bundle/bundle_tag/url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url/out.test.toml +++ b/acceptance/bundle/bundle_tag/url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url_ref/out.test.toml b/acceptance/bundle/bundle_tag/url_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.test.toml +++ b/acceptance/bundle/bundle_tag/url_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml +++ b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml +++ b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml index 4c2be3166c4..dc7c5353574 100644 --- a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml +++ b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml +++ b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_json/out.test.toml b/acceptance/bundle/config-remote-sync/output_json/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_json/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_json/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml +++ b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/target_override/out.test.toml b/acceptance/bundle/config-remote-sync/target_override/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/target_override/out.test.toml +++ b/acceptance/bundle/config-remote-sync/target_override/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml +++ b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/debug/list-targets/out.test.toml b/acceptance/bundle/debug/list-targets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/list-targets/out.test.toml +++ b/acceptance/bundle/debug/list-targets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/debug/out.test.toml b/acceptance/bundle/debug/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/out.test.toml +++ b/acceptance/bundle/debug/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/empty-bundle/out.test.toml b/acceptance/bundle/deploy/empty-bundle/out.test.toml index 72e8a7a4dfe..3054de89706 100644 --- a/acceptance/bundle/deploy/empty-bundle/out.test.toml +++ b/acceptance/bundle/deploy/empty-bundle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/experimental-python/out.test.toml b/acceptance/bundle/deploy/experimental-python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/experimental-python/out.test.toml +++ b/acceptance/bundle/deploy/experimental-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/force-lock-config/out.test.toml b/acceptance/bundle/deploy/force-lock-config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/force-lock-config/out.test.toml +++ b/acceptance/bundle/deploy/force-lock-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable/out.test.toml b/acceptance/bundle/deploy/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable/out.test.toml +++ b/acceptance/bundle/deploy/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/mlops-stacks/out.test.toml b/acceptance/bundle/deploy/mlops-stacks/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deploy/mlops-stacks/out.test.toml +++ b/acceptance/bundle/deploy/mlops-stacks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml +++ b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/python-notebook/out.test.toml b/acceptance/bundle/deploy/python-notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/python-notebook/out.test.toml +++ b/acceptance/bundle/deploy/python-notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml +++ b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/spark-jar-task/out.test.toml b/acceptance/bundle/deploy/spark-jar-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/spark-jar-task/out.test.toml +++ b/acceptance/bundle/deploy/spark-jar-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index 1d895a16c96..426690291a0 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index 9448f875df7..84742e9cd0a 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml +++ b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index f61486ff080..3f6826cd945 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresCluster = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/all-resources/out.test.toml b/acceptance/bundle/destroy/all-resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/all-resources/out.test.toml +++ b/acceptance/bundle/destroy/all-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml +++ b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 5b9ca490227..952dcb00a86 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 3b5da470cfa..6233f158974 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,7 +58,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 7e81865e11a..b593998dbd1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -20,7 +19,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 0307d57910b..47ffe89cb73 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/defau Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -16,7 +15,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/defau Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index e4c0341934b..8801d4179b1 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --get { @@ -51,7 +50,6 @@ Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index 8c36883a9db..f6984c72258 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -66,7 +65,6 @@ This action will result in the deletion or recreation of the following UC schema Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index a4084f82728..c567a12d66c 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 9a51c82e60e..a707e5e46d7 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -71,7 +70,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 64c4e6b8c4f..12ce1a298c4 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -25,7 +24,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index 13a86eb3a21..c792a60fcfa 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle summary -o json { @@ -19,7 +18,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> [CLI] bundle summary -o json { diff --git a/acceptance/bundle/empty_string_dropped/out.test.toml b/acceptance/bundle/empty_string_dropped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_dropped/out.test.toml +++ b/acceptance/bundle/empty_string_dropped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/empty_string_variable/out.test.toml b/acceptance/bundle/empty_string_variable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_variable/out.test.toml +++ b/acceptance/bundle/empty_string_variable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/environments/dependencies/out.test.toml b/acceptance/bundle/environments/dependencies/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/environments/dependencies/out.test.toml +++ b/acceptance/bundle/environments/dependencies/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml +++ b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert/out.test.toml b/acceptance/bundle/generate/alert/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/alert/out.test.toml +++ b/acceptance/bundle/generate/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml +++ b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_subfolders/out.test.toml b/acceptance/bundle/generate/app_subfolders/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_subfolders/out.test.toml +++ b/acceptance/bundle/generate/app_subfolders/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/auto-bind/out.test.toml b/acceptance/bundle/generate/auto-bind/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/generate/auto-bind/out.test.toml +++ b/acceptance/bundle/generate/auto-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard/out.test.toml b/acceptance/bundle/generate/dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard/out.test.toml +++ b/acceptance/bundle/generate/dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/designer_job/out.test.toml b/acceptance/bundle/generate/designer_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/designer_job/out.test.toml +++ b/acceptance/bundle/generate/designer_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/generate/git_job/out.test.toml b/acceptance/bundle/generate/git_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/git_job/out.test.toml +++ b/acceptance/bundle/generate/git_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/include_warning/out.test.toml b/acceptance/bundle/generate/include_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/include_warning/out.test.toml +++ b/acceptance/bundle/generate/include_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/ipynb_job/out.test.toml b/acceptance/bundle/generate/ipynb_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/ipynb_job/out.test.toml +++ b/acceptance/bundle/generate/ipynb_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml +++ b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml +++ b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline/out.test.toml b/acceptance/bundle/generate/pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline/out.test.toml +++ b/acceptance/bundle/generate/pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job/out.test.toml b/acceptance/bundle/generate/python_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/python_job/out.test.toml +++ b/acceptance/bundle/generate/python_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/spark_python_task_job/out.test.toml b/acceptance/bundle/generate/spark_python_task_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/spark_python_task_job/out.test.toml +++ b/acceptance/bundle/generate/spark_python_task_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/git-permerror/out.test.toml b/acceptance/bundle/git-permerror/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/git-permerror/out.test.toml +++ b/acceptance/bundle/git-permerror/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deploy/out.test.toml b/acceptance/bundle/help/bundle-deploy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deploy/out.test.toml +++ b/acceptance/bundle/help/bundle-deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment/out.test.toml b/acceptance/bundle/help/bundle-deployment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-destroy/out.test.toml b/acceptance/bundle/help/bundle-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-destroy/out.test.toml +++ b/acceptance/bundle/help/bundle-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-job/out.test.toml b/acceptance/bundle/help/bundle-generate-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-job/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate/out.test.toml b/acceptance/bundle/help/bundle-generate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate/out.test.toml +++ b/acceptance/bundle/help/bundle-generate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-init/out.test.toml b/acceptance/bundle/help/bundle-init/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-init/out.test.toml +++ b/acceptance/bundle/help/bundle-init/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-open/out.test.toml b/acceptance/bundle/help/bundle-open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-open/out.test.toml +++ b/acceptance/bundle/help/bundle-open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-run/out.test.toml b/acceptance/bundle/help/bundle-run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-run/out.test.toml +++ b/acceptance/bundle/help/bundle-run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-schema/out.test.toml b/acceptance/bundle/help/bundle-schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-schema/out.test.toml +++ b/acceptance/bundle/help/bundle-schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-summary/out.test.toml b/acceptance/bundle/help/bundle-summary/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-summary/out.test.toml +++ b/acceptance/bundle/help/bundle-summary/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-sync/out.test.toml b/acceptance/bundle/help/bundle-sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-sync/out.test.toml +++ b/acceptance/bundle/help/bundle-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-validate/out.test.toml b/acceptance/bundle/help/bundle-validate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-validate/out.test.toml +++ b/acceptance/bundle/help/bundle-validate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle/out.test.toml b/acceptance/bundle/help/bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle/out.test.toml +++ b/acceptance/bundle/help/bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/glob_in_root_path/out.test.toml b/acceptance/bundle/includes/glob_in_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/glob_in_root_path/out.test.toml +++ b/acceptance/bundle/includes/glob_in_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/include_outside_root/out.test.toml b/acceptance/bundle/includes/include_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/include_outside_root/out.test.toml +++ b/acceptance/bundle/includes/include_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml +++ b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/yml_outside_root/out.test.toml b/acceptance/bundle/includes/yml_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/yml_outside_root/out.test.toml +++ b/acceptance/bundle/includes/yml_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index 19068d43e0a..5d56f06c3a2 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless/out.test.toml b/acceptance/bundle/integration_whl/serverless/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index c294b244621..c9d202227e3 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 8560caa0ee5..cf188cbc54b 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/libraries/maven/out.test.toml b/acceptance/bundle/libraries/maven/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/maven/out.test.toml +++ b/acceptance/bundle/libraries/maven/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml +++ b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/pypi/out.test.toml b/acceptance/bundle/libraries/pypi/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/pypi/out.test.toml +++ b/acceptance/bundle/libraries/pypi/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml index b37ee45aed6..931153dacc9 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.test.toml +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/lifecycle/started/out.test.toml +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/local_state_staleness/out.test.toml b/acceptance/bundle/local_state_staleness/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/local_state_staleness/out.test.toml +++ b/acceptance/bundle/local_state_staleness/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/multi_profile/auto_select/out.test.toml b/acceptance/bundle/multi_profile/auto_select/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/auto_select/out.test.toml +++ b/acceptance/bundle/multi_profile/auto_select/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml +++ b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml +++ b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml +++ b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/open/out.test.toml b/acceptance/bundle/open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/open/out.test.toml +++ b/acceptance/bundle/open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/clusters/out.test.toml b/acceptance/bundle/override/clusters/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/clusters/out.test.toml +++ b/acceptance/bundle/override/clusters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster/out.test.toml b/acceptance/bundle/override/job_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster/out.test.toml +++ b/acceptance/bundle/override/job_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster_var/out.test.toml b/acceptance/bundle/override/job_cluster_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster_var/out.test.toml +++ b/acceptance/bundle/override/job_cluster_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_tasks/out.test.toml b/acceptance/bundle/override/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_tasks/out.test.toml +++ b/acceptance/bundle/override/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/merge-string-map/out.test.toml b/acceptance/bundle/override/merge-string-map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/merge-string-map/out.test.toml +++ b/acceptance/bundle/override/merge-string-map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/pipeline_cluster/out.test.toml b/acceptance/bundle/override/pipeline_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/pipeline_cluster/out.test.toml +++ b/acceptance/bundle/override/pipeline_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/designer_notebook/out.test.toml b/acceptance/bundle/paths/designer_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/designer_notebook/out.test.toml +++ b/acceptance/bundle/paths/designer_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/fallback/out.test.toml b/acceptance/bundle/paths/fallback/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/fallback/out.test.toml +++ b/acceptance/bundle/paths/fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/git_source_jobs/out.test.toml b/acceptance/bundle/paths/git_source_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/git_source_jobs/out.test.toml +++ b/acceptance/bundle/paths/git_source_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/nominal/out.test.toml b/acceptance/bundle/paths/nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/nominal/out.test.toml +++ b/acceptance/bundle/paths/nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml +++ b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_globs/out.test.toml b/acceptance/bundle/paths/pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml +++ b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_translation/out.test.toml b/acceptance/bundle/paths/relative_path_translation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_translation/out.test.toml +++ b/acceptance/bundle/paths/relative_path_translation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/plan/no_upload/out.test.toml b/acceptance/bundle/plan/no_upload/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/plan/no_upload/out.test.toml +++ b/acceptance/bundle/plan/no_upload/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml +++ b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility/out.test.toml b/acceptance/bundle/python/experimental-compatibility/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/grants-aliases/out.test.toml b/acceptance/bundle/python/grants-aliases/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/grants-aliases/out.test.toml +++ b/acceptance/bundle/python/grants-aliases/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/mutator-ordering/out.test.toml b/acceptance/bundle/python/mutator-ordering/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/mutator-ordering/out.test.toml +++ b/acceptance/bundle/python/mutator-ordering/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml index 4c48a83f25b..256d0941b25 100644 --- a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml +++ b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/pipelines-support/out.test.toml b/acceptance/bundle/python/pipelines-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/pipelines-support/out.test.toml +++ b/acceptance/bundle/python/pipelines-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/propagates-auth-env/out.test.toml +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resolve-variable/out.test.toml b/acceptance/bundle/python/resolve-variable/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resolve-variable/out.test.toml +++ b/acceptance/bundle/python/resolve-variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resource-loading/out.test.toml b/acceptance/bundle/python/resource-loading/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resource-loading/out.test.toml +++ b/acceptance/bundle/python/resource-loading/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/restricted-execution/out.test.toml b/acceptance/bundle/python/restricted-execution/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/restricted-execution/out.test.toml +++ b/acceptance/bundle/python/restricted-execution/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/schemas-support/out.test.toml b/acceptance/bundle/python/schemas-support/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/schemas-support/out.test.toml +++ b/acceptance/bundle/python/schemas-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/unicode-support/out.test.toml b/acceptance/bundle/python/unicode-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/unicode-support/out.test.toml +++ b/acceptance/bundle/python/unicode-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/volumes-support/out.test.toml b/acceptance/bundle/python/volumes-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/volumes-support/out.test.toml +++ b/acceptance/bundle/python/volumes-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/quality_monitor/out.test.toml b/acceptance/bundle/quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/quality_monitor/out.test.toml +++ b/acceptance/bundle/quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/refschema/out.test.toml b/acceptance/bundle/refschema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/refschema/out.test.toml +++ b/acceptance/bundle/refschema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/create_error/out.test.toml b/acceptance/bundle/resource_deps/create_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/create_error/out.test.toml +++ b/acceptance/bundle/resource_deps/create_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/id_chain/out.test.toml b/acceptance/bundle/resource_deps/id_chain/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.test.toml +++ b/acceptance/bundle/resource_deps/id_chain/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/id_star/out.test.toml b/acceptance/bundle/resource_deps/id_star/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_star/out.test.toml +++ b/acceptance/bundle/resource_deps/id_star/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id/out.test.toml b/acceptance/bundle/resource_deps/job_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.test.toml +++ b/acceptance/bundle/resource_deps/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update/out.test.toml b/acceptance/bundle/resource_deps/jobs_update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_self/out.test.toml b/acceptance/bundle/resource_deps/loop_self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_self/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml +++ b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/permission_ref/out.test.toml b/acceptance/bundle/resource_deps/permission_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/permission_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var/out.test.toml b/acceptance/bundle/resource_deps/resources_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml +++ b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index c45d8e76a8e..eaf2e7a7966 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file/out.test.toml b/acceptance/bundle/resources/alerts/with_file/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/alerts/with_file/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml +++ b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/default_description/out.test.toml b/acceptance/bundle/resources/apps/default_description/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/default_description/out.test.toml +++ b/acceptance/bundle/resources/apps/default_description/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/immutable/out.test.toml b/acceptance/bundle/resources/apps/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/immutable/out.test.toml +++ b/acceptance/bundle/resources/apps/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/inline_config/out.test.toml b/acceptance/bundle/resources/apps/inline_config/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/apps/inline_config/out.test.toml +++ b/acceptance/bundle/resources/apps/inline_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/resource-refs/out.test.toml b/acceptance/bundle/resources/apps/resource-refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/resource-refs/out.test.toml +++ b/acceptance/bundle/resources/apps/resource-refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/update/out.test.toml b/acceptance/bundle/resources/apps/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/update/out.test.toml +++ b/acceptance/bundle/resources/apps/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/basic/out.test.toml b/acceptance/bundle/resources/catalogs/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/catalogs/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml +++ b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml +++ b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml index 5a821e39edc..813b3a187d4 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudEnvs.aws = false CloudEnvs.azure = false CloudEnvs.gcp = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 475b179caed..ebb5db02455 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-name/out.test.toml b/acceptance/bundle/resources/dashboards/change-name/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-name/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index c2ac722e76a..ec9bbdf7292 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 1976bc173ca..65ad5749140 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -2,5 +2,6 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml index b5ce19512e3..2e27b38ff97 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index 1f9d1fc1b75..281190c64eb 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -4,4 +4,5 @@ CloudSlow = true RequiresUnityCatalog = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/recreate/out.test.toml b/acceptance/bundle/resources/database_instances/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_instances/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_instances/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/external_locations/out.test.toml +++ b/acceptance/bundle/resources/external_locations/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.test.toml b/acceptance/bundle/resources/grants/catalogs/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.test.toml +++ b/acceptance/bundle/resources/grants/catalogs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/registered_models/out.test.toml b/acceptance/bundle/resources/grants/registered_models/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.test.toml +++ b/acceptance/bundle/resources/grants/registered_models/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/volumes/out.test.toml b/acceptance/bundle/resources/grants/volumes/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/volumes/out.test.toml +++ b/acceptance/bundle/resources/grants/volumes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/independent/out.test.toml b/acceptance/bundle/resources/independent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/independent/out.test.toml +++ b/acceptance/bundle/resources/independent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/instance_pools/out.test.toml +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/basic/out.test.toml +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml +++ b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/alert-task/out.test.toml b/acceptance/bundle/resources/jobs/alert-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/alert-task/out.test.toml +++ b/acceptance/bundle/resources/jobs/alert-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index 71970b719d4..310be221793 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt index 82ec469ca83..c539d037f79 100644 --- a/acceptance/bundle/resources/jobs/big_id/output.txt +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", @@ -70,7 +70,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 0b803cb1b8d..6f0e1215c6a 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index 6dd4efd85d3..467d0855fa4 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,6 +3,12 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Repls]] Old = '9223372036854775807' New = '[MAX_INT_64]' diff --git a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/create-error/out.test.toml b/acceptance/bundle/resources/jobs/create-error/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/create-error/out.test.toml +++ b/acceptance/bundle/resources/jobs/create-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.test.toml b/acceptance/bundle/resources/jobs/delete_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index c9242b9e209..c3ce3c66802 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/num_workers/out.test.toml +++ b/acceptance/bundle/resources/jobs/num_workers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index f61ab034296..702558444e9 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 83d9321bcc4..674a820061c 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index d7593e7b8f7..37a37b0059f 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index c97c93273cb..45a761af80e 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -21,4 +21,4 @@ Updating deployment state... Deployment complete! === No delete API calls for resources that are already gone remotely ->>> print_requests.py //jobs/delete //pipelines/ +>>> print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script index 2b032406a00..45098c8a690 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script @@ -18,4 +18,4 @@ trace $CLI bundle deploy trace $CLI bundle summary &> out.summary.$DATABRICKS_BUNDLE_ENGINE.txt title "No delete API calls for resources that are already gone remotely" -trace print_requests.py //jobs/delete //pipelines/ +trace print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index 9ce9dba13c1..d1f2a5cf7d5 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 972181ba497..b528539bf92 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,9 +13,9 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml +++ b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/task-source/out.test.toml b/acceptance/bundle/resources/jobs/task-source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.test.toml +++ b/acceptance/bundle/resources/jobs/task-source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml +++ b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update/output.txt b/acceptance/bundle/resources/jobs/update/output.txt index eea83d272d5..f6a44af17c6 100644 --- a/acceptance/bundle/resources/jobs/update/output.txt +++ b/acceptance/bundle/resources/jobs/update/output.txt @@ -8,7 +8,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -19,7 +19,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -33,7 +33,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 270efcaea6d..c15b425741c 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -2,14 +2,14 @@ echo "*" > .gitignore trace $CLI bundle plan $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) -trace print_requests.py //jobs > out.create.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS @@ -17,7 +17,7 @@ trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -trace print_requests.py //jobs | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json trace $CLI bundle plan @@ -30,7 +30,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index ff8a66c196e..a8a9a1e90b9 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml +++ b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index aba6e239b86..ec195ebc965 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [FOO_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 55ce937b978..d822881fe20 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,17 +1,17 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan @@ -19,12 +19,12 @@ title "Fetch job ID and verify remote state" ppid=`read_id.py foo` -trace $CLI jobs get $ppid | jq 'del(.settings.run_as)' +trace $CLI jobs get $ppid | jq 'del(.settings.run_as, .settings.deployment.deployment_id, .settings.deployment.version_id)' rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index 31a759d0c51..2185aedcef6 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index fda553c3cea..7d9103aca52 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/empty-name/out.test.toml b/acceptance/bundle/resources/models/empty-name/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/models/empty-name/out.test.toml +++ b/acceptance/bundle/resources/models/empty-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml index 7845c49f70c..973ce7c68cf 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index 581c975b773..64851222e54 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml index 887e4650a78..0b68a00d7ba 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/out.test.toml b/acceptance/bundle/resources/permissions/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/resources/permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml +++ b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml +++ b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/update/out.test.toml b/acceptance/bundle/resources/pipelines/update/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/update/out.test.toml b/acceptance/bundle/resources/postgres_databases/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index c5b8e7c8a71..67797faa7b2 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index c5b8e7c8a71..67797faa7b2 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/update/out.test.toml b/acceptance/bundle/resources/postgres_roles/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/out.test.toml b/acceptance/bundle/resources/schemas/recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/recreate/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/update/out.test.toml b/acceptance/bundle/resources/schemas/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/update/out.test.toml +++ b/acceptance/bundle/resources/schemas/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 6fc644d5164..aa7060abb02 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index 6b858c4df47..5ab6742ca97 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/secrets/basic/out.test.toml +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/secrets/direct-only/out.test.toml +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/update-value/out.test.toml +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml index 5bbfaf5e65a..fe1401b93c2 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml index 91e30e807cf..82f3670f861 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/out.test.toml b/acceptance/bundle/resources/sql_warehouses/out.test.toml index 355ae0775bc..b1754ac936e 100644 --- a/acceptance/bundle/resources/sql_warehouses/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index e991fce9180..dbcf84075ed 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index a8d852b9821..df0bcf4b84a 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,34 +1,27 @@ RecordRequests = true -# Recording adds two things to a deploy's output, and both are normalized away so the -# DMS run asserts the same goldens as the engine runs. That is the point of the run: -# every test then checks that recording does not change what a deploy does, rather than -# needing a second copy of 600-odd output files. They live here rather than in the parent so -# bundle/dms, which asserts the recording itself, does not inherit them. +# These normalize what deployment history recording adds to a deploy's output, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They live here +# rather than in the parent so bundle/dms, which asserts the recording itself, does not +# inherit them. # -# The link printed after a deploy: +# The stamp on jobs and pipelines, which the plan reports as a change of its own. It shows +# up at whatever depth the enclosing object sits at, so the indent is matched loosely; the +# body lines are matched as `"key": value` pairs rather than `.*` so the match stops at the +# entry's own closing brace instead of running into its siblings. (Go's regexp is RE2, so +# the indent cannot be captured and back-referenced.) [[Repls]] -Old = '(?m)^Deployment history: .*\n' +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' New = '' -# And the stamp on jobs and pipelines, which the plan reports as a change of its own. -# Matched with the trailing comma and without, since it can be the only entry - in which -# case the whole "changes" object exists only because of recording, and goes too. +# Same entry when it is the last one in the object, so the comma is on the line before. [[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\}\n *\},?\n' -New = '' - -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\},\n' -New = '' - -# The stamp itself, where it appears inside a serialized deployment block (plan JSON, -# state dumps). Both orderings are covered: the pair can sit before or after the fields -# that stay, so the comma may be on this line or the one before. -[[Repls]] -Old = '(?m)^( *)"(deployment_id|version_id)": "[^"]*",\n' -New = '' +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = ''' +''' +# And when it is the only entry, the whole "changes" object exists because of recording. [[Repls]] -Old = ',(\n *"(deployment_id|version_id)": "[^"]*")+' +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/change-comment/out.test.toml b/acceptance/bundle/resources/volumes/change-comment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-comment/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-comment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-name/out.test.toml b/acceptance/bundle/resources/volumes/change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/root/env-not-a-directory/out.test.toml b/acceptance/bundle/root/env-not-a-directory/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-a-directory/out.test.toml +++ b/acceptance/bundle/root/env-not-a-directory/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/env-not-found/out.test.toml b/acceptance/bundle/root/env-not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-found/out.test.toml +++ b/acceptance/bundle/root/env-not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/not-found/out.test.toml b/acceptance/bundle/root/not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/not-found/out.test.toml +++ b/acceptance/bundle/root/not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/real-empty-dir/out.test.toml b/acceptance/bundle/root/real-empty-dir/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/real-empty-dir/out.test.toml +++ b/acceptance/bundle/root/real-empty-dir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run/app-with-job/out.test.toml b/acceptance/bundle/run/app-with-job/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/run/app-with-job/out.test.toml +++ b/acceptance/bundle/run/app-with-job/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/basic/out.test.toml b/acceptance/bundle/run/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/basic/out.test.toml +++ b/acceptance/bundle/run/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/diagnostics/out.test.toml b/acceptance/bundle/run/diagnostics/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/diagnostics/out.test.toml +++ b/acceptance/bundle/run/diagnostics/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/basic/out.test.toml b/acceptance/bundle/run/inline-script/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/basic/out.test.toml +++ b/acceptance/bundle/run/inline-script/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/cwd/out.test.toml b/acceptance/bundle/run/inline-script/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/cwd/out.test.toml +++ b/acceptance/bundle/run/inline-script/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-auth/out.test.toml b/acceptance/bundle/run/inline-script/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-auth/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-separator/out.test.toml b/acceptance/bundle/run/inline-script/no-separator/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-separator/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-separator/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/jobs/partial_run/out.test.toml b/acceptance/bundle/run/jobs/partial_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/jobs/partial_run/out.test.toml +++ b/acceptance/bundle/run/jobs/partial_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/no-state/out.test.toml b/acceptance/bundle/run/no-state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/no-state/out.test.toml +++ b/acceptance/bundle/run/no-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/refresh-flags/out.test.toml b/acceptance/bundle/run/refresh-flags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/refresh-flags/out.test.toml +++ b/acceptance/bundle/run/refresh-flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/basic/out.test.toml b/acceptance/bundle/run/scripts/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/basic/out.test.toml +++ b/acceptance/bundle/run/scripts/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/cwd/out.test.toml b/acceptance/bundle/run/scripts/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/cwd/out.test.toml +++ b/acceptance/bundle/run/scripts/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml +++ b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-precedence/out.test.toml b/acceptance/bundle/run/scripts/env-precedence/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-precedence/out.test.toml +++ b/acceptance/bundle/run/scripts/env-precedence/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-section/out.test.toml b/acceptance/bundle/run/scripts/env-section/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-section/out.test.toml +++ b/acceptance/bundle/run/scripts/env-section/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/exit_code/out.test.toml b/acceptance/bundle/run/scripts/exit_code/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/exit_code/out.test.toml +++ b/acceptance/bundle/run/scripts/exit_code/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/io/out.test.toml b/acceptance/bundle/run/scripts/io/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/io/out.test.toml +++ b/acceptance/bundle/run/scripts/io/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-auth/out.test.toml b/acceptance/bundle/run/scripts/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-auth/out.test.toml +++ b/acceptance/bundle/run/scripts/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml +++ b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no_content/out.test.toml b/acceptance/bundle/run/scripts/no_content/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no_content/out.test.toml +++ b/acceptance/bundle/run/scripts/no_content/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/math/out.test.toml b/acceptance/bundle/run/scripts/shell/math/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/math/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/math/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/state-wiped/out.test.toml b/acceptance/bundle/run/state-wiped/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/state-wiped/out.test.toml +++ b/acceptance/bundle/run/state-wiped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/dashboard_embed/out.test.toml b/acceptance/bundle/run_as/dashboard_embed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/dashboard_embed/out.test.toml +++ b/acceptance/bundle/run_as/dashboard_embed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_override/out.test.toml b/acceptance/bundle/run_as/empty_override/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_override/out.test.toml +++ b/acceptance/bundle/run_as/empty_override/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as/out.test.toml b/acceptance/bundle/run_as/empty_run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_sp/out.test.toml b/acceptance/bundle/run_as/empty_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user/out.test.toml b/acceptance/bundle/run_as/empty_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user/out.test.toml +++ b/acceptance/bundle/run_as/empty_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml +++ b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/job_default/out.test.toml b/acceptance/bundle/run_as/job_default/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/run_as/job_default/out.test.toml +++ b/acceptance/bundle/run_as/job_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run_as/model_serving_different/out.test.toml b/acceptance/bundle/run_as/model_serving_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_different/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/model_serving_matching/out.test.toml b/acceptance/bundle/run_as/model_serving_matching/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_matching/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_matching/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/out.test.toml +++ b/acceptance/bundle/run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml +++ b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml +++ b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/out.test.toml b/acceptance/bundle/scripts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/out.test.toml +++ b/acceptance/bundle/scripts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/restricted-execution/out.test.toml b/acceptance/bundle/scripts/restricted-execution/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/restricted-execution/out.test.toml +++ b/acceptance/bundle/scripts/restricted-execution/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/ambiguous/out.test.toml b/acceptance/bundle/select/ambiguous/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/ambiguous/out.test.toml +++ b/acceptance/bundle/select/ambiguous/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index 8b995e4d177..9c22c36f16e 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index 55ed5ee6619..c96f9a9f6c9 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/missing/out.test.toml b/acceptance/bundle/select/missing/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/missing/out.test.toml +++ b/acceptance/bundle/select/missing/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/rejected/out.test.toml b/acceptance/bundle/select/rejected/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/select/rejected/out.test.toml +++ b/acceptance/bundle/select/rejected/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/state/bad_env/out.test.toml b/acceptance/bundle/state/bad_env/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_env/out.test.toml +++ b/acceptance/bundle/state/bad_env/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/basic/out.test.toml b/acceptance/bundle/state/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/basic/out.test.toml +++ b/acceptance/bundle/state/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/engine_default/out.test.toml b/acceptance/bundle/state/engine_default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/engine_default/out.test.toml +++ b/acceptance/bundle/state/engine_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml +++ b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/modified_status/out.test.toml b/acceptance/bundle/summary/modified_status/out.test.toml index 7f4e2c0ca80..262e580a832 100644 --- a/acceptance/bundle/summary/modified_status/out.test.toml +++ b/acceptance/bundle/summary/modified_status/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"] diff --git a/acceptance/bundle/sync/dryrun/out.test.toml b/acceptance/bundle/sync/dryrun/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/dryrun/out.test.toml +++ b/acceptance/bundle/sync/dryrun/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/sync/out.test.toml b/acceptance/bundle/sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/out.test.toml +++ b/acceptance/bundle/sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-git/out.test.toml b/acceptance/bundle/syncroot/dotdot-git/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-git/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-git/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 4e136c6838f..4e97b0db661 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helper_username/out.test.toml b/acceptance/bundle/templates-machinery/helper_username/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_username/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_username/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/number-precision/out.test.toml b/acceptance/bundle/templates-machinery/number-precision/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/number-precision/out.test.toml +++ b/acceptance/bundle/templates-machinery/number-precision/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/supported-url/out.test.toml b/acceptance/bundle/templates-machinery/supported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/supported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/supported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 2f44fc0b7cc..99483caeee6 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 9f9b4934ffe..79230bd2367 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 9f9b4934ffe..79230bd2367 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index 50677b5f636..ed19028b891 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.UV_PYTHON = [ "3.9", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index 88bd948e0a9..464dbdb3ab7 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index 88bd948e0a9..464dbdb3ab7 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index c9acc1e0635..2ff57477252 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -1,11 +1,38 @@ # This allows recording per-deployment output files, e.g. $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" +# Runs the whole bundle suite a second time with deployment history recording on, so the +# deployment metadata service (DMS) is exercised by every test rather than only the +# handful under bundle/dms. Empty is the default pair of engine runs. +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] + +# DMS is only supported by the direct engine, and only against the local testserver: +# the service runs in dev and staging, so a cloud run has nothing to record to. +EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] +EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] + +# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and +# refuses a bundle whose state already tracks resources - which most tests here seed. +# Both are forced on: these tests assert what a deploy does, so the resource duplication +# the refusal guards against cannot bite them. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "1" + +# The DMS run asserts the same golden files as the engine runs. +EnvRepl.DATABRICKS_BUNDLE_DMS = false + Ignore = ["databricks.yml"] # The lowest Python version we support. Alternative to "uv run --python 3.10" Env.UV_PYTHON = "3.10" +# The link a recorded deploy prints, dropped so a test asserts the same output whether or +# not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and +# the calls behind it by bundle/dms/record. +[[Repls]] +Old = '(?m)^Deployment history: .*\n' +New = '' + # User-agent: [[Repls]] Old = 'os/darwin' diff --git a/acceptance/bundle/trampoline/warning_message/out.test.toml b/acceptance/bundle/trampoline/warning_message/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/undefined_resources/out.test.toml b/acceptance/bundle/undefined_resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/undefined_resources/out.test.toml +++ b/acceptance/bundle/undefined_resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/internal_server_error/out.test.toml b/acceptance/bundle/upload/internal_server_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/internal_server_error/out.test.toml +++ b/acceptance/bundle/upload/internal_server_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/timeout/out.test.toml b/acceptance/bundle/upload/timeout/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/timeout/out.test.toml +++ b/acceptance/bundle/upload/timeout/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/anchor_containers/out.test.toml +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/validate/dashboard_defaults/out.test.toml b/acceptance/bundle/validate/dashboard_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_defaults/out.test.toml +++ b/acceptance/bundle/validate/dashboard_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_name/out.test.toml b/acceptance/bundle/validate/dashboard_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_name/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml +++ b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/null/out.test.toml b/acceptance/bundle/validate/empty_resources/null/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/null/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/null/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_tasks/out.test.toml b/acceptance/bundle/validate/empty_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_tasks/out.test.toml +++ b/acceptance/bundle/validate/empty_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/engine-config-valid/out.test.toml b/acceptance/bundle/validate/engine-config-valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/engine-config-valid/out.test.toml +++ b/acceptance/bundle/validate/engine-config-valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum/out.test.toml b/acceptance/bundle/validate/enum/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum/out.test.toml +++ b/acceptance/bundle/validate/enum/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum_resource_refs/out.test.toml b/acceptance/bundle/validate/enum_resource_refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum_resource_refs/out.test.toml +++ b/acceptance/bundle/validate/enum_resource_refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/grants_required_principal/out.test.toml +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml +++ b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/include_locations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/include_locations/out.test.toml +++ b/acceptance/bundle/validate/include_locations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-target/out.test.toml b/acceptance/bundle/validate/invalid-engine-target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-target/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/job-references/out.test.toml b/acceptance/bundle/validate/job-references/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/job-references/out.test.toml +++ b/acceptance/bundle/validate/job-references/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml +++ b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_conversion/out.test.toml b/acceptance/bundle/validate/model_serving_conversion/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_conversion/out.test.toml +++ b/acceptance/bundle/validate/model_serving_conversion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/missing_name/out.test.toml b/acceptance/bundle/validate/models/missing_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/missing_name/out.test.toml +++ b/acceptance/bundle/validate/models/missing_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/user_id/out.test.toml b/acceptance/bundle/validate/models/user_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/user_id/out.test.toml +++ b/acceptance/bundle/validate/models/user_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml +++ b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/permissions/out.test.toml b/acceptance/bundle/validate/permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions/out.test.toml +++ b/acceptance/bundle/validate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/permissions_overlap/out.test.toml b/acceptance/bundle/validate/permissions_overlap/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions_overlap/out.test.toml +++ b/acceptance/bundle/validate/permissions_overlap/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml +++ b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix/out.test.toml b/acceptance/bundle/validate/presets_name_prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_name_prefix/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/presets_tags/out.test.toml b/acceptance/bundle/validate/presets_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_tags/out.test.toml +++ b/acceptance/bundle/validate/presets_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/required/out.test.toml b/acceptance/bundle/validate/required/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/required/out.test.toml +++ b/acceptance/bundle/validate/required/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml +++ b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/strict/out.test.toml b/acceptance/bundle/validate/strict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/strict/out.test.toml +++ b/acceptance/bundle/validate/strict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sync_patterns/out.test.toml b/acceptance/bundle/validate/sync_patterns/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sync_patterns/out.test.toml +++ b/acceptance/bundle/validate/sync_patterns/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml +++ b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/volume_defaults/out.test.toml b/acceptance/bundle/validate/volume_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/volume_defaults/out.test.toml +++ b/acceptance/bundle/validate/volume_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/arg-repeat/out.test.toml b/acceptance/bundle/variables/arg-repeat/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/arg-repeat/out.test.toml +++ b/acceptance/bundle/variables/arg-repeat/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cross-ref/out.test.toml b/acceptance/bundle/variables/complex-cross-ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cross-ref/out.test.toml +++ b/acceptance/bundle/variables/complex-cross-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle-self/out.test.toml b/acceptance/bundle/variables/complex-cycle-self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle-self/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle-self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle/out.test.toml b/acceptance/bundle/variables/complex-cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-simple/out.test.toml b/acceptance/bundle/variables/complex-simple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-simple/out.test.toml +++ b/acceptance/bundle/variables/complex-simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive/out.test.toml b/acceptance/bundle/variables/complex-transitive/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml +++ b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-within-complex/out.test.toml b/acceptance/bundle/variables/complex-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-within-complex/out.test.toml +++ b/acceptance/bundle/variables/complex-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex/out.test.toml b/acceptance/bundle/variables/complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex/out.test.toml +++ b/acceptance/bundle/variables/complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex_multiple_files/out.test.toml b/acceptance/bundle/variables/complex_multiple_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex_multiple_files/out.test.toml +++ b/acceptance/bundle/variables/complex_multiple_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/cycle/out.test.toml b/acceptance/bundle/variables/cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/cycle/out.test.toml +++ b/acceptance/bundle/variables/cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/double_underscore/out.test.toml b/acceptance/bundle/variables/double_underscore/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/double_underscore/out.test.toml +++ b/acceptance/bundle/variables/double_underscore/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/empty/out.test.toml b/acceptance/bundle/variables/empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/empty/out.test.toml +++ b/acceptance/bundle/variables/empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/env_overrides/out.test.toml b/acceptance/bundle/variables/env_overrides/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/env_overrides/out.test.toml +++ b/acceptance/bundle/variables/env_overrides/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/file-defaults/out.test.toml b/acceptance/bundle/variables/file-defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/file-defaults/out.test.toml +++ b/acceptance/bundle/variables/file-defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/git-branch/out.test.toml b/acceptance/bundle/variables/git-branch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/git-branch/out.test.toml +++ b/acceptance/bundle/variables/git-branch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/host/out.test.toml b/acceptance/bundle/variables/host/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/host/out.test.toml +++ b/acceptance/bundle/variables/host/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/int/out.test.toml b/acceptance/bundle/variables/int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/int/out.test.toml +++ b/acceptance/bundle/variables/int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_2436/out.test.toml b/acceptance/bundle/variables/issue_2436/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_2436/out.test.toml +++ b/acceptance/bundle/variables/issue_2436/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/lookup/out.test.toml b/acceptance/bundle/variables/lookup/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/variables/lookup/out.test.toml +++ b/acceptance/bundle/variables/lookup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml +++ b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-builtin/out.test.toml b/acceptance/bundle/variables/resolve-builtin/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-builtin/out.test.toml +++ b/acceptance/bundle/variables/resolve-builtin/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-empty/out.test.toml b/acceptance/bundle/variables/resolve-empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-empty/out.test.toml +++ b/acceptance/bundle/variables/resolve-empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml +++ b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml +++ b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml +++ b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/unicode_reference/out.test.toml b/acceptance/bundle/variables/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/unicode_reference/out.test.toml +++ b/acceptance/bundle/variables/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/vanilla/out.test.toml b/acceptance/bundle/variables/vanilla/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/vanilla/out.test.toml +++ b/acceptance/bundle/variables/vanilla/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/var_in_var/out.test.toml b/acceptance/bundle/variables/var_in_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/var_in_var/out.test.toml +++ b/acceptance/bundle/variables/var_in_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml +++ b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml +++ b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/without_definition/out.test.toml b/acceptance/bundle/variables/without_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/without_definition/out.test.toml +++ b/acceptance/bundle/variables/without_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_file/out.test.toml b/acceptance/bundle/volume_path/invalid_file/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_file/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_resource/out.test.toml b/acceptance/bundle/volume_path/invalid_resource/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_resource/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_resource/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_root/out.test.toml b/acceptance/bundle/volume_path/invalid_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_root/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_state/out.test.toml b/acceptance/bundle/volume_path/invalid_state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_state/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/valid/out.test.toml b/acceptance/bundle/volume_path/valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/valid/out.test.toml +++ b/acceptance/bundle/volume_path/valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] From f49c0ce95f3b9b5a221a5deb44267832ea1b6ca1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 18:16:19 +0000 Subject: [PATCH 53/56] bundle: extend the DMS acceptance run across bundle/resources The jobs tests were converted first; this does the rest of bundle/resources the same way - job and pipeline request assertions drop deployment_id and version_id with print_requests.py --del-body, and the tests that capture every request exclude the deployment metadata service's own calls, which bundle/dms asserts instead. bundle/resources now passes with recording both on and off, apart from 23 tests whose plan or state dumps still carry the stamp. Co-authored-by: Isaac --- .../bundle/resources/apps/lifecycle-started-omitted/script | 2 +- .../clusters/deploy/update-and-resize-autoscale/script | 2 +- .../bundle/resources/clusters/deploy/update-and-resize/script | 2 +- .../bundle/resources/dashboards/unpublish-out-of-band/script | 2 +- .../permissions/genie_spaces/current_can_manage/script | 2 +- .../resources/permissions/jobs/current_can_manage_run/script | 4 ++-- .../resources/permissions/jobs/other_can_manage_run/script | 4 ++-- .../resources/permissions/models/current_can_manage/script | 2 +- .../bundle/resources/permissions/target_permissions/script | 2 +- .../resources/pipelines/allow-duplicate-names/output.txt | 2 +- .../bundle/resources/pipelines/allow-duplicate-names/script | 2 +- .../bundle/resources/pipelines/remote_matches_config/script | 2 +- acceptance/bundle/resources/pipelines/update/script | 2 +- .../resources/quality_monitors/change_assets_dir/output.txt | 2 +- .../resources/quality_monitors/change_assets_dir/script | 2 +- .../quality_monitors/change_output_schema_name/output.txt | 2 +- .../quality_monitors/change_output_schema_name/script | 2 +- .../resources/quality_monitors/change_table_name/output.txt | 2 +- .../resources/quality_monitors/change_table_name/script | 2 +- .../bundle/resources/quality_monitors/create/output.txt | 2 +- acceptance/bundle/resources/quality_monitors/create/script | 2 +- .../secret_scopes/delete_scope/out.deploy.requests.txt | 2 +- acceptance/bundle/resources/secret_scopes/delete_scope/script | 2 +- acceptance/bundle/resources/volumes/change-name/script | 2 +- acceptance/bundle/resources/volumes/remote-change-name/script | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index c836adff587..79a1ff2e93b 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json > LOG.planjson +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 1370846fe4b..16928c71889 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index f2d80d05de0..06cdfc7caaf 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index 8631b921342..e6267ead0a8 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index ae805caebbc..22a34cb117e 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 96eaf8a4c75..3cfdd63e806 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -2,10 +2,10 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json # check plan to ensure there is not drift trace $CLI bundle plan -o json -t green > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script index 4c406d91cfa..6d9f9c0453f 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 9ac6f2cd41a..eb4446a728d 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 1a07dad637e..67a775cfb49 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..7f517420608 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipelines +>>> print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script index d121e073a3d..9055e4ba02b 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script @@ -16,4 +16,4 @@ export PIPELINE_ID # Deploy the bundle that has a pipeline with the same name: trace $CLI bundle deploy -trace print_requests.py //pipelines +trace print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 5b9c37402c7..ccbc1936b57 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index 255dea2e00d..f48e025aba2 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -4,7 +4,7 @@ touch bar.py trace $CLI bundle deploy print_requests() { - print_requests.py //pipelines + print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id read_state.py pipelines my id name } diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 08f6c53ae17..fba01abcffb 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 6caf49a7f4e..99e80f5f70a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -26,5 +26,5 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index d67ee41975f..c3e638e9907 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index 72c57e08401..a14879d79f1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index 879222aa8a4..9ee37b2f03d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 891aece1c11..6cf47e95c59 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -26,7 +26,7 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 8037d5ec9cc..5390fd078e6 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 78c7853b264..22aaeec97aa 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 2d469e4abce..e21f54fcf1f 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index b12e98775a3..c7bcc9bf0a5 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt trace $CLI bundle deploy -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index ba4d63c4032..a897616ae12 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 2485737fdf4..b4ca369abe4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' From 98c50c5907787c6735f37aa6bf12172e07d37da1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 18:22:00 +0000 Subject: [PATCH 54/56] bundle: drop the deployment stamp from get responses in the DMS run The last of bundle/resources: `jobs get` and `pipelines get` print the deployment block of the live resource, so with recording on it carries deployment_id and version_id. Each pattern anchors on the "kind" or "metadata_file_path" line that always sits beside them, so it cannot match an unrelated field of the same name. bundle/resources now passes with recording both on and off. Co-authored-by: Isaac --- acceptance/bundle/resources/test.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index df0bcf4b84a..7d509deb132 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -25,3 +25,16 @@ New = ''' [[Repls]] Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines +# get` and in state dumps. Both keys always sit alongside "kind" and +# "metadata_file_path", so each pattern anchors on one of those - that keeps it from +# matching an unrelated field named deployment_id elsewhere in the output. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n(?= *"kind": )' +New = '' + +[[Repls]] +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "\d+"\n' +New = '''$1 +''' From 518434f94f9f600de6796c72d5ef89f0cefa40a3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 19:21:11 +0000 Subject: [PATCH 55/56] bundle: run the resources acceptance tests with deployment history on Record an emptied-out resource as a delete rather than an update. DMS drops a resource from the deployment only for a delete, so an update left it listed with an id but no state, and the next plan failed to unmarshal it ("unexpected end of JSON input" on an emptied grants node). The rest is test-only. The stamp normalization needs Order = 20 so it runs after the root's numeric rules have turned the id into [NUMID], and a variant for gron.py's flattened output. bind/unbind is not supported yet, so the three postgres tests that stage state through unbind opt out of the DMS run. Co-authored-by: Isaac --- .../resources/apps/lifecycle-started/output.txt | 2 +- .../resources/apps/lifecycle-started/script | 2 +- .../replace_existing/out.test.toml | 2 +- .../replace_existing/test.toml | 3 +++ .../inherited-role-conflict/out.test.toml | 2 +- .../inherited-role-conflict/test.toml | 4 ++++ .../replace_existing/out.test.toml | 2 +- .../postgres_roles/replace_existing/test.toml | 3 +++ acceptance/bundle/resources/test.toml | 17 ++++++++++++++--- bundle/direct/apply.go | 6 +++++- 10 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_databases/replace_existing/test.toml create mode 100644 acceptance/bundle/resources/postgres_roles/replace_existing/test.toml diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index 4562a6b9b22..910f5722e38 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //deployments +>>> print_requests.py //deployments ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script index 710dec5a10a..15a9197c3c5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/script +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -15,7 +15,7 @@ rm -f out.requests.txt title "Re-deploy with description change: code deployed again" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION trace errcode $CLI bundle deploy -trace print_requests.py //deployments +trace print_requests.py //deployments ^//api/2.0/bundle rm -f out.requests.txt title "Stop app externally while config says started=true: plan detects drift" diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 2adb592001c..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml new file mode 100644 index 00000000000..31e3a00a75e --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the database as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index 67797faa7b2..50ada34cac8 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -3,5 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml index 3e475e74819..18183d0f8f0 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml @@ -5,3 +5,7 @@ Cloud = false # Deploy error wording differs between engines; the conflict itself is engine-agnostic. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the role staged above still looks tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 2adb592001c..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml new file mode 100644 index 00000000000..2123f2f1544 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the role as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 7d509deb132..d9d6f59b631 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -30,11 +30,22 @@ New = '' # get` and in state dumps. Both keys always sit alongside "kind" and # "metadata_file_path", so each pattern anchors on one of those - that keeps it from # matching an unrelated field named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. [[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n(?= *"kind": )' -New = '' +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 [[Repls]] -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "\d+"\n' +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^\n]*"\n' New = '''$1 ''' +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index e8f95daa48a..d615b5c3566 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -162,7 +162,11 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(ctx, d.ResourceKey, deployplan.Update) + // + // Recorded as a delete, not the update that caused it: the resource is no longer + // tracked, and DMS drops it from the deployment only for a delete. Recording an + // update would leave it listed with no state, which the next plan cannot read. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } From 9a0a8009f7675d719aceaaad4a2eb5c027fadbab Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 20:41:34 +0000 Subject: [PATCH 56/56] bundle: run the whole bundle acceptance suite with deployment history on The DATABRICKS_BUNDLE_DMS matrix variable now produces zero failures across acceptance/bundle. Recording is exercised by every test rather than only the handful under bundle/dms. Recording adds a deployment stamp to every job and pipeline, so each subtree that asserts a plan, a state dump, or a request body normalizes those two keys. The rules are repeated per subtree rather than living in the shared parent: bundle/dms asserts the stamp itself and would inherit them. Every pattern is anchored on an adjacent key so it cannot match an unrelated deployment_id, and version_id is required to be non-empty - a terraform state dump carries "version_id": "" for a job it never stamped. Opted out, each with the reason in its test.toml: the saved-plan path (a plan written before the deployment exists carries no stamp), terraform-to-direct migration and continue_293 (resources a pre-DMS deploy created are recorded nowhere), a 1000-task job (state exceeds the 64 KB per-operation limit), bind/unbind, templates (minutes of runtime for no new coverage), and two tests whose assertion is a state byte size or the User-Agent of every request. Co-authored-by: Isaac --- acceptance/bundle/ai_runtime_task/test.toml | 55 ++++++++++++++++++ acceptance/bundle/artifacts/test.toml | 56 +++++++++++++++++++ acceptance/bundle/bundle_tag/test.toml | 56 +++++++++++++++++++ .../deploy/readplan/basic/out.test.toml | 2 +- .../cli-version-mismatch/out.test.toml | 2 +- .../grants-remove-principal/out.test.toml | 2 +- .../readplan/invalid-plan/out.test.toml | 2 +- .../readplan/lineage-mismatch/out.test.toml | 2 +- .../readplan/plan-not-found/out.test.toml | 2 +- .../plan-version-mismatch/out.test.toml | 2 +- .../readplan/postgres_role/out.test.toml | 2 +- .../readplan/serial-mismatch/out.test.toml | 2 +- .../readplan/terraform-error/out.test.toml | 2 +- acceptance/bundle/deploy/readplan/test.toml | 6 ++ .../readplan/unknown-field/out.test.toml | 2 +- acceptance/bundle/deploy/test.toml | 56 +++++++++++++++++++ .../deployment/bind/alert/out.test.toml | 2 +- .../deployment/bind/catalog/out.test.toml | 2 +- .../deployment/bind/cluster/out.test.toml | 2 +- .../deployment/bind/dashboard/out.test.toml | 2 +- .../bind/dashboard/recreation/out.test.toml | 2 +- .../bind/database_instance/out.test.toml | 2 +- .../deployment/bind/experiment/out.test.toml | 2 +- .../bind/external_location/out.test.toml | 2 +- .../deployment/bind/genie_space/out.test.toml | 2 +- .../already-managed-different/out.test.toml | 2 +- .../job/already-managed-same/out.test.toml | 2 +- .../bind/job/engine-from-config/out.test.toml | 2 +- .../bind/job/generate-and-bind/out.test.toml | 2 +- .../bind/job/job-abort-bind/out.test.toml | 2 +- .../job/job-spark-python-task/out.test.toml | 2 +- .../bind/job/noop-job/out.test.toml | 2 +- .../bind/job/python-job/out.test.toml | 2 +- .../bind/job/stale-state/out.test.toml | 2 +- .../bind/model-serving-endpoint/out.test.toml | 2 +- .../bind/pipelines/recreate/out.test.toml | 2 +- .../bind/pipelines/update/out.test.toml | 2 +- .../bind/postgres_database/out.test.toml | 2 +- .../bind/postgres_role/out.test.toml | 2 +- .../bind/quality-monitor/out.test.toml | 2 +- .../bind/registered-model/out.test.toml | 2 +- .../deployment/bind/schema/out.test.toml | 2 +- .../bind/secret-scope/out.test.toml | 2 +- .../bind/sql_warehouse/out.test.toml | 2 +- acceptance/bundle/deployment/bind/test.toml | 2 + .../bind/vector_search_endpoint/out.test.toml | 2 +- .../bind/vector_search_index/out.test.toml | 2 +- .../deployment/bind/volume/out.test.toml | 2 +- acceptance/bundle/deployment/test.toml | 56 +++++++++++++++++++ .../unbind/engine-from-config/out.test.toml | 2 +- .../deployment/unbind/grants/out.test.toml | 2 +- .../deployment/unbind/job/out.test.toml | 2 +- .../unbind/permissions/out.test.toml | 2 +- .../unbind/python-job/out.test.toml | 2 +- acceptance/bundle/deployment/unbind/test.toml | 2 + acceptance/bundle/destroy/test.toml | 55 ++++++++++++++++++ acceptance/bundle/empty_string_dropped/script | 7 ++- .../bundle/empty_string_dropped/test.toml | 56 +++++++++++++++++++ acceptance/bundle/environments/test.toml | 55 ++++++++++++++++++ .../invariant/continue_293/out.test.toml | 2 +- .../bundle/invariant/continue_293/test.toml | 6 ++ .../bundle/invariant/no_drift/test.toml | 6 ++ acceptance/bundle/invariant/test.toml | 56 +++++++++++++++++++ acceptance/bundle/migrate/added/out.test.toml | 2 +- .../migrate/auto-migrate-clean/out.test.toml | 2 +- .../auto-migrate-empty-tfstate/out.test.toml | 2 +- .../migrate/auto-migrate-envvar/out.test.toml | 2 +- .../auto-migrate-push-failure/out.test.toml | 2 +- .../out.test.toml | 2 +- acceptance/bundle/migrate/basic/out.test.toml | 2 +- .../bundle/migrate/dashboards/out.test.toml | 2 +- .../migrate/default-python/out.test.toml | 2 +- .../engine-config-direct/out.test.toml | 2 +- .../engine-config-terraform/out.test.toml | 2 +- .../bundle/migrate/grants/out.test.toml | 2 +- .../bundle/migrate/permissions/out.test.toml | 2 +- .../bundle/migrate/profile_arg/out.test.toml | 2 +- .../bundle/migrate/removed/out.test.toml | 2 +- acceptance/bundle/migrate/runas/out.test.toml | 2 +- acceptance/bundle/migrate/test.toml | 6 ++ .../bundle/migrate/var_arg/out.test.toml | 2 +- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- acceptance/bundle/resource_deps/test.toml | 56 +++++++++++++++++++ acceptance/bundle/resources/test.toml | 44 ++++++++------- acceptance/bundle/run_as/test.toml | 55 ++++++++++++++++++ acceptance/bundle/select/test.toml | 56 +++++++++++++++++++ acceptance/bundle/state/test.toml | 55 ++++++++++++++++++ acceptance/bundle/summary/test.toml | 55 ++++++++++++++++++ .../config-remote-sync-error/out.test.toml | 2 +- .../config-remote-sync-recreate/out.test.toml | 2 +- .../config-remote-sync-save/out.test.toml | 2 +- .../config-remote-sync/out.test.toml | 2 +- .../out.test.toml | 2 +- .../deploy-artifact-path-type/out.test.toml | 2 +- .../deploy-artifacts-variables/out.test.toml | 2 +- .../deploy-compute-type/out.test.toml | 2 +- .../deploy-config-file-count/out.test.toml | 2 +- .../deploy-error-message/out.test.toml | 2 +- .../telemetry/deploy-error/out.test.toml | 2 +- .../deploy-experimental/out.test.toml | 2 +- .../telemetry/deploy-mode/out.test.toml | 2 +- .../deploy-name-prefix/custom/out.test.toml | 2 +- .../mode-development/out.test.toml | 2 +- .../telemetry/deploy-no-uuid/out.test.toml | 2 +- .../telemetry/deploy-run-as/out.test.toml | 2 +- .../deploy-target-count/out.test.toml | 2 +- .../deploy-variable-count/out.test.toml | 2 +- .../deploy-whl-artifacts/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/telemetry/deploy/out.test.toml | 2 +- acceptance/bundle/telemetry/test.toml | 4 ++ .../bundle/templates/dbt-sql/out.test.toml | 2 +- .../default-minimal/python/out.test.toml | 2 +- .../default-minimal/skip/out.test.toml | 2 +- .../default-minimal/sql/out.test.toml | 2 +- .../azure-government/out.test.toml | 2 +- .../default-python/classic/out.test.toml | 2 +- .../combinations/classic/out.test.toml | 2 +- .../combinations/serverless/out.test.toml | 2 +- .../fail-missing-uv/out.test.toml | 2 +- .../integration_classic/out.test.toml | 2 +- .../default-python/no-uc/out.test.toml | 2 +- .../serverless-customcatalog/out.test.toml | 2 +- .../default-python/serverless/out.test.toml | 2 +- .../templates/default-scala/out.test.toml | 2 +- .../templates/default-sql/out.test.toml | 2 +- .../lakeflow-integrations/out.test.toml | 2 +- .../lakeflow-pipelines/python/out.test.toml | 2 +- .../lakeflow-pipelines/sql/out.test.toml | 2 +- .../templates/nested-output/out.test.toml | 2 +- .../pydabs/check-consistency/out.test.toml | 2 +- .../pydabs/check-formatting/out.test.toml | 2 +- .../pydabs/deploy-classic/out.test.toml | 2 +- .../pydabs/init-classic/out.test.toml | 2 +- .../telemetry/custom-template/out.test.toml | 2 +- .../templates/telemetry/dbt-sql/out.test.toml | 2 +- .../telemetry/default-python/out.test.toml | 2 +- .../telemetry/default-sql/out.test.toml | 2 +- acceptance/bundle/templates/test.toml | 6 ++ acceptance/bundle/test.toml | 7 +++ acceptance/bundle/user_agent/out.test.toml | 2 +- .../bundle/user_agent/simple/out.test.toml | 2 +- acceptance/bundle/user_agent/test.toml | 5 ++ 144 files changed, 979 insertions(+), 144 deletions(-) create mode 100644 acceptance/bundle/ai_runtime_task/test.toml create mode 100644 acceptance/bundle/deploy/readplan/test.toml create mode 100644 acceptance/bundle/deployment/bind/test.toml create mode 100644 acceptance/bundle/deployment/unbind/test.toml create mode 100644 acceptance/bundle/destroy/test.toml create mode 100644 acceptance/bundle/environments/test.toml create mode 100644 acceptance/bundle/run_as/test.toml create mode 100644 acceptance/bundle/state/test.toml create mode 100644 acceptance/bundle/summary/test.toml diff --git a/acceptance/bundle/ai_runtime_task/test.toml b/acceptance/bundle/ai_runtime_task/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/ai_runtime_task/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index 61bf8345e7b..a8051bc8ca6 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -29,3 +29,59 @@ Response.Body = ''' "spark_version": "13.3.x-scala2.12" } ''' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/bundle_tag/test.toml b/acceptance/bundle/bundle_tag/test.toml index 8540f9500e6..ea76209cc5a 100644 --- a/acceptance/bundle/bundle_tag/test.toml +++ b/acceptance/bundle/bundle_tag/test.toml @@ -1 +1,57 @@ Badness = "configs with id and url should be rejected" + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 2962c9963cc..310be221793 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 76ce926fd59..25ad1a52fcf 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml new file mode 100644 index 00000000000..d5628ffe5fe --- /dev/null +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -0,0 +1,6 @@ +# These tests apply a saved plan, which does not carry the deployment stamp: on a first +# deploy there is no deployment to resolve when `bundle plan` runs, so the plan it writes +# leaves the field unset and applying it plans an update the next time. Same reason as +# EnvMatrixExclude.dms_no_readplan in acceptance/bundle/test.toml, which only covers the +# tests that take the saved-plan path through the READPLAN matrix variable. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/test.toml b/acceptance/bundle/deploy/test.toml index 84e8a4a1990..10c2541173a 100644 --- a/acceptance/bundle/deploy/test.toml +++ b/acceptance/bundle/deploy/test.toml @@ -3,3 +3,59 @@ Ignore = [ '.databricks', '__pycache__', ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index ce8dec17c30..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index 3f6826cd945..ea5b0803e06 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresCluster = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index c35c189b0af..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index c35c189b0af..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/test.toml b/acceptance/bundle/deployment/bind/test.toml new file mode 100644 index 00000000000..10bef2f1ccb --- /dev/null +++ b/acceptance/bundle/deployment/bind/test.toml @@ -0,0 +1,2 @@ +# Bind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index ce8dec17c30..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 8c71b922b55..a250199143a 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,5 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index c7c6f58ed6e..32ecf0fa454 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1 +1,57 @@ Cloud = true + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/test.toml b/acceptance/bundle/deployment/unbind/test.toml new file mode 100644 index 00000000000..2be1dcf74aa --- /dev/null +++ b/acceptance/bundle/deployment/unbind/test.toml @@ -0,0 +1,2 @@ +# Unbind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/destroy/test.toml b/acceptance/bundle/destroy/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/destroy/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index 973e436617c..f856c966e20 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -10,11 +10,12 @@ # fix in the initialize phase would drop them, and this golden would show that. $CLI bundle validate -o json -t direct | jq .resources > out.validate.json -# Exclude non-create traffic: workspace file ops and telemetry (nondeterministic). +# Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the +# deployment history calls the DMS run adds. trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t tf -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.terraform.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.terraform.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -t direct -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.direct.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.direct.json $TESTDIR/empty_sent.py diff --git a/acceptance/bundle/empty_string_dropped/test.toml b/acceptance/bundle/empty_string_dropped/test.toml index 51e7bc13e23..ebc74f5b64f 100644 --- a/acceptance/bundle/empty_string_dropped/test.toml +++ b/acceptance/bundle/empty_string_dropped/test.toml @@ -10,3 +10,59 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ ".databricks", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/environments/test.toml b/acceptance/bundle/environments/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/environments/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index c9d202227e3..663a49fa779 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,7 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index c6fba9c43fb..5daa377779c 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,3 +1,9 @@ +# The seed deploy runs an old CLI that knows nothing about the deployment metadata +# service, so the resources it creates are recorded nowhere. Reading state from the +# service then finds none and plans a create on top of them. Adopting resources a +# pre-DMS CLI deployed is a migration story of its own, not something this test covers. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + # $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index ff8a66c196e..ddcf203eb9a 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] + +# A 1000-task job serializes to ~110 KB, over the 64 KB per-operation state limit the +# deployment metadata service accepts. Recording skips the resource with a warning, so it +# is absent from the state the service reports and the next plan wants to create it again. +# Raising the limit or splitting the state is a service-side decision. +EnvMatrixExclude.dms_state_too_large = ["DATABRICKS_BUNDLE_DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 1d0d883f6d5..d20dc220ff5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -125,3 +125,59 @@ Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"col [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 2962c9963cc..310be221793 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 8c4484f983a..964175b8b01 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -7,3 +7,9 @@ Ignore = [".databricks"] # matrix to ["direct"] so CI's engine filter includes these tests without # also running the same script twice per engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# These tests deploy on terraform first and then migrate to direct. Recording is a +# direct-engine feature, so the resources the terraform half creates are recorded nowhere +# and the migration reads state the service does not have. Migrating a deployment onto the +# service is a story of its own; these tests are not it. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..2f48d384b3f 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -29,7 +29,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort ^//import-file/ +>>> print_requests.py --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index d38692366b3..2ce8c3e3c5e 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' +trace print_requests.py --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index dc29b70c320..405ffae696a 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -4,3 +4,59 @@ Ignore = [ ".databricks", ".gitignore", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index d9d6f59b631..3d68723dbe7 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,16 +1,16 @@ RecordRequests = true -# These normalize what deployment history recording adds to a deploy's output, so a test +# These normalize the deployment stamp recording adds to every job and pipeline, so a test # asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They live here -# rather than in the parent so bundle/dms, which asserts the recording itself, does not -# inherit them. +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. # -# The stamp on jobs and pipelines, which the plan reports as a change of its own. It shows -# up at whatever depth the enclosing object sits at, so the indent is matched loosely; the -# body lines are matched as `"key": value` pairs rather than `.*` so the match stops at the -# entry's own closing brace instead of running into its siblings. (Go's regexp is RE2, so -# the indent cannot be captured and back-referenced.) +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) [[Repls]] Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' New = '' @@ -18,18 +18,23 @@ New = '' # Same entry when it is the last one in the object, so the comma is on the line before. [[Repls]] Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = ''' -''' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" -# And when it is the only entry, the whole "changes" object exists because of recording. [[Repls]] Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines -# get` and in state dumps. Both keys always sit alongside "kind" and -# "metadata_file_path", so each pattern anchors on one of those - that keeps it from -# matching an unrelated field named deployment_id elsewhere in the output. +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. # # Order puts these after the root's numeric rules (Order = 10), which have by then turned # the id into [NUMID]. @@ -39,9 +44,10 @@ New = '${1}' Order = 20 [[Repls]] -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^\n]*"\n' -New = '''$1 -''' +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" Order = 20 # Same two keys in gron.py's flattened form, where each is its own line. diff --git a/acceptance/bundle/run_as/test.toml b/acceptance/bundle/run_as/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/run_as/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/select/test.toml b/acceptance/bundle/select/test.toml index 85ce448afd3..792257f226f 100644 --- a/acceptance/bundle/select/test.toml +++ b/acceptance/bundle/select/test.toml @@ -1,3 +1,59 @@ Local = true Cloud = false Ignore = [".databricks", ".gitignore"] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/state/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/summary/test.toml b/acceptance/bundle/summary/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/summary/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 4e97b0db661..ef2b279f225 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 14453c07f92..92804cc8ed9 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,6 +1,10 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] +# Telemetry reports the byte size of the serialized state, which the deployment stamp +# legitimately grows. The number is the assertion here, so there is nothing to normalize. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 99483caeee6..8a113e1dfd4 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 79230bd2367..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 79230bd2367..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index ed19028b891..e5eb55f33db 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.UV_PYTHON = [ "3.9", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index b827ff3f062..40a9dbfa26b 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index 464dbdb3ab7..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index 464dbdb3ab7..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 977f3725d01..5d7fc7bfa7a 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,5 +1,11 @@ # Local-only: At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +# A template test materializes a whole project and deploys it, taking tens of seconds each, +# and some diff against a sibling test's output directory. Running all of that a second time +# for deployment history recording costs minutes and adds no coverage the rest of the suite +# does not already give, so these opt out. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Server]] Pattern = "POST /telemetry-ext" Response.Body = ''' diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 2ff57477252..32ccd18e452 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -11,6 +11,13 @@ EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] +# A saved plan does not carry the deployment stamp. On a first deploy there is no +# deployment to resolve when `bundle plan` runs, so the plan it writes leaves the field +# unset; `deploy --plan` then creates the resources without it and the next plan reports +# drift. Stamping at plan time would mean `bundle plan` creating the deployment record, +# which is a design decision, so the saved-plan path is left out of the DMS run for now. +EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_DMS=true", "READPLAN=1"] + # Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and # refuses a bundle whose state already tracks resources - which most tests here seed. # Both are forced on: these tests assert what a deploy does, so the resource duplication diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index b827ff3f062..40a9dbfa26b 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index a9f876b6b77..59b0f354509 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -3,5 +3,10 @@ RecordRequests = true Local = true IncludeRequestHeaders = ["User-Agent"] +# This test asserts the User-Agent on every single request the CLI makes, so recording's +# extra calls belong in the golden rather than being filtered out - but they are the same +# header the existing requests already cover, so the DMS run only adds entries to maintain. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false'