direct: Warn when state was written by a newer CLI version

.nextchanges/bundles/state-newer-cli-version-warning.md

@@ -0,0 +1 @@
+Warn when the deployment state was last written by a newer CLI version than the one running.

.nextchanges/cli/dev-build-version.md

@@ -0,0 +1 @@
+A locally built CLI (`go build`, without release flags) now reports the next release version with a `-dev` prerelease, e.g. `1.12.0-dev+abcdef123456`, instead of `0.0.0-dev+abcdef123456`. The old string sorted below every published release even though a local build is newer than the latest release; the new one sorts above the latest release and below the release it will become, matching what goreleaser already produces for snapshot builds.

.nextchanges/nextversion.go

@@ -0,0 +1,21 @@
+// Package nextchanges exposes the next release version to the build.
+//
+// It lives in this directory because go:embed cannot reach a parent directory:
+// embedding the version file from internal/build would require a second copy of
+// the value, which could then drift from this one.
+package nextchanges
+
+import (
+	_ "embed"
+	"strings"
+)
+
+// versionFile is the raw contents of the version file. It has a trailing newline
+// (the whitespace linter requires one), so it is not exported directly.
+//
+//go:embed version
+var versionFile string
+
+// Version is the next release version, e.g. "1.12.0". The release tooling bumps
+// the embedded file after each release; see README.md.
+var Version = strings.TrimSpace(versionFile)

acceptance/bundle/state/newer_cli_version/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: "my job"

acceptance/bundle/state/newer_cli_version/output.txt

@@ -0,0 +1,15 @@
+
+=== State written by a newer CLI: warn, but proceed
+
+>>> [CLI] bundle plan
+Warn: State was last deployed with CLI version 99.0.0 but current version is [CLI_VERSION]
+create jobs.my_job
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+
+=== State written by this CLI: no warning
+
+>>> [CLI] bundle plan
+create jobs.my_job
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged

acceptance/bundle/state/newer_cli_version/resources.newer.json

@@ -0,0 +1,7 @@
+{
+ "state_version": 2,
+ "cli_version": "99.0.0",
+ "lineage": "test-lineage",
+ "serial": 1,
+ "state": {}
+}

acceptance/bundle/state/newer_cli_version/script

@@ -0,0 +1,11 @@
+title "State written by a newer CLI: warn, but proceed\n"
+mkdir -p .databricks/bundle/default
+cp resources.newer.json .databricks/bundle/default/resources.json
+trace $CLI bundle plan
+
+title "State written by this CLI: no warning\n"
+# The running version is only known at test time, so derive the state's
+# cli_version from the binary itself rather than committing it as a fixture.
+version=$($CLI version | sed 's/^Databricks CLI v//')
+jq --arg v "$version" '.cli_version = $v' resources.newer.json > .databricks/bundle/default/resources.json
+trace $CLI bundle plan

acceptance/bundle/state/newer_cli_version/test.toml

@@ -0,0 +1,4 @@
+Ignore = [".databricks"]
+
+# The warning is emitted when reading the direct engine's resources.json.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/state/permission_level_migration/output.txt

@@ -13,7 +13,7 @@ Deployment complete!
 >>> print_state.py
 {
  "state_version": 2,
- "cli_version": "[CLI_VERSION]",
+ "cli_version": "0.0.0-test",
  "lineage": "test-lineage",
  "serial": 2,
  "state": {

acceptance/bundle/state/permission_level_migration/resources.v1.json

@@ -1,6 +1,6 @@
 {
  "state_version": 1,
- "cli_version": "0.0.0-dev",
+ "cli_version": "0.0.0-test",
  "lineage": "test-lineage",
  "serial": 1,
  "state": {

acceptance/bundle/telemetry/deploy/script

@@ -13,7 +13,7 @@ trace cat telemetry.json | jq ' .entry.databricks_cli_log.bundle_deploy_event.ex
 # omit it from the engine-agnostic out.telemetry.txt below. Sizes are deterministic
 # for a fixed config and asserted exactly. state_file_size_bytes is dropped because
 # it is os.Stat of resources.json, whose header embeds the CLI version string
-# (0.0.0-dev+<sha> on linux/macos vs 0.0.0-dev on windows).
+# (<version>-dev+<sha> on linux/macos vs <version>-dev on windows).
 cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources_metadata | if . then del(.state_file_size_bytes) else . end' > out.resources_metadata.$DATABRICKS_BUNDLE_ENGINE.txt
 
 # The dry-run migration to the direct engine runs only after a terraform deploy,

acceptance/cmd/bundle/dms-read-only/output.txt

@@ -32,7 +32,7 @@
 >>> [CLI] bundle-deployments list-versions deployments/abc
 [
   {
-    "cli_version": "[CLI_VERSION]",
+    "cli_version": "0.0.0-test",
     "name": "deployments/abc/versions/v1",
     "status": "VERSION_STATUS_COMPLETED",
     "version_type": ""
@@ -41,7 +41,7 @@
 
 >>> [CLI] bundle-deployments get-version deployments/abc/versions/v1
 {
-  "cli_version": "[CLI_VERSION]",
+  "cli_version": "0.0.0-test",
   "git_info": {
     "branch": "main",
     "commit": "[COMMIT_SHA]",

acceptance/cmd/bundle/dms-read-only/test.toml

@@ -65,7 +65,7 @@ Response.Body = '''
   "versions": [
     {
       "name": "deployments/abc/versions/v1",
-      "cli_version": "0.0.0-dev",
+      "cli_version": "0.0.0-test",
       "status": "VERSION_STATUS_COMPLETED"
     }
   ]
@@ -77,7 +77,7 @@ Pattern = "GET /api/2.0/bundle/deployments/abc/versions/v1"
 Response.Body = '''
 {
   "name": "deployments/abc/versions/v1",
-  "cli_version": "0.0.0-dev",
+  "cli_version": "0.0.0-test",
   "status": "VERSION_STATUS_COMPLETED",
   "git_info": {
     "branch": "main",

bundle/config/mutator/verify_cli_version.go

@@ -39,7 +39,10 @@ func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
 	}
 
 	if !c.Check(version) {
-		if version.Prerelease() == "dev" && version.Major() == 0 {
+		// Development builds are built from main and may already carry the change
+		// the constraint requires, so the constraint is a warning rather than an
+		// error. Masterminds reports the prerelease without its leading dash.
+		if version.Prerelease() == "dev" {
 			return diag.Warningf("Ignoring Databricks CLI version constraint for development build. Required: %s, current: %s", constraint, currentVersion)
 		}
 

bundle/deploy/terraform/init_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/internal/build"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/hashicorp/go-version"
@@ -237,8 +238,9 @@ func TestSetUserAgentExtra_Python(t *testing.T) {
 	env := make(map[string]string, 0)
 	err := setUserAgentExtraEnvVar(env, b)
 	require.NoError(t, err)
+	// The CLI version is the test binary's own, which is the not-injected default.
 	assert.Equal(t, map[string]string{
-		"DATABRICKS_USER_AGENT_EXTRA": "cli/0.0.0-dev databricks-pydabs/0.7.0",
+		"DATABRICKS_USER_AGENT_EXTRA": "cli/" + build.DefaultSemver + " databricks-pydabs/0.7.0",
 	}, env)
 }
 

bundle/direct/dstate/state.go

@@ -184,6 +184,18 @@ func (db *DeploymentState) GetResourceID(key string) string {
 	return db.stateIDs[key]
 }
 
+// StateCLIVersion returns the CLI version that last wrote the state, or an empty
+// string if the state does not record one (a fresh state that this CLI has not
+// written yet). It is the version stored in the on-disk header, not the running
+// build's version.
+func (db *DeploymentState) StateCLIVersion() string {
+	db.AssertOpenedForReadOrWrite()
+	db.mu.Lock()
+	defer db.mu.Unlock()
+
+	return db.Data.CLIVersion
+}
+
 // GetOrInitLineage returns the deployment lineage, generating and storing a new
 // one if the state does not have one yet. It is the single place the lineage is
 // initialized, shared so the direct deployment engine (when it writes state, via

cmd/bundle/utils/process.go

@@ -26,6 +26,7 @@ import (
 	"github.com/databricks/cli/libs/sync"
 	"github.com/databricks/cli/libs/telemetry/protos"
 	"github.com/spf13/cobra"
+	"golang.org/x/mod/semver"
 )
 
 type ProcessOptions struct {
@@ -215,6 +216,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
 				logdiag.LogError(ctx, err)
 				return b, stateDesc, root.ErrAlreadyPrinted
 			}
+
+			// Warn when the state was last written by a newer CLI than the one
+			// running now. The state schema version is a hard gate (dstate.Open
+			// rejects a too-new state_version), but a state can be written by a
+			// newer CLI that shares this schema; that is allowed, and this only
+			// hints that a downgrade may be unintended.
+			currentVersion := build.GetInfo().Version
+			if stateVersion := b.DeploymentBundle.StateDB.StateCLIVersion(); isNewerVersion(stateVersion, currentVersion) {
+				log.Warnf(ctx, "State was last deployed with CLI version %s but current version is %s", stateVersion, currentVersion)
+			}
 		}
 
 		// These are not safe in plan/deploy because they insert empty config settings for deleted resources.
@@ -396,6 +407,19 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS
 	return engine.EngineSetting{}, nil
 }
 
+// isNewerVersion reports whether the state's recorded CLI version is strictly
+// newer than the running build. Both are bare versions without a leading "v".
+// An empty stateVersion (state not written by any CLI yet) or an unparseable
+// version returns false, so we never warn on missing or malformed data.
+func isNewerVersion(stateVersion, currentVersion string) bool {
+	sv := "v" + stateVersion
+	cv := "v" + currentVersion
+	if !semver.IsValid(sv) || !semver.IsValid(cv) {
+		return false
+	}
+	return semver.Compare(sv, cv) > 0
+}
+
 func rejectDefinitions(ctx context.Context, b *bundle.Bundle) {
 	if b.Config.Definitions != nil {
 		v := dyn.GetValue(b.Config.Value(), "definitions")

cmd/bundle/utils/process_test.go

@@ -0,0 +1,44 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsNewerVersion(t *testing.T) {
+	tests := []struct {
+		name    string
+		state   string
+		current string
+		want    bool
+	}{
+		{"state newer major", "1.0.0", "0.300.0", true},
+		{"state newer minor", "0.301.0", "0.300.0", true},
+		{"state newer patch", "0.300.1", "0.300.0", true},
+		{"same version", "0.300.0", "0.300.0", false},
+		{"state older", "0.299.0", "0.300.0", false},
+		// A dev build is built from main, so its version is the next release with a
+		// -dev prerelease: newer than the last release, older than the release it
+		// will become. Deploying with a dev build after a state written by the last
+		// release is the normal case for a CLI developer and must not warn.
+		{"state from last release, dev current", "0.300.0", "0.301.0-dev+abc123", false},
+		// A released CLI reading a state written by a dev build of the same upcoming
+		// release does warn: that build may have written fields this CLI lacks.
+		{"dev state, released current", "0.301.0-dev+abc123", "0.300.0", true},
+		// A prerelease sorts below its own release per semver.
+		{"prerelease below release", "0.300.0-rc1", "0.300.0", false},
+		{"release above prerelease", "0.300.0", "0.300.0-rc1", true},
+		// Missing or malformed data must never produce a warning.
+		{"empty state version", "", "0.300.0", false},
+		{"empty current version", "0.300.0", "", false},
+		{"malformed state version", "not-a-version", "0.300.0", false},
+		{"malformed current version", "0.300.0", "not-a-version", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, isNewerVersion(tt.state, tt.current))
+		})
+	}
+}

internal/build/info.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	nextchanges "github.com/databricks/cli/.nextchanges"
 	"golang.org/x/mod/semver"
 )
 
@@ -42,7 +43,65 @@ func (i Info) GetSanitizedVersion() string {
 	return version
 }
 
-const DefaultSemver = "0.0.0-dev"
+// devIdentifier is the prerelease identifier marking a build that was not
+// produced from a release tag.
+const devIdentifier = "dev"
+
+// DefaultSemver is the version reported when buildVersion was not injected,
+// i.e. a plain "go build" rather than a goreleaser build. It is the next release
+// version with a dev prerelease identifier, so it sorts above the latest release
+// and below the release it will become:
+//
+//	Compare(v1.11.0, v1.12.0-dev+sha) = -1
+//	Compare(v1.12.0, v1.12.0-dev+sha) = +1
+//
+// A bare "0.0.0-dev" would instead sort below every published release, even
+// though a local build is built from main and is therefore newer than the
+// latest release. This matches what goreleaser produces for snapshot builds
+// (see snapshot.version_template in .goreleaser.yaml).
+var DefaultSemver = devVersion(nextchanges.Version)
+
+// devVersion returns the development-build version for an upcoming release,
+// normally a plain version with a dev prerelease ("1.12.0" -> "1.12.0-dev").
+//
+// The prerelease branch is for completeness with next_release_version() in
+// internal/genkit/tagging.py, which bumps the prerelease instead of the minor
+// when .nextchanges/version already carries one; it is not because this repo
+// publishes prereleases (it never has). Appending "-dev" to "1.13.0-rc.2" would
+// produce the prerelease "rc.2-dev", a single identifier that merely ends in
+// "dev", so IsDevelopmentVersion would report false and every dev-build
+// exemption would silently switch off. A dot-separated identifier
+// ("1.13.0-rc.2.dev") keeps detection working and still sorts after the rc and
+// before the final release.
+func devVersion(next string) string {
+	if semver.Prerelease("v"+next) != "" {
+		return next + "." + devIdentifier
+	}
+	return next + "-" + devIdentifier
+}
+
+// IsDevelopmentVersion reports whether a version string is a development build's.
+// It keys off the trailing dev prerelease identifier rather than a specific
+// version number, so it keeps working as the next release version changes. The
+// version may be given with or without a leading "v".
+//
+// Prefer Info.IsDevelopment when you have the running build's Info; this is for
+// callers holding only a version string (e.g. one read from a file or a
+// parameter), so the definition of "development build" lives in one place.
+func IsDevelopmentVersion(version string) bool {
+	prerelease := strings.TrimPrefix(semver.Prerelease("v"+strings.TrimPrefix(version, "v")), "-")
+	if prerelease == "" {
+		return false
+	}
+	identifiers := strings.Split(prerelease, ".")
+	return identifiers[len(identifiers)-1] == devIdentifier
+}
+
+// IsDevelopment reports whether this binary was built from a development or
+// snapshot build rather than a release tag.
+func (i Info) IsDevelopment() bool {
+	return i.IsSnapshot || IsDevelopmentVersion(i.Version)
+}
 
 // getDefaultBuildVersion uses build information stored by Go itself
 // to synthesize a build version if one wasn't set.