Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions acceptance/localenv/merge-warnings-json/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 76 additions & 0 deletions acceptance/localenv/merge-warnings-json/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@

>>> [CLI] environments setup-local --serverless-version 4 --dry-run --output json
{
"schemaVersion": 1,
"command": "environments setup-local",
"ok": true,
"mode": "default",
"dryRun": true,
"compute": {
"source": "serverless",
"serverlessVersion": "v4",
"envKey": "serverless/serverless-v4"
},
"resolved": {
"pythonVersion": "3.12",
"dbconnectVersion": "17.2.0",
"artifactSource": "network"
},
"greenfield": false,
"plan": {
"wouldWrite": "[TEST_TMP_DIR]/pyproject.toml",
"wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak",
"wouldInstallPython": "3.12",
"diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,17 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n spark = [\"databricks-connect==15.0.0\"]\n qa = [\"pandas==4.0.0\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n"
},
"phases": [
{
"phase": "preflight",
"status": "ok"
},
{
"phase": "resolve",
"status": "ok"
},
{
"phase": "fetch",
"status": "ok"
},
{
"phase": "merge",
"status": "ok"
},
{
"phase": "provision",
"status": "ok"
},
{
"phase": "validate",
"status": "ok"
}
],
"warnings": [
{
"code": "W_REQUIRES_PYTHON_OVERRIDDEN",
"message": "requires-python \"\u003e=3.10\" is replaced by the environment's \"\u003e=3.12\""
},
{
"code": "W_DBCONNECT_PIN_OVERRIDDEN",
"message": "databricks-connect \"databricks-connect~=16.0.0\" is replaced by the environment's \"databricks-connect~=17.2.0\""
},
{
"code": "W_DBCONNECT_PIN_DUPLICATED",
"message": "databricks-connect \"databricks-connect==15.0.0\" is not rewritten by the merge; the environment's \"databricks-connect~=17.2.0\" sits in \"dev\" alongside it, and no version satisfies both"
},
{
"code": "W_USER_CONSTRAINT_CONFLICT",
"message": "dependency \"pyarrow==21.0.0\" conflicts with the environment constraint \"pyarrow\u003c19\""
},
{
"code": "W_USER_CONSTRAINT_CONFLICT",
"message": "dependency \"pandas==4.0.0\" conflicts with the environment constraint \"pandas\u003c3\""
}
],
"error": null,
"durationMs": 0
}
16 changes: 16 additions & 0 deletions acceptance/localenv/merge-warnings-json/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# An existing project whose pins the merge overrides, plus a dependency the env's
# constraints exclude and a pin in a non-dev group uv still locks. All four warning
# conditions in one run, so warnings[] is exercised end to end.
cat > pyproject.toml <<'PY'
[project]
name = "demo"
requires-python = ">=3.10"
dependencies = ["pyarrow==21.0.0"]

[dependency-groups]
dev = ["databricks-connect~=16.0.0", {include-group = "spark"}]
spark = ["databricks-connect==15.0.0"]
qa = ["pandas==4.0.0"]
PY

trace $CLI environments setup-local --serverless-version 4 --dry-run --output json
24 changes: 24 additions & 0 deletions acceptance/localenv/merge-warnings-json/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

# The script writes pyproject.toml as the merge input; --dry-run leaves it unchanged.
Ignore = ["pyproject.toml"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /serverless/serverless-v4/pyproject.toml"
Response.Body = '''
[project]
requires-python = ">=3.12"

[dependency-groups]
dev = ["databricks-connect~=17.2.0"]

[tool.uv]
constraint-dependencies = ["pyarrow<19", "pandas<3"]
'''

[[Repls]]
Old = 'uv uv \S+(?: \([^)]+\))?'
New = 'uv [UV_VERSION]'
3 changes: 3 additions & 0 deletions acceptance/localenv/merge-warnings/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 18 additions & 0 deletions acceptance/localenv/merge-warnings/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@

>>> [CLI] environments setup-local --serverless-version 4 --dry-run
preflight ok check
resolve ok source=serverless envKey=serverless/serverless-v4
fetch ok source=[DATABRICKS_URL]/serverless/serverless-v4/pyproject.toml fromCache=false
merge ok
provision ok
validate ok
warning: requires-python ">=3.10" is replaced by the environment's ">=3.12"
warning: databricks-connect "databricks-connect~=16.0.0" is replaced by the environment's "databricks-connect~=17.2.0"
warning: databricks-connect "databricks-connect==15.0.0" is not rewritten by the merge; the environment's "databricks-connect~=17.2.0" sits in "dev" alongside it, and no version satisfies both
warning: dependency "pyarrow==21.0.0" conflicts with the environment constraint "pyarrow<19"
warning: dependency "pandas==4.0.0" conflicts with the environment constraint "pandas<3"
Plan: [TEST_TMP_DIR]/pyproject.toml
changed region: requires-python
changed region: databricks-connect
changed region: tool.uv.constraint-dependencies
Check complete. No files were modified.
15 changes: 15 additions & 0 deletions acceptance/localenv/merge-warnings/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# The text renderer's warning lines, which the --json variant does not exercise.
# See merge-warnings-json for the same input through --output json.
cat > pyproject.toml <<'PY'
[project]
name = "demo"
requires-python = ">=3.10"
dependencies = ["pyarrow==21.0.0"]

[dependency-groups]
dev = ["databricks-connect~=16.0.0", {include-group = "spark"}]
spark = ["databricks-connect==15.0.0"]
qa = ["pandas==4.0.0"]
PY

trace $CLI environments setup-local --serverless-version 4 --dry-run
24 changes: 24 additions & 0 deletions acceptance/localenv/merge-warnings/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

# The script writes pyproject.toml as the merge input; --dry-run leaves it unchanged.
Ignore = ["pyproject.toml"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /serverless/serverless-v4/pyproject.toml"
Response.Body = '''
[project]
requires-python = ">=3.12"

[dependency-groups]
dev = ["databricks-connect~=17.2.0"]

[tool.uv]
constraint-dependencies = ["pyarrow<19", "pandas<3"]
'''

[[Repls]]
Old = 'uv uv \S+(?: \([^)]+\))?'
New = 'uv [UV_VERSION]'
84 changes: 57 additions & 27 deletions libs/localenv/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"regexp"
"slices"
"strings"
)

Expand Down Expand Up @@ -44,6 +45,30 @@ var (
requiresPythonRe = regexp.MustCompile(`^(\s*)requires-python\s*=`)
)

// replacedDBConnectPin returns the databricks-connect requirement that merging
// target would rewrite in place, or "" when the merge would instead insert the
// managed pin (leaving anything already there untouched).
//
// detectMergeWarnings uses this rather than re-deciding which pins the merge
// recognizes: the merge matches double-quoted elements within the array devKeyRe
// finds, and any second implementation of that rule drifts from it. A detector that
// disagrees reports an override for a pin the merge never touched, which is a false
// claim about the user's file rather than a missed warning.
func replacedDBConnectPin(target []byte, c Constraints) string {
if c.DatabricksConnect == "" {
return ""
}
// Mirror MergeManaged's own preprocessing so the same lines are inspected.
lines := strings.Split(strings.ReplaceAll(string(target), "\r\n", "\n"), "\n")
if containsMultilineString(lines) {
return ""
}
// mergeDatabricksConnect rewrites element lines in place, so hand it a copy: this
// probe must not disturb the caller's view of the pre-merge file.
_, replaced, _ := mergeDatabricksConnect(slices.Clone(lines), c.DatabricksConnect)
return replaced
}

// MergeManaged applies the three managed transforms to target, preserving every other
// byte (comments, ordering, whitespace). It returns the merged bytes and the list of
// regions that actually changed. The operation is idempotent: feeding its own output
Expand Down Expand Up @@ -87,7 +112,7 @@ func MergeManaged(target []byte, c Constraints) (merged []byte, regions []string
regions = append(regions, regionRequiresPython)
}

lines, dbcChanged := mergeDatabricksConnect(lines, c.DatabricksConnect)
lines, _, dbcChanged := mergeDatabricksConnect(lines, c.DatabricksConnect)
if dbcChanged {
regions = append(regions, regionDatabricksConnect)
}
Expand Down Expand Up @@ -237,16 +262,21 @@ var devKeyRe = regexp.MustCompile(`^\s*dev\s*=`)
// databricks-connect pin sitting in a sibling group (e.g. docs/test) or inside a
// trailing comment on some other line is never clobbered. The insert path is
// idempotent: a subsequent merge finds the element and rewrites it in place.
func mergeDatabricksConnect(lines []string, value string) ([]string, bool) {
//
// replacedPin is the requirement it rewrote in place, empty when it inserted the
// managed pin instead. That distinction is what detectMergeWarnings needs: only a
// rewrite means the user's pin is gone, and only the merge itself can say which
// spellings it recognizes.
func mergeDatabricksConnect(lines []string, value string) (out []string, replacedPin string, changed bool) {
if value == "" {
return lines, false
return lines, "", false
}
elem := `"` + value + `"`

header, end, found := tableBounds(lines, "[dependency-groups]")
if !found {
// No [dependency-groups] table: append a fresh managed dev group.
return appendManagedBlock(lines, []string{"[dependency-groups]", "dev = [", " " + elem + ",", "]"}), true
return appendManagedBlock(lines, []string{"[dependency-groups]", "dev = [", " " + elem + ",", "]"}), "", true
}

// Locate the dev assignment and the line span of its array value.
Expand All @@ -264,7 +294,7 @@ func mergeDatabricksConnect(lines []string, value string) ([]string, bool) {
out = append(out, lines[:header+1]...)
out = append(out, insert...)
out = append(out, lines[header+1:]...)
return out, true
return out, "", true
}
arrayLast, _ := arrayLineSpan(lines, devStart, end)

Expand All @@ -274,27 +304,27 @@ func mergeDatabricksConnect(lines []string, value string) ([]string, bool) {
if devStart == arrayLast {
line := lines[devStart]
arrayPart, commentPart := splitAtArrayClose(line)
if replaced, ok := replaceDbconnectElement(arrayPart, elem); ok {
newLine := replaced + commentPart
if rewritten, replaced, ok := replaceDbconnectElement(arrayPart, elem); ok {
newLine := rewritten + commentPart
if newLine == line {
return lines, false
return lines, replaced, false
}
lines[devStart] = newLine
return lines, true
return lines, replaced, true
}
// No databricks-connect element: insert one as the first array element.
open := strings.Index(arrayPart, "[")
closeIdx := strings.LastIndex(arrayPart, "]")
if open < 0 || closeIdx < open {
return lines, false
return lines, "", false
}
inner := strings.TrimSpace(arrayPart[open+1 : closeIdx])
newInner := elem
if inner != "" {
newInner = elem + ", " + inner
}
lines[devStart] = arrayPart[:open+1] + newInner + arrayPart[closeIdx:] + commentPart
return lines, true
return lines, "", true
}

// Multi-line form: the array spans devStart..arrayLast. An existing
Expand All @@ -310,15 +340,15 @@ func mergeDatabricksConnect(lines []string, value string) ([]string, bool) {
if c := commentStart(code); c >= 0 {
code, comment = code[:c], code[c:]
}
if replaced, ok := replaceDbconnectElement(code, elem); ok {
if rewritten, replaced, ok := replaceDbconnectElement(code, elem); ok {
// Rewrite only the code portion; a trailing comment is user content and
// must be preserved byte-for-byte, even if it contains a quoted token.
newLine := replaced + comment
newLine := rewritten + comment
if newLine == lines[i] {
return lines, false
return lines, replaced, false
}
lines[i] = newLine
return lines, true
return lines, replaced, true
}
if dbconnectQuotedRe.MatchString(code) {
lastElem = i
Expand All @@ -340,11 +370,11 @@ func mergeDatabricksConnect(lines []string, value string) ([]string, bool) {
if lastElem >= 0 && lastElem < arrayLast {
lines[lastElem] = ensureTrailingComma(lines[lastElem])
}
out := make([]string, 0, len(lines)+1)
out = append(out, lines[:arrayLast]...)
out = append(out, indent+elem+",")
out = append(out, lines[arrayLast:]...)
return out, true
inserted := make([]string, 0, len(lines)+1)
inserted = append(inserted, lines[:arrayLast]...)
inserted = append(inserted, indent+elem+",")
inserted = append(inserted, lines[arrayLast:]...)
return inserted, "", true
}

// dbconnectQuotedRe matches any double-quoted array element token.
Expand All @@ -353,18 +383,18 @@ var dbconnectQuotedRe = regexp.MustCompile(`"[^"]*"`)
// replaceDbconnectElement replaces the first quoted element in code whose package
// name is databricks-connect (compared under PEP 503 normalization, so
// "databricks_connect" / "Databricks-Connect" / "databricks.connect" all match)
// with elem. It returns the rewritten code and whether a replacement was made.
// Matching mirrors the artifact side (isDatabricksConnectDep) so a differently
// spelled existing pin is rewritten in place rather than left for the insert path
// to duplicate.
func replaceDbconnectElement(code, elem string) (string, bool) {
// with elem. It returns the rewritten code, the requirement it replaced, and
// whether a replacement was made. Matching mirrors the artifact side
// (isDatabricksConnectDep) so a differently spelled existing pin is rewritten in
// place rather than left for the insert path to duplicate.
func replaceDbconnectElement(code, elem string) (out, replaced string, ok bool) {
for _, m := range dbconnectQuotedRe.FindAllStringIndex(code, -1) {
inner := code[m[0]+1 : m[1]-1]
if isDatabricksConnectDep(inner) {
return code[:m[0]] + elem + code[m[1]:], true
return code[:m[0]] + elem + code[m[1]:], inner, true
}
}
return code, false
return code, "", false
}

// ensureTrailingComma appends a "," after the last non-space code character of
Expand Down
8 changes: 8 additions & 0 deletions libs/localenv/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,14 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints,
if err != nil {
return nil, greenfield, p.fail(PhaseMerge, false, NewError(ErrMerge, err, "merge managed regions failed"))
}
// Surface merge-quality warnings (overridden or duplicated pins, conflicting
// user constraints) from the pre-merge file. Greenfield has nothing of the
// user's to override, so it is skipped. This runs for both dry-run and real
// runs so the --json consumer sees the same warnings either way. The pin the
// merge rewrote comes from the merge itself, so the warning can never claim a
// replacement that did not happen.
p.res.Warnings = append(p.res.Warnings,
detectMergeWarnings(baseBytes, effective, replacedDBConnectPin(baseBytes, effective))...)
}

// Under --dry-run, build the plan (with a diff) for reporting. A real run does
Expand Down
Loading
Loading