Speed up loading bundles with many included files

.nextchanges/bundles/include-load-performance.md

@@ -0,0 +1 @@
+Improved configuration load time for bundles with many included files ([#6195](https://github.com/databricks/cli/pull/6195)).

bundle/config/loader/process_root_includes.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
 )
 
 type processRootIncludes struct{}
@@ -130,13 +131,35 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		}
 	}
 
-	// Swap out the original includes list with the expanded globs.
-	b.Config.Include = files
+	// Swap out the original includes list with the expanded globs. This goes through
+	// Mutate so the dynamic tree is updated too: the includes below are applied without
+	// their own mutator scope, so nothing converts the typed field back into the dynamic
+	// tree afterwards, and the next ToTyped would otherwise restore the raw patterns.
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		// Include is omitempty in the typed configuration, so an empty list must stay
+		// absent from the dynamic tree rather than be written as [].
+		if len(files) == 0 {
+			return dyn.DropKeys(root, []string{"include"})
+		}
+
+		includeValues := make([]dyn.Value, 0, len(files))
+		for _, file := range files {
+			includeValues = append(includeValues, dyn.V(file))
+		}
+		return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations()))
+	})
+	if err != nil {
+		return diag.FromErr(err)
+	}
 
 	// Track number of bundle YAML (or JSON) files in the configuration. The +1 is there
 	// to account for the root databricks.yaml file.
 	b.Metrics.ConfigurationFileCount = int64(len(files)) + 1
 
-	bundle.ApplySeqContext(ctx, b, out...)
+	// ProcessInclude merges into the configuration via [config.Root.Merge], so it does
+	// not need its own mutator scope. Giving each included file one would re-convert the
+	// whole accumulated configuration per file, making load quadratic in the number of
+	// included files (~20 minutes for 6000 files).
+	bundle.ApplySeqInScopeContext(ctx, b, out...)
 	return nil
 }

bundle/config/loader/process_root_includes_test.go

@@ -1,6 +1,7 @@
 package loader_test
 
 import (
+	"path/filepath"
 	"runtime"
 	"testing"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config/loader"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -98,6 +100,104 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
 	assert.Equal(t, []string{"a.yml"}, b.Config.Include)
 }
 
+// The expanded include list must be visible in both the typed and the dynamic
+// configuration: the per-file includes are applied without their own mutator scope, so
+// nothing converts the typed field back into the dynamic tree afterwards.
+func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) {
+	b := &bundle.Bundle{
+		BundleRootPath: t.TempDir(),
+		Config: config.Root{
+			Include: []string{
+				"*.yml",
+			},
+		},
+	}
+
+	testutil.Touch(t, b.BundleRootPath, "databricks.yml")
+	testutil.Touch(t, b.BundleRootPath, "a.yml")
+
+	diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes())
+	require.NoError(t, diags.Error())
+	assert.Equal(t, []string{"a.yml"}, b.Config.Include)
+
+	assert.Equal(t, []any{"a.yml"}, b.Config.Value().Get("include").AsAny())
+}
+
+// An empty include list must stay absent from the dynamic tree: the typed field is
+// omitempty, so writing [] would add an empty "include" to `bundle validate -o json`.
+func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) {
+	b := &bundle.Bundle{
+		BundleRootPath: t.TempDir(),
+		Config: config.Root{
+			Include: []string{
+				"*.yml",
+			},
+		},
+	}
+
+	testutil.Touch(t, b.BundleRootPath, "databricks.yml")
+
+	diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes())
+	require.NoError(t, diags.Error())
+	assert.Empty(t, b.Config.Include)
+	assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind())
+}
+
+// Merge semantics across included files must be unaffected by how the per-file includes
+// are applied: maps merge per key with the later file winning, sequences concatenate, and
+// locations accumulate (UniqueResourceKeys reports duplicates by counting locations).
+func TestProcessRootIncludesMergesAcrossFiles(t *testing.T) {
+	b := &bundle.Bundle{
+		BundleRootPath: t.TempDir(),
+		Config: config.Root{
+			Include: []string{
+				"*.yml",
+			},
+		},
+	}
+
+	testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), `
+resources:
+  jobs:
+    shared:
+      max_concurrent_runs: 1
+      tags:
+        from_a: yes_a
+      tasks:
+        - task_key: task_a
+`)
+
+	testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "b.yml"), `
+resources:
+  jobs:
+    shared:
+      tags:
+        from_b: yes_b
+      tasks:
+        - task_key: task_b
+`)
+
+	diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes())
+	require.NoError(t, diags.Error())
+
+	job := b.Config.Value().Get("resources").Get("jobs").Get("shared")
+
+	// Set only in a.yml: a per-key map merge must not drop it.
+	assert.Equal(t, int64(1), job.Get("max_concurrent_runs").MustInt())
+
+	// Maps merge per key across both files.
+	assert.Equal(t, map[string]any{"from_a": "yes_a", "from_b": "yes_b"}, job.Get("tags").AsAny())
+
+	// Sequences concatenate rather than overwrite.
+	assert.Equal(t, []any{
+		map[string]any{"task_key": "task_a"},
+		map[string]any{"task_key": "task_b"},
+	}, job.Get("tasks").AsAny())
+
+	// Both definitions must remain visible, otherwise duplicate keys go unreported.
+	assert.Len(t, job.Locations(), 2)
+}
+
 func TestProcessRootIncludesNotExists(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: t.TempDir(),

bundle/mutator.go

@@ -104,6 +104,34 @@ func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) {
 	}
 }
 
+// ApplySeqInScopeContext applies mutators without opening a mutator scope per mutator,
+// reusing the caller's scope instead.
+//
+// [ApplyContext] converts the whole configuration tree between its typed and dynamic
+// representations on entry and exit (see [config.Root.MarkMutatorEntry]). That cost is
+// proportional to the size of the accumulated configuration, so applying N mutators this
+// way is quadratic in N. For a bundle with thousands of included files that dominates
+// load time, hence this variant.
+//
+// Only use it for mutators that modify the configuration through [config.Root.Mutate]
+// (which keeps both representations in sync). A mutator that assigns to a typed field
+// directly relies on the scope entry to carry that value into the dynamic tree, and
+// would lose it here.
+func ApplySeqInScopeContext(ctx context.Context, b *Bundle, mutators ...Mutator) {
+	for _, m := range mutators {
+		mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name()))
+		log.Debugf(mctx, "Apply")
+
+		for _, d := range m.Apply(mctx, b) {
+			logdiag.LogDiag(mctx, d)
+		}
+
+		if logdiag.HasError(ctx) {
+			break
+		}
+	}
+}
+
 type funcMutator struct {
 	fn func(context.Context, *Bundle)
 }