Add pipelines preview-datasets command

acceptance/pipelines/datasets/flag-conflict/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-datasets-flag-conflict
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-datasets-flag-conflict
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/datasets/flag-conflict/output.txt

@@ -0,0 +1,3 @@
+
+=== datasets rejects --force-dry-run and --no-dry-run together (before any bundle work)
+Error: --force-dry-run and --no-dry-run cannot be used together

acceptance/pipelines/datasets/flag-conflict/pipeline_file.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(10)

acceptance/pipelines/datasets/flag-conflict/script

@@ -0,0 +1,2 @@
+title "datasets rejects --force-dry-run and --no-dry-run together (before any bundle work)\n"
+musterr $CLI pipelines datasets --force-dry-run --no-dry-run

acceptance/pipelines/datasets/force/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-datasets-force
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-datasets-force
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/datasets/force/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] pipelines deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-datasets-force/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]
+
+=== datasets --force-dry-run always triggers a fresh dry-run (announces the billed cluster)
+>>> [CLI] pipelines datasets --force-dry-run
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Triggering a fresh dry-run (this starts a billed cluster). Press Ctrl-C to cancel.
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+Update ID: [UUID]
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK

acceptance/pipelines/datasets/force/pipeline_file.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(10)

acceptance/pipelines/datasets/force/script

@@ -0,0 +1,4 @@
+trace $CLI pipelines deploy
+
+title "datasets --force-dry-run always triggers a fresh dry-run (announces the billed cluster)"
+trace $CLI pipelines datasets --force-dry-run

acceptance/pipelines/datasets/happy/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-datasets
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-datasets
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/datasets/happy/output.txt

@@ -0,0 +1,89 @@
+
+>>> [CLI] pipelines deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-datasets/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]
+
+=== datasets: default triggers a dry-run, then lists the pipeline's datasets
+>>> [CLI] pipelines datasets
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+No readable dry-run found; triggering one (this starts a billed cluster). Press Ctrl-C to cancel.
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+Update ID: [UUID]
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK
+
+=== datasets with an explicit pipeline KEY (same result)
+>>> [CLI] pipelines datasets my_pipeline
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK
+
+=== datasets as JSON
+>>> [CLI] pipelines datasets --output json
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+[
+  {
+    "dataset_ref": "n1",
+    "name": "main.demo.source",
+    "full_name": "`main`.`demo`.`source`",
+    "dataset_type": "MATERIALIZED_VIEW"
+  },
+  {
+    "dataset_ref": "n2",
+    "name": "main.demo.filtered",
+    "full_name": "`main`.`demo`.`filtered`",
+    "dataset_type": "MATERIALIZED_VIEW"
+  },
+  {
+    "dataset_ref": "n3",
+    "name": "main.demo.aggregated",
+    "full_name": "`main`.`demo`.`aggregated`",
+    "dataset_type": "MATERIALIZED_VIEW"
+  },
+  {
+    "dataset_ref": "n4",
+    "name": "recent",
+    "full_name": "",
+    "dataset_type": "VIEW"
+  },
+  {
+    "dataset_ref": "s1",
+    "name": "",
+    "full_name": "main.demo.archive",
+    "dataset_type": "SINK"
+  }
+]
+
+=== datasets --no-dry-run reads the existing dry-run (no re-trigger)
+>>> [CLI] pipelines datasets --no-dry-run
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK

acceptance/pipelines/datasets/happy/pipeline_file.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(10)

acceptance/pipelines/datasets/happy/script

@@ -0,0 +1,13 @@
+trace $CLI pipelines deploy
+
+title "datasets: default triggers a dry-run, then lists the pipeline's datasets"
+trace $CLI pipelines datasets
+
+title "datasets with an explicit pipeline KEY (same result)"
+trace $CLI pipelines datasets my_pipeline
+
+title "datasets as JSON"
+trace $CLI pipelines datasets --output json
+
+title "datasets --no-dry-run reads the existing dry-run (no re-trigger)"
+trace $CLI pipelines datasets --no-dry-run

acceptance/pipelines/datasets/no-dry-run-missing/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-datasets-no-dry-run-missing
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-datasets-no-dry-run-missing
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/datasets/no-dry-run-missing/output.txt

@@ -0,0 +1,14 @@
+
+>>> [CLI] pipelines deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-datasets-no-dry-run-missing/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]
+
+=== datasets --no-dry-run with no prior dry-run reports the graph as unavailable
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Error: datasets for my_pipeline: no dry-run found; re-run without --no-dry-run to trigger one

acceptance/pipelines/datasets/no-dry-run-missing/pipeline_file.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(10)

acceptance/pipelines/datasets/no-dry-run-missing/script

@@ -0,0 +1,4 @@
+trace $CLI pipelines deploy
+
+title "datasets --no-dry-run with no prior dry-run reports the graph as unavailable\n"
+musterr $CLI pipelines datasets --no-dry-run

acceptance/pipelines/datasets/no-retrigger/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-pipeline-datasets-no-retrigger
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-datasets-no-retrigger
+      libraries:
+        - file:
+            path: pipeline_file.py

acceptance/pipelines/datasets/no-retrigger/output.txt

@@ -0,0 +1,44 @@
+
+>>> [CLI] pipelines deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-datasets-no-retrigger/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]
+
+>>> [CLI] pipelines datasets
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+No readable dry-run found; triggering one (this starts a billed cluster). Press Ctrl-C to cancel.
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+Update ID: [UUID]
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK
+
+>>> [CLI] pipelines datasets
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Name                  Type
+main.demo.source      MATERIALIZED_VIEW
+main.demo.filtered    MATERIALIZED_VIEW
+main.demo.aggregated  MATERIALIZED_VIEW
+recent                VIEW
+main.demo.archive     SINK
+
+=== only one dry-run was triggered across two reads
+{
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID]/updates",
+  "body": {
+    "validate_only": true
+  }
+}

acceptance/pipelines/datasets/no-retrigger/pipeline_file.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(10)

acceptance/pipelines/datasets/no-retrigger/script

@@ -0,0 +1,10 @@
+trace $CLI pipelines deploy
+rm out.requests.txt
+
+# First call triggers a dry-run (POST .../updates); the second reuses it. print_requests.py
+# excludes GETs, so the recorded POSTs show exactly one StartUpdate across both reads.
+trace $CLI pipelines datasets
+trace $CLI pipelines datasets
+
+title "only one dry-run was triggered across two reads\n"
+print_requests.py //pipelines --sort

acceptance/pipelines/datasets/no-retrigger/test.toml

@@ -0,0 +1 @@
+RecordRequests = true

cmd/pipelines/commands.go

@@ -17,6 +17,7 @@ func Commands() []*cobra.Command {
 		destroyCommand(),
 		runCommand(),
 		dryRunCommand(),
+		datasetsCommand(),
 		historyCommand(),
 		logsCommand(),
 		openCommand(),