fix: only import pyspark if using databricks connect (#2432)

Author: eakmanrq

sqlmesh/core/engine_adapter/databricks.py

@@ -137,11 +137,11 @@ def fetchdf(
         return df
 
     def get_current_catalog(self) -> t.Optional[str]:
-        from py4j.protocol import Py4JError
-        from pyspark.errors.exceptions.connect import SparkConnectGrpcException
-
-        # Update the Dataframe API is we have a spark session
+        # Update the Dataframe API if we have a spark session
         if self._use_spark_session:
+            from py4j.protocol import Py4JError
+            from pyspark.errors.exceptions.connect import SparkConnectGrpcException
+
             try:
                 # Note: Spark 3.4+ Only API
                 return super().get_current_catalog()
@@ -153,15 +153,15 @@ def get_current_catalog(self) -> t.Optional[str]:
         return None
 
     def set_current_catalog(self, catalog_name: str) -> None:
-        from py4j.protocol import Py4JError
-        from pyspark.errors.exceptions.connect import SparkConnectGrpcException
-
         # Since Databricks splits commands across the Dataframe API and the SQL Connector
         # (depending if databricks-connect is installed and a Dataframe is used) we need to ensure both
         # are set to the same catalog since they maintain their default catalog seperately
         self.execute(exp.Use(this=exp.to_identifier(catalog_name), kind="CATALOG"))
         # Update the Dataframe API is we have a spark session
         if self._use_spark_session:
+            from py4j.protocol import Py4JError
+            from pyspark.errors.exceptions.connect import SparkConnectGrpcException
+
             try:
                 # Note: Spark 3.4+ Only API
                 super().set_current_catalog(catalog_name)