Fix: iceberg tables have linebreaks that mess up column descriptions (#1677)

Author: tobymao

sqlmesh/core/engine_adapter/base.py

@@ -740,6 +740,7 @@ def columns(
                 lambda t: not t[0].startswith("#"),
                 describe_output,
             )
+            if column_name and column_name.strip() and column_type and column_type.strip()
         }
 
     @set_catalog()

sqlmesh/core/model/definition.py

@@ -1779,7 +1779,6 @@ def _create_model(
     physical_schema_override: t.Optional[t.Dict[str, str]] = None,
     **kwargs: t.Any,
 ) -> Model:
-
     _validate_model_fields(klass, {"name", *kwargs} - {"grain"}, path)
 
     dialect = dialect or ""

sqlmesh/utils/pydantic.py

@@ -72,7 +72,6 @@ def _expression_encoder(e: exp.Expression) -> str:
 
 class PydanticModel(pydantic.BaseModel):
     if PYDANTIC_MAJOR_VERSION >= 2:
-
         model_config = pydantic.ConfigDict(  # type: ignore
             arbitrary_types_allowed=True,
             extra="forbid",  # type: ignore
@@ -239,7 +238,6 @@ def positive_int_validator(v: t.Any) -> int:
     SQLGlotBool = Annotated[bool, BeforeValidator(bool_validator)]
     SQLGlotPositiveInt = Annotated[int, BeforeValidator(positive_int_validator)]
 else:
-
     T = t.TypeVar("T")
 
     class PydanticTypeProxy(t.Generic[T]):

tests/core/engine_adapter/test_base.py

@@ -134,6 +134,18 @@ def test_columns(make_mocked_engine_adapter: t.Callable):
     adapter.cursor.execute.assert_called_once_with('DESCRIBE "test_table"')
 
 
+def test_iceberg_corrupt(make_mocked_engine_adapter: t.Callable):
+    adapter = make_mocked_engine_adapter(EngineAdapter)
+    adapter.cursor.fetchall.return_value = [
+        ("id", "int"),
+        (" ", ""),
+        ("", b"\n"),
+    ]
+    assert adapter.columns("test_table") == {
+        "id": exp.DataType.build("int"),
+    }
+
+
 def test_table_exists(make_mocked_engine_adapter: t.Callable):
     adapter = make_mocked_engine_adapter(EngineAdapter)
     assert adapter.table_exists("test_table")

tests/core/test_scheduler.py

@@ -31,7 +31,7 @@ def test_interval_params(scheduler: Scheduler, sushi_context_fixed_date: Context
     start_ds = "2022-01-01"
     end_ds = "2022-02-05"
 
-    assert compute_interval_params([orders, waiter_revenue], start=start_ds, end=end_ds,) == {
+    assert compute_interval_params([orders, waiter_revenue], start=start_ds, end=end_ds) == {
         orders: [
             (to_datetime(start_ds), to_datetime("2022-01-31")),
             (to_datetime("2022-01-31"), to_datetime("2022-02-06")),

tests/core/test_snapshot.py

@@ -586,7 +586,6 @@ def test_fingerprint_seed_model():
 
 
 def test_fingerprint_jinja_macros(model: Model):
-
     model = SqlModel(
         **{
             **model.dict(),