add transform_async for automapper

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: (^.idea/)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
         types: [ python ]

Pipfile

@@ -8,17 +8,19 @@ name = "pypi"
 py4j="==0.10.9.7" # https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies
 # pyspark is needed for working with Spark
 pyspark="==3.5.1"  # should match the version of spark we use for testing
+# logger is needed for logging
 logger = ">=1.4"
-sparkdataframecomparer = ">=2.0.6"
+# sparkdataframecomparer is needed for comparing Spark DataFrames
+sparkdataframecomparer = ">=2.0.13"
 Deprecated = ">=1.2.12"
 numpy = ">=1.15"
 
 [dev-packages]
 # setuptools is needed for building the package
 setuptools=">=72.1.0"
-packaging=">=24.1"
 # wheel is needed for building the package
 wheel = ">=0.43.0"
+# twine is needed for uploading the package to PyPI
 twine=">=5.1.1"
 # pre-commit is needed for running code quality checks
 pre-commit=">=3.7.1"
@@ -27,21 +29,22 @@ autoflake=">=2.3.1"
 # mypy is needed for type checking
 mypy = ">=1.11.1"
 # pytest is needed for running tests
-pytest = ">=8.2.2"
+pytest = ">=8.3.2"
 # black is needed for formatting code
-black = ">=24.4.2"
-# importlib-metadata is needed for working with metadata
-importlib-metadata = ">=5.2.0"
-pygments = ">=2.18.0" # not directly required, pinned by Snyk to avoid a vulnerability
-types-Deprecated=">=1.2.9.20240311"
+black = ">=24.8.0"
+# pygments is needed for syntax highlighting
+pygments=">=2.18.0" # not directly required, pinned by Snyk to avoid a vulnerability
 # Sphinx is needed for generating documentation
-Sphinx="==4.1.2"
+Sphinx="==7.4.7"
 # sphinx-autoapi is needed for generating API documentation
-sphinx-autoapi="==2.0.0"
+sphinx-autoapi="==3.2.1"
 # sphinx-rtd-theme is needed for the Read the Docs theme
-sphinx-rtd-theme="==1.0.0"
+sphinx-rtd-theme="==2.0.0"
 # myst-parser is needed for parsing Markdown
-myst-parser="==0.17.2"
+myst-parser="==3.0.1"
+# recodoc is needed for generating documentation
+recommonmark="==0.7.1"
+types-Deprecated=">=1.2.9.20240311"
 
 # These dependencies are required for pipenv-setup.  They conflict with ones above, so we install these
 # only when running pipenv-setup

pre-commit.Dockerfile

@@ -1,14 +1,14 @@
-FROM python:3.12-slim
+FROM imranq2/helix.spark:3.5.1.11-precommit-slim
 
 RUN apt-get update && \
     apt-get install -y git && \
     pip install pipenv
 
-COPY ${project_root}/Pipfile* ./
+COPY Pipfile* ./
 
-RUN pipenv lock --dev && \
-    pipenv sync --dev --system --verbose
+ARG TARGETPLATFORM
+RUN pipenv sync --dev --system --extra-pip-args="--prefer-binary"
 
 WORKDIR /sourcecode
 RUN git config --global --add safe.directory /sourcecode
-CMD pre-commit run --all-files
+CMD ["pre-commit", "run", "--all-files"]

setup.py

@@ -25,7 +25,7 @@
         "py4j==0.10.9.7",
         "pyspark==3.5.1",
         "logger>=1.4",
-        "sparkdataframecomparer>=2.0.6",
+        "sparkdataframecomparer>=2.0.13",
         "deprecated>=1.2.12",
         "numpy>=1.15",
     ],

spark.Dockerfile

@@ -1,4 +1,4 @@
-FROM imranq2/helix.spark:3.5.1.5-slim
+FROM imranq2/helix.spark:3.5.1.11-slim
 # https://github.com/icanbwell/helix.spark
 USER root
 

spark_auto_mapper/automappers/automapper.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 from logging import Logger, getLogger, StreamHandler, Formatter
 from pathlib import Path
@@ -253,12 +254,6 @@ def _transform_with_data_frame_single_select(
             # run all the selects
             df = source_df.alias("b").select(*column_specs)
 
-            if self.log_level and self.log_level == "DEBUG":
-                print(
-                    f"------------  Start Execution Plan for view {self.view} -----------"
-                )
-                df.explain(extended="cost")
-                print("------------  End Execution Plan -----------")
             # write out final checkpoint for this automapper
             if self.checkpoint_path:
                 checkpoint_path = (
@@ -502,6 +497,24 @@ def _get_message_for_exception(
         return msg
 
     def transform(self, df: DataFrame) -> DataFrame:
+        """
+        Override this method to implement transformation
+
+        :param df: input dataframe
+        :return: transformed dataframe
+        """
+        return asyncio.run(self._transform_async(df))
+
+    async def transform_async(self, df: DataFrame) -> DataFrame:
+        """
+        Override this method to implement transformation
+
+        :param df: input dataframe
+        :return: transformed dataframe
+        """
+        return await self._transform_async(df)
+
+    async def _transform_async(self, df: DataFrame) -> DataFrame:
         """
         Uses this AutoMapper to transform the specified data frame and return the new data frame