Add py-files support for function python dependency management (#77)

* Function Support for pyfiles

* Complete function pyfiles support with pip install and error handling

* Code refactor

* Fix py-files directory handling - add existence check before rmtree

* fixing dependencies

Author: sarasavilli-sf

src/datacustomcode/cli.py

@@ -134,9 +134,12 @@ def auth(profile: str):
 @click.option("--network", default="default")
 def zip(path: str, network: str):
     from datacustomcode.deploy import zip
+    from datacustomcode.scan import find_base_directory, get_package_type
 
     logger.debug("Zipping project")
-    zip(path, network)
+    base_directory = find_base_directory(path)
+    package_type = get_package_type(base_directory)
+    zip(path, network, package_type)
 
 
 @cli.command()

src/datacustomcode/deploy.py

@@ -279,10 +279,13 @@ def create_deployment(
 DEPENDENCIES_ARCHIVE_PATH = os.path.join(
     "payload", "archives", DEPENDENCIES_ARCHIVE_FULL_NAME
 )
+PY_FILES_PATH = os.path.join("payload", "py-files")
 ZIP_FILE_NAME = "deployment.zip"
 
 
-def prepare_dependency_archive(directory: str, docker_network: str) -> None:
+def prepare_dependency_archive(
+    directory: str, docker_network: str, package_type: str
+) -> None:
     cmd = f"docker images -q {DOCKER_IMAGE_NAME}"
     image_exists = cmd_output(cmd)
 
@@ -305,11 +308,28 @@ def prepare_dependency_archive(directory: str, docker_network: str) -> None:
         shutil.copy("build_native_dependencies.sh", temp_dir)
         cmd = docker_run_cmd(docker_network, temp_dir)
         cmd_output(cmd, env=docker_env)
-        archives_temp_path = os.path.join(temp_dir, DEPENDENCIES_ARCHIVE_FULL_NAME)
-        os.makedirs(os.path.dirname(DEPENDENCIES_ARCHIVE_PATH), exist_ok=True)
-        shutil.copy(archives_temp_path, DEPENDENCIES_ARCHIVE_PATH)
-
-        logger.info(f"Dependencies archived to {DEPENDENCIES_ARCHIVE_PATH}")
+        if package_type == "function":
+            source_py_files = os.path.join(temp_dir, "py-files")
+            if os.path.exists(source_py_files):
+                logger.info(
+                    f"py-files directory found at {source_py_files}. "
+                    "Copying to payload directory..."
+                )
+                os.makedirs(os.path.dirname(PY_FILES_PATH), exist_ok=True)
+                if os.path.exists(PY_FILES_PATH):
+                    shutil.rmtree(PY_FILES_PATH)
+                shutil.copytree(source_py_files, PY_FILES_PATH)
+                logger.info(f"py-files copied to {PY_FILES_PATH}")
+            else:
+                logger.info(
+                    f"No py-files directory found at {source_py_files}. "
+                    "Skipping py-files copy."
+                )
+        else:
+            archives_temp_path = os.path.join(temp_dir, DEPENDENCIES_ARCHIVE_FULL_NAME)
+            os.makedirs(os.path.dirname(DEPENDENCIES_ARCHIVE_PATH), exist_ok=True)
+            shutil.copy(archives_temp_path, DEPENDENCIES_ARCHIVE_PATH)
+            logger.info(f"Dependencies archived to {DEPENDENCIES_ARCHIVE_PATH}")
 
 
 def docker_build_cmd(network: str) -> str:
@@ -516,13 +536,14 @@ def upload_zip(file_upload_url: str) -> None:
 def zip(
     directory: str,
     docker_network: str,
+    package_type: str,
 ):
     # Create a zip file excluding .DS_Store files
     import zipfile
 
     # prepare payload only if requirements.txt is non-empty
     if has_nonempty_requirements_file(directory):
-        prepare_dependency_archive(directory, docker_network)
+        prepare_dependency_archive(directory, docker_network, package_type)
     else:
         logger.info(
             f"Skipping dependency archive: requirements.txt is missing or empty "
@@ -561,7 +582,7 @@ def deploy_full(
 
     # create deployment and upload payload
     deployment = create_deployment(access_token, metadata)
-    zip(directory, docker_network)
+    zip(directory, docker_network, metadata.codeType)
     upload_zip(deployment.fileUploadUrl)
     wait_for_deployment(access_token, metadata, callback)
 

src/datacustomcode/templates/function/build_native_dependencies.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 set -e
 
-# Description: build native dependencies
+# Description: build native dependencies for function (unpacked pip install to py-files)
 
 python3.11 -m venv --copies .venv
 source .venv/bin/activate
-pip install -r requirements.txt
-venv-pack -o native_dependencies.tar.gz -f
+pip install --target ./py-files -r requirements.txt

tests/test_deploy.py

@@ -93,7 +93,7 @@ def test_prepare_dependency_archive_image_exists(
         mock_docker_build_cmd.return_value = "mock build command"
         mock_docker_run_cmd.return_value = "mock run command"
 
-        prepare_dependency_archive("/test/dir", "default")
+        prepare_dependency_archive("/test/dir", "default", "script")
 
         # Verify docker images command was called
         mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
@@ -153,7 +153,7 @@ def test_prepare_dependency_archive_build_image(
         mock_docker_build_cmd.return_value = "mock build command"
         mock_docker_run_cmd.return_value = "mock run command"
 
-        prepare_dependency_archive("/test/dir", "default")
+        prepare_dependency_archive("/test/dir", "default", "script")
 
         # Verify docker images command was called
         mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
@@ -214,7 +214,7 @@ def test_prepare_dependency_archive_docker_build_failure(
         ]
 
         with pytest.raises(CalledProcessError, match="Build failed"):
-            prepare_dependency_archive("/test/dir", "default")
+            prepare_dependency_archive("/test/dir", "default", "script")
 
         # Verify docker images command was called
         mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
@@ -257,7 +257,7 @@ def test_prepare_dependency_archive_docker_run_failure(
         ]
 
         with pytest.raises(CalledProcessError, match="Run failed"):
-            prepare_dependency_archive("/test/dir", "default")
+            prepare_dependency_archive("/test/dir", "default", "script")
 
         # Verify docker images command was called
         mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
@@ -300,14 +300,150 @@ def test_prepare_dependency_archive_file_copy_failure(
         mock_copy.side_effect = FileNotFoundError("File not found")
 
         with pytest.raises(FileNotFoundError, match="File not found"):
-            prepare_dependency_archive("/test/dir", "default")
+            prepare_dependency_archive("/test/dir", "default", "script")
 
         # Verify docker images command was called
         mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
 
         # Verify files were attempted to be copied
         mock_copy.assert_any_call("requirements.txt", "/tmp/test_dir")
 
+    @patch("datacustomcode.deploy.cmd_output")
+    @patch("datacustomcode.deploy.shutil.copytree")
+    @patch("datacustomcode.deploy.shutil.rmtree")
+    @patch("datacustomcode.deploy.shutil.copy")
+    @patch("datacustomcode.deploy.tempfile.TemporaryDirectory")
+    @patch("datacustomcode.deploy.os.path.exists")
+    @patch("datacustomcode.deploy.os.path.join")
+    @patch("datacustomcode.deploy.os.makedirs")
+    @patch("datacustomcode.deploy.docker_build_cmd")
+    @patch("datacustomcode.deploy.docker_run_cmd")
+    def test_prepare_dependency_archive_function_type(
+        self,
+        mock_docker_run_cmd,
+        mock_docker_build_cmd,
+        mock_makedirs,
+        mock_join,
+        mock_exists,
+        mock_temp_dir,
+        mock_copy,
+        mock_rmtree,
+        mock_copytree,
+        mock_cmd_output,
+    ):
+        """Test prepare_dependency_archive with function package type."""
+        # Mock the temporary directory context manager
+        mock_temp_dir_instance = MagicMock()
+        mock_temp_dir_instance.__enter__.return_value = "/tmp/test_dir"
+        mock_temp_dir_instance.__exit__.return_value = None
+        mock_temp_dir.return_value = mock_temp_dir_instance
+
+        # Mock cmd_output to return image ID (indicating image exists)
+        mock_cmd_output.return_value = "abc123"
+
+        # Mock os.path.join for py-files paths
+        def join_side_effect(*args):
+            if args == ("/tmp/test_dir", "py-files"):
+                return "/tmp/test_dir/py-files"
+            return "/".join(args)
+
+        mock_join.side_effect = join_side_effect
+
+        # Mock os.path.exists
+        def exists_side_effect(path):
+            if path == "/tmp/test_dir/py-files":
+                return True
+            if path == "payload/py-files":
+                return False
+            return False
+
+        mock_exists.side_effect = exists_side_effect
+
+        # Mock the docker command functions
+        mock_docker_build_cmd.return_value = "mock build command"
+        mock_docker_run_cmd.return_value = "mock run command"
+
+        prepare_dependency_archive("/test/dir", "default", "function")
+
+        # Verify docker images command was called
+        mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
+
+        # Verify docker build command was not called (since image already exists)
+        mock_docker_build_cmd.assert_not_called()
+
+        # Verify files were copied to temp directory
+        mock_copy.assert_any_call("requirements.txt", "/tmp/test_dir")
+        mock_copy.assert_any_call("build_native_dependencies.sh", "/tmp/test_dir")
+
+        # Verify docker run command was called
+        mock_docker_run_cmd.assert_called_once_with("default", "/tmp/test_dir")
+        mock_cmd_output.assert_any_call("mock run command", env=ANY)
+
+        # Verify payload directory was created
+        mock_makedirs.assert_called_once_with("payload", exist_ok=True)
+
+        # Verify py-files was NOT removed (doesn't exist yet)
+        mock_rmtree.assert_not_called()
+
+        # Verify py-files directory was copied
+        mock_copytree.assert_called_once_with(
+            "/tmp/test_dir/py-files", "payload/py-files"
+        )
+
+    @patch("datacustomcode.deploy.cmd_output")
+    @patch("datacustomcode.deploy.shutil.copy")
+    @patch("datacustomcode.deploy.tempfile.TemporaryDirectory")
+    @patch("datacustomcode.deploy.os.path.exists")
+    @patch("datacustomcode.deploy.os.path.join")
+    @patch("datacustomcode.deploy.os.makedirs")
+    @patch("datacustomcode.deploy.docker_build_cmd")
+    @patch("datacustomcode.deploy.docker_run_cmd")
+    def test_prepare_dependency_archive_function_type_missing_pyfiles(
+        self,
+        mock_docker_run_cmd,
+        mock_docker_build_cmd,
+        mock_makedirs,
+        mock_join,
+        mock_exists,
+        mock_temp_dir,
+        mock_copy,
+        mock_cmd_output,
+    ):
+        """
+        Test prepare_dependency_archive with function type when py-files is missing.
+        Should log and continue without error.
+        """
+        # Mock the temporary directory context manager
+        mock_temp_dir_instance = MagicMock()
+        mock_temp_dir_instance.__enter__.return_value = "/tmp/test_dir"
+        mock_temp_dir_instance.__exit__.return_value = None
+        mock_temp_dir.return_value = mock_temp_dir_instance
+
+        # Mock cmd_output to return image ID (indicating image exists)
+        mock_cmd_output.return_value = "abc123"
+
+        # Mock os.path.join for py-files path
+        def join_side_effect(*args):
+            if args == ("/tmp/test_dir", "py-files"):
+                return "/tmp/test_dir/py-files"
+            return "/".join(args)
+
+        mock_join.side_effect = join_side_effect
+
+        # Mock os.path.exists to return False for py-files (doesn't exist)
+        mock_exists.return_value = False
+
+        # Mock the docker command functions
+        mock_docker_build_cmd.return_value = "mock build command"
+        mock_docker_run_cmd.return_value = "mock run command"
+
+        # Should complete successfully without raising an error
+        prepare_dependency_archive("/test/dir", "default", "function")
+
+        # Verify docker commands were called
+        mock_cmd_output.assert_any_call(self.EXPECTED_DOCKER_IMAGES_CMD)
+        mock_docker_run_cmd.assert_called_once_with("default", "/tmp/test_dir")
+
 
 class TestHasNonemptyRequirementsFile:
     @patch("datacustomcode.deploy.os.path.dirname")
@@ -609,10 +745,10 @@ def test_zip_with_requirements(
             ("/test/dir/subdir", [], ["file3.py"]),
         ]
 
-        zip("/test/dir", "default")
+        zip("/test/dir", "default", "script")
 
         mock_has_requirements.assert_called_once_with("/test/dir")
-        mock_prepare.assert_called_once_with("/test/dir", "default")
+        mock_prepare.assert_called_once_with("/test/dir", "default", "script")
         mock_zipfile.assert_called_once_with(
             "deployment.zip", "w", zipfile.ZIP_DEFLATED
         )
@@ -637,7 +773,7 @@ def test_zip_without_requirements(
             ("/test/dir/subdir", [], ["file3.py"]),
         ]
 
-        zip("/test/dir", "default")
+        zip("/test/dir", "default", "script")
 
         mock_has_requirements.assert_called_once_with("/test/dir")
         mock_prepare.assert_not_called()
@@ -646,6 +782,38 @@ def test_zip_without_requirements(
         )
         assert mock_zipfile_instance.write.call_count == 3  # One call per file
 
+    @patch("datacustomcode.deploy.has_nonempty_requirements_file")
+    @patch("datacustomcode.deploy.prepare_dependency_archive")
+    @patch("zipfile.ZipFile")
+    @patch("os.walk")
+    def test_zip_with_function_package_type(
+        self,
+        mock_walk,
+        mock_zipfile,
+        mock_prepare,
+        mock_has_requirements,
+    ):
+        """Test zipping a directory with function package type."""
+        mock_has_requirements.return_value = True
+        mock_zipfile_instance = MagicMock()
+        mock_zipfile.return_value.__enter__.return_value = mock_zipfile_instance
+        mock_zipfile_instance.write = MagicMock()
+
+        # Mock os.walk to return some test files
+        mock_walk.return_value = [
+            ("/test/dir", ["subdir"], ["file1.py", "file2.py"]),
+            ("/test/dir/subdir", [], ["file3.py"]),
+        ]
+
+        zip("/test/dir", "default", "function")
+
+        mock_has_requirements.assert_called_once_with("/test/dir")
+        mock_prepare.assert_called_once_with("/test/dir", "default", "function")
+        mock_zipfile.assert_called_once_with(
+            "deployment.zip", "w", zipfile.ZIP_DEFLATED
+        )
+        assert mock_zipfile_instance.write.call_count == 3  # One call per file
+
 
 class TestUploadZip:
     @patch("datacustomcode.deploy.requests.put")
@@ -934,7 +1102,7 @@ def test_deploy_full(
         mock_retrieve_token.assert_called_once_with(credentials)
         mock_get_config.assert_called_once_with("/test/dir")
         mock_create_deployment.assert_called_once_with(access_token, metadata)
-        mock_zip.assert_called_once_with("/test/dir", "default")
+        mock_zip.assert_called_once_with("/test/dir", "default", "script")
         mock_upload_zip.assert_called_once_with("https://upload.example.com")
         mock_wait.assert_called_once_with(access_token, metadata, callback)
         mock_create_transform.assert_called_once_with(
@@ -998,7 +1166,7 @@ def test_deploy_full_client_credentials(
         mock_retrieve_token.assert_called_once_with(credentials)
         mock_get_config.assert_called_once_with("/test/dir")
         mock_create_deployment.assert_called_once_with(access_token, metadata)
-        mock_zip.assert_called_once_with("/test/dir", "default")
+        mock_zip.assert_called_once_with("/test/dir", "default", "script")
         mock_upload_zip.assert_called_once_with("https://upload.example.com")
         mock_wait.assert_called_once_with(access_token, metadata, callback)
         mock_create_transform.assert_called_once_with(
@@ -1104,7 +1272,7 @@ def test_deploy_full_happy_path(
         mock_retrieve_token.assert_called_once_with(credentials)
         mock_get_config.assert_called_once_with("/test/dir")
         mock_create_deployment.assert_called_once_with(access_token, metadata)
-        mock_zip.assert_called_once_with("/test/dir", "default")
+        mock_zip.assert_called_once_with("/test/dir", "default", "script")
         mock_upload_zip.assert_called_once_with("https://upload.example.com")
         mock_wait.assert_called_once_with(access_token, metadata, callback)
         mock_create_transform.assert_called_once_with(

tests/test_scan.py

@@ -9,6 +9,7 @@
 
 from datacustomcode.scan import (
     SDK_CONFIG_DIR,
+    SDK_CONFIG_FILE,
     DataAccessLayerCalls,
     dc_config_json_from_file,
     scan_file,
@@ -40,7 +41,7 @@ def create_sdk_config(base_directory: str, package_type: str = "script") -> str:
     """
     sdk_config = {"type": package_type}
     write_sdk_config(base_directory, sdk_config)
-    return os.path.join(base_directory, SDK_CONFIG_DIR, "config.json")
+    return os.path.join(base_directory, SDK_CONFIG_DIR, SDK_CONFIG_FILE)
 
 
 class TestClientMethodVisitor: