From 69633031e9c1057e7b6ef85f9a2ca73173542332 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Mon, 27 Jan 2025 10:21:07 +0100 Subject: [PATCH 1/3] Add integration testing for Snowflake notebooks --- .../workflows/snowflake-integration-tests.yml | 46 +++++++++++++++++++ python-wrapper/pyproject.toml | 1 + python-wrapper/tests/conftest.py | 10 ++++ python-wrapper/tests/pytest.ini | 1 + python-wrapper/tests/test_notebooks.py | 32 +++++++------ 5 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/snowflake-integration-tests.yml mode change 100755 => 100644 python-wrapper/tests/test_notebooks.py diff --git a/.github/workflows/snowflake-integration-tests.yml b/.github/workflows/snowflake-integration-tests.yml new file mode 100644 index 00000000..e82028e0 --- /dev/null +++ b/.github/workflows/snowflake-integration-tests.yml @@ -0,0 +1,46 @@ +name: Run Snowflake integration tests + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the "main" branch + push: + branches: [ "main" ] + # Skip on this check PR to reduce number of AuraDS instances created + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + tests: + # The type of runner that the job will run on + runs-on: ${{ matrix.os}} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + defaults: + run: + working-directory: python-wrapper + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: 'pip' + cache-dependency-path: pyproject.toml + - run: pip install ".[dev]" + - run: pip install ".[pandas]" + + - name: Run tests + env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ACCOUNTADMIN + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + run: pytest tests/ --include-snowflake \ No newline at end of file diff --git a/python-wrapper/pyproject.toml b/python-wrapper/pyproject.toml index 27b556c8..edbf6d90 100644 --- a/python-wrapper/pyproject.toml +++ b/python-wrapper/pyproject.toml @@ -66,6 +66,7 @@ notebook = [ "ipywidgets>=8.0.0", "palettable==3.3.3", "matplotlib==3.10.0", + "snowflake-snowpark-python==1.26.0", ] [project.urls] diff --git a/python-wrapper/tests/conftest.py b/python-wrapper/tests/conftest.py index 27c1a551..ecaa0add 100644 --- a/python-wrapper/tests/conftest.py +++ b/python-wrapper/tests/conftest.py @@ -10,6 +10,11 @@ def pytest_addoption(parser: Any) -> None: action="store_true", help="include tests requiring a Neo4j instance with GDS running", ) + parser.addoption( + "--include-snowflake", + action="store_true", + help="include tests requiring a Snowflake connection", + ) def pytest_collection_modifyitems(config: Any, items: Any) -> None: @@ -18,6 +23,11 @@ def pytest_collection_modifyitems(config: Any, items: Any) -> None: for item in items: if "requires_neo4j_and_gds" in item.keywords: item.add_marker(skip) + if not config.getoption("--include-snowflake"): + skip = pytest.mark.skip(reason="skipping since requiring a Snowflake connection") + for item in items: + if "requires_snowflake" in item.keywords: + item.add_marker(skip) @pytest.fixture(scope="package") diff --git a/python-wrapper/tests/pytest.ini b/python-wrapper/tests/pytest.ini index 8fd6d0ec..dd8fa56e 100644 --- a/python-wrapper/tests/pytest.ini +++ b/python-wrapper/tests/pytest.ini @@ -1,6 +1,7 @@ [pytest] markers = requires_neo4j_and_gds: mark a test as a requiring a Neo4j instance with GDS running + requires_snowflake: mark a test as a requiring a Snowflake connection filterwarnings = error ignore:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning diff --git a/python-wrapper/tests/test_notebooks.py b/python-wrapper/tests/test_notebooks.py old mode 100755 new mode 100644 index 68fcf331..299c817c --- a/python-wrapper/tests/test_notebooks.py +++ b/python-wrapper/tests/test_notebooks.py @@ -120,18 +120,20 @@ def filter_func(notebook: str) -> bool: run_notebooks(filter_func) -# def test_snowflake() -> None: -# snowflake_notebooks = ["snowflake-nvl-example.ipynb"] -# -# def filter_func(notebook: str) -> bool: -# return notebook in snowflake_notebooks -# -# run_notebooks(filter_func) -# -# def test_simple() -> None: -# simple_notebooks = ["simple-nvl-example.ipynb"] -# -# def filter_func(notebook: str) -> bool: -# return notebook in simple_notebooks -# -# run_notebooks(filter_func) +@pytest.mark.requires_snowflake +def test_snowflake() -> None: + snowflake_notebooks = ["snowpark-nvl-example.ipynb"] + + def filter_func(notebook: str) -> bool: + return notebook in snowflake_notebooks + + run_notebooks(filter_func) + + +def test_simple() -> None: + simple_notebooks = ["simple-nvl-example.ipynb"] + + def filter_func(notebook: str) -> bool: + return notebook in simple_notebooks + + run_notebooks(filter_func) From c1cf4b111cc7c805d5642172ef1b2d3cfe8a5022 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Mon, 27 Jan 2025 11:22:55 +0100 Subject: [PATCH 2/3] Add testing info to `CONTRIBUTING.md` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- CONTRIBUTING.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 087f5768..889ca094 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,7 @@ We can't guarantee that we'll accept pull requests and may ask you to make some Occasionally, we might also have logistical, commercial, or legal reasons why we can't accept your work but we'll try to find an alternative way for you to contribute in that case. Remember that many community members have become regular contributors and some are now even Neo employees! + ## Building the project locally To build the Python packages, run inside the `python-wrapper` folder: @@ -67,9 +68,13 @@ yarn build # Build JavaScript resources to be used by Python code This will build the app and copy the relevant files to the python wrapper + ## Specifically for this project -Setting up the Python development environment: +In this section, we will provide some more specific information about how to work with this particular project. + + +### Python development environment * Install Python 3.9+ * [Install pip](https://pip.pypa.io/en/stable/installation/) @@ -79,6 +84,28 @@ Setting up the Python development environment: pip install ".[dev]" ``` +### Testing + +To run unit tests, execute: + +```sh +pytest python-wrapper/tests +``` + +Additionally, there are integration tests that require an external data source. +These require additional setup and configuration, such as environment variables specifying connection details. +To run tests requiring a Neo4j DB instance with GDS installed, execute: + +```sh +pytest python-wrapper/tests --include-neo4j-and-gds +``` + +To run tests requiring a Snowflake connection, execute: + +```sh +pytest python-wrapper/tests --include-snowflake +``` + ### Project structure From 844efb4bc88d78df9952327c766881beb51c2dbd Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Mon, 27 Jan 2025 14:00:32 +0100 Subject: [PATCH 3/3] Fix comment in snowflake ci workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .github/workflows/snowflake-integration-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snowflake-integration-tests.yml b/.github/workflows/snowflake-integration-tests.yml index e82028e0..71e2a179 100644 --- a/.github/workflows/snowflake-integration-tests.yml +++ b/.github/workflows/snowflake-integration-tests.yml @@ -5,7 +5,8 @@ on: # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main" ] - # Skip on this check PR to reduce number of AuraDS instances created + # Skip on this check PR to minimize the load against Snowflake (and keep PR checks fast) + # Allows you to run this workflow manually from the Actions tab workflow_dispatch: