tox.ini

[coverage:report]
skip_covered = False
show_missing = True
# TODO: SNOW-1453027 remove omit for config
# TODO: SNOW-1826001 remove mock folder, and bring up coverage.
omit = */src/snowflake/snowpark/modin/config/*
       */src/snowflake/snowpark/modin/plugin/docstrings/*
       */src/snowflake/snowpark/mock/*
       */src/snowflake/snowpark/_internal/data_source/datasource_typing.py
       tests/integ/datasource/*.py
       tests/integ/test_data_source_api.py

[coverage:run]
relative_files = true
branch = true
parallel = true
concurrency = multiprocessing, thread
# TODO: SNOW-1453027 remove omit for config
omit = */src/snowflake/snowpark/modin/config/*
       */src/snowflake/snowpark/modin/plugin/docstrings/*
       */src/snowflake/snowpark/_internal/data_source/datasource_typing.py
       tests/parameters.py
[coverage:paths]
source = src/snowflake/snowpark
         */.tox/*/lib/python*/site-packages/snowflake/snowpark
         */.tox\*\Lib\site-packages\snowflake\snowpark
         */src/snowflake/snowpark
         *\src\snowflake\snowpark
         */fips_env/lib/python*/site-packages/snowflake/snowpark
[coverage:html]
show_contexts = true

[tox]
minversion = 3.7
envlist = fix_lint,
          py39,
          coverage
          nopandas
skip_missing_interpreters = true
setenv = SNOWPARK_LOCAL_TESTING_INTERNAL_TELEMETRY=1

[testenv]
allowlist_externals = bash
description = run the tests with pytest under {basepython}
deps =
    {env:SNOWFLAKE_PYTEST_MODIN_JENKINS_PYARROW_CAP}
    {env:SNOWFLAKE_PYTEST_MODIN_JENKINS_PANDAS_CAP}
    pip >= 19.3.1
    pytest-xdist
    pytest-timeout
    pytest-rerunfailures < 16.0.0
    llvmlite >= 0.43.0  # for uv, https://github.com/astral-sh/uv/issues/9413
    numba >= 0.60.0  # for uv, same as above
    .[pandas]
    .[development]
    .[opentelemetry]
    {env:SNOWFLAKE_PYTEST_MODIN_DEPS}
    {env:SNOWFLAKE_PYTEST_MODIN_PIN}
    {env:SNOWFLAKE_PYTEST_PANDAS_DEPS}
install_command = bash ./scripts/tox_install_cmd.sh {opts} {packages}
setenv =
    COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}}
    # Leave verbosity at the default level, since printing test names adds I/O overhead latency that
    # is trivial in a single job run, but adds up over the course of many CI runs.
    # -v also causes pytest-xdist to print the name of each test case twice (once when it's scheduled
    # and once when it's completed), while default verbosity just represents each test with a . or X.
    ci: SNOWFLAKE_PYTEST_VERBOSITY =
    SNOWFLAKE_PYTEST_COLOR = --color=no
    # Do not run doctests in parallel so coverage works
    # Snowpark uses all available logical cores to accelerate testing in merge gate.
    # Note that using all available physical cores with `-n auto` seems to
    # spawn fewer than 64 pytest processes on the 64-core github runners.
    # For AST tests, use just 36 cores, because using more cores sometimes
    # causes the test job to fail with exit code 143, which supposedly means
    # that it uses too many machine resources.
    !doctest-!ast: SNOWFLAKE_PYTEST_PARALLELISM = -n logical
    ast: SNOWFLAKE_PYTEST_PARALLELISM = -n 36
    # Snowpark uses 4 workers for daily testing since some of its test jobs use weak MacOS instances.
    !doctest: SNOWFLAKE_PYTEST_DAILY_PARALLELISM = -n 6
    # Set test type, either notset, unit, integ, or both
    unit-integ-doctest: SNOWFLAKE_TEST_TYPE = (unit or integ or doctest)
    !unit-!integ-!doctest: SNOWFLAKE_TEST_TYPE = (unit or integ or doctest)
    unit: SNOWFLAKE_TEST_TYPE = unit
    integ: SNOWFLAKE_TEST_TYPE = integ
    doctest: SNOWFLAKE_TEST_TYPE = doctest
    ast: SNOWFLAKE_TEST_TYPE = ast
    # Add common parts into pytest command
    SNOWFLAKE_PYTEST_COV_LOCATION = {env:JUNIT_REPORT_DIR:{toxworkdir}}/junit.{envname}-{env:cloud_provider:dev}.xml
    SNOWFLAKE_PYTEST_COV_CMD = --cov snowflake.snowpark --junitxml {env:SNOWFLAKE_PYTEST_COV_LOCATION} --cov-report=
    SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD = --ignore=src/snowflake/snowpark/modin --ignore=tests/integ/modin --ignore=tests/unit/modin
    SNOWFLAKE_PYTEST_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COLOR:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources {env:SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD}
    SNOWFLAKE_PYTEST_DAILY_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COLOR:} {env:SNOWFLAKE_PYTEST_DAILY_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources {env:SNOWFLAKE_PYTEST_IGNORE_MODIN_CMD}
    # This configures the extra dependency required by modin test
    modin: SNOWFLAKE_PYTEST_MODIN_DEPS = [modin-development]
    modin_pandas_version: SNOWFLAKE_PYTEST_PANDAS_DEPS = pandas=={env:MODIN_PANDAS_PATCH_VERSION}
    modin_previous_version: SNOWFLAKE_PYTEST_MODIN_PIN = modin==0.36.0
    RERUN_FLAGS = --reruns 5 --reruns-delay 5
    SNOW_1314507_WORKAROUND_RERUN_FLAGS = {env:RERUN_FLAGS} --only-rerun "Insufficient resource during interleaved execution."
    MODIN_PYTEST_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COLOR:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources
    MODIN_PYTEST_DAILY_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COLOR:} {env:SNOWFLAKE_PYTEST_DAILY_PARALLELISM:} {env:SNOWFLAKE_PYTEST_COV_CMD} --ignore=tests/resources
    MODIN_PYTEST_NO_COV_CMD = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COLOR:} {env:SNOWFLAKE_PYTEST_PARALLELISM:} --ignore=tests/resources
    # Cap pyarrow<21 for Snowpark pandas tests on Jenkins due to SNOW-2266293
    snowparkpandasjenkins: SNOWFLAKE_PYTEST_MODIN_JENKINS_PYARROW_CAP = pyarrow<21
    pyarrowcap: SNOWFLAKE_PYTEST_MODIN_JENKINS_PYARROW_CAP = pyarrow<21
    pandascap: SNOWFLAKE_PYTEST_MODIN_JENKINS_PANDAS_CAP = pandas<=2.3.2

passenv =
    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY
    SF_PROJECT_ROOT
    cloud_provider
    SF_REGRESS_LOGS
    ; Github Actions provided environmental variables
    GITHUB_ACTIONS
    JENKINS_HOME
    ; This is required on windows. Otherwise pwd module won't be imported successfully,
    ; see https://github.com/tox-dev/tox/issues/1455
    USERNAME
    CLIENT_LOG_DIR_PATH_DOCKER
    PYTEST_ADDOPTS
    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
    snowflake_path
    ; Below only used in AST tests
    TZ
    GITHUB_ENV
    SNOWPARK_PYTHON_API_TEST_BUCKET_PATH
    SNOWPARK_PYTHON_API_S3_STORAGE_INTEGRATION
commands =
    notudf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests
    udf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests
    notdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} {env:RERUN_FLAGS} tests
    notudfdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} {env:RERUN_FLAGS} tests
    local: {env:SNOWFLAKE_PYTEST_CMD} --local_testing_mode -m "integ or unit or mock" {posargs:} tests
    dailynotdoctest: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} tests
    dailynotdoctestnotudf: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
    # Use a shell script for Snowpark pandas not-doctests because they are too
    # complex to run in a single bash command, and adding multiple bash
    # commands here is difficult. pass SNOW_1314507_WORKAROUND_RERUN_FLAGS here
    # instead of getting it within the script because escaping it in the script
    # is difficult.
    snowparkpandasnotdoctest: bash scripts/run_snowparkpandasnotdoctest.sh {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} {posargs:}
    # This one only run doctest but we still need to include the tests folder to let tests/conftest.py to mark the doctest files for us
    snowparkpandasdoctest: {env:MODIN_PYTEST_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} src/snowflake/snowpark/modin/ tests/unit/modin
    # This one is used by daily_modin_precommit_py39_py310.yml
    snowparkpandasdailynotdoctest: {env:MODIN_PYTEST_DAILY_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin tests/integ/test_df_to_snowpark_pandas.py
    # This one is only called by jenkins job and the only difference from `snowparkpandasnotdoctest` is that it uses
    # MODIN_PYTEST_NO_COV_CMD instead of MODIN_PYTEST_CMD
    snowparkpandasjenkins: {env:MODIN_PYTEST_NO_COV_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin
    # These jobs run outside pytest since we manipulate AutoSwitchBackend during pytest setup and we want to test a fresh session.
    snowparkpandashybridcheck_sessionfirst: python -c "from tests.parameters import CONNECTION_PARAMETERS; from snowflake.snowpark.session import Session; Session.builder.configs(CONNECTION_PARAMETERS).create(); import snowflake.snowpark.modin.plugin; from modin.config import AutoSwitchBackend; assert AutoSwitchBackend.get() is True"
    snowparkpandashybridcheck_pluginfirst: python -c "from tests.parameters import CONNECTION_PARAMETERS; import snowflake.snowpark.modin.plugin; from snowflake.snowpark.session import Session; Session.builder.configs(CONNECTION_PARAMETERS).create(); from modin.config import AutoSwitchBackend; assert AutoSwitchBackend.get() is True"
    # Snowpark IR commands:
    ast: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} tests

[testenv:nopandas]
allowlist_externals = bash
description = run the tests with pytest under {basepython} with no pandas installed
deps =
    pip >= 19.3.1
    pytest-xdist
    pytest-timeout
    .[development]
    .[opentelemetry]
commands = {env:SNOWFLAKE_PYTEST_DAILY_CMD} -vvv -m "integ or unit" {posargs:}  tests

[testenv:modin_extra_without_pandas_extra]
allowlist_externals = bash
description = SNOW-1345421: test that we can use modin by specifying the modin extra but not the pandas extra.
deps = .[modin-development]
commands = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COV_CMD} {posargs:}  tests/integ/modin/test_modin_extra_without_pandas_extra.py

[testenv:coverage]
description = [run locally after tests]: combine coverage data and create report
deps = {[testenv]deps}
       coverage!=7.6.5
skip_install = True
passenv =
    DIFF_AGAINST
    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
setenv = COVERAGE_FILE={toxworkdir}/.coverage
commands = coverage combine
           coverage report -m
           coverage xml -o {env:COV_REPORT_DIR:{toxworkdir}}/coverage.xml
           coverage html -d {env:COV_REPORT_DIR:{toxworkdir}}/htmlcov --show-contexts
depends = py39, py310, py311, py312, py313, py314

[testenv:docs]
basepython = python3.9
description = build docs for the project
skip_install = false
deps =
    sphinx
    .[modin-development]
extras = pandas
commands =
    sphinx-build -d "{envtmpdir}{/}doctree" docs/source "{toxworkdir}{/}docs_out" --color -b html {posargs:-W --keep-going}
    python -c 'print(r"documentation available under file://{toxworkdir}{/}docs_out{/}index.html")'

[testenv:flake8]
description = check code style with flake8
skip_install = true
deps = flake8
commands = flake8 {posargs}

[testenv:fix_lint]
allowlist_externals = bash
description = format the code base to adhere to our styles, and complain about what we cannot do automatically
basepython = python3.9
passenv =
    PROGRAMDATA
deps =
    pre-commit >= 3.5.0
skip_install = True
commands = pre-commit run -a -v
           python -c 'import pathlib; print("hint: run \{\} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))'

[testenv:pyright]
description = static type checking with pyright
deps = pyright==1.1.338
commands =
    pyright src/snowflake/snowpark/_internal/analyzer
    pyright src/snowflake/snowpark/_internal/compiler
    pyright src/snowflake/snowpark/stored_procedure_profiler.py
    pyright src/snowflake/snowpark/_internal/debug_utils.py

[testenv:protoc]
description = generate python code from protobuf
allowlist_externals = bash, protoc
deps =
    protoc-wheel-0==21.1
    mypy-protobuf
    protobuf
commands =
    protoc --proto_path=src/snowflake/snowpark/_internal/proto/ --python_out=src/snowflake/snowpark/_internal/proto/generated --mypy_out=src/snowflake/snowpark/_internal/proto/generated/ src/snowflake/snowpark/_internal/proto/ast.proto

[testenv:dev]
description = create dev environment
extras = pandas, development, sso
usedevelop = True
commands = python -m pip list --format=columns
           python -c "print(r'{envpython}')"

[testenv:snowpark_pandas_modin_pandas_import_error]
deps = .[modin-development]
description = test error messages when importing unsupported modin or pandas versions
basepython = python3.9
setenv = TEST_INCORRECT_MODIN_PANDAS_VERSIONS=True
commands =
    python -m pytest --noconftest tests/unit/modin/test_pandas_version.py
    python -m pytest --noconftest tests/unit/modin/test_pandas_version_notebooks.py
    python -m pytest --noconftest tests/unit/modin/test_modin_version.py
    python -m pytest --noconftest tests/unit/modin/test_modin_version_notebooks.py

[testenv:snowpark_pandas_old_np]
description = test that Snowpark pandas works with numpy 1.26
deps = .[modin-development]
       numpy<2.0.0
commands = python -m pytest tests/integ/modin/test_old_numpy_aliases.py

[testenv:datasource]
description = run the datasource tests that requires DBMS drivers
deps =
    {[testenv]deps}
    databricks-sql-connector > 4.0.0
    oracledb
    psycopg2-binary
    pymysql
    pyodbc
commands =
    {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} tests/integ/datasource -n 8

[pytest]
# Keep log_cli disabled but set log_cli_level set to DEBUG. This prevents log messages from clogging
# test output while running, but still displays captured query text on test failure/error.
# log_cli = True
log_cli_level = DEBUG
markers =
    # Optional dependency groups markers
    udf: Snowpark UDF tests
    # Test type markers
    integ: integration tests
    unit: unit tests
    doctest: doctest tests
    # Other markers
    timeout: tests that need a timeout time
    modin_sp_precommit: modin precommit tests run in sproc
    ast: tests run from the ast directory
    compiler: tests run from the compiler directory
    scala: tests run from the scala directory
    mock: tests run from the mock directory
    skip_hybrid: tests that cannot run under hybrid mode
addopts = --doctest-modules --timeout=1200
norecursedirs = tests/perf/data_source

[flake8]
# E203: Whitespace before ':'
# E501: Line too long
# W503: Line break occurred before a binary operator
ignore = E203,E501,W503
exclude=
    build,.tox,parameters.py,
# Disable checking virtualenv contents
    *venv*
max-line-length = 88
show-source = true