diff --git a/dev/run-tests.py b/dev/run-tests.py index d50307513bb01..7b057ed6f5af9 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -25,7 +25,7 @@ import subprocess from contextlib import contextmanager -from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES +from sparktestsupport import SPARK_HOME, USER_HOME from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which from sparktestsupport.utils import ( determine_dangling_python_tests, @@ -69,43 +69,42 @@ def determine_java_executable(): # ------------------------------------------------------------------------------------------------- -def set_title_and_block(title, err_block): - os.environ["CURRENT_BLOCK"] = str(ERROR_CODES[err_block]) - line_str = "=" * 72 - - print("") - print(line_str) - print(title) - print(line_str) +_in_titled_block = False @contextmanager -def group_in_github_actions(title): +def titled_block(title): + global _in_titled_block + if _in_titled_block: + raise RuntimeError(f"titled_block({title!r}) cannot be nested") + _in_titled_block = True + line_str = "=" * 72 if "GITHUB_ACTIONS" in os.environ: print(f"::group::{title}", flush=True) - try: - yield - finally: - print("::endgroup::", flush=True) - else: + print("") + print(line_str) + print(title) + print(line_str) + try: yield + finally: + _in_titled_block = False + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::", flush=True) def run_apache_rat_checks(): - set_title_and_block("Running Apache RAT checks", "BLOCK_RAT") run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")]) def run_scala_style_checks(extra_profiles): build_profiles = extra_profiles + modules.root.build_profile_flags - set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE") profiles = " ".join(build_profiles) print("[info] Checking Scala style using SBT with these profiles: ", profiles) run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles]) def run_java_style_checks(build_profiles): - set_title_and_block("Running Java style checks", "BLOCK_JAVA_STYLE") # The same profiles used for building are used to run Checkstyle by SBT as well because # the previous build looks reused for Checkstyle and affecting Checkstyle. See SPARK-27130. profiles = " ".join(build_profiles) @@ -114,13 +113,10 @@ def run_java_style_checks(build_profiles): def run_python_style_checks(): - set_title_and_block("Running Python style checks", "BLOCK_PYTHON_STYLE") run_cmd([os.path.join(SPARK_HOME, "dev", "lint-python")]) def run_sparkr_style_checks(): - set_title_and_block("Running R style checks", "BLOCK_R_STYLE") - if which("R"): # R style check should be executed after `install-dev.sh`. # Since warnings about `no visible global function definition` appear @@ -130,27 +126,6 @@ def run_sparkr_style_checks(): print("Ignoring SparkR style check as R was not found in PATH") -def build_spark_documentation(): - set_title_and_block("Building Spark Documentation", "BLOCK_DOCUMENTATION") - os.environ["PRODUCTION"] = "1" - - os.chdir(os.path.join(SPARK_HOME, "docs")) - - bundle_bin = which("bundle") - - if not bundle_bin: - print( - "[error] Cannot find a version of `bundle` on the system; please", - " install one with `gem install bundler` and retry to build documentation.", - ) - sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) - else: - run_cmd([bundle_bin, "install"]) - run_cmd([bundle_bin, "exec", "jekyll", "build"]) - - os.chdir(SPARK_HOME) - - def exec_maven(mvn_args=()): """Will call Maven in the current directory with the list of mvn_args passed in and returns the subprocess for any further processing""" @@ -208,16 +183,15 @@ def get_scala_profiles(scala_version): " are", sbt_maven_scala_profiles.keys(), ) - sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) + sys.exit(1) def switch_scala_version(scala_version): """ Switch the code base to use the given Scala version. """ - set_title_and_block("Switch the Scala version to %s" % scala_version, "BLOCK_SCALA_VERSION") - assert scala_version is not None + print("[info] Switching to Scala version", scala_version) ver_num = scala_version[-4:] # Simply extract. e.g.) 2.13 from scala2.13 command = [os.path.join(SPARK_HOME, "dev", "change-scala-version.sh"), ver_num] run_cmd(command) @@ -243,7 +217,7 @@ def get_hadoop_profiles(hadoop_version): " are", sbt_maven_hadoop_profiles.keys(), ) - sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) + sys.exit(1) def build_spark_maven(extra_profiles): @@ -269,12 +243,10 @@ def build_spark_sbt(extra_profiles): print("[info] Building Spark using SBT with these arguments: ", " ".join(profiles_and_goals)) - with group_in_github_actions("sbt build spark"): - exec_sbt(profiles_and_goals) + exec_sbt(profiles_and_goals) def build_spark_unidoc_sbt(extra_profiles): - set_title_and_block("Building Unidoc API Documentation", "BLOCK_DOCUMENTATION") # Enable all of the profiles for the build: build_profiles = extra_profiles + modules.root.build_profile_flags sbt_goals = ["unidoc"] @@ -288,7 +260,7 @@ def build_spark_unidoc_sbt(extra_profiles): exec_sbt(profiles_and_goals) -def build_spark_assembly_sbt(extra_profiles, checkstyle=False): +def build_spark_assembly_sbt(extra_profiles): # Enable all of the profiles for the build: build_profiles = extra_profiles + modules.root.build_profile_flags sbt_goals = ["assembly/package"] @@ -298,22 +270,13 @@ def build_spark_assembly_sbt(extra_profiles, checkstyle=False): " ".join(profiles_and_goals), ) - with group_in_github_actions("sbt build spark assembly"): - exec_sbt(profiles_and_goals) - - if checkstyle: - run_java_style_checks(build_profiles) - - if not os.environ.get("SKIP_UNIDOC"): - build_spark_unidoc_sbt(extra_profiles) + exec_sbt(profiles_and_goals) def build_apache_spark(build_tool, extra_profiles): """Will build Spark with the extra profiles and the passed in build tool (either `sbt` or `maven`). Defaults to using `sbt`.""" - set_title_and_block("Building Spark", "BLOCK_BUILD") - rm_r("lib_managed") if build_tool == "maven": @@ -324,7 +287,6 @@ def build_apache_spark(build_tool, extra_profiles): def detect_binary_inop_with_mima(extra_profiles): build_profiles = extra_profiles + modules.root.build_profile_flags - set_title_and_block("Detecting binary incompatibilities with MiMa", "BLOCK_MIMA") profiles = " ".join(build_profiles) print( "[info] Detecting binary incompatibilities with MiMa using SBT with these profiles: ", @@ -364,7 +326,6 @@ def run_scala_tests_sbt(test_modules, test_profiles): def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags): """Function to properly execute all tests passed in as a set from the `determine_test_suites` function""" - set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS") # Remove duplicates while keeping the test module order test_modules = list(dict.fromkeys(test_modules)) @@ -393,8 +354,6 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, inc def run_python_tests(test_modules, test_pythons, parallelism, with_coverage=False): - set_title_and_block("Running PySpark tests", "BLOCK_PYSPARK_UNIT_TESTS") - if with_coverage: # Coverage makes the PySpark tests flaky due to heavy parallelism. # When we run PySpark tests with coverage, it uses 4 for now as @@ -412,20 +371,15 @@ def run_python_tests(test_modules, test_pythons, parallelism, with_coverage=Fals def run_python_packaging_tests(): - if os.environ.get("SKIP_PACKAGING", "false") != "true": - set_title_and_block("Running PySpark packaging tests", "BLOCK_PYSPARK_PIP_TESTS") - command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")] - run_cmd(command) + command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")] + run_cmd(command) def run_build_tests(): - set_title_and_block("Running build tests", "BLOCK_BUILD_TESTS") run_cmd([os.path.join(SPARK_HOME, "dev", "test-dependencies.sh")]) def run_sparkr_tests(): - set_title_and_block("Running SparkR tests", "BLOCK_SPARKR_UNIT_TESTS") - if which("R"): run_cmd([os.path.join(SPARK_HOME, "R", "run-tests.sh")]) else: @@ -498,8 +452,6 @@ def main(): rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "local", "org.apache.spark")) rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "cache", "org.apache.spark")) - os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"]) - java_exe = determine_java_executable() if not java_exe: @@ -509,7 +461,6 @@ def main(): ) sys.exit(2) - # Install SparkR should_only_test_modules = opts.modules is not None test_modules = [] if should_only_test_modules: @@ -520,7 +471,8 @@ def main(): # If tests modules are specified, we will not run R linter. # SparkR needs the manual SparkR installation. if which("R"): - run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")]) + with titled_block("Installing SparkR"): + run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")]) else: print("Cannot install SparkR as R was not found in PATH") @@ -610,18 +562,21 @@ def main(): if scala_version is not None: # If not set, assume this is default and doesn't need to change. - switch_scala_version(scala_version) + with titled_block("Switching Scala version"): + switch_scala_version(scala_version) should_run_java_style_checks = False if not should_only_test_modules: # license checks - run_apache_rat_checks() + with titled_block("Running Apache RAT checks"): + run_apache_rat_checks() # style checks if not changed_files or any( f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files ): - run_scala_style_checks(extra_profiles) + with titled_block("Running Scala style checks"): + run_scala_style_checks(extra_profiles) if not changed_files or any( f.endswith(".java") or f.endswith("checkstyle.xml") @@ -634,43 +589,60 @@ def main(): f.endswith("lint-python") or f.endswith("pyproject.toml") or f.endswith(".py") for f in changed_files ): - run_python_style_checks() + with titled_block("Running Python style checks"): + run_python_style_checks() if not changed_files or any( f.endswith(".R") or f.endswith("lint-r") or f.endswith(".lintr") for f in changed_files ): - run_sparkr_style_checks() + with titled_block("Running R style checks"): + run_sparkr_style_checks() if any(m.should_run_build_tests for m in test_modules): - run_build_tests() + with titled_block("Running build tests"): + run_build_tests() # spark build if os.environ.get("SKIP_SCALA_BUILD", "false") != "true": - build_apache_spark(build_tool, extra_profiles) + with titled_block("Building Spark"): + build_apache_spark(build_tool, extra_profiles) # backwards compatibility checks if build_tool == "sbt": # Note: compatibility tests only supported in sbt for now if not os.environ.get("SKIP_MIMA"): - detect_binary_inop_with_mima(extra_profiles) + with titled_block("Detecting binary incompatibilities with MiMa"): + detect_binary_inop_with_mima(extra_profiles) # Since we did not build assembly/package before running dev/mima, we need to # do it here because the tests still rely on it; see SPARK-13294 for details. if os.environ.get("SKIP_SCALA_BUILD", "false") != "true": - build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks) + with titled_block("Building Spark assembly"): + build_spark_assembly_sbt(extra_profiles) + if should_run_java_style_checks: + with titled_block("Running Java style checks"): + run_java_style_checks(extra_profiles + modules.root.build_profile_flags) + if not os.environ.get("SKIP_UNIDOC"): + with titled_block("Building Unidoc API Documentation"): + build_spark_unidoc_sbt(extra_profiles) # run the test suites - run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags) + with titled_block("Running Spark unit tests"): + run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags) modules_with_python_tests = [m for m in test_modules if m.python_test_goals] if modules_with_python_tests and not os.environ.get("SKIP_PYTHON"): - run_python_tests( - modules_with_python_tests, - opts.python_executables, - opts.parallelism, - with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true", - ) - run_python_packaging_tests() + with titled_block("Running PySpark tests"): + run_python_tests( + modules_with_python_tests, + opts.python_executables, + opts.parallelism, + with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true", + ) + if os.environ.get("SKIP_PACKAGING", "false") != "true": + with titled_block("Running PySpark packaging tests"): + run_python_packaging_tests() if any(m.should_run_r_tests for m in test_modules) and not os.environ.get("SKIP_R"): - run_sparkr_tests() + with titled_block("Running SparkR tests"): + run_sparkr_tests() def _test(): diff --git a/dev/sparktestsupport/__init__.py b/dev/sparktestsupport/__init__.py index b4edb6b62c20b..12696d98fb988 100644 --- a/dev/sparktestsupport/__init__.py +++ b/dev/sparktestsupport/__init__.py @@ -19,21 +19,3 @@ SPARK_HOME = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")) USER_HOME = os.environ.get("HOME") -ERROR_CODES = { - "BLOCK_GENERAL": 10, - "BLOCK_RAT": 11, - "BLOCK_SCALA_STYLE": 12, - "BLOCK_PYTHON_STYLE": 13, - "BLOCK_R_STYLE": 14, - "BLOCK_DOCUMENTATION": 15, - "BLOCK_BUILD": 16, - "BLOCK_MIMA": 17, - "BLOCK_SPARK_UNIT_TESTS": 18, - "BLOCK_PYSPARK_UNIT_TESTS": 19, - "BLOCK_SPARKR_UNIT_TESTS": 20, - "BLOCK_JAVA_STYLE": 21, - "BLOCK_BUILD_TESTS": 22, - "BLOCK_PYSPARK_PIP_TESTS": 23, - "BLOCK_SCALA_VERSION": 24, - "BLOCK_TIMEOUT": 124, -} diff --git a/dev/sparktestsupport/shellutils.py b/dev/sparktestsupport/shellutils.py index 1d40ae9f2718c..662b8c8cc0ec3 100644 --- a/dev/sparktestsupport/shellutils.py +++ b/dev/sparktestsupport/shellutils.py @@ -28,7 +28,7 @@ def exit_from_command_with_retcode(cmd, retcode): print("[error] running", " ".join(cmd), "; process was terminated by signal", -retcode) else: print("[error] running", " ".join(cmd), "; received return code", retcode) - sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) + sys.exit(1) def rm_r(path):