[SPARK-58573][INFRA][BUILD] Replace defunct Jenkins test grouping with modern GitHub Actions grouping

dev/run-tests.py

@@ -25,7 +25,7 @@
 import subprocess
 from contextlib import contextmanager
 
-from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES
+from sparktestsupport import SPARK_HOME, USER_HOME
 from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which
 from sparktestsupport.utils import (
     determine_dangling_python_tests,
@@ -69,43 +69,42 @@ def determine_java_executable():
 # -------------------------------------------------------------------------------------------------
 
 
-def set_title_and_block(title, err_block):
-    os.environ["CURRENT_BLOCK"] = str(ERROR_CODES[err_block])
-    line_str = "=" * 72
-
-    print("")
-    print(line_str)
-    print(title)
-    print(line_str)
+_in_titled_block = False
 
 
 @contextmanager
-def group_in_github_actions(title):
+def titled_block(title):
+    global _in_titled_block
+    if _in_titled_block:
+        raise RuntimeError(f"titled_block({title!r}) cannot be nested")
+    _in_titled_block = True
+    line_str = "=" * 72
     if "GITHUB_ACTIONS" in os.environ:
         print(f"::group::{title}", flush=True)
-        try:
-            yield
-        finally:
-            print("::endgroup::", flush=True)
-    else:
+    print("")
+    print(line_str)
+    print(title)
+    print(line_str)
+    try:
         yield
+    finally:
+        _in_titled_block = False
+        if "GITHUB_ACTIONS" in os.environ:
+            print("::endgroup::", flush=True)
 
 
 def run_apache_rat_checks():
-    set_title_and_block("Running Apache RAT checks", "BLOCK_RAT")
     run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")])
 
 
 def run_scala_style_checks(extra_profiles):
     build_profiles = extra_profiles + modules.root.build_profile_flags
-    set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE")
     profiles = " ".join(build_profiles)
     print("[info] Checking Scala style using SBT with these profiles: ", profiles)
     run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles])
 
 
 def run_java_style_checks(build_profiles):
-    set_title_and_block("Running Java style checks", "BLOCK_JAVA_STYLE")
     # The same profiles used for building are used to run Checkstyle by SBT as well because
     # the previous build looks reused for Checkstyle and affecting Checkstyle. See SPARK-27130.
     profiles = " ".join(build_profiles)
@@ -114,13 +113,10 @@ def run_java_style_checks(build_profiles):
 
 
 def run_python_style_checks():
-    set_title_and_block("Running Python style checks", "BLOCK_PYTHON_STYLE")
     run_cmd([os.path.join(SPARK_HOME, "dev", "lint-python")])
 
 
 def run_sparkr_style_checks():
-    set_title_and_block("Running R style checks", "BLOCK_R_STYLE")
-
     if which("R"):
         # R style check should be executed after `install-dev.sh`.
         # Since warnings about `no visible global function definition` appear
@@ -130,27 +126,6 @@ def run_sparkr_style_checks():
         print("Ignoring SparkR style check as R was not found in PATH")
 
 
-def build_spark_documentation():
-    set_title_and_block("Building Spark Documentation", "BLOCK_DOCUMENTATION")
-    os.environ["PRODUCTION"] = "1"
-
-    os.chdir(os.path.join(SPARK_HOME, "docs"))
-
-    bundle_bin = which("bundle")
-
-    if not bundle_bin:
-        print(
-            "[error] Cannot find a version of `bundle` on the system; please",
-            " install one with `gem install bundler` and retry to build documentation.",
-        )
-        sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
-    else:
-        run_cmd([bundle_bin, "install"])
-        run_cmd([bundle_bin, "exec", "jekyll", "build"])
-
-    os.chdir(SPARK_HOME)
-
-
 def exec_maven(mvn_args=()):
     """Will call Maven in the current directory with the list of mvn_args passed
     in and returns the subprocess for any further processing"""
@@ -208,16 +183,15 @@ def get_scala_profiles(scala_version):
             " are",
             sbt_maven_scala_profiles.keys(),
         )
-        sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+        sys.exit(1)
 
 
 def switch_scala_version(scala_version):
     """
     Switch the code base to use the given Scala version.
     """
-    set_title_and_block("Switch the Scala version to %s" % scala_version, "BLOCK_SCALA_VERSION")
-
     assert scala_version is not None
+    print("[info] Switching to Scala version", scala_version)
     ver_num = scala_version[-4:]  # Simply extract. e.g.) 2.13 from scala2.13
     command = [os.path.join(SPARK_HOME, "dev", "change-scala-version.sh"), ver_num]
     run_cmd(command)
@@ -243,7 +217,7 @@ def get_hadoop_profiles(hadoop_version):
             " are",
             sbt_maven_hadoop_profiles.keys(),
         )
-        sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+        sys.exit(1)
 
 
 def build_spark_maven(extra_profiles):
@@ -269,12 +243,10 @@ def build_spark_sbt(extra_profiles):
 
     print("[info] Building Spark using SBT with these arguments: ", " ".join(profiles_and_goals))
 
-    with group_in_github_actions("sbt build spark"):
-        exec_sbt(profiles_and_goals)
+    exec_sbt(profiles_and_goals)
 
 
 def build_spark_unidoc_sbt(extra_profiles):
-    set_title_and_block("Building Unidoc API Documentation", "BLOCK_DOCUMENTATION")
     # Enable all of the profiles for the build:
     build_profiles = extra_profiles + modules.root.build_profile_flags
     sbt_goals = ["unidoc"]
@@ -288,7 +260,7 @@ def build_spark_unidoc_sbt(extra_profiles):
     exec_sbt(profiles_and_goals)
 
 
-def build_spark_assembly_sbt(extra_profiles, checkstyle=False):
+def build_spark_assembly_sbt(extra_profiles):
     # Enable all of the profiles for the build:
     build_profiles = extra_profiles + modules.root.build_profile_flags
     sbt_goals = ["assembly/package"]
@@ -298,22 +270,13 @@ def build_spark_assembly_sbt(extra_profiles, checkstyle=False):
         " ".join(profiles_and_goals),
     )
 
-    with group_in_github_actions("sbt build spark assembly"):
-        exec_sbt(profiles_and_goals)
-
-    if checkstyle:
-        run_java_style_checks(build_profiles)
-
-    if not os.environ.get("SKIP_UNIDOC"):
-        build_spark_unidoc_sbt(extra_profiles)
+    exec_sbt(profiles_and_goals)
 
 
 def build_apache_spark(build_tool, extra_profiles):
     """Will build Spark with the extra profiles and the passed in build tool
     (either `sbt` or `maven`). Defaults to using `sbt`."""
 
-    set_title_and_block("Building Spark", "BLOCK_BUILD")
-
     rm_r("lib_managed")
 
     if build_tool == "maven":
@@ -324,7 +287,6 @@ def build_apache_spark(build_tool, extra_profiles):
 
 def detect_binary_inop_with_mima(extra_profiles):
     build_profiles = extra_profiles + modules.root.build_profile_flags
-    set_title_and_block("Detecting binary incompatibilities with MiMa", "BLOCK_MIMA")
     profiles = " ".join(build_profiles)
     print(
         "[info] Detecting binary incompatibilities with MiMa using SBT with these profiles: ",
@@ -364,7 +326,6 @@ def run_scala_tests_sbt(test_modules, test_profiles):
 def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags):
     """Function to properly execute all tests passed in as a set from the
     `determine_test_suites` function"""
-    set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
 
     # Remove duplicates while keeping the test module order
     test_modules = list(dict.fromkeys(test_modules))
@@ -393,8 +354,6 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, inc
 
 
 def run_python_tests(test_modules, test_pythons, parallelism, with_coverage=False):
-    set_title_and_block("Running PySpark tests", "BLOCK_PYSPARK_UNIT_TESTS")
-
     if with_coverage:
         # Coverage makes the PySpark tests flaky due to heavy parallelism.
         # When we run PySpark tests with coverage, it uses 4 for now as
@@ -412,20 +371,15 @@ def run_python_tests(test_modules, test_pythons, parallelism, with_coverage=Fals
 
 
 def run_python_packaging_tests():
-    if os.environ.get("SKIP_PACKAGING", "false") != "true":
-        set_title_and_block("Running PySpark packaging tests", "BLOCK_PYSPARK_PIP_TESTS")
-        command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")]
-        run_cmd(command)
+    command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")]
+    run_cmd(command)
 
 
 def run_build_tests():
-    set_title_and_block("Running build tests", "BLOCK_BUILD_TESTS")
     run_cmd([os.path.join(SPARK_HOME, "dev", "test-dependencies.sh")])
 
 
 def run_sparkr_tests():
-    set_title_and_block("Running SparkR tests", "BLOCK_SPARKR_UNIT_TESTS")
-
     if which("R"):
         run_cmd([os.path.join(SPARK_HOME, "R", "run-tests.sh")])
     else:
@@ -498,8 +452,6 @@ def main():
     rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "local", "org.apache.spark"))
     rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "cache", "org.apache.spark"))
 
-    os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"])
-
     java_exe = determine_java_executable()
 
     if not java_exe:
@@ -509,7 +461,6 @@ def main():
         )
         sys.exit(2)
 
-    # Install SparkR
     should_only_test_modules = opts.modules is not None
     test_modules = []
     if should_only_test_modules:
@@ -520,7 +471,8 @@ def main():
         # If tests modules are specified, we will not run R linter.
         # SparkR needs the manual SparkR installation.
         if which("R"):
-            run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
+            with titled_block("Installing SparkR"):
+                run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
         else:
             print("Cannot install SparkR as R was not found in PATH")
 
@@ -610,18 +562,21 @@ def main():
 
     if scala_version is not None:
         # If not set, assume this is default and doesn't need to change.
-        switch_scala_version(scala_version)
+        with titled_block("Switching Scala version"):
+            switch_scala_version(scala_version)
 
     should_run_java_style_checks = False
     if not should_only_test_modules:
         # license checks
-        run_apache_rat_checks()
+        with titled_block("Running Apache RAT checks"):
+            run_apache_rat_checks()
 
         # style checks
         if not changed_files or any(
             f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files
         ):
-            run_scala_style_checks(extra_profiles)
+            with titled_block("Running Scala style checks"):
+                run_scala_style_checks(extra_profiles)
         if not changed_files or any(
             f.endswith(".java")
             or f.endswith("checkstyle.xml")
@@ -634,43 +589,60 @@ def main():
             f.endswith("lint-python") or f.endswith("pyproject.toml") or f.endswith(".py")
             for f in changed_files
         ):
-            run_python_style_checks()
+            with titled_block("Running Python style checks"):
+                run_python_style_checks()
         if not changed_files or any(
             f.endswith(".R") or f.endswith("lint-r") or f.endswith(".lintr") for f in changed_files
         ):
-            run_sparkr_style_checks()
+            with titled_block("Running R style checks"):
+                run_sparkr_style_checks()
 
     if any(m.should_run_build_tests for m in test_modules):
-        run_build_tests()
+        with titled_block("Running build tests"):
+            run_build_tests()
 
     # spark build
     if os.environ.get("SKIP_SCALA_BUILD", "false") != "true":
-        build_apache_spark(build_tool, extra_profiles)
+        with titled_block("Building Spark"):
+            build_apache_spark(build_tool, extra_profiles)
 
     # backwards compatibility checks
     if build_tool == "sbt":
         # Note: compatibility tests only supported in sbt for now
         if not os.environ.get("SKIP_MIMA"):
-            detect_binary_inop_with_mima(extra_profiles)
+            with titled_block("Detecting binary incompatibilities with MiMa"):
+                detect_binary_inop_with_mima(extra_profiles)
         # Since we did not build assembly/package before running dev/mima, we need to
         # do it here because the tests still rely on it; see SPARK-13294 for details.
         if os.environ.get("SKIP_SCALA_BUILD", "false") != "true":
-            build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks)
+            with titled_block("Building Spark assembly"):
+                build_spark_assembly_sbt(extra_profiles)
+            if should_run_java_style_checks:
+                with titled_block("Running Java style checks"):
+                    run_java_style_checks(extra_profiles + modules.root.build_profile_flags)
+            if not os.environ.get("SKIP_UNIDOC"):
+                with titled_block("Building Unidoc API Documentation"):
+                    build_spark_unidoc_sbt(extra_profiles)
 
     # run the test suites
-    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags)
+    with titled_block("Running Spark unit tests"):
+        run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags)
 
     modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
     if modules_with_python_tests and not os.environ.get("SKIP_PYTHON"):
-        run_python_tests(
-            modules_with_python_tests,
-            opts.python_executables,
-            opts.parallelism,
-            with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true",
-        )
-        run_python_packaging_tests()
+        with titled_block("Running PySpark tests"):
+            run_python_tests(
+                modules_with_python_tests,
+                opts.python_executables,
+                opts.parallelism,
+                with_coverage=os.environ.get("PYSPARK_CODECOV", "false") == "true",
+            )
+        if os.environ.get("SKIP_PACKAGING", "false") != "true":
+            with titled_block("Running PySpark packaging tests"):
+                run_python_packaging_tests()
     if any(m.should_run_r_tests for m in test_modules) and not os.environ.get("SKIP_R"):
-        run_sparkr_tests()
+        with titled_block("Running SparkR tests"):
+            run_sparkr_tests()
 
 
 def _test():

dev/sparktestsupport/__init__.py

@@ -19,21 +19,3 @@
 
 SPARK_HOME = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
 USER_HOME = os.environ.get("HOME")
-ERROR_CODES = {
-    "BLOCK_GENERAL": 10,
-    "BLOCK_RAT": 11,
-    "BLOCK_SCALA_STYLE": 12,
-    "BLOCK_PYTHON_STYLE": 13,
-    "BLOCK_R_STYLE": 14,
-    "BLOCK_DOCUMENTATION": 15,
-    "BLOCK_BUILD": 16,
-    "BLOCK_MIMA": 17,
-    "BLOCK_SPARK_UNIT_TESTS": 18,
-    "BLOCK_PYSPARK_UNIT_TESTS": 19,
-    "BLOCK_SPARKR_UNIT_TESTS": 20,
-    "BLOCK_JAVA_STYLE": 21,
-    "BLOCK_BUILD_TESTS": 22,
-    "BLOCK_PYSPARK_PIP_TESTS": 23,
-    "BLOCK_SCALA_VERSION": 24,
-    "BLOCK_TIMEOUT": 124,
-}

dev/sparktestsupport/shellutils.py

@@ -28,7 +28,7 @@ def exit_from_command_with_retcode(cmd, retcode):
         print("[error] running", " ".join(cmd), "; process was terminated by signal", -retcode)
     else:
         print("[error] running", " ".join(cmd), "; received return code", retcode)
-    sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+    sys.exit(1)
 
 
 def rm_r(path):