Migrate tests/buildTools to JUnit5 (#914)

Author: jupblb

build.sbt

@@ -567,18 +567,31 @@ lazy val unit = project
 
 lazy val buildTools = project
   .in(file("tests/buildTools"))
+  .enablePlugins(KotlinPlugin)
   .settings(
-    testSettings,
-    Test / envVars ++=
-      Map(
-        "SCIP_JAVA_CLI" -> ((cli / pack).value / "bin" / "scip-java").toString
-      ),
+    crossPaths := false,
+    autoScalaLibrary := false,
+    publish / skip := true,
+    kotlinVersion := V.kotlinVersion,
+    kotlincJvmTarget := "11",
     Test / fork := true,
     // Our CI set up is a couple of measly vCPUs so parallelising tests there makes
     // everything worse
-    Test / testForkedParallel := !sys.env.contains("CI")
+    Test / testForkedParallel := !sys.env.contains("CI"),
+    // The SCIP build tool drives javac in-process; on JDK 17+ this requires
+    // opening the JDK-internal javac packages.
+    Test / javaOptions ++= javacModuleOptions.map(_.stripPrefix("-J")),
+    // Pin the JDK version embedded in stdlib SCIP symbols so output is stable.
+    Test / javaOptions += "-Dscip.jdk.version=11",
+    libraryDependencies ++=
+      Seq(
+        "org.jetbrains.kotlin" % "kotlin-test" % V.kotlinVersion % Test,
+        "org.jetbrains.kotlin" % "kotlin-test-junit5" % V.kotlinVersion % Test,
+        "com.github.sbt.junit" % "jupiter-interface" %
+          JupiterKeys.jupiterVersion.value % Test
+      )
   )
-  .dependsOn(unit)
+  .dependsOn(cli)
 
 lazy val snapshots = project
   .in(file("tests/snapshots"))

docs/getting-started.md

@@ -68,11 +68,6 @@ curl -fLo coursier https://git.io/coursier-cli \
   && chmod +x coursier \
   && ./coursier launch com.sourcegraph:scip-java_2.13:@STABLE_VERSION@ -- --help
 
-# Windows
-bitsadmin /transfer downloadCoursierCli https://git.io/coursier-cli "%cd%\coursier"
-bitsadmin /transfer downloadCoursierBat https://git.io/coursier-bat "%cd%\coursier.bat"
-./coursier launch com.sourcegraph:scip-java_2.13:@STABLE_VERSION@ -- --help
-
 # macOS Homebrew
 brew install coursier/formulas/coursier \
  && coursier launch com.sourcegraph:scip-java_2.13:@STABLE_VERSION@ -- --help
@@ -99,12 +94,6 @@ curl -fLo coursier https://git.io/coursier-cli \
   && chmod +x coursier \
   && ./coursier bootstrap --standalone -o scip-java com.sourcegraph:scip-java_2.13:@STABLE_VERSION@ --main com.sourcegraph.scip_java.ScipJava
 ./scip-java --help
-
-# Windows
-bitsadmin /transfer downloadCoursierCli https://git.io/coursier-cli "%cd%\coursier"
-bitsadmin /transfer downloadCoursierBat https://git.io/coursier-bat "%cd%\coursier.bat"
-./coursier bootstrap --standalone -o scip-java com.sourcegraph:scip-java_2.13:@STABLE_VERSION@
-./scip-java --help
 ```
 
 The fat jar uses the [Coursier](https://get-coursier.io/docs/cli-installation)

flake.nix

@@ -53,23 +53,26 @@
           javafmt = pkgs.runCommand "check-javafmt" { } ''
             cd ${./.}
             # Exclude minimized fixtures and generated SCIP snapshot goldens
-            # (coupled to exact line/column annotations) plus the standalone
-            # example projects: their layout must not be reformatted here.
+            # (coupled to exact line/column annotations), the standalone example
+            # projects, and the buildTools test fixtures: these all mirror
+            # real-world project layouts that must not be reformatted here.
             find . -name '*.java' \
               -not -path './examples/*' \
               -not -path './tests/minimized/*' \
               -not -path './tests/snapshots/*' \
+              -not -path './tests/buildTools/src/test/resources/fixtures/*' \
               -not -path './scip-kotlinc/minimized/*' \
               -exec ${pkgs.google-java-format}/bin/google-java-format --dry-run --set-exit-if-changed {} +
             touch $out
           '';
           ktfmt = pkgs.runCommand "check-ktfmt" { } ''
             cd ${./.}
-            # Exclude minimized Kotlin snapshots: the input fixtures are coupled
-            # to generated SCIP goldens with exact line/column annotations, so
-            # their layout must not be reformatted here.
+            # Exclude minimized Kotlin snapshots (coupled to generated SCIP
+            # goldens with exact line/column annotations) and the buildTools test
+            # fixtures (real-world project layouts): neither may be reformatted.
             find . -name '*.kt' \
               -not -path './scip-kotlinc/minimized/*' \
+              -not -path './tests/buildTools/src/test/resources/fixtures/*' \
               -exec ${pkgs.ktfmt}/bin/ktfmt --kotlinlang-style --dry-run --set-exit-if-changed {} +
             touch $out
           '';

scip-aggregator/src/main/java/com/sourcegraph/scip_aggregator/OperatingSystem.java

@@ -16,15 +16,11 @@ public class ScipWriter implements AutoCloseable {
   private final ScipAggregatorOptions options;
 
   public ScipWriter(ScipAggregatorOptions options) throws IOException {
-    if (OperatingSystem.isWindows()) {
-      this.tmp = Files.createTempFile("scip-aggregator", "index.scip");
-    } else {
-      this.tmp =
-          Files.createTempFile(
-              "scip-aggregator",
-              "index.scip",
-              PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
-    }
+    this.tmp =
+        Files.createTempFile(
+            "scip-aggregator",
+            "index.scip",
+            PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
     this.output = new ScipOutputStream(new BufferedOutputStream(Files.newOutputStream(tmp)));
     this.options = options;
   }

scip-aggregator/src/main/java/com/sourcegraph/scip_aggregator/ScipWriter.java

@@ -74,10 +74,7 @@ This means our SCIP compiler plugin was not attached to one or more JavaCompile
     private val defaultTargetroot: Path = Paths.get("build", "scip-targetroot")
 
     private fun runBuild(): ProcessResult {
-        val gradleWrapperName =
-            if (System.getProperty("os.name").lowercase().contains("win")) "gradlew.bat"
-            else "gradlew"
-        val gradleWrapper = index.workingDirectory.resolve(gradleWrapperName)
+        val gradleWrapper = index.workingDirectory.resolve("gradlew")
         val gradleCommand =
             if (Files.isRegularFile(gradleWrapper) && Files.isExecutable(gradleWrapper))
                 gradleWrapper.toString()

scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/GradleBuildTool.kt

@@ -0,0 +1,19 @@
+package tests
+
+import org.junit.jupiter.api.DynamicTest
+import org.junit.jupiter.api.TestFactory
+
+class AutoBuildToolTest : BuildToolHarness() {
+
+    @TestFactory
+    fun tests(): List<DynamicTest> =
+        listOf(
+            checkErrorOutput(
+                "no-tools-found",
+                listOf("index", "--build-tool", "auto"),
+                expectedOutput =
+                    "error: Build tool mode set to `auto`, but no supported build tools " +
+                        "were detected",
+            )
+        )
+}

tests/buildTools/src/test/kotlin/tests/AutoBuildToolTest.kt

@@ -0,0 +1,209 @@
+package tests
+
+import com.sourcegraph.scip_java.CliEnvironment
+import com.sourcegraph.scip_java.ScipJavaApp
+import com.sourcegraph.scip_java.buildtools.ClasspathEntry
+import java.io.ByteArrayOutputStream
+import java.io.PrintStream
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
+import java.util.stream.Collectors
+import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
+import kotlin.test.fail
+import org.junit.jupiter.api.Assumptions.assumeTrue
+import org.junit.jupiter.api.DynamicTest
+import org.junit.jupiter.api.DynamicTest.dynamicTest
+
+/**
+ * Base class for build-tool integration tests. Each `check*` helper returns a JUnit 5
+ * [DynamicTest]; suites expose them from a `@TestFactory` method.
+ */
+abstract class BuildToolHarness {
+
+    /** Run `scip-java` in-process with stdout/stderr redirected into a buffer. */
+    private fun runScipJava(workingDirectory: Path, arguments: List<String>): Pair<Int, String> {
+        val buffer = ByteArrayOutputStream()
+        val stream = PrintStream(buffer, true, StandardCharsets.UTF_8.name())
+        val app = ScipJavaApp()
+        app.env =
+            CliEnvironment(
+                workingDirectory = workingDirectory,
+                standardOutput = stream,
+                standardError = stream,
+            )
+        val exit = app.run(arguments)
+        return exit to buffer.toString(StandardCharsets.UTF_8.name())
+    }
+
+    private fun listScipShards(targetroot: Path): List<Path> {
+        if (!Files.isDirectory(targetroot)) return emptyList()
+        Files.walk(targetroot).use { stream ->
+            return stream.filter { it.toString().endsWith(".scip") }.collect(Collectors.toList())
+        }
+    }
+
+    /**
+     * Fresh temp directory for one test, with symlinks resolved. `toRealPath` matters on macOS
+     * where the system temp dir lives under `/var` (a symlink to `/private/var`): build tools write
+     * canonical paths into their output, so the sourceroot must be canonical too or path prefix
+     * checks fail.
+     */
+    private fun newTempBase(): Path = Files.createTempDirectory("buildtools").toRealPath()
+
+    /**
+     * Materialize a test project from the `fixtures/<name>` directory on the test classpath into
+     * [target], overwriting existing files (e.g. a stub `build.gradle` left behind by wrapper
+     * generation). When [substitutions] is non-empty, each token is textually replaced in every
+     * file, which lets a single fixture cover parametrized variants (e.g. the JDK in a toolchain).
+     */
+    protected fun copyFixture(
+        fixture: String,
+        target: Path,
+        substitutions: Map<String, String> = emptyMap(),
+    ) {
+        val resource = "/fixtures/$fixture"
+        val url =
+            BuildToolHarness::class.java.getResource(resource)
+                ?: error("Fixture not found on test classpath: $resource")
+        val source = Paths.get(url.toURI())
+        Files.walk(source).use { stream ->
+            stream.forEach { path ->
+                val dest = target.resolve(source.relativize(path).toString())
+                if (Files.isDirectory(path)) {
+                    Files.createDirectories(dest)
+                } else {
+                    Files.createDirectories(dest.parent)
+                    if (substitutions.isEmpty()) {
+                        Files.copy(path, dest, StandardCopyOption.REPLACE_EXISTING)
+                    } else {
+                        var text = Files.readString(path)
+                        for ((token, value) in substitutions) {
+                            text = text.replace(token, value)
+                        }
+                        Files.writeString(dest, text)
+                    }
+                }
+            }
+        }
+    }
+
+    /** Compare two strings ignoring trailing whitespace. */
+    protected fun assertNoDiff(obtained: String, expected: String) {
+        if (obtained.trimEnd() != expected.trimEnd()) {
+            fail(
+                "diff:\n--- obtained ---\n${obtained.trimEnd()}\n" +
+                    "--- expected ---\n${expected.trimEnd()}"
+            )
+        }
+    }
+
+    protected fun checkBuild(
+        name: String,
+        fixture: String,
+        expectedScipFiles: Int = 0,
+        extraArguments: List<String> = emptyList(),
+        expectedError: ((String) -> Unit)? = null,
+        expectedPackages: String = "",
+        initCommand: (Path) -> List<String> = { emptyList() },
+        prepare: (Path) -> Unit = {},
+        targetRoot: String? = null,
+        maxJdk: Int? = null,
+        substitutions: Map<String, String> = emptyMap(),
+    ): DynamicTest =
+        dynamicTest(name) {
+            // Some build tools cap the JDK they support (e.g. Gradle 8.10 tops
+            // out at JDK 21); skip rather than fail when the JDK is too new.
+            assumeTrue(
+                maxJdk == null || currentJavaVersion <= maxJdk,
+                "Test $name ignored: JDK $currentJavaVersion exceeds max $maxJdk",
+            )
+
+            val base = newTempBase()
+            try {
+                val workingDirectory = Files.createDirectories(base.resolve("workingDirectory"))
+                val cacheDirectory = Files.createDirectories(base.resolve("cache"))
+
+                val init = initCommand(workingDirectory)
+                if (init.isNotEmpty()) exec(init, workingDirectory)
+
+                copyFixture(fixture, workingDirectory, substitutions)
+                prepare(workingDirectory)
+
+                val targetroot = workingDirectory.resolve(targetRoot ?: "targetroot")
+                val arguments =
+                    listOf(
+                        "index",
+                        "--temporary-directory",
+                        cacheDirectory.toString(),
+                        "--targetroot",
+                        targetroot.toString(),
+                    ) + extraArguments
+
+                val (exit, output) = runScipJava(workingDirectory, arguments)
+
+                if (expectedError != null) {
+                    assertNotEquals(0, exit, "expected failure, output:\n$output")
+                    expectedError(output)
+                } else {
+                    assertEquals(0, exit, "expected success, output:\n$output")
+                }
+
+                val shards = listScipShards(targetroot)
+                if (shards.size != expectedScipFiles) {
+                    fail(
+                        "Expected $expectedScipFiles SCIP shard(s), got ${shards.size}: " +
+                            "$shards\noutput:\n$output"
+                    )
+                }
+
+                if (expectedPackages.isNotEmpty()) {
+                    val obtained =
+                        ClasspathEntry.fromTargetroot(targetroot, workingDirectory)
+                            .map { it.mavenCoordinate() }
+                            .sorted()
+                            .distinct()
+                            .joinToString("\n")
+                    assertNoDiff(obtained, expectedPackages)
+                }
+            } finally {
+                base.toFile().deleteRecursively()
+            }
+        }
+
+    protected fun checkErrorOutput(
+        name: String,
+        arguments: List<String>,
+        expectedOutput: String,
+        fixture: String? = null,
+    ): DynamicTest =
+        dynamicTest(name) {
+            val base = newTempBase()
+            try {
+                val workingDirectory = Files.createDirectories(base.resolve("workingDirectory"))
+                if (fixture != null) {
+                    copyFixture(fixture, workingDirectory, emptyMap())
+                }
+                val (exit, rawOutput) = runScipJava(workingDirectory, arguments)
+                val output = rawOutput.replace(base.toString(), "")
+                assertEquals(1, exit, "output:\n$output")
+                assertNoDiff(output, expectedOutput)
+            } finally {
+                base.toFile().deleteRecursively()
+            }
+        }
+
+    companion object {
+        /** Major version of the JVM running these tests (e.g. 17, 21). */
+        private val currentJavaVersion: Int = Runtime.version().feature()
+
+        /** Run an external command, inheriting stdio. Throws if it exits non-zero. */
+        internal fun exec(command: List<String>, cwd: Path) {
+            val exit = ProcessBuilder(command).directory(cwd.toFile()).inheritIO().start().waitFor()
+            check(exit == 0) { "Command failed ($exit): ${command.joinToString(" ")}" }
+        }
+    }
+}