diff --git a/affected-tests-gradle/build.gradle b/affected-tests-gradle/build.gradle index 176a0d4..f67a92e 100644 --- a/affected-tests-gradle/build.gradle +++ b/affected-tests-gradle/build.gradle @@ -47,12 +47,40 @@ configurations { functionalTestRuntimeOnly.extendsFrom testRuntimeOnly } +dependencies { + // Cucumber-JVM + JUnit Platform Suite power the human-readable e2e + // scenario layer under src/functionalTest/resources/.../features. + // Every feature scenario spawns a real Gradle TestKit build so the + // assertions cover the exact plugin surface consumers like + // security-service see, not just the decision engine. See + // RunCucumberFunctionalTest for the JUnit Platform Suite entrypoint. + functionalTestImplementation 'io.cucumber:cucumber-java:7.25.0' + functionalTestImplementation 'io.cucumber:cucumber-junit-platform-engine:7.25.0' + // Required so Cucumber can constructor-inject the shared `World` + // into step-def classes; without a DI container, CucumberException + // "No suitable constructor" blows up at scenario discovery time. + functionalTestImplementation 'io.cucumber:cucumber-picocontainer:7.25.0' + functionalTestImplementation 'org.junit.platform:junit-platform-suite:1.14.3' + // JGit mirrors the way the core engine tests construct temp repos — + // reusing the same dependency means the feature-file scenarios build + // history with the exact semantics the engine will later parse. + functionalTestImplementation 'org.eclipse.jgit:org.eclipse.jgit:7.6.0.202603022253-r' +} + tasks.register('functionalTest', Test) { - description = 'Runs functional tests with Gradle TestKit.' + description = 'Runs functional tests with Gradle TestKit, including Cucumber e2e scenarios.' group = 'verification' testClassesDirs = sourceSets.functionalTest.output.classesDirs classpath = sourceSets.functionalTest.runtimeClasspath useJUnitPlatform() + + // Cucumber-JVM's JUnit Platform engine scans the classpath resources of + // the running test JVM for .feature files. Without this, feature files + // placed under src/functionalTest/resources are packaged into the + // classpath but the engine still needs glue-code packages pointed at + // explicitly so step definitions get picked up deterministically. + systemProperty 'cucumber.glue', 'io.affectedtests.gradle.e2e.steps' + systemProperty 'cucumber.publish.quiet', 'true' } tasks.named('check') { diff --git a/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/RunCucumberFunctionalTest.java b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/RunCucumberFunctionalTest.java new file mode 100644 index 0000000..16e592e --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/RunCucumberFunctionalTest.java @@ -0,0 +1,37 @@ +package io.affectedtests.gradle.e2e; + +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; + +/** + * JUnit 5 Platform Suite entrypoint for the Cucumber e2e feature files. + * + *

Every feature file under {@code src/functionalTest/resources/io/affectedtests/gradle/e2e/features} + * is discovered by the Cucumber JUnit Platform engine at test time. Each + * scenario spawns a real Gradle TestKit build via {@link TestProject}, so + * the assertions cover the exact plugin surface that consumer builds like + * {@code security-service} see in production, not just the decision + * engine in isolation. + * + *

The suite is deliberately split across multiple {@code .feature} + * files (pilot scenarios, the Mode x Situation matrix, security-service + * consumer shape, DSL migration errors, edge cases). Splitting by theme + * keeps feature files self-contained — a reviewer landing in + * {@code 02-mode-situation-matrix.feature} sees every default action + * cell in one place without having to cross-reference other files. + * + *

Glue packages point at {@code steps/} so step definitions are + * discovered deterministically regardless of package scanning order. + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource("io/affectedtests/gradle/e2e/features") +@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "io.affectedtests.gradle.e2e.steps") +@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty,summary") +public class RunCucumberFunctionalTest { +} diff --git a/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/TestProject.java b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/TestProject.java new file mode 100644 index 0000000..f5b1bb1 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/TestProject.java @@ -0,0 +1,333 @@ +package io.affectedtests.gradle.e2e; + +import org.eclipse.jgit.api.Git; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.GradleRunner; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * A scratch consumer project for one Cucumber scenario. + * + *

Combines a JGit-managed working tree, a minimal Gradle build that + * applies {@code io.github.vedanthvdev.affectedtests} via TestKit's + * {@code withPluginClasspath()}, and helpers for committing an initial + * baseline then layering diffs on top. Instances are one-shot per + * scenario — Cucumber's {@code @Before} creates one and {@code @After} + * cleans it up, so scenarios never see state from prior runs. + * + *

The shape mirrors how a real consumer adopts the plugin: a tiny + * root project with the plugin applied and a {@code baseRef} + * configured. Everything else (file layout, diff contents, out-of-scope + * dirs) comes from the feature-file steps. + */ +public final class TestProject { + + private final Path projectDir; + private final Git git; + private String baselineCommit; + private final List gradleArguments = new ArrayList<>(); + private String affectedTestsBlock = ""; + private BuildResult lastBuildResult; + private String lastBuildOutput = ""; + private boolean lastBuildFailed; + + private TestProject(Path projectDir, Git git) { + this.projectDir = projectDir; + this.git = git; + } + + /** + * Initialises a fresh temp project with a single {@code project init} + * commit that contains only the settings file, README, and the + * plugin-applied build script. The baseline commit — i.e. the ref + * the engine will diff against — is captured later via + * {@link #captureBaseline()}, *after* Given steps have added the + * pre-existing production / test classes the scenario starts from. + * + *

This two-phase setup mirrors how a real MR looks in git: the + * pre-existing code base is the baseline, the MR's commits are the + * diff, and the engine resolves test selection from that delta. + * Capturing the baseline before Given-step files exist would make + * them appear as part of the diff, which is the opposite of what + * every scenario wants to assert. + */ + public static TestProject createEmptyBaseline(Path projectDir) throws Exception { + Files.createDirectories(projectDir); + Files.writeString(projectDir.resolve("settings.gradle"), + "rootProject.name = 'affected-tests-e2e'\n"); + Files.writeString(projectDir.resolve("README.md"), "# initial\n"); + // TestKit leaves `.gradle/` and `build/` trees in the working + // directory after each invocation. Without this .gitignore, + // any scenario that runs affectedTest twice — or enables + // includeUncommitted/includeStaged — would see those + // TestKit-produced directories surface as unmapped non-Java + // files in the diff, flipping real DISCOVERY_SUCCESS runs into + // spurious UNMAPPED_FILE → FULL_SUITE outcomes. + Files.writeString(projectDir.resolve(".gitignore"), + ".gradle/\nbuild/\n"); + + Git git = Git.init().setDirectory(projectDir.toFile()).call(); + TestProject p = new TestProject(projectDir, git); + p.writeBuildScript(); + + git.add().addFilepattern(".").call(); + git.commit().setMessage("project init").call(); + return p; + } + + /** + * Commits any uncommitted work the Given steps have written and + * captures the resulting SHA as {@link #baselineCommit()}. The + * baseRef is passed to each Gradle invocation via + * {@code -PaffectedTestsBaseRef=} (see {@link #runAffectedTests()}) + * instead of being baked into {@code build.gradle} — that avoids a + * chicken-and-egg where rewriting the build script after capturing + * the SHA leaves an uncommitted edit that the next + * {@link #commit(String)} would silently drag into the diff, + * classifying scenarios as {@code UNMAPPED_FILE} instead of the + * pure production change they set up. + */ + public String captureBaseline() throws Exception { + git.add().addFilepattern(".").call(); + git.add().setUpdate(true).addFilepattern(".").call(); + if (!git.status().call().isClean()) { + git.commit().setMessage("baseline").call(); + } + this.baselineCommit = git.log().call().iterator().next().getName(); + return baselineCommit; + } + + /** + * Appends a snippet of Groovy DSL inside the {@code affectedTests {}} + * block before the next Gradle invocation. Scenario steps use this + * to set {@code mode}, {@code onXxx} overrides, {@code ignorePaths}, + * {@code outOfScopeTestDirs}, etc., without re-authoring the whole + * build script for every scenario. + */ + public void extendAffectedTestsBlock(String groovySnippet) throws IOException { + this.affectedTestsBlock = (this.affectedTestsBlock.isBlank() ? "" : this.affectedTestsBlock + "\n") + groovySnippet; + writeBuildScript(); + } + + /** + * Extends this single-module test project into a multi-module layout + * shaped like security-service's: a root project applying the plugin + * plus N sub-projects each with their own {@code java} plugin. Used + * by scenarios that need to assert on cross-module routing (a prod + * change in {@code :module-a} should select tests in + * {@code :module-a:test} and not {@code :module-b:test}). + * + *

This must be called before {@link #captureBaseline()} so the + * {@code settings.gradle} with {@code include} directives is part + * of the baseline, not the diff — otherwise every multi-module + * scenario would classify as {@code UNMAPPED_FILE} due to the + * root-level settings edit. + */ + public void convertToMultiModule(String... moduleNames) throws IOException { + StringBuilder settings = new StringBuilder(); + settings.append("rootProject.name = 'affected-tests-e2e'\n"); + for (String mod : moduleNames) { + settings.append("include '").append(mod).append("'\n"); + Path moduleDir = projectDir.resolve(mod); + Files.createDirectories(moduleDir); + // Minimal per-module build: `java` plugin so :module:test + // exists as a dispatch target. The root project's + // affectedTests block does all the heavy lifting. + Files.writeString(moduleDir.resolve("build.gradle"), + "plugins {\n id 'java'\n}\n"); + } + Files.writeString(projectDir.resolve("settings.gradle"), settings.toString()); + } + + /** + * Creates (or overwrites) a file under the project root and schedules + * it to be staged on the next {@link #commit(String)} call. Parent + * directories are created as needed — tests shouldn't have to author + * boilerplate directory setup inline. + */ + public void writeFile(String relativePath, String content) throws IOException { + Path file = projectDir.resolve(relativePath); + Files.createDirectories(file.getParent()); + Files.writeString(file, content, StandardCharsets.UTF_8); + } + + /** + * Deletes a file from the working tree. The deletion will be picked + * up by the next {@code git add .} inside {@link #commit(String)}. + */ + public void deleteFile(String relativePath) throws IOException { + Files.deleteIfExists(projectDir.resolve(relativePath)); + } + + /** + * Stages every change under the working tree (additions, modifications, + * deletions) and commits with the given message. Returns the resulting + * commit SHA so scenarios can chain diffs against arbitrary checkpoints. + */ + public String commit(String message) throws Exception { + git.add().addFilepattern(".").call(); + // addFilepattern(".") doesn't stage deletions until setUpdate(true) + // is called, because the legacy libgit2-style pathspec semantics + // treat deletion as a rename candidate. Without this pass, a + // "delete the production class" scenario would commit the + // additions from this round but keep the deleted files in the + // index, which completely hides the S07 scenario from the + // engine's diff path. + git.add().setUpdate(true).addFilepattern(".").call(); + return git.commit().setMessage(message).call().getName(); + } + + /** Returns the SHA of the baseline commit created by {@link #createEmptyBaseline(Path)}. */ + public String baselineCommit() { return baselineCommit; } + + /** + * Renames a file on disk and stages the rename via git — JGit's + * {@code git mv} equivalent. Used by rename-detection scenarios in + * {@code 05-edge-cases.feature}: the diff must be recognised as a + * rename (not an add + delete) so the engine can route the new + * filename to the same test class. + */ + public void renameFile(String fromPath, String toPath) throws Exception { + Path from = projectDir.resolve(fromPath); + Path to = projectDir.resolve(toPath); + Files.createDirectories(to.getParent()); + Files.move(from, to); + git.add().addFilepattern(".").call(); + git.add().setUpdate(true).addFilepattern(".").call(); + git.commit().setMessage("rename " + fromPath + " -> " + toPath).call(); + } + + /** + * Writes content to a file without committing. Scenarios that + * exercise {@code includeUncommitted = true} use this to leave + * working-tree edits in place, then run affectedTest — the engine + * should see those changes as part of the diff. + */ + public void writeUncommitted(String relativePath, String content) throws IOException { + writeFile(relativePath, content); + } + + /** + * Stages a new write to the index but does not commit. Scenarios + * that exercise {@code includeStaged = true} — the intermediate + * diff-visibility tier between {@code includeUncommitted=true} and + * pure committed-only — land here. + */ + public void writeStaged(String relativePath, String content) throws Exception { + writeFile(relativePath, content); + git.add().addFilepattern(relativePath).call(); + } + + /** Directory the build was initialised in. Useful for crafting diff assertions. */ + public Path projectDir() { return projectDir; } + + /** + * Adds a Gradle CLI argument for the next invocation (e.g. + * {@code --explain}, {@code -PaffectedTests.baseRef=HEAD~1}). Arguments + * are reset after every {@link #runAffectedTests()} call to keep + * scenarios from leaking flags into unrelated assertions. + */ + public void addGradleArgument(String argument) { + this.gradleArguments.add(argument); + } + + /** + * Invokes {@code affectedTest} via TestKit with the arguments queued + * by {@link #addGradleArgument(String)}. The task is run in isolation + * from compile tasks ({@code -x compileJava -x compileTestJava -x + * processResources -x processTestResources}) because the scenario + * project never produces real class files — mirroring the pilot's + * --explain-only mode and keeping per-scenario wall time + * to a few seconds instead of a full Java build. + * + *

Both the success and failure paths are supported: the caller + * decides via {@link #lastBuildFailed()} whether a non-zero exit was + * the expected outcome (e.g. DSL migration errors) or a genuine + * regression. + */ + public void runAffectedTests() { + List args = new ArrayList<>(); + args.add("affectedTest"); + args.add("--stacktrace"); + args.add("-x"); args.add("compileJava"); + args.add("-x"); args.add("compileTestJava"); + args.add("-x"); args.add("processResources"); + args.add("-x"); args.add("processTestResources"); + if (baselineCommit != null) { + // Pass baseRef as a Gradle property instead of baking it + // into build.gradle. See captureBaseline() for the full + // reasoning on why the baked-in approach silently poisons + // the diff with the build.gradle rewrite itself. + args.add("-PaffectedTestsBaseRef=" + baselineCommit); + } + args.addAll(gradleArguments); + + GradleRunner runner = GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withArguments(args) + .withPluginClasspath() + .forwardOutput(); + try { + lastBuildResult = runner.build(); + lastBuildFailed = false; + } catch (RuntimeException buildFailure) { + lastBuildResult = runner.buildAndFail(); + lastBuildFailed = true; + } + lastBuildOutput = lastBuildResult.getOutput(); + gradleArguments.clear(); + } + + /** + * Runs a non-{@code affectedTest} task (typically {@code help}) used + * by scenarios that only care about configuration-time failures — the + * legacy-DSL-knob migration tests in 04-dsl-migration-errors.feature + * hit this path. Running {@code help} is the lightest way to force + * {@code project.afterEvaluate} to fire without accidentally doing + * real build work. + */ + public void runHelpExpectingFailure() { + GradleRunner runner = GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withArguments("help", "--stacktrace") + .withPluginClasspath() + .forwardOutput(); + lastBuildResult = runner.buildAndFail(); + lastBuildOutput = lastBuildResult.getOutput(); + lastBuildFailed = true; + } + + public String lastOutput() { return lastBuildOutput; } + public boolean lastBuildFailed() { return lastBuildFailed; } + public BuildResult lastBuildResult() { return lastBuildResult; } + + private void writeBuildScript() throws IOException { + // The `java` plugin is applied so a real `test` task exists for + // the plugin to dispatch into. Compilation is skipped via the + // `-x compileJava -x compileTestJava ...` arguments on every + // invocation, so scenarios don't pay the cost of resolving a + // toolchain or producing class files — we only need the task + // graph to be shaped the way a real consumer's is. + // + // baseRef intentionally omitted from the DSL block — it's + // passed per-invocation via -PaffectedTestsBaseRef. See + // captureBaseline() for the rationale. + String body = """ + plugins { + id 'java' + id 'io.github.vedanthvdev.affectedtests' + } + + affectedTests { + """ + + affectedTestsBlock + + "\n}\n"; + Files.writeString(projectDir.resolve("build.gradle"), body); + } +} diff --git a/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/World.java b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/World.java new file mode 100644 index 0000000..9fe773b --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/World.java @@ -0,0 +1,67 @@ +package io.affectedtests.gradle.e2e; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Per-scenario shared state. Cucumber creates one instance per scenario + * via its dependency-injection container; the {@code steps/*.java} + * classes take a {@code World} constructor argument and Cucumber wires + * them up. This eliminates the usual BDD pitfall of "did this step + * modify the same state the next step reads?" — every step file sees + * the same {@code World}, and the {@code World} owns the single + * {@link TestProject} a scenario interacts with. + */ +public final class World { + + private final Path rootTempDir; + private TestProject project; + + public World() throws IOException { + this.rootTempDir = Files.createTempDirectory("affected-tests-e2e-"); + } + + /** + * Lazily initialises the underlying {@link TestProject} on first + * access. Keeps Given steps free to add DSL snippets or files + * before the project is "concrete", matching the way feature files + * read: "Given a project with X" is a single declaration, not a + * setup-then-mutate sequence. + * + *

Any failure to bootstrap the TestKit project wraps into an + * {@code IllegalStateException} so step-def call sites don't have + * to declare {@code throws Exception} on every assertion. The only + * way this actually fails at runtime is if the temp-dir write path + * is unwritable, which is a catastrophic CI environment problem, + * not a scenario-level concern. + */ + public TestProject project() { + if (project == null) { + try { + project = TestProject.createEmptyBaseline(rootTempDir.resolve("project")); + } catch (Exception e) { + throw new IllegalStateException("Failed to initialise e2e TestProject", e); + } + } + return project; + } + + public Path rootTempDir() { return rootTempDir; } + + /** + * Recursively removes the scratch temp dir. Scenarios that fail + * still run this via Cucumber's {@code @After} hook so CI runners + * don't accumulate dangling repos across retries. + */ + public void cleanup() throws IOException { + if (rootTempDir != null && Files.exists(rootTempDir)) { + try (var stream = Files.walk(rootTempDir)) { + stream.sorted((a, b) -> b.compareTo(a)) + .forEach(p -> { + try { Files.deleteIfExists(p); } catch (IOException ignore) { } + }); + } + } + } +} diff --git a/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/steps/CommonSteps.java b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/steps/CommonSteps.java new file mode 100644 index 0000000..be7d672 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/java/io/affectedtests/gradle/e2e/steps/CommonSteps.java @@ -0,0 +1,386 @@ +package io.affectedtests.gradle.e2e.steps; + +import io.affectedtests.gradle.e2e.World; +import io.cucumber.java.After; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Cucumber step definitions shared across every feature file. + * + *

Each {@code @Given/@When/@Then} maps one line of a feature file to + * a Java method. The methods delegate all real work to the + * {@link io.affectedtests.gradle.e2e.TestProject} on the shared + * {@link World} — step defs intentionally stay thin so the feature + * files themselves remain the spec of record. + * + *

Grouping note: everything lives in one file on purpose. + * Splitting step defs across many classes is the usual BDD anti-pattern + * — it makes it much harder to track which step phrasings already + * exist, encouraging near-duplicate definitions. A single ~300-line + * file with contextually-named sections stays indexable by any IDE + * outline and stays the honest single source of truth for the DSL the + * feature files are written in. + */ +public class CommonSteps { + + private final World world; + + // Cucumber instantiates step classes per scenario and passes the + // shared World by constructor-injection (picocontainer is bundled + // with cucumber-java). + public CommonSteps(World world) { + this.world = world; + } + + @After + public void tearDown() throws Exception { + world.cleanup(); + } + + // ------------------------------------------------------------------ + // Given — project setup and diff construction + // ------------------------------------------------------------------ + + @Given("a freshly initialised project with a committed baseline") + public void aFreshlyInitialisedProjectWithACommittedBaseline() throws Exception { + // Force lazy initialisation so the baseline commit exists before + // any subsequent diff. No-op on repeated access. + world.project(); + } + + @Given("the affected-tests DSL contains:") + public void theAffectedTestsDslContains(String groovySnippet) throws Exception { + world.project().extendAffectedTestsBlock(groovySnippet); + } + + @Given("the project is multi-module with sub-projects {string} and {string}") + public void theProjectIsMultiModuleWith(String modA, String modB) throws Exception { + world.project().convertToMultiModule(modA, modB); + } + + @Given("the mode is {word}") + public void theModeIs(String mode) throws Exception { + // Lightweight sugar for the matrix scenario outlines — a full + // DSL doc-string for every row would bury the single token that + // actually varies (the mode). Keeping this inline leaves the + // Examples table self-explanatory. + world.project().extendAffectedTestsBlock(" mode = '" + mode + "'"); + } + + @Given("a canned diff that produces the {word} situation") + public void aCannedDiffThatProducesTheSituation(String situation) throws Exception { + // One set-up routine per situation so the matrix outlines don't + // have to re-author the per-situation diff shape in every row. + // Each branch produces the minimal diff that reliably trips the + // target situation on the current engine — the goal is to keep + // scenarios honest about the situation they're asserting on, not + // to stress-test every possible triggering path (pilot-scenarios + // already covers those variations explicitly). + switch (situation) { + case "EMPTY_DIFF" -> { + world.project().captureBaseline(); + // Empty diff — no additional commits after baseline. + } + case "ALL_FILES_IGNORED" -> { + world.project().extendAffectedTestsBlock(" ignorePaths = ['docs/**']"); + world.project().writeFile("docs/architecture.md", "# Arch\n"); + world.project().captureBaseline(); + world.project().writeFile("docs/architecture.md", "# Arch\nupdated\n"); + world.project().commit("diff: update docs"); + } + case "ALL_FILES_OUT_OF_SCOPE" -> { + world.project().extendAffectedTestsBlock(" outOfScopeTestDirs = ['api-test/**']"); + world.project().writeFile("api-test/com/example/SmokeApiTest.java", + "package com.example;\npublic class SmokeApiTest {}\n"); + world.project().captureBaseline(); + world.project().writeFile("api-test/com/example/SmokeApiTest.java", + "package com.example;\npublic class SmokeApiTest { /* tweak */ }\n"); + world.project().commit("diff: update api-test"); + } + case "UNMAPPED_FILE" -> { + world.project().writeFile("src/main/resources/application.yml", "server:\n port: 8080\n"); + world.project().captureBaseline(); + world.project().writeFile("src/main/resources/application.yml", "server:\n port: 9090\n"); + world.project().commit("diff: update yaml"); + } + case "DISCOVERY_EMPTY" -> { + // Production class with no matching test → discovery + // runs but finds nothing to route, and the diff is + // clean Java (not UNMAPPED_FILE). + world.project().writeFile("src/main/java/com/example/Lonely.java", + "package com.example;\npublic class Lonely {}\n"); + world.project().captureBaseline(); + world.project().writeFile("src/main/java/com/example/Lonely.java", + "package com.example;\npublic class Lonely { /* tweak */ }\n"); + world.project().commit("diff: update lonely"); + } + case "DISCOVERY_SUCCESS" -> { + world.project().writeFile("src/main/java/com/example/FooService.java", + "package com.example;\npublic class FooService {}\n"); + world.project().writeFile("src/test/java/com/example/FooServiceTest.java", + "package com.example;\npublic class FooServiceTest {}\n"); + world.project().captureBaseline(); + world.project().writeFile("src/main/java/com/example/FooService.java", + "package com.example;\npublic class FooService { /* tweak */ }\n"); + world.project().commit("diff: update foo"); + } + case "DISCOVERY_INCOMPLETE" -> { + // Malformed Java that JavaParser cannot parse. The + // engine reports a parse failure → DISCOVERY_INCOMPLETE. + // Shape matters: the `broken(` below is a truncated + // parameter list that JavaParser treats as a hard + // syntax error (vs missing semicolons which it tends + // to recover from). + world.project().writeFile("src/main/java/com/example/Broken.java", + "package com.example;\npublic class Broken {\n public void broken(\n}\n"); + world.project().captureBaseline(); + world.project().writeFile("src/main/java/com/example/Broken.java", + "package com.example;\npublic class Broken {\n public void broken(\n /* tweak */\n}\n"); + world.project().commit("diff: update broken"); + } + default -> throw new IllegalArgumentException( + "No canned setup for situation " + situation + + " — extend CommonSteps#aCannedDiffThatProducesTheSituation."); + } + } + + @Given("a production class {string} with its matching test {string}") + public void aProductionClassWithItsMatchingTest(String prodFqn, String testFqn) throws Exception { + writeJavaClass(prodFqn, "src/main/java", "public class " + simpleName(prodFqn) + " {}"); + writeJavaClass(testFqn, "src/test/java", "public class " + simpleName(testFqn) + " {}"); + } + + @Given("a production class {string} with no matching test on disk") + public void aProductionClassWithNoMatchingTestOnDisk(String prodFqn) throws Exception { + writeJavaClass(prodFqn, "src/main/java", "public class " + simpleName(prodFqn) + " {}"); + } + + @Given("a file at {string} with content:") + public void aFileAtWithContent(String relativePath, String content) throws Exception { + world.project().writeFile(relativePath, content); + } + + @Given("the baseline commit is captured") + public void theBaselineCommitIsCaptured() throws Exception { + world.project().captureBaseline(); + } + + // ------------------------------------------------------------------ + // And — diff layering (additive over Given) + // ------------------------------------------------------------------ + + @And("the diff modifies {string}") + public void theDiffModifies(String relativePath) throws Exception { + // Read the current file and append a no-op tweak so git + // records a content change. If the file doesn't exist we + // create it — matching the intuition of "modifies" in the + // feature file when the scenario's Given didn't pre-create it. + io.affectedtests.gradle.e2e.TestProject p = world.project(); + java.nio.file.Path path = p.projectDir().resolve(relativePath); + String existing = java.nio.file.Files.exists(path) + ? java.nio.file.Files.readString(path) + : ""; + p.writeFile(relativePath, existing + "\n// e2e modification\n"); + p.commit("diff: modify " + relativePath); + } + + @And("the diff adds a new file at {string} with content:") + public void theDiffAddsANewFileAtWithContent(String relativePath, String content) throws Exception { + world.project().writeFile(relativePath, content); + world.project().commit("diff: add " + relativePath); + } + + @And("the diff deletes the file {string}") + public void theDiffDeletesTheFile(String relativePath) throws Exception { + world.project().deleteFile(relativePath); + world.project().commit("diff: delete " + relativePath); + } + + @And("the diff renames {string} to {string}") + public void theDiffRenames(String from, String to) throws Exception { + world.project().renameFile(from, to); + } + + @And("the working tree has an uncommitted modification to {string}") + public void theWorkingTreeHasAnUncommittedModificationTo(String path) throws Exception { + io.affectedtests.gradle.e2e.TestProject p = world.project(); + java.nio.file.Path file = p.projectDir().resolve(path); + String existing = java.nio.file.Files.exists(file) + ? java.nio.file.Files.readString(file) + : ""; + p.writeUncommitted(path, existing + "\n// uncommitted edit\n"); + } + + @And("the working tree has a staged modification to {string}") + public void theWorkingTreeHasAStagedModificationTo(String path) throws Exception { + io.affectedtests.gradle.e2e.TestProject p = world.project(); + java.nio.file.Path file = p.projectDir().resolve(path); + String existing = java.nio.file.Files.exists(file) + ? java.nio.file.Files.readString(file) + : ""; + p.writeStaged(path, existing + "\n// staged edit\n"); + } + + @And("the diff contains no committed changes on top of baseline") + public void theDiffContainsNoCommittedChangesOnTopOfBaseline() { + // Intentional no-op. The project at this point has only the + // baseline commit, so `baseRef = ` vs HEAD produces + // an empty diff — exactly the S01 EMPTY_DIFF pre-condition. + } + + // ------------------------------------------------------------------ + // When — execution + // ------------------------------------------------------------------ + + @When("the affected-tests task runs") + public void theAffectedTestsTaskRuns() throws Exception { + world.project().runAffectedTests(); + } + + @When("the affected-tests task runs with {string}") + public void theAffectedTestsTaskRunsWith(String extraArg) throws Exception { + world.project().addGradleArgument(extraArg); + world.project().runAffectedTests(); + } + + @When("any Gradle task is configured") + public void anyGradleTaskIsConfigured() throws Exception { + // Scenarios that assert on configuration-time failures run + // `help` — the cheapest task that still forces afterEvaluate + // to fire. TestProject.runHelpExpectingFailure handles the + // non-zero exit path. + world.project().runHelpExpectingFailure(); + } + + // ------------------------------------------------------------------ + // Then — assertions on the TestKit build output + // ------------------------------------------------------------------ + + @Then("the task succeeds") + public void theTaskSucceeds() { + assertFalse(world.project().lastBuildFailed(), + "Expected a green build, got:\n" + world.project().lastOutput()); + } + + @Then("the task fails at configuration time") + public void theTaskFailsAtConfigurationTime() { + assertTrue(world.project().lastBuildFailed(), + "Expected a configuration-time failure, build was green:\n" + world.project().lastOutput()); + assertTrue(world.project().lastOutput().contains("A problem occurred configuring") + || world.project().lastOutput().contains("A problem occurred evaluating"), + "Failure must come from configuration / evaluation phase, got:\n" + world.project().lastOutput()); + } + + @Then("the situation is {word}") + public void theSituationIs(String situation) { + // The --explain trace prints the ACTUAL situation on the + // "Situation: DISCOVERY_SUCCESS" header line, but the + // downstream matrix block lists *every* situation by name as a + // reference table — so a bare contains(situation) would pass + // trivially regardless of the actual classification. Anchor on + // the exact header substring to assert on the resolved call, + // not on the reference matrix. + String expected = "Situation: " + situation; + assertTrue(world.project().lastOutput().contains(expected), + "Expected header '" + expected + "' in --explain output, got:\n" + + world.project().lastOutput()); + } + + @Then("the action is {word}") + public void theActionIs(String action) { + // Same matrix-block-poisoning hazard as theSituationIs — the + // reference matrix always contains every Action name, so match + // the "Action: (source: ...)" header line + // which is the only line that names the ACTUAL resolved action. + String expected = "Action: " + action + " ("; + assertTrue(world.project().lastOutput().contains(expected), + "Expected header '" + expected + "...)' in --explain output, got:\n" + + world.project().lastOutput()); + } + + @Then("the action source is {word}") + public void theActionSourceIs(String source) { + // The `Action:` header is the only line that names the *actual* + // resolved action source for this run (matrix rows include every + // situation's hypothetical source, which would make a bare + // `contains("[mode default]")` trivially true on every run). + // Anchor on the full `(source: ...)` substring so we're asserting + // the resolved call, not the reference matrix. + String expected = switch (source.toUpperCase()) { + case "MODE_DEFAULT" -> "(source: mode default)"; + case "EXPLICIT" -> "(source: explicit onXxx setting)"; + default -> throw new IllegalArgumentException( + "Unknown action source in feature file: " + source + + " (use MODE_DEFAULT or EXPLICIT)"); + }; + assertTrue(world.project().lastOutput().contains(expected), + "Expected action source " + expected + " in --explain output, got:\n" + + world.project().lastOutput()); + } + + @Then("the output contains {string}") + public void theOutputContains(String needle) { + assertTrue(world.project().lastOutput().contains(needle), + "Expected output to contain '" + needle + "', got:\n" + + world.project().lastOutput()); + } + + @Then("the output does not contain {string}") + public void theOutputDoesNotContain(String needle) { + assertFalse(world.project().lastOutput().contains(needle), + "Expected output NOT to contain '" + needle + "', got:\n" + + world.project().lastOutput()); + } + + @Then("the selected tests include {string}") + public void theSelectedTestsInclude(String testFqn) { + // FQN assertions only work on non-`--explain` runs where the + // dispatch preview is actually emitted. On `--explain` runs the + // task exits after the decision trace and no dispatch happens, + // so scenarios that specifically want to verify FQN-level + // routing must omit the --explain flag. + assertTrue(world.project().lastOutput().contains(testFqn), + "Expected selected tests to include " + testFqn + ", got:\n" + + world.project().lastOutput()); + } + + @Then("the outcome is {string}") + public void theOutcomeIs(String outcome) { + // The `Outcome:` line in the --explain trace is the canonical + // single-line summary of the resolved action, combining the + // action name and any action-specific detail (e.g. "SELECTED — + // 1 test class(es) will run" or "FULL_SUITE — + // onUnmappedFile=FULL_SUITE — non-Java or unmapped file in + // diff"). Matching on the full substring pins both what + // happened and why, which together are a stronger spec than + // action-alone. + String expected = "Outcome: " + outcome; + assertTrue(world.project().lastOutput().contains(expected), + "Expected outcome line '" + expected + "' in --explain output, got:\n" + + world.project().lastOutput()); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + private void writeJavaClass(String fqn, String sourceRoot, String body) throws Exception { + int lastDot = fqn.lastIndexOf('.'); + String pkg = fqn.substring(0, lastDot); + String simple = fqn.substring(lastDot + 1); + String relativePath = sourceRoot + "/" + pkg.replace('.', '/') + "/" + simple + ".java"; + String source = "package " + pkg + ";\n" + body + "\n"; + world.project().writeFile(relativePath, source); + } + + private String simpleName(String fqn) { + int lastDot = fqn.lastIndexOf('.'); + return lastDot < 0 ? fqn : fqn.substring(lastDot + 1); + } +} diff --git a/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/01-pilot-scenarios.feature b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/01-pilot-scenarios.feature new file mode 100644 index 0000000..318f26c --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/01-pilot-scenarios.feature @@ -0,0 +1,190 @@ +Feature: Pilot scenarios — the situations security-service-like consumers hit in real MRs + These scenarios replay the end-to-end contract the `affected-tests` plugin + is meant to honour for every real-world consumer — a security-service- + shaped micro-service that wires `io.github.vedanthvdev.affectedtests` + into its MR pipeline and expects deterministic test-selection + behaviour across EMPTY_DIFF, DISCOVERY_SUCCESS, UNMAPPED_FILE and the + rest of the v2 situation taxonomy. Every scenario spawns a real + Gradle TestKit build, so the assertions cover the same surface the + consumer sees — task log lines, `--explain` trace, exit code — not + just the decision engine in isolation. + + Background: + Given a freshly initialised project with a committed baseline + + Scenario: S01 — empty diff on top of baseline skips tests in LOCAL mode + # LOCAL is the default mode when `CI` env isn't set. The EMPTY_DIFF + # default there is SKIPPED so a developer running `./gradlew + # affectedTest` with no committed changes pays zero wall-time. + Given a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And the baseline commit is captured + And the diff contains no committed changes on top of baseline + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is EMPTY_DIFF + And the action is SKIPPED + And the action source is MODE_DEFAULT + + Scenario: S02 — production change with a matching naming-strategy test selects only that test + Given a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And the baseline commit is captured + And the diff modifies "src/main/java/com/example/FooService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the action is SELECTED + # DISCOVERY_SUCCESS sources as EXPLICIT by design — running the discovered + # tests is the plugin's definitional purpose, not a mode-default. + And the action source is EXPLICIT + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: S03 — test-only diff selects only the touched test class + # Tests in the diff are themselves "affected tests" — no production- + # mapping needed, the test file IS the signal. Guards against a + # regression where test-file-only diffs accidentally fell through + # to UNMAPPED_FILE. + Given a production class "com.example.BarService" with its matching test "com.example.BarServiceTest" + And the baseline commit is captured + And the diff modifies "src/test/java/com/example/BarServiceTest.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the action is SELECTED + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: S04 — yaml-only diff escalates to FULL_SUITE under UNMAPPED_FILE + # Non-Java files have no safe routing; LOCAL default for UNMAPPED_FILE + # is FULL_SUITE so consumers never silently miss tests touched by a + # config change. CI stays aligned with this default. + Given a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And a file at "src/main/resources/application.yml" with content: + """ + server: + port: 8080 + """ + And the baseline commit is captured + And the diff modifies "src/main/resources/application.yml" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is UNMAPPED_FILE + And the action is FULL_SUITE + And the action source is MODE_DEFAULT + And the output contains "onUnmappedFile=FULL_SUITE" + + Scenario: S05 — diff fully inside ignorePaths short-circuits to SKIPPED in LOCAL + # ignorePaths is the "paper-cut" knob: flaky test outputs, docs, etc. + # A diff of only-ignored-files in LOCAL should not run anything. + Given the affected-tests DSL contains: + """ + ignorePaths = ['docs/**'] + """ + And a file at "docs/architecture.md" with content: + """ + # Architecture + """ + And the baseline commit is captured + And the diff modifies "docs/architecture.md" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is ALL_FILES_IGNORED + And the action is SKIPPED + And the action source is MODE_DEFAULT + + Scenario: S06 — diff fully inside outOfScopeTestDirs short-circuits to SKIPPED in LOCAL + # outOfScopeTestDirs is what security-service uses to quarantine + # flaky api-test/** and performance-test/** suites out of normal + # selection. A diff of only-out-of-scope files is by construction + # untouched by the rest of the suite, so SKIPPED is correct. + Given the affected-tests DSL contains: + """ + outOfScopeTestDirs = ['api-test/**'] + """ + And a file at "api-test/com/example/SmokeApiTest.java" with content: + """ + package com.example; + public class SmokeApiTest {} + """ + And the baseline commit is captured + And the diff modifies "api-test/com/example/SmokeApiTest.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is ALL_FILES_OUT_OF_SCOPE + And the action is SKIPPED + And the action source is MODE_DEFAULT + + Scenario: S07 — CI mode escalates DISCOVERY_EMPTY to FULL_SUITE + # A production change with no matching test exercises DISCOVERY_EMPTY. + # CI's default for DISCOVERY_EMPTY is FULL_SUITE because trusting a + # green result from an empty selection would under-test real MRs. + # This is the exact safety net security-service relies on when its + # pipeline sets `mode = 'ci'`. + Given the affected-tests DSL contains: + """ + mode = 'ci' + """ + And a production class "com.example.BazService" with no matching test on disk + And the baseline commit is captured + And the diff modifies "src/main/java/com/example/BazService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_EMPTY + And the action is FULL_SUITE + And the action source is MODE_DEFAULT + + Scenario: S08 — explicit onEmptyDiff override wins over mode default + # Consumers who want LOCAL mode behaviour for most situations but + # prefer FULL_SUITE on empty diffs must be able to pin that single + # row. This is the "override wins" contract: action source flips + # to EXPLICIT to make the cause traceable in --explain. + Given the affected-tests DSL contains: + """ + onEmptyDiff = 'full_suite' + """ + And the baseline commit is captured + And the diff contains no committed changes on top of baseline + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is EMPTY_DIFF + And the action is FULL_SUITE + And the action source is EXPLICIT + + Scenario: S09 — deleted production class lands in DISCOVERY_EMPTY, not a crash + # S07 in the pilot surfaced a bug where deleting a file silently + # crashed file scanning. The contract today: deletion is just + # another diff entry — the engine routes it through DISCOVERY_EMPTY + # (no production → no mapping → no tests) without blowing up. + # + # Mode pinned to LOCAL because DISCOVERY_EMPTY is one of the two + # situations where LOCAL and CI defaults diverge (LOCAL=SKIPPED, + # CI=FULL_SUITE). Leaving it on AUTO would make this scenario + # pass locally but FULL_SUITE on GitHub Actions. The 21-row + # matrix in 02-mode-situation-matrix.feature exercises the + # CI/STRICT columns explicitly; this scenario only needs to + # pin the LOCAL default-action behaviour. + Given the mode is local + And a production class "com.example.ObsoleteService" with no matching test on disk + And the baseline commit is captured + And the diff deletes the file "src/main/java/com/example/ObsoleteService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_EMPTY + And the action is SKIPPED + And the action source is MODE_DEFAULT + + Scenario: S10 — mixed yaml + production diff classifies as UNMAPPED_FILE and escalates + # Order-of-evaluation scenario. UNMAPPED_FILE beats DISCOVERY_* so a + # single yaml file alongside a hundred happy prod changes still + # trips full-suite — the conservative choice consumers depend on. + Given a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And a file at "src/main/resources/application.yml" with content: + """ + server: + port: 8080 + """ + And the baseline commit is captured + And the diff modifies "src/main/java/com/example/FooService.java" + And the diff modifies "src/main/resources/application.yml" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is UNMAPPED_FILE + And the action is FULL_SUITE diff --git a/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/02-mode-situation-matrix.feature b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/02-mode-situation-matrix.feature new file mode 100644 index 0000000..fc34507 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/02-mode-situation-matrix.feature @@ -0,0 +1,130 @@ +Feature: Mode × Situation default action matrix + The action the plugin takes for each Situation depends on the + effective Mode (LOCAL / CI / STRICT). This feature pins every cell of + the default decision matrix as documented in README.md and + AffectedTestsConfig#defaultFor. Changing any cell must be a + deliberate decision that updates both the README table and this + scenario outline — a drifted matrix is an invisible behaviour + change for every consumer. + + The 21 cells (7 situations × 3 modes) break down to: + * Situations producing clear SKIPPED in LOCAL (EMPTY_DIFF, + ALL_FILES_IGNORED, ALL_FILES_OUT_OF_SCOPE, DISCOVERY_EMPTY) + escalate in CI/STRICT because CI cannot afford to silently + skip coverage on a merge-gate. + * UNMAPPED_FILE and DISCOVERY_INCOMPLETE escalate in every mode + because the engine cannot prove coverage; LOCAL only relaxes + DISCOVERY_INCOMPLETE to SELECTED to keep dev iteration fast. + * DISCOVERY_SUCCESS is always SELECTED — the plugin's whole + reason to exist. + + Background: + Given a freshly initialised project with a committed baseline + + Scenario Outline: default for EMPTY_DIFF is + Given the mode is + And a canned diff that produces the EMPTY_DIFF situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is EMPTY_DIFF + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | SKIPPED | + | ci | SKIPPED | + | strict | FULL_SUITE | + + Scenario Outline: default for ALL_FILES_IGNORED is + Given the mode is + And a canned diff that produces the ALL_FILES_IGNORED situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is ALL_FILES_IGNORED + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | SKIPPED | + | ci | SKIPPED | + | strict | FULL_SUITE | + + Scenario Outline: default for ALL_FILES_OUT_OF_SCOPE is + Given the mode is + And a canned diff that produces the ALL_FILES_OUT_OF_SCOPE situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is ALL_FILES_OUT_OF_SCOPE + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | SKIPPED | + | ci | SKIPPED | + | strict | SKIPPED | + + Scenario Outline: default for UNMAPPED_FILE is + Given the mode is + And a canned diff that produces the UNMAPPED_FILE situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is UNMAPPED_FILE + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | FULL_SUITE | + | ci | FULL_SUITE | + | strict | FULL_SUITE | + + Scenario Outline: default for DISCOVERY_EMPTY is + Given the mode is + And a canned diff that produces the DISCOVERY_EMPTY situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_EMPTY + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | SKIPPED | + | ci | FULL_SUITE | + | strict | FULL_SUITE | + + Scenario Outline: default for DISCOVERY_INCOMPLETE is + Given the mode is + And a canned diff that produces the DISCOVERY_INCOMPLETE situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_INCOMPLETE + And the action is + And the action source is MODE_DEFAULT + + Examples: + | mode | action | + | local | SELECTED | + | ci | FULL_SUITE | + | strict | FULL_SUITE | + + Scenario Outline: default for DISCOVERY_SUCCESS is SELECTED (EXPLICIT) + Given the mode is + And a canned diff that produces the DISCOVERY_SUCCESS situation + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the action is SELECTED + # DISCOVERY_SUCCESS is the only cell sourced as EXPLICIT regardless + # of mode — running discovered tests is the definitional purpose + # of the plugin, not a defaulted choice. + And the action source is EXPLICIT + + Examples: + | mode | + | local | + | ci | + | strict | diff --git a/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/03-security-service-shape.feature b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/03-security-service-shape.feature new file mode 100644 index 0000000..3e99660 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/03-security-service-shape.feature @@ -0,0 +1,116 @@ +Feature: security-service-shape consumer configurations + These scenarios mirror the exact shape of security-service's + adoption: a multi-module Gradle build with out-of-scope quarantine + dirs expressed as ant-style globs, a custom test-suffix list, and a + bounded transitiveDepth. Breaking any of these assumptions breaks + security-service's MR gate — so the scenarios here are a direct + integration contract with the primary consumer. + + Background: + Given a freshly initialised project with a committed baseline + + Scenario: nested-glob outOfScopeTestDirs matches files at any depth + # security-service's build isolates both api-test/** and + # performance-test/** across many modules. The globs in their + # DSL look like "api-test/**" — that pattern must match files + # nested arbitrarily deep under that directory. + Given the affected-tests DSL contains: + """ + outOfScopeTestDirs = ['api-test/**', 'performance-test/**'] + """ + And a file at "api-test/com/example/very/deep/NestedSmokeTest.java" with content: + """ + package com.example.very.deep; + public class NestedSmokeTest {} + """ + And the baseline commit is captured + And the diff modifies "api-test/com/example/very/deep/NestedSmokeTest.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is ALL_FILES_OUT_OF_SCOPE + And the action is SKIPPED + + Scenario: custom testSuffixes picks up IT-suffixed integration tests + # security-service uses both *Test.java and *IT.java suffixes. + # A production change whose only test counterpart uses the IT + # suffix must still be discovered — otherwise integration-only + # services are silently untested. + Given the affected-tests DSL contains: + """ + testSuffixes = ['Test', 'IT', 'IntegrationTest'] + """ + And a file at "src/main/java/com/example/PaymentGateway.java" with content: + """ + package com.example; + public class PaymentGateway {} + """ + And a file at "src/test/java/com/example/PaymentGatewayIT.java" with content: + """ + package com.example; + public class PaymentGatewayIT {} + """ + And the baseline commit is captured + And the diff modifies "src/main/java/com/example/PaymentGateway.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the action is SELECTED + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: transitiveDepth caps the strategy chain so large blast radii don't run everything + # Default transitiveDepth is 4 — deep enough for most service + # call graphs. Consumers who see their call graph explode when a + # core util changes can clamp it. A transitiveDepth of 0 disables + # transitive inclusion entirely: direct tests only. + Given the affected-tests DSL contains: + """ + transitiveDepth = 0 + """ + And a production class "com.example.CoreUtil" with its matching test "com.example.CoreUtilTest" + And the baseline commit is captured + And the diff modifies "src/main/java/com/example/CoreUtil.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + # With transitiveDepth=0 we still get the direct match via the + # naming strategy, just no transitive expansion. Result: the one + # direct test is selected. + And the situation is DISCOVERY_SUCCESS + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: multi-module prod change routes to the correct sub-module's tests + # security-service is a multi-module Gradle build. A change in + # module-a/src/main/java/... must route the discovered test to + # :module-a:test, not :module-b:test. Cross-module leakage is + # the single most expensive failure mode we protect against — + # dispatch cost scales linearly in the number of modules and + # routing the wrong module wastes CI minutes on a green build + # that proves nothing. + Given the project is multi-module with sub-projects "module-a" and "module-b" + And a file at "module-a/src/main/java/com/example/a/ServiceA.java" with content: + """ + package com.example.a; + public class ServiceA {} + """ + And a file at "module-a/src/test/java/com/example/a/ServiceATest.java" with content: + """ + package com.example.a; + public class ServiceATest {} + """ + And a file at "module-b/src/main/java/com/example/b/ServiceB.java" with content: + """ + package com.example.b; + public class ServiceB {} + """ + And a file at "module-b/src/test/java/com/example/b/ServiceBTest.java" with content: + """ + package com.example.b; + public class ServiceBTest {} + """ + And the baseline commit is captured + And the diff modifies "module-a/src/main/java/com/example/a/ServiceA.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + # Only one test was discovered — confirms the engine did NOT + # select module-b tests for a module-a-only diff. + And the outcome is "SELECTED — 1 test class(es) will run" diff --git a/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/04-dsl-migration-errors.feature b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/04-dsl-migration-errors.feature new file mode 100644 index 0000000..4a26a07 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/04-dsl-migration-errors.feature @@ -0,0 +1,76 @@ +Feature: v1 → v2 DSL migration errors + v2.0.0 removed three configuration knobs (runAllIfNoMatches, + runAllOnNonJavaChange, excludePaths) and replaced them with + situation-scoped onXxx settings. Consumers migrating from v1 must + see targeted, actionable error messages at configuration time — not + silent no-ops that leave their MR gate misconfigured. + + v2.1.0 added configuration-time validation for gradlewTimeoutSeconds + so that a negative value fails the build before the task runs, + instead of crashing mid-dispatch with a less obvious error. + + Each scenario here asserts both that configuration-time failure + happens and that the error message names the specific v2 + replacement so an operator can make the migration in one step. + + Background: + Given a freshly initialised project with a committed baseline + + Scenario: legacy runAllIfNoMatches throws with v2 migration hint + # v1 knob: runAllIfNoMatches = true + # v2 equivalent: onEmptyDiff = 'full_suite' and/or + # onDiscoveryEmpty = 'full_suite' (or mode = 'ci') + Given the affected-tests DSL contains: + """ + runAllIfNoMatches = true + """ + When any Gradle task is configured + Then the task fails at configuration time + And the output contains "runAllIfNoMatches was removed in v2.0.0" + And the output contains "onEmptyDiff" + And the output contains "onDiscoveryEmpty" + # Pointer to the migration table saves operators a grep. + And the output contains "CHANGELOG.md v2.0" + + Scenario: legacy runAllOnNonJavaChange throws with v2 migration hint + # v1 knob: runAllOnNonJavaChange = true + # v2 equivalent: onUnmappedFile = 'full_suite' (or mode = 'ci') + Given the affected-tests DSL contains: + """ + runAllOnNonJavaChange = true + """ + When any Gradle task is configured + Then the task fails at configuration time + And the output contains "runAllOnNonJavaChange was removed in v2.0.0" + And the output contains "onUnmappedFile" + And the output contains "CHANGELOG.md v2.0" + + Scenario: legacy excludePaths throws with v2 migration hint + # v1 knob: excludePaths = ['docs/**'] + # v2 equivalents: ignorePaths (quiet drop) OR outOfScopeTestDirs + # (explicitly-quarantined test dirs). The error must mention both + # so operators pick the right one for their case. + Given the affected-tests DSL contains: + """ + excludePaths = ['docs/**'] + """ + When any Gradle task is configured + Then the task fails at configuration time + And the output contains "excludePaths was removed in v2.0.0" + And the output contains "ignorePaths" + And the output contains "outOfScopeTestDirs" + + Scenario: negative gradlewTimeoutSeconds fails at configuration time (v2.1.0 polish) + # Before v2.1.0 this would crash mid-dispatch with an IllegalStateException + # from ProcessBuilder. v2.1.0 adds afterEvaluate validation so the + # error surfaces before the task runs. + Given the affected-tests DSL contains: + """ + gradlewTimeoutSeconds = -1L + """ + When any Gradle task is configured + Then the task fails at configuration time + And the output contains "gradlewTimeoutSeconds must be >= 0" + # The message must explicitly call out that 0 is valid, so operators + # who want "no timeout" don't misread the message as "must be > 0". + And the output contains "0 disables the timeout" diff --git a/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/05-edge-cases.feature b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/05-edge-cases.feature new file mode 100644 index 0000000..ae370a9 --- /dev/null +++ b/affected-tests-gradle/src/functionalTest/resources/io/affectedtests/gradle/e2e/features/05-edge-cases.feature @@ -0,0 +1,79 @@ +Feature: edge cases — rename detection, working-tree visibility, STRICT mode + These scenarios cover the narrow edges where the engine's diff + semantics matter most: what git diff modes we respect, what file + operations we treat as diffable, and what STRICT mode forces the + fail-closed behaviour we promise consumers who opt in. + + Background: + Given a freshly initialised project with a committed baseline + + Scenario: renaming a production class still selects its original test + # A rename shows up in the diff as `delete old path + add new + # path`. Crucially the delete half still maps to the test that + # matched the old class name — so the engine keeps selecting + # LegacyNameTest even though LegacyName.java no longer exists. + # That's the protective behaviour we want: renames don't silently + # orphan tests from the selected set. + Given a production class "com.example.LegacyName" with its matching test "com.example.LegacyNameTest" + And the baseline commit is captured + And the diff renames "src/main/java/com/example/LegacyName.java" to "src/main/java/com/example/NewName.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: uncommitted working-tree changes are invisible by default + # The default `includeUncommitted=false` is load-bearing for CI/dev + # parity: running `./gradlew affectedTest` locally must pick the + # same tests CI will pick on the same HEAD, regardless of what's + # sitting uncommitted in the working tree. + Given a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And the baseline commit is captured + And the working tree has an uncommitted modification to "src/main/java/com/example/FooService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + # No committed changes on top of baseline, uncommitted edit ignored + # by default → EMPTY_DIFF. + And the situation is EMPTY_DIFF + + Scenario: includeUncommitted=true surfaces working-tree changes into the diff + Given the affected-tests DSL contains: + """ + includeUncommitted = true + """ + And a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And the baseline commit is captured + And the working tree has an uncommitted modification to "src/main/java/com/example/FooService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: includeStaged=true surfaces staged-but-uncommitted changes + # Staged is the useful middle tier — a dev has run `git add` but + # not yet committed. Enables iterative "what tests will run when + # I commit?" exploration without requiring an actual commit. + Given the affected-tests DSL contains: + """ + includeStaged = true + """ + And a production class "com.example.FooService" with its matching test "com.example.FooServiceTest" + And the baseline commit is captured + And the working tree has a staged modification to "src/main/java/com/example/FooService.java" + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is DISCOVERY_SUCCESS + And the outcome is "SELECTED — 1 test class(es) will run" + + Scenario: STRICT mode runs full suite even on EMPTY_DIFF + # STRICT is the "paranoid release gate" mode: when you care more + # about coverage than speed. An empty diff on STRICT still runs + # everything — the fail-closed counterpart to LOCAL's fast path. + Given the mode is strict + And the baseline commit is captured + And the diff contains no committed changes on top of baseline + When the affected-tests task runs with "--explain" + Then the task succeeds + And the situation is EMPTY_DIFF + And the action is FULL_SUITE + And the action source is MODE_DEFAULT