diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 1588e8dfafa97d..ac5711f7e72319 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -153,8 +153,10 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.Phaser; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -1809,36 +1811,111 @@ public void uploadOutputs( if (remoteOptions.getRemoteCacheAsync() && !action.getSpawn().getResourceOwner().mayModifySpawnOutputsAfterExecution()) { - var uploadDone = new CountDownLatch(1); - var future = - backgroundTaskExecutor.submit( - () -> { - try { - doUploadOutputs(action, spawnResult, onUploadComplete); - } catch (ExecException e) { - reportUploadError(e); - } catch (InterruptedException ignored) { - // ThreadPerTaskExecutor does not care about interrupt status. - } finally { - uploadDone.countDown(); - } - }); - - if (outputService instanceof RemoteOutputService remoteOutputService - && remoteOutputService.getRewoundActionSynchronizer() - instanceof RemoteRewoundActionSynchronizer remoteRewoundActionSynchronizer) { - remoteRewoundActionSynchronizer.registerOutputUploadTask( - action.getRemoteActionExecutionContext().getSpawnOwner(), - () -> { - future.cancel(true); - uploadDone.await(); - }); - } + new OutputUploadTask(action, spawnResult, onUploadComplete).start(backgroundTaskExecutor); } else { doUploadOutputs(action, spawnResult, onUploadComplete); } } + @Nullable + private RemoteRewoundActionSynchronizer getRewoundActionSynchronizer() { + if (outputService instanceof RemoteOutputService remoteOutputService + && remoteOutputService.getRewoundActionSynchronizer() + instanceof RemoteRewoundActionSynchronizer rewoundActionSynchronizer) { + return rewoundActionSynchronizer; + } + return null; + } + + /** + * A cancellable background upload of an action's outputs. + * + *
Registers itself with the {@link RemoteRewoundActionSynchronizer}, if there is one, before + * the upload starts and unregisters itself when it is done, so that a rewinding of the action + * waits for uploads that are still in flight. + * + *
Ensures that the completion callback runs exactly once and that {@link #cancel} only returns + * once the upload no longer accesses the action's outputs. + */ + private final class OutputUploadTask implements RemoteRewoundActionSynchronizer.Cancellable { + private final RemoteAction action; + private final SpawnResult spawnResult; + private final Runnable onUploadComplete; + private final ActionExecutionMetadata spawnOwner; + @Nullable private final RemoteRewoundActionSynchronizer rewoundActionSynchronizer; + private final CountDownLatch done = new CountDownLatch(1); + + // The thread running the upload, published by the upload itself before it reads cancelled. + @Nullable private volatile Thread thread; + // Set by cancel before it reads thread. + private volatile boolean cancelled; + + OutputUploadTask(RemoteAction action, SpawnResult spawnResult, Runnable onUploadComplete) { + this.action = action; + this.spawnResult = spawnResult; + this.onUploadComplete = onUploadComplete; + this.spawnOwner = action.getRemoteActionExecutionContext().getSpawnOwner(); + this.rewoundActionSynchronizer = getRewoundActionSynchronizer(); + } + + /** Registers the task for cancellation and starts the upload on the given executor. */ + void start(Executor executor) { + // Register before starting the upload so that it can't unregister itself before it has been + // registered. + if (rewoundActionSynchronizer != null) { + rewoundActionSynchronizer.registerOutputUploadTask(spawnOwner, this); + } + try { + // Runs the upload rather than submitting it: the body of a task submitted to an + // ExecutorService is skipped entirely if its future is cancelled before it starts, which + // would leave the completion callback unrun and cancel waiting forever. + executor.execute(this::upload); + } catch (RejectedExecutionException e) { + // The upload will never run, so complete the task on its behalf. + finish(); + throw e; + } + } + + private void upload() { + // Publish the thread before reading cancelled, which cancel writes before it reads the + // thread. At least one of the two thus observes the other and the upload is either skipped + // or interrupted. + thread = Thread.currentThread(); + try { + if (cancelled) { + onUploadComplete.run(); + } else { + doUploadOutputs(action, spawnResult, onUploadComplete); + } + } catch (ExecException e) { + reportUploadError(e); + } catch (InterruptedException ignored) { + // ThreadPerTaskExecutor does not care about interrupt status. + } finally { + finish(); + } + } + + @Override + public void cancel() throws InterruptedException { + cancelled = true; + var localThread = thread; + if (localThread != null) { + localThread.interrupt(); + } + done.await(); + } + + /** Signals that the upload no longer accesses the action's outputs. */ + private void finish() { + done.countDown(); + if (rewoundActionSynchronizer != null) { + rewoundActionSynchronizer.unregisterOutputUploadTask(spawnOwner, this); + } + } + } + private void doUploadOutputs( RemoteAction action, SpawnResult spawnResult, Runnable onUploadComplete) throws ExecException, InterruptedException { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteRewoundActionSynchronizer.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteRewoundActionSynchronizer.java index b14263d6ddb3c7..c59a4c3a8e046c 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteRewoundActionSynchronizer.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteRewoundActionSynchronizer.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.remote; + import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.ImmutableList; @@ -41,13 +42,22 @@ * while they are being read. */ final class RemoteRewoundActionSynchronizer implements RewoundActionSynchronizer { - /** A task with a cancellation callback. */ + /** A cancellable task that operates on the outputs of an action. */ public interface Cancellable { + /** + * Cancels the task and returns only once it no longer accesses the outputs of the action it has + * been registered for, which are about to be regenerated by a rewound execution of that action. + * + *
May be called after the task has completed, in which case it returns immediately.
+ */
void cancel() throws InterruptedException;
}
private final RemoteActionInputFetcher actionInputFetcher;
- private final ConcurrentHashMap Must be paired with a call to {@link #unregisterOutputUploadTask} once the upload has
+ * completed so that the task doesn't remain registered (and thus retained) for the rest of the
+ * build.
*/
public void registerOutputUploadTask(ActionExecutionMetadata action, Cancellable task) {
- // We don't expect to have multiple output upload tasks for the same action registered at the
- // same time.
- outputUploadTasks.merge(
- action,
- task,
- (oldTask, newTask) -> {
- throw new IllegalStateException(
- "Attempted to register multiple output upload tasks for %s: %s and %s"
- .formatted(action, oldTask, newTask));
+ // The task is added to the list inside the mapping function so that registration is atomic
+ // with respect to the removal of the entry in prepareOutputsForRewinding.
+ outputUploadTasks.compute(
+ actionKeyFor(action),
+ (unusedKey, tasks) ->
+ tasks == null
+ ? ImmutableList.of(task)
+ : ImmutableList. This is the artifact's own generating action key except for the outputs of an {@link
+ * com.google.devtools.build.lib.actions.ActionTemplate} expansion, which are guarded by the key
+ * of the template: they are only ever consumed as part of a tree artifact the template declares,
+ * either by actions outside the expansion, which depend on that tree artifact, or by other
+ * actions of the same expansion, which depend on individual files in it.
+ */
+ private static ActionLookupData lockKeyFor(DerivedArtifact artifact) {
+ var outermost = artifact;
+ for (var parent = artifact.getParent(); parent != null; parent = parent.getParent()) {
+ outermost = parent;
+ }
+ return outermost.getGeneratingActionKey();
+ }
+
+ /**
+ * Returns the key of the lock that guards the outputs of the given action, which is the key its
+ * consumers acquire the read lock of.
+ */
+ private static ActionLookupData outputKeyFor(Action action) {
+ return lockKeyFor((DerivedArtifact) action.getPrimaryOutput());
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
index 956f51e6b9c3cd..ddc2b97d3b42ac 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -1141,8 +1141,9 @@ public ActionStepOrResult run(Environment env)
}
env.getListener().post(event);
var rewoundActionSynchronizer = outputService.getRewoundActionSynchronizer();
+ boolean wasRewound = wasRewound(action);
try (SilentCloseable outerLock =
- rewoundActionSynchronizer.enterActionPreparation(action, wasRewound(action))) {
+ rewoundActionSynchronizer.enterActionPreparation(action, wasRewound)) {
if (actionFileSystemType().shouldDoEagerActionPrep()) {
try (SilentCloseable d =
Profiler.instance().profile(ProfilerTask.INFO, "action.prepare")) {
@@ -1177,7 +1178,7 @@ public ActionStepOrResult run(Environment env)
try (SilentCloseable innerLock =
rewoundActionSynchronizer.enterActionExecution(
- action, actionExecutionContext.getInputMetadataProvider())) {
+ action, wasRewound, actionExecutionContext.getInputMetadataProvider())) {
return executeAction(env.getListener(), action);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
index 6b034956d90076..4b93b730e75e09 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
@@ -291,7 +291,8 @@ SilentCloseable enterActionPreparation(Action action, boolean wasRewound)
throws InterruptedException;
/** Guards an action from the beginning to the end of its {@link Action#execute execution}. */
- SilentCloseable enterActionExecution(Action action, InputMetadataProvider metadataProvider)
+ SilentCloseable enterActionExecution(
+ Action action, boolean wasRewound, InputMetadataProvider metadataProvider)
throws InterruptedException;
/**
@@ -307,7 +308,7 @@ public SilentCloseable enterActionPreparation(Action action, boolean wasRewound)
@Override
public SilentCloseable enterActionExecution(
- Action action, InputMetadataProvider metadataProvider) {
+ Action action, boolean wasRewound, InputMetadataProvider metadataProvider) {
return () -> {};
}
};
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD
index a96f75249d4487..7296b03461aa2f 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD
@@ -38,6 +38,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/skyframe:aspect_key_creator",
"//src/main/java/com/google/devtools/build/lib/skyframe/rewinding",
"//src/main/java/com/google/devtools/build/lib/skyframe/rewinding:lost_important_output_handler_module",
+ "//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/build/skyframe",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java
index dc12784dafd741..7467ca06ce1218 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java
@@ -266,6 +266,11 @@ public void treeArtifactRewound_oneFileLost() throws Exception {
helper.runTreeArtifactRewound_oneFileLost_spawnFailed();
}
+ @Test
+ public void actionTemplateExpansionRewound_notConcurrentWithTreeConsumers() throws Exception {
+ helper.runActionTemplateExpansionRewound_notConcurrentWithTreeConsumers();
+ }
+
@Test
public void generatedRunfilesRewound_allFilesLost() throws Exception {
helper.runGeneratedRunfilesRewound_allFilesLost_spawnFailed();
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java
index 142d1bcfd17834..fb7be69562a599 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java
@@ -23,6 +23,7 @@
import static com.google.devtools.build.lib.vfs.FileSystemUtils.readContentAsLatin1;
import static com.google.devtools.build.lib.vfs.FileSystemUtils.writeContent;
import static java.util.Arrays.stream;
+import static java.util.stream.Collectors.joining;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
@@ -33,6 +34,7 @@
import com.google.common.collect.ImmutableMultiset;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Iterables;
+import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.google.common.flogger.GoogleLogger;
import com.google.common.util.concurrent.Uninterruptibles;
@@ -49,6 +51,7 @@
import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
import com.google.devtools.build.lib.actions.LostInputsExecException;
import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnExecutedEvent;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.analysis.AspectCompleteEvent;
import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
@@ -79,6 +82,7 @@
import com.google.devtools.build.lib.testutil.SpawnController.SpawnShim;
import com.google.devtools.build.lib.testutil.SpawnInputUtils;
import com.google.devtools.build.lib.testutil.TestConstants;
+import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.NodeEntry.DirtyType;
@@ -1664,6 +1668,259 @@ final void runTreeArtifactRewoundWhenTreeFilesLost(
assertArtifactKey(rewoundKeys.get(2), "tree/_pic_objs/consumes_tree/make_cc_dir");
}
+ /**
+ * The number of actions that consume the tree artifact populated by an action template expansion
+ * concurrently in {@link #runActionTemplateExpansionRewound_notConcurrentWithTreeConsumers}.
+ *
+ * The more of them are reading it when the expansion is rewound, the more likely it is that a
+ * missing mutual exclusion is detected.
+ */
+ private static final int TREE_CONSUMER_COUNT = 8;
+
+ /**
+ * A tool that copies its second argument to its first.
+ *
+ * On Windows, {@code cmd.exe} treats forward slashes as option prefixes, so the paths have to
+ * be translated before they are passed to {@code copy}.
+ */
+ private static final String COPY_TOOL_SCRIPT =
+ OS.getCurrent() == OS.WINDOWS
+ ? """
+ @echo off
+ set "OUT=%~1"
+ set "IN=%~2"
+ copy /Y "%IN:/=\\%" "%OUT:/=\\%" >NUL
+ """
+ : """
+ #!/bin/bash
+ cp "$2" "$1"
+ """;
+
+ /**
+ * Verifies that an action generated by an {@link
+ * com.google.devtools.build.lib.actions.ActionTemplate} does not prepare for its re-execution
+ * while a consumer of the tree artifact it populates is running.
+ *
+ * The outputs of such an action are {@link
+ * com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact}s owned by the expansion,
+ * whereas its consumers depend on the parent tree artifact, which is owned by the template. A
+ * rewound expansion action must therefore synchronize on the template rather than on itself.
+ *
+ * The rewound action also consumes the output of another action of the same expansion, which
+ * is a special case that only happens with expansions and must not result in deadlocks.
+ */
+ public final void runActionTemplateExpansionRewound_notConcurrentWithTreeConsumers() throws Exception {
+ // All consumers and the action that reports the lost input have to run concurrently for the
+ // expansion to be rewound while the tree artifact is being read.
+ ensureMinimumJobs(TREE_CONSUMER_COUNT + 1);
+ testCase.addOptions("--experimental_allow_map_directory");
+ testCase.write(
+ "foo/defs.bzl",
+ """
+ def _copy_tool_impl(ctx):
+ tool = ctx.actions.declare_file(ctx.attr.name + ".bat")
+ ctx.actions.write(tool, r\"\"\"COPY_TOOL_SCRIPT\"\"\", is_executable = True)
+ return DefaultInfo(files = depset([tool]), executable = tool)
+
+ copy_tool = rule(implementation = _copy_tool_impl, executable = True)
+
+ def _map_impl(template_ctx, input_directories, output_directories, tools, **kwargs):
+ for child in input_directories["seed"].children:
+ # The two actions form a chain within the expansion, so the second one consumes a
+ # file of the very tree artifact it populates.
+ mid = template_ctx.declare_file(
+ child.basename + ".mid",
+ directory = output_directories["mapped"],
+ )
+ args = template_ctx.args()
+ args.add_all([mid, child])
+ template_ctx.run(
+ inputs = [child],
+ outputs = [mid],
+ executable = tools["copy_tool"],
+ arguments = [args],
+ progress_message = "Mapping foo/mapped_dir (1)",
+ )
+ out = template_ctx.declare_file(
+ child.basename + ".out",
+ directory = output_directories["mapped"],
+ )
+ args = template_ctx.args()
+ args.add_all([out, mid])
+ template_ctx.run(
+ inputs = [mid],
+ outputs = [out],
+ executable = tools["copy_tool"],
+ arguments = [args],
+ progress_message = "Mapping foo/mapped_dir (2)",
+ )
+
+ def _mapped_tree_impl(ctx):
+ seed = ctx.actions.declare_directory("seed_dir")
+ ctx.actions.run_shell(
+ outputs = [seed],
+ command = "echo seed > $1/f1",
+ arguments = [seed.path],
+ progress_message = "Seeding foo/seed_dir",
+ )
+ mapped = ctx.actions.declare_directory("mapped_dir")
+ ctx.actions.map_directory(
+ implementation = _map_impl,
+ input_directories = {"seed": seed},
+ output_directories = {"mapped": mapped},
+ tools = {"copy_tool": ctx.attr._copy_tool.files_to_run},
+ # Ensure that the rewound expansion action re-executes its spawn instead of
+ # picking up the result of its first execution from the cache.
+ execution_requirements = {"no-cache": "1"},
+ )
+ return DefaultInfo(files = depset([mapped]))
+
+ mapped_tree = rule(
+ implementation = _mapped_tree_impl,
+ attrs = {
+ "_copy_tool": attr.label(
+ default = ":copy_tool",
+ executable = True,
+ cfg = "exec",
+ ),
+ },
+ )
+
+ def _consumer_impl(ctx):
+ out = ctx.actions.declare_file(ctx.attr.name + ".out")
+ ctx.actions.run_shell(
+ inputs = ctx.files.srcs,
+ outputs = [out],
+ command = "echo consumed > $1",
+ arguments = [out.path],
+ progress_message = "Consuming //foo:" + ctx.attr.name,
+ )
+ return DefaultInfo(files = depset([out]))
+
+ consumer = rule(
+ implementation = _consumer_impl,
+ attrs = {"srcs": attr.label_list(allow_files = True)},
+ )
+ """
+ .replace("COPY_TOOL_SCRIPT", COPY_TOOL_SCRIPT));
+ testCase.write(
+ "foo/BUILD",
+ """
+ load(":defs.bzl", "consumer", "copy_tool", "mapped_tree")
+
+ copy_tool(name = "copy_tool")
+
+ mapped_tree(name = "mapped_tree")
+
+ genrule(
+ name = "warmup_gen",
+ outs = ["warmup.out"],
+ cmd = "echo warmup > $@",
+ tags = ["no-cache"],
+ )
+
+ genrule(
+ name = "warmup_consumer",
+ srcs = ["warmup.out"],
+ outs = ["warmup_consumed.out"],
+ cmd = "cp $< $@",
+ )
+
+ consumer(
+ name = "losing_consumer",
+ srcs = [
+ "warmup_consumed.out",
+ ":mapped_tree",
+ ],
+ )
+ """
+ + IntStream.range(0, TREE_CONSUMER_COUNT)
+ .mapToObj(
+ i ->
+ """
+ consumer(
+ name = "consumer_%d",
+ srcs = [
+ "warmup_consumed.out",
+ ":mapped_tree",
+ ],
+ )
+ """
+ .formatted(i))
+ .collect(joining("\n")));
+
+ // The first rewound action of a build waits for all in-flight actions to finish before it
+ // prepares for its re-execution, which would mask the behavior under test. Rewind an unrelated
+ // action first; all consumers depend on its output and thus can't start any earlier.
+ addSpawnShim(
+ "Executing genrule //foo:warmup_consumer",
+ (spawn, context) -> createLostInputsExecException(spawn, context, "warmup.out"));
+
+ // A consumer holds the read lock on the tree artifact from before its spawn shim runs until
+ // after its spawn has been executed, so the count is a lower bound on the number of consumers
+ // that are reading the tree artifact.
+ AtomicInteger consumersReadingTree = new AtomicInteger();
+ AtomicInteger maxConsumersReadingDuringExpansion = new AtomicInteger();
+ testCase.getRuntimeWrapper()
+ .registerSubscriber(
+ new Object() {
+ @Subscribe
+ @AllowConcurrentEvents
+ @SuppressWarnings("unused")
+ public void accept(SpawnExecutedEvent event) {
+ if (event.getActionMetadata().describe().startsWith("Consuming //foo:consumer_")) {
+ consumersReadingTree.decrementAndGet();
+ }
+ }
+ });
+ for (int i = 0; i < TREE_CONSUMER_COUNT; i++) {
+ addSpawnShim(
+ "Consuming //foo:consumer_" + i,
+ (spawn, context) -> {
+ consumersReadingTree.incrementAndGet();
+ return ExecResult.delegate();
+ });
+ }
+ // The expansion actions are the only writers, so they must never observe a reader. Both of them
+ // run twice: once initially and once after the expansion has been rewound.
+ for (String expansionAction :
+ ImmutableList.of("Mapping foo/mapped_dir (1)", "Mapping foo/mapped_dir (2)")) {
+ for (int run = 0; run < 2; run++) {
+ addSpawnShim(
+ expansionAction,
+ (spawn, context) -> {
+ maxConsumersReadingDuringExpansion.accumulateAndGet(
+ consumersReadingTree.get(), Math::max);
+ return ExecResult.delegate();
+ });
+ }
+ }
+ // Fails without executing a spawn, so that the expansion is rewound while the other consumers
+ // are as likely as possible to still be reading the tree artifact. The consumers deliberately
+ // don't wait for each other: making them do so would deadlock whenever Skyframe can't run all
+ // of them concurrently.
+ addSpawnShim(
+ "Consuming //foo:losing_consumer",
+ (spawn, context) -> {
+ SpecialArtifact mappedTree = SpawnInputUtils.getTreeArtifactWithName(spawn, "mapped_dir");
+ return createLostInputsExecException(
+ context, SpawnInputUtils.getExpandedToArtifact("f1.out", mappedTree, spawn, context));
+ });
+
+ testCase.buildTarget("//foo:all");
+
+ verifyAllSpawnShimsConsumed();
+ assertWithMessage(
+ "the rewound action template expansion prepared for its re-execution while consumers"
+ + " were still reading the tree artifact it populates")
+ .that(maxConsumersReadingDuringExpansion.get())
+ .isEqualTo(0);
+ // Rewinding an expanded action re-expands the template, so both actions of the chain re-run.
+ var executedSpawns = ImmutableMultiset.copyOf(getExecutedSpawnDescriptions());
+ assertThat(executedSpawns).hasCount("Mapping foo/mapped_dir (1)", 2);
+ assertThat(executedSpawns).hasCount("Mapping foo/mapped_dir (2)", 2);
+ }
+
public final void runGeneratedRunfilesRewound_allFilesLost_spawnFailed() throws Exception {
// This test demonstrates that rewinding works when an action fails due to lost inputs which are
// generated files in the action's runfiles. Rewinding must propagate across the runfiles tree
@@ -3316,10 +3573,20 @@ static boolean isActionExecutionKey(Object key, Label label) {
* CPU.
*/
private void ensureMultipleJobs() throws Exception {
+ ensureMinimumJobs(2);
+ }
+
+ /**
+ * Ensures that the value of the {@code --jobs} flag is at least {@code minJobs}.
+ *
+ * Note that the default value for {@code --jobs} is automatically calculated based on host
+ * CPU.
+ */
+ private void ensureMinimumJobs(int minJobs) throws Exception {
int autoJobs = new JobsConverter().convert("auto");
- if (autoJobs == 1) {
- logger.atInfo().log("Setting --jobs=2 (was 1)");
- testCase.addOptions("--jobs=2");
+ if (autoJobs < minJobs) {
+ logger.atInfo().log("Setting --jobs=%s (was %s)", minJobs, autoJobs);
+ testCase.addOptions("--jobs=" + minJobs);
} else {
logger.atInfo().log("Keeping default value of --jobs=%s", autoJobs);
}