From 6f0be557ec65916a4fa1b282177a88a99a4227e5 Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Mon, 27 Jul 2026 15:35:32 +0200 Subject: [PATCH] Harden MTP artifact post-processing and add an opt-out Artifact post-processing runs after a test run has already completed and reported its results, so it must never be able to change that outcome. Two ways it could are fixed: - Any exception outside a small hand-picked set escaped ExecuteAsync, was not caught by the caller, and reached the top-level CLI handler, turning a finished run into a crash with a different exit code. Every failure is now logged to trace and downgraded to the existing warning. - Cancelling mid-job kills the post-processing process, and the resulting non-zero exit code was then reported as a post-processing failure. Both warning sites now route through ReportFailureUnlessCancelled. Also adds '--no-artifact-post-processing', which skips post-processing so each test application keeps its own artifacts instead of test applications being relaunched to merge them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e002463b-c925-4c2f-90c3-b56d480faa7d --- .../CommandDefinitionStrings.resx | 3 + ...mandDefinition.MicrosoftTestingPlatform.cs | 7 + .../xlf/CommandDefinitionStrings.cs.xlf | 5 + .../xlf/CommandDefinitionStrings.de.xlf | 5 + .../xlf/CommandDefinitionStrings.es.xlf | 5 + .../xlf/CommandDefinitionStrings.fr.xlf | 5 + .../xlf/CommandDefinitionStrings.it.xlf | 5 + .../xlf/CommandDefinitionStrings.ja.xlf | 5 + .../xlf/CommandDefinitionStrings.ko.xlf | 5 + .../xlf/CommandDefinitionStrings.pl.xlf | 5 + .../xlf/CommandDefinitionStrings.pt-BR.xlf | 5 + .../xlf/CommandDefinitionStrings.ru.xlf | 5 + .../xlf/CommandDefinitionStrings.tr.xlf | 5 + .../xlf/CommandDefinitionStrings.zh-Hans.xlf | 5 + .../xlf/CommandDefinitionStrings.zh-Hant.xlf | 5 + .../Test/MTP/ArtifactPostProcessingManager.cs | 37 +++-- .../MicrosoftTestingPlatformTestCommand.cs | 5 +- .../ArtifactPostProcessingManagerTests.cs | 149 +++++++++++++++++- ...tBuildsAndRunsArtifactPostProcessingMTP.cs | 39 ++++- ...shotTests.VerifyMTPHelpOutput.verified.txt | 1 + 20 files changed, 281 insertions(+), 25 deletions(-) diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx b/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx index 9645faf1db54..225cf839c4b0 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx @@ -494,6 +494,9 @@ This is equivalent to deleting project.assets.json. Disable Http Caching for packages. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Disable progress reporting. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs b/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs index 8862525d8418..61ff9e11af13 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs @@ -133,6 +133,12 @@ public sealed partial class MicrosoftTestingPlatform : TestCommandDefinition, IC Arity = ArgumentArity.Zero }; + public readonly Option NoArtifactPostProcessingOption = new("--no-artifact-post-processing") + { + Description = CommandDefinitionStrings.CmdNoArtifactPostProcessingDescription, + Arity = ArgumentArity.Zero + }; + public readonly Option OutputOption = new("--output") { Description = CommandDefinitionStrings.CmdTestOutputDescription, @@ -215,6 +221,7 @@ public MicrosoftTestingPlatform() Options.Add(UseCurrentRuntimeOption); Options.Add(NoAnsiOption); Options.Add(NoProgressOption); + Options.Add(NoArtifactPostProcessingOption); Options.Add(OutputOption); Options.Add(ListTestsOption); Options.Add(NoLaunchProfileOption); diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf index 06000cb8f219..7aec231e2ddb 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf @@ -439,6 +439,11 @@ Jedná se o ekvivalent odstranění project.assets.json. Zakáže výstup ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Nesestavujte projekt, dokud ho neotestujete. Implikuje možnost --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf index 83979a8c32b6..7d88f7c7a435 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf @@ -439,6 +439,11 @@ Dies entspricht dem Löschen von "project.assets.json". Deaktivieren Sie die ANSI-Ausgabe. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Erstellt das Projekt nicht vor dem Testen. Impliziert "--no-restore". diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf index e85ecdec78c8..38b8bad076ec 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf @@ -439,6 +439,11 @@ Esta acción es equivalente a eliminar project.assets.json. Deshabilite la salida ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. No compile el proyecto antes de probarlo. Implica --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf index 48890e4a1dbf..9acc94604593 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf @@ -439,6 +439,11 @@ Cela équivaut à supprimer project.assets.json. Désactivez la sortie ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Ne pas générer le projet avant les tests. Implique --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf index b48974079e2b..3e1d85e7e905 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf @@ -439,6 +439,11 @@ Equivale a eliminare project.assets.json. Disabilita l'output ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Non compila il progetto prima del test. Implica --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf index 789645e7aeff..4628143f0a2c 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf @@ -439,6 +439,11 @@ This is equivalent to deleting project.assets.json. ANSI 出力を無効にします。 + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. テストする前にプロジェクトをビルドしないでください。--no-restore を意味します。 diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf index 39a7e14d7881..6720cec7cbf1 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf @@ -439,6 +439,11 @@ project.assets.json을 삭제하는 것과 동일합니다. ANSI 출력을 비활성화합니다. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. 테스트하기 전에 프로젝트를 빌드하지 않습니다. 복원 없음을 의미합니다. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf index 68aac5510e09..700dc031935d 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf @@ -439,6 +439,11 @@ Jest to równoważne usunięciu pliku project.assets.json. Wyłącz dane wyjściowe ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Nie kompiluj tego projektu przed testowaniem. Powoduje przyjęcie, że podano parametr --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf index 7b285cd2bd81..b56b9668c9e2 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf @@ -439,6 +439,11 @@ Isso equivale a excluir o project.assets.json. Desabilite a saída ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Não compile o projeto antes de testar. Implica em --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf index c6f28a7d1221..a70352c370ba 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf @@ -439,6 +439,11 @@ This is equivalent to deleting project.assets.json. Отключить вывод ANSI. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Сборка проекта перед тестированием не выполняется. Подразумевает --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf index bce3188757f2..eb7fd2553d10 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf @@ -439,6 +439,11 @@ project.assets.json öğesini silmeyle eşdeğerdir. ANSI çıkışını devre dışı bırakın. + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. Test etmeden önce projeyi derlemeyin. --no-restore anlamına gelir. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf index 15b4af011561..b5d44694133a 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf @@ -439,6 +439,11 @@ This is equivalent to deleting project.assets.json. 禁用 ANSI 输出。 + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. 测试之前不要生成项目。Implies --no-restore. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf index 702ac70b0e3d..137e54b25716 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf @@ -439,6 +439,11 @@ This is equivalent to deleting project.assets.json. 停用 ANSI 輸出。 + + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. + + Do not build the project before testing. Implies --no-restore. 請勿在測試之前建置專案。提示：-no-restore。 diff --git a/src/Cli/dotnet/Commands/Test/MTP/ArtifactPostProcessingManager.cs b/src/Cli/dotnet/Commands/Test/MTP/ArtifactPostProcessingManager.cs index 45029f9aed6b..ed60d21429cb 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/ArtifactPostProcessingManager.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/ArtifactPostProcessingManager.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.ComponentModel; using System.Globalization; using System.Text.Json; using Microsoft.DotNet.Cli.Commands.Test.IPC.Models; @@ -112,7 +111,7 @@ public async Task ExecuteAsync( if (invocation.FailureMessage is { } failureMessage) { - output.WriteWarningMessage(string.Format( + ReportFailureUnlessCancelled(output, ctrlC, string.Format( CultureInfo.CurrentCulture, CliCommandStrings.ArtifactPostProcessingFailed, job.Application.Module.TargetPath, @@ -120,21 +119,21 @@ public async Task ExecuteAsync( } else if (exitCode != ExitCode.Success) { - output.WriteWarningMessage(string.Format( + ReportFailureUnlessCancelled(output, ctrlC, string.Format( CultureInfo.CurrentCulture, CliCommandStrings.ArtifactPostProcessingProcessFailed, job.Application.Module.TargetPath, exitCode)); } } - catch (Exception ex) when (ex is IOException - or UnauthorizedAccessException - or InvalidOperationException - or Win32Exception - or NotSupportedException - or TimeoutException) + catch (Exception ex) { - output.WriteWarningMessage(string.Format( + // Post-processing is a best-effort convenience on top of a completed test run: the + // original artifacts are always still on disk and still reported, so no failure here + // may escape and turn a finished run into a CLI crash with a different exit code. + Logger.LogTrace($"Artifact post-processing with '{job.Application.Module.TargetPath}' failed: {ex}"); + + ReportFailureUnlessCancelled(output, ctrlC, string.Format( CultureInfo.CurrentCulture, CliCommandStrings.ArtifactPostProcessingFailed, job.Application.Module.TargetPath, @@ -157,6 +156,24 @@ or NotSupportedException } } + /// + /// Reports a post-processing failure, unless the user cancelled the run. Cancellation kills the + /// post-processing process the same way it kills a test application, so the resulting failure is + /// the cancellation the user asked for rather than a post-processing problem worth reporting. + /// + internal static void ReportFailureUnlessCancelled( + TerminalTestReporter output, + CtrlCCancellationManager ctrlC, + string message) + { + if (ctrlC.Token.IsCancellationRequested) + { + return; + } + + output.WriteWarningMessage(message); + } + internal IReadOnlyList SnapshotApplications() { lock (_lock) diff --git a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs index b0fa329d6aa8..6a4f98087c43 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs @@ -116,7 +116,10 @@ public int Run(ParseResult parseResult, bool isHelp) exitCode = ExitCode.TestSessionAborted; } - if (!testOptions.IsHelp && !testOptions.IsDiscovery && !ctrlC.Token.IsCancellationRequested) + if (!testOptions.IsHelp + && !testOptions.IsDiscovery + && !parseResult.GetValue(definition.NoArtifactPostProcessingOption) + && !ctrlC.Token.IsCancellationRequested) { artifactPostProcessingManager.ExecuteAsync(buildOptions, output, ctrlC).GetAwaiter().GetResult(); } diff --git a/test/dotnet.Tests/CommandTests/Test/ArtifactPostProcessingManagerTests.cs b/test/dotnet.Tests/CommandTests/Test/ArtifactPostProcessingManagerTests.cs index 1de00153a755..f546c93c8d93 100644 --- a/test/dotnet.Tests/CommandTests/Test/ArtifactPostProcessingManagerTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/ArtifactPostProcessingManagerTests.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.DotNet.Cli.Commands; using Microsoft.DotNet.Cli.Commands.Run; using Microsoft.DotNet.Cli.Commands.Test; +using Microsoft.DotNet.Cli.Commands.Test.IPC.Models; using Microsoft.DotNet.Cli.Commands.Test.Terminal; using TestExitCode = Microsoft.DotNet.Cli.Commands.Test.ExitCode; @@ -129,6 +131,135 @@ public void GetArtifactPostProcessingLaunchArguments_AppHost_UsesNoTestArguments arguments.Should().BeEmpty(); } + [TestMethod] + public async Task ExecuteAsync_WhenJobFailsUnexpectedly_ReportsWarningWithoutThrowing() + { + var console = new CapturingConsole(); + using var reporter = CreateReporter(console); + // A NUL character makes Path.GetFullPath throw ArgumentException while planning the job. + // ArgumentException is outside the exception set this code used to catch, so before the + // catch-all it escaped ExecuteAsync and crashed a 'dotnet test' run that had already + // completed, replacing its exit code. + ArtifactPostProcessingManager manager = CreateManagerWithMergeableArtifacts("first\0.trx", "second\0.trx"); + using var ctrlC = CreateCancellationManager(); + + await manager.ExecuteAsync(CreateBuildOptions(), reporter, ctrlC); + reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, TestExitCode.Success); + + console.GetOutput().Should().Contain( + FormatPrefix(CliCommandStrings.ArtifactPostProcessingFailed, "A.dll"), + "the failure must degrade to a warning instead of escaping"); + } + + /// + /// Formats a two-placeholder message up to its second placeholder, so an assertion can match the + /// reported message without depending on the exception text that fills the placeholder. + /// + private static string FormatPrefix(string format, string firstArgument) + => string.Format(format, firstArgument, "\u0001").Split('\u0001')[0]; + + [TestMethod] + public async Task ExecuteAsync_WhenCancelledBeforeStarting_RunsNoJobs() + { + var console = new CapturingConsole(); + using var reporter = CreateReporter(console); + ArtifactPostProcessingManager manager = CreateManagerWithMergeableArtifacts("first.trx", "second.trx"); + using var ctrlC = CreateCancellationManager(); + ctrlC.SimulateCtrlC(); + // Running a job creates its results directory before anything else is attempted, so the + // directory staying absent is the observable proof that no job ran. Asserting on the absence + // of a warning would not be: a job that ran and failed is silent too, because the failure of + // a cancelled run is deliberately not reported. + string resultsDirectory = Path.Combine( + Path.GetTempPath(), + $"dotnet-test-postproc-tests-{Guid.NewGuid():N}"); + + try + { + await manager.ExecuteAsync(CreateBuildOptions(resultsDirectory), reporter, ctrlC); + + Directory.Exists(resultsDirectory).Should().BeFalse( + "a cancelled run must not start the jobs it planned"); + } + finally + { + if (Directory.Exists(resultsDirectory)) + { + Directory.Delete(resultsDirectory, recursive: true); + } + } + } + + [TestMethod] + public void ReportFailureUnlessCancelled_WhenNotCancelled_WritesWarning() + { + var console = new CapturingConsole(); + using var reporter = CreateReporter(console); + using var ctrlC = CreateCancellationManager(); + + ArtifactPostProcessingManager.ReportFailureUnlessCancelled(reporter, ctrlC, "post-processing warning"); + + console.GetOutput().Should().Contain("post-processing warning"); + } + + [TestMethod] + public void ReportFailureUnlessCancelled_WhenCancelled_WritesNothing() + { + var console = new CapturingConsole(); + using var reporter = CreateReporter(console); + using var ctrlC = CreateCancellationManager(); + // Cancellation kills the post-processing process, so its failure is the cancellation the + // user asked for rather than a post-processing problem worth reporting. + ctrlC.SimulateCtrlC(); + + ArtifactPostProcessingManager.ReportFailureUnlessCancelled(reporter, ctrlC, "post-processing warning"); + + console.GetOutput().Should().NotContain("post-processing warning"); + } + + private static ArtifactPostProcessingManager CreateManagerWithMergeableArtifacts(params string[] artifactPaths) + { + var manager = new ArtifactPostProcessingManager(); + TestModule module = CreateModule(); + manager.RecordCapabilities( + module, + "net10.0", + "x64", + new HandshakeMessage(new Dictionary + { + [HandshakeMessagePropertyNames.SupportedPostProcessorKinds] = "microsoft.testing.trx", + })); + + foreach (string artifactPath in artifactPaths) + { + manager.RecordArtifact( + module, + "net10.0", + "x64", + "execution-1", + new FileArtifactMessage(artifactPath, "TRX", null, null, null, null, "microsoft.testing.trx")); + } + + return manager; + } + + private static CtrlCCancellationManager CreateCancellationManager() + => new(onFirstCtrlC: () => { }, exitAction: _ => { }, subscribeToConsole: false); + + private static BuildOptions CreateBuildOptions(string? resultsDirectory = null) + => new( + new PathOptions(null, null, null, ResultsDirectoryPath: resultsDirectory, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: [], + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: new Dictionary()); + private static TerminalTestReporter CreateReporter(CapturingConsole console) { var reporter = new TerminalTestReporter(console, new TerminalTestReporterOptions @@ -147,7 +278,16 @@ private static TerminalTestReporter CreateReporter(CapturingConsole console) private static ArtifactPostProcessingApplication CreateApplication() { - var module = new TestModule( + return new ArtifactPostProcessingApplication( + CreateModule(), + "net10.0", + "x64", + new HashSet(StringComparer.Ordinal) { "microsoft.testing.trx", "microsoft.codecoverage" }, + new HashSet(StringComparer.Ordinal)); + } + + private static TestModule CreateModule() + => new( new RunProperties("dotnet", "A.dll", null), ProjectFullPath: null, TargetFramework: "net10.0", @@ -156,13 +296,6 @@ private static ArtifactPostProcessingApplication CreateApplication() TargetPath: "A.dll", DotnetRootArchVariableName: null, EnvironmentVariables: new Dictionary()); - return new ArtifactPostProcessingApplication( - module, - "net10.0", - "x64", - new HashSet(StringComparer.Ordinal) { "microsoft.testing.trx", "microsoft.codecoverage" }, - new HashSet(StringComparer.Ordinal)); - } private static ArtifactPostProcessingArtifact CreateArtifact(string path, string? kind) => new(path, kind, "A.dll", "net10.0", "x64", Guid.NewGuid().ToString("N")); diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsArtifactPostProcessingMTP.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsArtifactPostProcessingMTP.cs index 68a9731b1d15..1bb91fe9f9d3 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsArtifactPostProcessingMTP.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsArtifactPostProcessingMTP.cs @@ -44,13 +44,40 @@ public void MultiProjectRun_MergesTrxArtifacts() mergedTrx.Descendants(ns + "Counters").Single().Attribute("total")!.Value.Should().Be("5"); } - private CommandResult Run(string workingDirectory, string resultsDirectory) - => new DotnetTestCommand(Log, disableNewOutput: false) + [TestMethod] + public void MultiProjectRun_WithNoArtifactPostProcessing_KeepsOneReportPerTestApplication() + { + TestAsset testInstance = TestAssetsManager + .CopyTestAsset("MultiTestProjectSolutionWithTests", Guid.NewGuid().ToString()) + .WithSource(); + EnableTrxReport(testInstance.Path); + string resultsDirectory = Path.Combine(testInstance.Path, "TestResults"); + + CommandResult result = Run(testInstance.Path, resultsDirectory, "--no-artifact-post-processing"); + + result.ExitCode.Should().Be( + ExitCodes.AtLeastOneTestFailed, + $"the test output was:{Environment.NewLine}{result.StdOut}{Environment.NewLine}{result.StdErr}"); + + string[] trxReports = Directory.GetFiles(resultsDirectory, "*.trx", SearchOption.AllDirectories); + trxReports.Should().HaveCount(2, "no test application is relaunched to merge the reports"); + trxReports.Should().NotContain(path => Path.GetFileName(path).StartsWith("merged-", StringComparison.Ordinal)); + } + + private CommandResult Run(string workingDirectory, string resultsDirectory, params string[] additionalArguments) + { + string[] arguments = + [ + "--report-trx", + "--results-directory", resultsDirectory, + "--configuration", TestingConstants.Debug, + .. additionalArguments + ]; + + return new DotnetTestCommand(Log, disableNewOutput: false) .WithWorkingDirectory(workingDirectory) - .Execute( - "--report-trx", - "--results-directory", resultsDirectory, - "--configuration", TestingConstants.Debug); + .Execute(arguments); + } private static string GetMergedTrxPath(CommandResult result) { diff --git a/test/dotnet.Tests/CommandTests/Test/snapshots/MTPHelpSnapshotTests.VerifyMTPHelpOutput.verified.txt b/test/dotnet.Tests/CommandTests/Test/snapshots/MTPHelpSnapshotTests.VerifyMTPHelpOutput.verified.txt index 5e3f844bf320..8ec0dd17f5b8 100644 --- a/test/dotnet.Tests/CommandTests/Test/snapshots/MTPHelpSnapshotTests.VerifyMTPHelpOutput.verified.txt +++ b/test/dotnet.Tests/CommandTests/Test/snapshots/MTPHelpSnapshotTests.VerifyMTPHelpOutput.verified.txt @@ -41,6 +41,7 @@ Options: --ucr, --use-current-runtime Use current runtime as the target runtime. [default: False] --no-ansi Disable ANSI output. [default: False] --no-progress Disable progress reporting. [default: False] + --no-artifact-post-processing Do not merge compatible artifacts, such as TRX reports or code coverage files, produced by different test applications. [default: False] --output Verbosity of test output. --list-tests List the discovered tests instead of running the tests. Optionally accepts a format: 'text' (default) for human-readable output or 'json' for machine-readable output. --no-launch-profile Do not attempt to use launchSettings.json or [app].run.json to configure the application. [default: False]