diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 57c290e1fc2..61e808acd3d 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -448,10 +448,10 @@ stages: _testKind: testCoreclr TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var. transparentCompiler: TransparentCompiler - # inttests_release: - # _configuration: Release - # _testKind: testIntegration - # setupVsHive: true + inttests_release: + _configuration: Release + _testKind: testIntegration + setupVsHive: true steps: - checkout: self clean: true diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 41a52df6395..960642bac95 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -402,6 +402,41 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore +# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects +# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which +# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked). +# The runner is restored via a on the +# test project, so it is present in the NuGet global packages folder after a normal slnx restore. +function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) { + + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) + $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?" + } + + # Get-PackageVersion (eng\build-utils.ps1) reads from eng\Versions.props, + # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value. + $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim() + $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe" + if (-not (Test-Path $xunitConsole)) { + throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)." + } + + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml" + + # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false). + $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo" + + Write-Host("$xunitConsole $xunit_args") + + Exec-Console $xunitConsole $xunit_args +} + function Prepare-TempDir() { Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir @@ -665,8 +700,8 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } - if ($testIntegration) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework + if ($testIntegration -and -not $noVisualStudio) { + TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } if ($testAOT) { diff --git a/eng/Versions.props b/eng/Versions.props index 08cde86aa77..f93e881a823 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,7 +130,7 @@ $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) 17.14.0 - 0.1.800-beta + 5.3.0-2.26055.8 $(MicrosoftVisualStudioExtensibilityTestingVersion) @@ -161,6 +161,8 @@ 13.0.4 3.2.2 3.2.2 + + 2.9.3 8.0.0 diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs index 8a42bed252b..751fc461729 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using FSharp.Editor.IntegrationTests.Helpers; using Microsoft.CodeAnalysis.Testing.InProcess; using Microsoft.VisualStudio.Extensibility.Testing; using System.Threading; @@ -9,9 +10,14 @@ namespace Microsoft.CodeAnalysis.Testing { - [IdeSettings(MinVersion = VisualStudioVersion.VS2022)] + // RoslynWaiter* env vars enable Roslyn's async-operation listener tracking in devenv from launch, so the + // deterministic AsyncOperationWaiter drains actually wait (otherwise they no-op). See AsyncOperationWaiter. + [IdeSettings(MinVersion = VisualStudioVersion.VS18, MaxVersion = VisualStudioVersion.VS18, + EnvironmentVariables = new[] { "RoslynWaiterEnabled=1", "RoslynWaiterDiagnosticTokenEnabled=1" })] public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest { + protected AbstractIntegrationTest() => AsyncOperationWaiter.EnableTracking(); + protected CancellationToken TestToken => HangMitigatingCancellationToken; internal SolutionExplorerInProcess SolutionExplorer => TestServices.SolutionExplorer; diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs index ca89983189e..6605eda0199 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs @@ -43,7 +43,7 @@ module Test let answer = """; var expectedBuildSummary = "========== Build: 0 succeeded, 1 failed, 0 up-to-date, 0 skipped =========="; - var expectedError = "(Compiler) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding"; + var expectedError = "(Fsc) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding"; await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken); await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken); diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs index 7d902dbab6d..5aa5f121c01 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs @@ -83,13 +83,23 @@ private async Task PrepareProjectAndBuildAsync() await SolutionExplorer.CreateSingleProjectSolutionAsync(ProjectName, SolutionExplorerInProcess.ExistingProjectTemplate, TestToken); await SolutionExplorer.SetStartupProjectAsync(ProjectName, TestToken); + // Write the fixture directly to disk *before* opening it in VS, instead of using + // SetTextAsync + "File.SaveAll" command. Reason: the debugger binds breakpoints by + // matching the PDB's recorded source-file hash against the hash of Program.fs on disk. + // If the edit-buffer is built first and SaveAll lands a moment later, the PDB hash + // points at one snapshot while disk holds another -- breakpoints stay unbound + // (children=0) and never fire. Writing to disk first keeps the two in lockstep. + await SolutionExplorer.WriteFileAsync(ProjectName, "Program.fs", File.ReadAllText(GetFixturePath()), TestToken); await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken); - await Editor.SetTextAsync(File.ReadAllText(GetFixturePath()), TestToken); - await Shell.ExecuteCommandAsync("File.SaveAll", TestToken); var buildSummary = await SolutionExplorer.BuildSolutionAsync(TestToken); Assert.NotNull(buildSummary); Assert.Contains("Build: 1 succeeded, 0 failed", string.Join(Environment.NewLine, buildSummary)); + + // BuildSolutionAsync leaves the Build Output pane as the active text view; re-open Program.fs + // so the subsequent PlaceCaretAsync / ToggleBreakpointAtMarkerAsync operate on the F# source + // (rather than searching the build log for "BP_..." markers). + await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken); } private static string GetFixturePath() diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj index 374c8164a5b..9b04ed904fd 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj @@ -35,8 +35,21 @@ + + + + + + + diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs index 66bb33eccd9..dfbdfe14644 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs @@ -1,8 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using Microsoft.CodeAnalysis.Testing; +using System; +using System.Threading; using System.Threading.Tasks; using Xunit; using static Microsoft.VisualStudio.VSConstants; @@ -11,6 +13,52 @@ namespace FSharp.Editor.IntegrationTests; public class GoToDefinitionTests : AbstractIntegrationTest { + // GoToDefn resolves at invocation time: if the F# checker has not produced a result for the active + // document yet, the command no-ops, and only re-issuing it (a retry) or getting lucky with a sleep + // recovers -- either of which is brittle or masks a real regression. We avoid that by giving every test + // the setup the F# checker handles cleanly on the first try: the code is written to disk, the project is + // built, and the file is opened FRESH (never an already-open, buffer-edited document). With that a + // single GotoDefn resolves, and the only thing left to wait for is the navigation itself -- + // FSharpNavigation.NavigateToItem schedules the caret move asynchronously (via JoinableTaskFactory), so + // the caret has NOT moved by the time Shell.ExecuteCommandAsync(GotoDefn) returns. + // + // CI data behind this: with SetTextAsync on the auto-opened buffer, GoesToDefinition's first invocation + // deterministically no-ops even after a 10s settle (build 1463623), whereas FsiAndFs -- which adds files + // to disk and opens them fresh -- passes single-invoke. Editing an already-open buffer is the difference. + private static readonly TimeSpan GoToDefinitionNavigationTimeout = TimeSpan.FromSeconds(10); + private static readonly TimeSpan GoToDefinitionNavigationPollDelay = TimeSpan.FromMilliseconds(100); + + private async Task GoToDefinitionAsync(string caretMarker, CancellationToken cancellationToken) + { + // Wait for the project system to load the project into the workspace. + await Workspace.WaitForProjectSystemAsync(cancellationToken); + + // Make the editor the active command context (PlaceCaretAsync's dte.Find moves VS's active selection + // off the editor) and anchor the caret on the call site. + await Editor.ActivateAsync(cancellationToken); + await Editor.PlaceCaretAsync(caretMarker, cancellationToken); + + var before = await Editor.GetCurrentLineTextAsync(cancellationToken); + await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, cancellationToken); + + // Wait for the single asynchronous navigation to move the caret off the call site. This is not a + // retry of the command -- it just observes the completion of the one navigation we triggered, and + // fails the test if it never happens (so a real regression is caught). + for (var elapsed = TimeSpan.Zero; elapsed < GoToDefinitionNavigationTimeout; elapsed += GoToDefinitionNavigationPollDelay) + { + var after = await Editor.GetCurrentLineTextAsync(cancellationToken); + if (!string.Equals(before, after, StringComparison.Ordinal)) + { + return; + } + + await Task.Delay(GoToDefinitionNavigationPollDelay, cancellationToken); + } + + throw new InvalidOperationException( + $"GoToDefn did not navigate away from '{caretMarker}' within {GoToDefinitionNavigationTimeout.TotalSeconds:0}s."); + } + [IdeFact] public async Task GoesToDefinition() { @@ -27,10 +75,14 @@ module Test await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken); await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken); - await Editor.SetTextAsync(code, TestToken); - - await Editor.PlaceCaretAsync("add 1", TestToken); - await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); + // Add the code as a real file on disk and open it fresh after building (rather than editing the + // auto-opened buffer with SetTextAsync), so the F# checker resolves on the first GotoDefn -- see the + // note on GoToDefinitionAsync. Mirrors FsiAndFsFilesGoToCorrespondentDefinitions. + await SolutionExplorer.AddFileAsync("Library", "Test.fs", code, TestToken); + await SolutionExplorer.BuildSolutionAsync(TestToken); + await SolutionExplorer.OpenFileAsync("Library", "Test.fs", TestToken); + + await GoToDefinitionAsync("add 1", TestToken); var actualText = await Editor.GetCurrentLineTextAsync(TestToken); Assert.Contains(expectedText, actualText); @@ -72,8 +124,7 @@ let id (t: SomeType) = t await SolutionExplorer.BuildSolutionAsync(TestToken); await SolutionExplorer.OpenFileAsync("Library", "Module.fsi", TestToken); - await Editor.PlaceCaretAsync("SomeType ->", TestToken); - await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); + await GoToDefinitionAsync("SomeType ->", TestToken); var expectedText = "type SomeType ="; var expectedWindow = "Module.fsi"; var actualText = await Editor.GetCurrentLineTextAsync(TestToken); @@ -82,8 +133,7 @@ let id (t: SomeType) = t Assert.Equal(expectedWindow, actualWindow); await SolutionExplorer.OpenFileAsync("Library", "Module.fs", TestToken); - await Editor.PlaceCaretAsync("SomeType)", TestToken); - await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); + await GoToDefinitionAsync("SomeType)", TestToken); expectedText = "type SomeType ="; expectedWindow = "Module.fs"; actualText = await Editor.GetCurrentLineTextAsync(TestToken); @@ -91,4 +141,4 @@ let id (t: SomeType) = t Assert.Equal(expectedText, actualText); Assert.Equal(expectedWindow, actualWindow); } -} \ No newline at end of file +} diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/AsyncOperationWaiter.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/AsyncOperationWaiter.cs new file mode 100644 index 00000000000..b0970505e7e --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/AsyncOperationWaiter.cs @@ -0,0 +1,133 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.VisualStudio.ComponentModelHost; +using Microsoft.VisualStudio.Threading; +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +namespace FSharp.Editor.IntegrationTests.Helpers +{ + // Deterministic, focus-independent synchronization via Roslyn's IAsynchronousOperationListener waiters - the + // same mechanism Roslyn's own VS integration tests and the TypeScript-VS Apex tests use. Reached by reflection + // because the provider is internal; Roslyn exposes a non-InternalsVisibleTo path (RoslynWaiterEnabled env var + // + static Enable). Feature names are the stable literal strings from FeatureAttribute. + internal static class AsyncOperationWaiter + { + public const string Workspace = "Workspace"; + public const string SolutionCrawlerLegacy = "SolutionCrawlerLegacy"; + public const string DiagnosticService = "DiagnosticService"; + public const string LightBulb = "LightBulb"; + + private const string ProviderTypeName = "Microsoft.CodeAnalysis.Shared.TestHooks.AsynchronousOperationListenerProvider"; + private const string WorkspaceTypeName = "Microsoft.VisualStudio.LanguageServices.VisualStudioWorkspace"; + + private static readonly object s_gate = new object(); + private static bool s_enableAttempted; + + // Enable the listener tracking so the waiters below actually wait. The env vars are the robust path (read + // lazily by Roslyn, honored even for non-IVT teams); the static Enable covers an already-cached state. + public static void EnableTracking() + { + lock (s_gate) + { + if (s_enableAttempted) + { + return; + } + + s_enableAttempted = true; + + Environment.SetEnvironmentVariable("RoslynWaiterEnabled", "1"); + Environment.SetEnvironmentVariable("RoslynWaiterDiagnosticTokenEnabled", "1"); + + var enable = TryGetProviderType()?.GetMethod("Enable", new[] { typeof(bool), typeof(bool?) }); + enable?.Invoke(null, new object?[] { true, true }); + } + } + + // Returns whether Roslyn currently reports listener tracking as enabled (s_enabled), for diagnostics: a + // zero-duration drain on a disabled provider is a silent no-op, so we surface this in failure messages. + public static bool IsTrackingEnabled() + { + var field = TryGetProviderType()?.GetField("s_enabled", BindingFlags.Public | BindingFlags.Static); + return field?.GetValue(null) is bool enabled && enabled; + } + + // Awaits completion of all queued async operations for the given Roslyn features. Focus-independent and + // deterministic, unlike polling the lightbulb UI session. + public static async Task WaitForFeaturesAsync(IComponentModel componentModel, string[] featureNames, CancellationToken cancellationToken) + { + var providerType = TryGetProviderType(); + if (providerType is null) + { + return; + } + + var provider = TryGetService(componentModel, providerType); + if (provider is null) + { + return; + } + + var workspaceType = TryGetWorkspaceType(); + var workspace = workspaceType is null ? null : TryGetService(componentModel, workspaceType); + + var waitAll = providerType.GetMethod("WaitAllAsync"); + if (waitAll is null) + { + return; + } + + Task task; + try + { + task = (Task)waitAll.Invoke(provider, new object?[] { workspace, featureNames, null, null })!; + } + catch (TargetInvocationException ex) + { + throw new InvalidOperationException( + $"Roslyn waiter invocation failed for [{string.Join(", ", featureNames)}] (trackingEnabled={IsTrackingEnabled()}).", + ex.InnerException ?? ex); + } + + try + { + await task.WithCancellation(cancellationToken); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new InvalidOperationException( + $"Roslyn waiter timed out for [{string.Join(", ", featureNames)}] (trackingEnabled={IsTrackingEnabled()})."); + } + } + + private static object? TryGetService(IComponentModel componentModel, Type serviceType) + { + try + { + var getService = typeof(IComponentModel).GetMethod("GetService")!.MakeGenericMethod(serviceType); + return getService.Invoke(componentModel, null); + } + catch + { + return null; + } + } + + private static Type? TryGetProviderType() => FindType("Microsoft.CodeAnalysis.Workspaces", ProviderTypeName); + + private static Type? TryGetWorkspaceType() => FindType("Microsoft.VisualStudio.LanguageServices", WorkspaceTypeName); + + private static Type? FindType(string assemblyName, string typeName) + { + var assembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, assemblyName, StringComparison.OrdinalIgnoreCase)); + return assembly?.GetType(typeName); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/LightBulbHelper.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/LightBulbHelper.cs index d40528cea76..52c3bca3b44 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/LightBulbHelper.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/Helpers/LightBulbHelper.cs @@ -1,66 +1,223 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using Microsoft.VisualStudio.Language.Intellisense; using Microsoft.VisualStudio.Text.Editor; using Microsoft.VisualStudio.Threading; +using Microsoft.VisualStudio.Utilities; using System; +using System.Collections; using System.Collections.Generic; -using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; namespace FSharp.Editor.IntegrationTests.Helpers { - // I stole this voodoo from Razor and removed the obscurest bits + // Reads code actions from the editor's OWN lightbulb session (triggered via the real ShowQuickFixes command), + // exactly like the TypeScript-VS Apex tests and Roslyn's integration tests. We do NOT create a broker-owned + // session: that gets superseded/dismissed by the editor's real lightbulb within ~20ms headless. The real + // session is producer-agnostic (aggregates Roslyn today, the VS LSP CodeActionSource tomorrow). internal static class LightBulbHelper { - public static async Task> WaitForItemsAsync( + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(60); + private static readonly TimeSpan s_perAttemptTimeout = TimeSpan.FromSeconds(15); + private static readonly TimeSpan s_activeWait = TimeSpan.FromSeconds(5); + + // PopulateWithDataAsync returns Task> and ActionSets is ImmutableArray; + // System.Collections.Immutable skews between the NuGet ref and the in-proc VS runtime, so we invoke/read + // these via reflection through the non-generic IEnumerable and never name ImmutableArray in compiled IL. + private static readonly MethodInfo s_populateWithDataAsync = + typeof(IAsyncLightBulbSession).GetMethod( + "PopulateWithDataAsync", + new[] { typeof(ISuggestedActionCategorySet), typeof(IUIThreadOperationContext) }) + ?? throw new InvalidOperationException("IAsyncLightBulbSession.PopulateWithDataAsync not found."); + + public static async Task> GetCodeActionsAsync( + ILightBulbBroker broker, + IWpfTextView view, + JoinableTaskFactory joinableTaskFactory, + Func showLightBulbAsync, + Func drainLightBulbOperationsAsync, + CancellationToken cancellationToken) + { + var start = DateTime.UtcNow; + var deadline = start + s_timeout; + var attempt = 0; + var lastDetail = "no attempt completed"; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + if (DateTime.UtcNow > deadline) + { + throw new InvalidOperationException( + $"No code actions found after {s_timeout.TotalSeconds:F0}s ({attempt} attempts). Last: {lastDetail}"); + } + + attempt++; + var (sets, detail) = await TryGetFromRealSessionAsync( + broker, view, joinableTaskFactory, showLightBulbAsync, drainLightBulbOperationsAsync, cancellationToken); + lastDetail = $"attempt {attempt}, elapsed {(DateTime.UtcNow - start).TotalSeconds:F1}s: {detail}"; + + if (sets.Count > 0) + { + return sets; + } + + await Task.Delay(250, cancellationToken); + } + } + + private static async Task<(IReadOnlyList sets, string detail)> TryGetFromRealSessionAsync( ILightBulbBroker broker, IWpfTextView view, + JoinableTaskFactory joinableTaskFactory, + Func showLightBulbAsync, + Func drainLightBulbOperationsAsync, CancellationToken cancellationToken) { - var activeSession = broker.GetSession(view); - var asyncSession = (IAsyncLightBulbSession)activeSession; - var tcs = new TaskCompletionSource>(); + await joinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + + // Clean slate then trigger the editor's own lightbulb (Ctrl+.). Dismissing first avoids ShowQuickFixes + // toggling/collapsing an already-expanded session from a previous attempt. + if (broker.IsLightBulbSessionActive(view)) + { + broker.DismissSession(view); + } + + await showLightBulbAsync(); - void Handler(object s, SuggestedActionsUpdatedArgs e) + // Push-model diagnostics (e.g. background unused-opens) can lag, so the session may not appear at once. + var activeDeadline = DateTime.UtcNow + s_activeWait; + while (!broker.IsLightBulbSessionActive(view)) + { + if (DateTime.UtcNow > activeDeadline) + { + return (Array.Empty(), "no active lightbulb session"); + } + + await Task.Delay(100, cancellationToken); + } + + if (broker.GetSession(view) is not IAsyncLightBulbSession session) + { + return (Array.Empty(), "session active but GetSession not IAsyncLightBulbSession"); + } + + var eventTcs = new TaskCompletionSource<(IReadOnlyList sets, QuerySuggestedActionCompletionStatus status)>(); + + void OnUpdated(object sender, SuggestedActionsUpdatedArgs e) { - // ignore these. we care about when the lightbulb items are all completed. if (e.Status == QuerySuggestedActionCompletionStatus.InProgress) { return; } - if (e.Status == QuerySuggestedActionCompletionStatus.Completed || - e.Status == QuerySuggestedActionCompletionStatus.CompletedWithoutData) + eventTcs.TrySetResult((ReadEnumerableProperty(e, "ActionSets"), e.Status)); + } + + void OnDismissed(object sender, EventArgs e) + => eventTcs.TrySetException(new SessionDismissedException()); + + session.SuggestedActionsUpdated += OnUpdated; + session.Dismissed += OnDismissed; + try + { + if (session.IsDismissed) + { + return (Array.Empty(), "session already dismissed"); + } + + // Ensure the session fires SuggestedActionsUpdated at least once with the latest computed data. + string populateStatus; + try { - tcs.SetResult(e.ActionSets); + var populateTask = (Task)s_populateWithDataAsync.Invoke(session, new object?[] { null, null })!; + populateTask.Forget(); + populateStatus = "populate-invoked"; } - else + catch (Exception ex) { - tcs.SetException(new InvalidOperationException($"Light bulb transitioned to non-complete state: {e.Status}")); + populateStatus = $"populate-invoke-failed {ex.GetType().Name}: {ex.Message}"; } - asyncSession.SuggestedActionsUpdated -= Handler; + // Best-effort deterministic drain (no-op if F# work isn't tracked by Roslyn's LightBulb listener). + try + { + await drainLightBulbOperationsAsync(cancellationToken); + } + catch + { + // ignore - the terminal event below is the source of truth + } + + using var perAttempt = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + perAttempt.CancelAfter(s_perAttemptTimeout); + + try + { + var (sets, status) = await eventTcs.Task.WithCancellation(perAttempt.Token); + return (sets, $"{populateStatus}, event status={status}, sets={sets.Count}"); + } + catch (SessionDismissedException) + { + return (Array.Empty(), $"{populateStatus}, real session dismissed"); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return (Array.Empty(), $"{populateStatus}, no terminal event within {s_perAttemptTimeout.TotalSeconds:F0}s"); + } } + finally + { + session.SuggestedActionsUpdated -= OnUpdated; + session.Dismissed -= OnDismissed; + } + } - asyncSession.SuggestedActionsUpdated += Handler; + // Reads an ImmutableArray-typed member through the non-generic IEnumerable + // interface, never naming ImmutableArray (see class comment for why). + private static IReadOnlyList ReadEnumerableProperty(object source, string propertyName) + { + object? value; + try + { + value = source.GetType().GetProperty(propertyName)?.GetValue(source); + } + catch + { + return Array.Empty(); + } - asyncSession.Dismissed += (_, _) => tcs.TrySetCanceled(new CancellationToken(true)); + if (value is not IEnumerable sequence) + { + return Array.Empty(); + } - if (asyncSession.IsDismissed) + var list = new List(); + try + { + foreach (var item in sequence) + { + if (item is SuggestedActionSet set) + { + list.Add(set); + } + } + } + catch { - tcs.TrySetCanceled(new CancellationToken(true)); + // A default(ImmutableArray) throws on enumeration; treat as no data. + return Array.Empty(); } - // Calling PopulateWithDataAsync ensures the underlying session will call SuggestedActionsUpdated at least once - // with the latest data computed. This is needed so that if the lightbulb computation is already complete - // that we hear about the results. - await asyncSession.PopulateWithDataAsync(overrideRequestedActionCategories: null, operationContext: null).ConfigureAwait(false); + return list; + } - return await tcs.Task.WithCancellation(cancellationToken); + private sealed class SessionDismissedException : Exception + { } } } diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs index 2e9009ba9c6..3e2c43e4550 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs @@ -4,19 +4,26 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using FSharp.Editor.IntegrationTests.Extensions; using FSharp.Editor.IntegrationTests.Helpers; +using Microsoft.VisualStudio.ComponentModelHost; using Microsoft.VisualStudio.Language.Intellisense; -using Microsoft.VisualStudio.OLE.Interop; using Microsoft.VisualStudio.Shell.Interop; using Microsoft.VisualStudio.Text; +using Microsoft.VisualStudio.Text.Editor; namespace Microsoft.VisualStudio.Extensibility.Testing; internal partial class EditorInProcess { + // VSStd14CmdID command-set GUID + ShowQuickFixes (Ctrl+.) id; OLECMDEXECOPT_DONTPROMPTUSER = 2. + private static readonly Guid s_vsStd14CmdSet = new Guid("4c7763bf-5faf-4264-a366-b7e1f27ba958"); + private const uint ShowQuickFixesCmdId = 1; + private const uint OleCmdExecOptDontPromptUser = 2; + public async Task GetTextAsync(CancellationToken cancellationToken) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); @@ -73,22 +80,99 @@ public async Task PlaceCaretAsync(string marker, CancellationToken cancellationT view.Selection.Clear(); } + // dte.Find.Execute (used by PlaceCaretAsync) leaves VS's active selection on the Find feature rather + // than the editor, so shell commands dispatched afterwards through SUIHostCommandDispatcher (e.g. + // VSStd97CmdID.GotoDefn) are routed to the wrong command target and come back disabled (E_FAIL). + // Re-activate the document window so the editor is the active command context again. This uses VS's + // internal active-frame selection (IVsMonitorSelection) and does not depend on the OS foreground window. + public async Task ActivateAsync(CancellationToken cancellationToken) + { + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + + var dte = await GetRequiredGlobalServiceAsync(cancellationToken); + dte.ActiveDocument?.Activate(); + } + public async Task> InvokeCodeActionListAsync(CancellationToken cancellationToken) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + var view = await GetActiveTextViewAsync(cancellationToken); + var componentModel = await GetRequiredGlobalServiceAsync(cancellationToken); + var broker = componentModel.GetService(); var shell = await GetRequiredGlobalServiceAsync(cancellationToken); - var cmdGroup = typeof(VSConstants.VSStd14CmdID).GUID; - var cmdExecOpt = OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER; - var cmdID = VSConstants.VSStd14CmdID.ShowQuickFixes; - object? obj = null; - shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)cmdExecOpt, ref obj); + // PlaceCaretAsync leaves the active command target on the Find feature, so re-activate the document or the + // posted ShowQuickFixes command routes to the wrong target. + await ActivateAsync(cancellationToken); - var view = await GetActiveTextViewAsync(cancellationToken); - var broker = await GetComponentModelServiceAsync(cancellationToken); + // Best-effort deterministic wait for the analyzer/diagnostic work that produces the fixes (focus-independent). + await AsyncOperationWaiter.WaitForFeaturesAsync( + componentModel, + new[] { AsyncOperationWaiter.Workspace, AsyncOperationWaiter.SolutionCrawlerLegacy, AsyncOperationWaiter.DiagnosticService }, + cancellationToken); + + // Invoke the editor's real lightbulb (Ctrl+. / VSStd14CmdID.ShowQuickFixes) so we read the editor-owned + // session, which is not superseded the way a broker.CreateSession session is. Caller is on the main thread. + Task ShowLightBulbAsync() + { + var cmdGroup = s_vsStd14CmdSet; + object? inArg = null; + shell.PostExecCommand(ref cmdGroup, ShowQuickFixesCmdId, OleCmdExecOptDontPromptUser, ref inArg); + return Task.CompletedTask; + } + + Task DrainLightBulbAsync(CancellationToken token) + => AsyncOperationWaiter.WaitForFeaturesAsync(componentModel, new[] { AsyncOperationWaiter.LightBulb }, token); + + try + { + return await LightBulbHelper.GetCodeActionsAsync(broker, view, JoinableTaskFactory, ShowLightBulbAsync, DrainLightBulbAsync, cancellationToken); + } + catch (InvalidOperationException ex) + { + // Report the error-list contents, tracking state, and a producer-agnostic probe of what the broker + // sees at the caret (distinguishes "fix not offered" from "fix offered but no session"). + var entries = await TestServices.ErrorList.GetAllEntriesAsync(cancellationToken); + var categoryRegistry = componentModel.GetService(); + var probe = await ProbeAvailableActionsAsync(broker, view, categoryRegistry.Any, cancellationToken); + throw new InvalidOperationException( + $"{ex.Message}{Environment.NewLine}trackingEnabled={AsyncOperationWaiter.IsTrackingEnabled()}{Environment.NewLine}" + + $"probe: {probe}{Environment.NewLine}" + + $"--- Error List ({entries.Length} entries) ---{Environment.NewLine}" + + string.Join(Environment.NewLine, entries), + ex); + } + } + + // Asks the lightbulb broker (producer-agnostic) whether any suggested actions exist at the caret and which + // categories, without creating a session - so a failure tells us if the fix was simply never offered. + private async Task ProbeAvailableActionsAsync(ILightBulbBroker broker, IWpfTextView view, ISuggestedActionCategorySet requested, CancellationToken cancellationToken) + { + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + + var result = ""; + try + { + var has = await broker.HasSuggestedActionsAsync(requested, view, cancellationToken); + result += $"HasSuggestedActions={has}"; + } + catch (Exception ex) + { + result += $"HasSuggestedActions threw {ex.GetType().Name}: {ex.Message}"; + } + + try + { + var categories = await ((ILightBulbBroker2)broker).GetSuggestedActionCategoriesAsync(requested, view, cancellationToken); + var names = categories is null ? Array.Empty() : ((IEnumerable)categories).ToArray(); + result += $"; categories=[{string.Join(",", names)}]"; + } + catch (Exception ex) + { + result += $"; categories threw {ex.GetType().Name}: {ex.Message}"; + } - var lightbulbs = await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken); - return lightbulbs; + return result; } } diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ErrorListInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ErrorListInProcess.cs index ca81bfbd8a1..43d121d86fd 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ErrorListInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ErrorListInProcess.cs @@ -95,6 +95,32 @@ public async Task GetErrorCountAsync(__VSERRORCATEGORY minimumSeverity, Can return errorItems.Count(e => e.GetCategory() <= minimumSeverity); } + // Diagnostic instrumentation: dumps every error-list entry (all sources and severities) so a failing + // test can report whether a given diagnostic (e.g. unused-opens) was actually published headless. + public async Task> GetAllEntriesAsync(CancellationToken cancellationToken) + { + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + + var errorItems = await GetErrorItemsAsync(cancellationToken); + var list = new List(); + + foreach (var item in errorItems) + { + var source = item.TryGetValue(StandardTableKeyNames.ErrorSource, out ErrorSource errorSource) + ? errorSource.ToString() + : ""; + var tool = item.GetBuildTool(); + var document = Path.GetFileName(item.GetPath() ?? item.GetDocumentName()) ?? ""; + var line = item.GetLine() ?? -1; + var code = item.GetErrorCode() ?? ""; + var text = item.GetText() ?? ""; + + list.Add($"[{item.GetCategory()}] source={source} tool={tool} {document}({line + 1}) {code}: {text}"); + } + + return list.ToImmutableArray(); + } + private async Task> GetErrorItemsAsync(CancellationToken cancellationToken) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs index 2dac220006f..d123851d366 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs @@ -84,10 +84,12 @@ public async Task WaitForBreakpointHitAsync(TimeSpan timeout, bool continueOnSte dbg.Go(true); if (seenRun && mode == EnvDTE.dbgDebugMode.dbgDesignMode) - throw new InvalidOperationException("Debug session ended before breakpoint hit."); + throw new InvalidOperationException( + $"Debug session ended before breakpoint hit. Breakpoints={BreakpointSummary(dbg)}."); if (sw.Elapsed > timeout) - throw new TimeoutException($"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}."); + throw new TimeoutException( + $"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}."); await Task.Delay(100, cancellationToken); } @@ -122,7 +124,11 @@ private static string BreakpointSummary(EnvDTE.Debugger dbg) { var children = 0; foreach (EnvDTE.Breakpoint _ in bp.Children) children++; - items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children})"); + // CurrentHits + Children counts together let us distinguish unbound (children=0, hits=0) + // from "bound but never executed" (children>0, hits=0) from "bound and hit" (hits>0). + // Unbound means the PDB has no IL location for this source line; the disk-source hash + // didn't match the PDB's recorded hash; or the loaded module doesn't have this code. + items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children},hits={bp.CurrentHits})"); } return items.Count == 0 ? "none" : string.Join(",", items); } diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs index 43743484e96..0ef32c4ae0a 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs @@ -4,9 +4,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.OperationProgress; @@ -50,23 +51,58 @@ public async Task CreateSingleProjectSolutionAsync(string name, string template, await AddProjectAsync(name, template, cancellationToken); } - // Repo root from compile-time source path — no runtime resolution needed. - private static readonly string RepoRoot = Path.GetFullPath(Path.Combine( - Path.GetDirectoryName(GetSourceFilePath())!, "..", "..", "..", "..")); + // RepoRoot, LocalCompilerConfiguration: derived at runtime from the test assembly's location, + // NOT from [CallerFilePath]. Arcade builds with deterministic source-root mapping that rewrites + // CallerFilePath to "/_/..." (for symbol-server reproducibility); using it at runtime gives + // "D:\_" on Windows CI agents, which then breaks Process.Start(WorkingDirectory) and fsc.dll + // path resolution. Assembly.Location IS the real post-build path on disk in all environments. + // + // Layout: /artifacts/bin/FSharp.Editor.IntegrationTests///FSharp.Editor.IntegrationTests.dll + private static readonly string AssemblyDir = + Path.GetDirectoryName(typeof(SolutionExplorerInProcess).Assembly.Location)!; + + private static readonly string LocalCompilerConfiguration = + new DirectoryInfo(AssemblyDir).Parent!.Name; + + private static readonly string RepoRoot = Path.GetFullPath( + Path.Combine(AssemblyDir, "..", "..", "..", "..", "..")); + + // Repo-pinned dotnet host installed by Arcade. Falls back to PATH lookup if not present + // (developer scenarios that build the integration project outside the repo's eng infra). + private static readonly string DotnetExe = + File.Exists(Path.Combine(RepoRoot, ".dotnet", "dotnet.exe")) + ? Path.Combine(RepoRoot, ".dotnet", "dotnet.exe") + : "dotnet"; private static string CreateStandaloneProjectFile() { var propsPath = Path.Combine(RepoRoot, "UseLocalCompiler.Directory.Build.props"); + // Sanity-check the inferred configuration: matching fsc must exist before VS tries to build. + // Validated here (not in the static initializer) so tests that don't use the standalone path + // can still load this type when fsc hasn't been built locally. + var fscRoot = Path.Combine(RepoRoot, "artifacts", "bin", "fsc", LocalCompilerConfiguration); + if (!Directory.Exists(fscRoot)) + { + throw new InvalidOperationException( + $"Inferred LocalCompilerConfiguration='{LocalCompilerConfiguration}' but no built fsc found at '{fscRoot}'. " + + $"The synthesized standalone fsproj would fail to load fsc.dll -- build the F# compiler in this configuration first."); + } + return $@" - Debug + {LocalCompilerConfiguration} {RepoRoot} Exe - net8.0 + + net10.0 @@ -74,9 +110,6 @@ private static string CreateStandaloneProjectFile() "; } - private static string GetSourceFilePath([CallerFilePath] string path = "") - => path; - public async Task CreateSolutionAsync(string solutionName, CancellationToken cancellationToken) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); @@ -115,22 +148,108 @@ private async Task GetDirectoryNameAsync(CancellationToken cancellationT public async Task AddProjectAsync(string projectName, string projectTemplate, CancellationToken cancellationToken) { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + // dotnet new must run off the UI thread (it's a synchronous shell-out). + var solutionDir = await GetDirectoryNameAsync(cancellationToken); + var projectDir = Path.Combine(solutionDir, projectName); - var projectPath = Path.Combine(await GetDirectoryNameAsync(cancellationToken), projectName); - var projectTemplatePath = await GetProjectTemplatePathAsync(projectTemplate, cancellationToken); - var solution = await GetRequiredGlobalServiceAsync(cancellationToken); - ErrorHandler.ThrowOnFailure(solution.AddNewProjectFromTemplate(projectTemplatePath, null, null, projectPath, projectName, null, out _)); - } + await TaskScheduler.Default; + await RunDotnetNewAsync(projectTemplate, projectName, projectDir, cancellationToken); - private async Task GetProjectTemplatePathAsync(string projectTemplate, CancellationToken cancellationToken) - { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + var projectFilePath = Path.Combine(projectDir, $"{projectName}.fsproj"); + if (!File.Exists(projectFilePath)) + { + throw new InvalidOperationException( + $"'dotnet new {projectTemplate}' completed but did not produce '{projectFilePath}'."); + } + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); var dte = await GetRequiredGlobalServiceAsync(cancellationToken); var solution = (EnvDTE80.Solution2)dte.Solution; + _ = solution.AddFromFile(projectFilePath, false); + + // Auto-open the project's main .fs file. The previous AddNewProjectFromTemplate path opened + // the file marked OpenInEditor="true" in the .vstemplate; tests rely on this implicit open + // (e.g. CodeActionTests calls Editor.SetTextAsync immediately afterwards). + // SDK templates currently produce exactly one .fs file per project (Library.fs / Program.fs / Tests.fs). + // If that ever changes (e.g. xunit template growing a Program.fs), fail loudly rather than + // silently opening the wrong file and producing confusing content-diff failures downstream. + var fsFiles = Directory.EnumerateFiles(projectDir, "*.fs", SearchOption.TopDirectoryOnly) + .OrderBy(p => p, StringComparer.OrdinalIgnoreCase) + .ToList(); + if (fsFiles.Count != 1) + { + throw new InvalidOperationException( + $"Expected exactly one *.fs file in '{projectDir}' produced by 'dotnet new {projectTemplate}', " + + $"found {fsFiles.Count}: [{string.Join(", ", fsFiles.Select(Path.GetFileName))}]. " + + $"Update AddProjectAsync's auto-open logic to pick the correct main file for this template."); + } + await OpenFileAsync(projectName, Path.GetFileName(fsFiles[0]), cancellationToken); + } - return solution.GetProjectTemplate(projectTemplate, "FSharp"); + // Shells out to `dotnet new