Increase test reliability

packages/cursorless-engine/src/api/CursorlessEngineApi.ts

@@ -23,6 +23,7 @@ export interface CursorlessEngine {
   storedTargets: StoredTargetMap;
   hatTokenMap: HatTokenMap;
   injectIde: (ide: IDE) => void;
+  loadLanguage(languageId: string): Promise<void>;
   addCommandRunnerDecorator: (
     commandRunnerDecorator: CommandRunnerDecorator,
   ) => void;

packages/cursorless-engine/src/cursorlessEngine.ts

@@ -154,6 +154,7 @@ export async function createCursorlessEngine({
     storedTargets,
     hatTokenMap,
     injectIde: (ide) => injectedIde.setIde(ide),
+    loadLanguage: (languageId) => languageDefinitions.loadLanguage(languageId),
     addCommandRunnerDecorator: (decorator: CommandRunnerDecorator) => {
       commandRunnerDecorators.push(decorator);
     },

packages/cursorless-engine/src/languages/LanguageDefinitions.ts

@@ -122,15 +122,14 @@ export class LanguageDefinitionsImpl implements LanguageDefinitions {
       return;
     }
 
-    const definition =
-      (await LanguageDefinition.create(
-        this.ide,
-        this.treeSitterQueryProvider,
-        this.treeSitter,
-        languageId,
-      )) ?? LANGUAGE_UNDEFINED;
-
-    this.languageDefinitions.set(languageId, definition);
+    const definition = await LanguageDefinition.create(
+      this.ide,
+      this.treeSitterQueryProvider,
+      this.treeSitter,
+      languageId,
+    );
+
+    this.languageDefinitions.set(languageId, definition ?? LANGUAGE_UNDEFINED);
   }
 
   private async reloadLanguageDefinitions(): Promise<void> {

packages/cursorless-vscode-e2e/src/endToEndTestSetup.ts

@@ -1,6 +1,9 @@
-import type { IDE } from "@cursorless/common";
+import type { IDE, NormalizedIDE } from "@cursorless/common";
 import { shouldUpdateFixtures, sleep, SpyIDE } from "@cursorless/common";
-import { getCursorlessApi } from "@cursorless/vscode-common";
+import {
+  getCursorlessApi,
+  resetReusableEditor,
+} from "@cursorless/vscode-common";
 import type { Context } from "mocha";
 import * as sinon from "sinon";
 
@@ -29,7 +32,7 @@ export function endToEndTestSetup(
   suite.timeout(timeout);
   suite.retries(retries);
 
-  let originalIde: IDE;
+  let originalIde: NormalizedIDE;
   let injectIde: (ide: IDE) => void;
   let spyIde: SpyIDE | undefined;
 
@@ -50,6 +53,10 @@ export function endToEndTestSetup(
     injectIde(originalIde);
   });
 
+  suiteTeardown(() => {
+    resetReusableEditor();
+  });
+
   return {
     getSpy() {
       if (spyIde == null) {

packages/cursorless-vscode-e2e/src/suite/intraCellSetSelection.vscode.test.ts

@@ -1,5 +1,6 @@
-import { LATEST_VERSION } from "@cursorless/common";
+import { LATEST_VERSION, splitKey } from "@cursorless/common";
 import {
+  getCellIndex,
   getCursorlessApi,
   openNewNotebookEditor,
   runCursorlessCommand,
@@ -16,11 +17,22 @@ suite("Within cell set selection", async function () {
 });
 
 async function runTest() {
-  const { hatTokenMap } = (await getCursorlessApi()).testHelpers!;
+  const { hatTokenMap, toVscodeEditor } = (await getCursorlessApi())
+    .testHelpers!;
 
-  await openNewNotebookEditor(['"hello world"']);
+  const notebook = await openNewNotebookEditor(['"hello world"']);
 
   await hatTokenMap.allocateHats();
+  const hatMap = await hatTokenMap.getReadableMap(false);
+  const targetHat = hatMap.getEntries().find(([, token]) => {
+    const editor = toVscodeEditor(token.editor);
+    return (
+      getCellIndex(notebook, editor.document) === 0 && token.text === "world"
+    );
+  });
+
+  assert(targetHat != null, 'Expected a default hat for "world" in the cell');
+  const { hatStyle, character } = splitKey(targetHat[0]);
 
   await runCursorlessCommand({
     version: LATEST_VERSION,
@@ -31,8 +43,8 @@ async function runTest() {
         type: "primitive",
         mark: {
           type: "decoratedSymbol",
-          symbolColor: "default",
-          character: "r",
+          symbolColor: hatStyle,
+          character,
         },
       },
     },

packages/cursorless-vscode-e2e/src/suite/scopeProvider/assertCalledWithScopeInfo.ts

@@ -1,18 +1,27 @@
-import type { ScopeType, ScopeTypeInfo } from "@cursorless/common";
-import * as sinon from "sinon";
+import { type ScopeType, type ScopeTypeInfo } from "@cursorless/common";
 import { assert } from "chai";
-import { sleepWithBackoff } from "../../endToEndTestSetup";
 import { isEqual } from "lodash-es";
+import type { SinonSpy } from "sinon";
+import { waitFor } from "../waitFor";
 
 export async function assertCalledWithScopeInfo<T extends ScopeTypeInfo>(
-  fake: sinon.SinonSpy<[scopeInfos: T[]], void>,
+  fake: SinonSpy<[scopeInfos: T[]], void>,
   ...expectedScopeInfos: T[]
 ) {
-  await sleepWithBackoff(100);
-  sinon.assert.called(fake);
+  const scopeInfos = await waitForScopeInfos(fake, (scopeInfos) =>
+    expectedScopeInfos.every((expectedScopeInfo) => {
+      const actualScopeInfo = scopeInfos.find((scopeInfo) =>
+        isEqual(scopeInfo.scopeType, expectedScopeInfo.scopeType),
+      );
+
+      return (
+        actualScopeInfo != null && isEqual(actualScopeInfo, expectedScopeInfo)
+      );
+    }),
+  );
 
   for (const expectedScopeInfo of expectedScopeInfos) {
-    const actualScopeInfo = fake.lastCall.args[0].find((scopeInfo) =>
+    const actualScopeInfo = scopeInfos.find((scopeInfo) =>
       isEqual(scopeInfo.scopeType, expectedScopeInfo.scopeType),
     );
     assert.isDefined(actualScopeInfo);
@@ -23,19 +32,36 @@ export async function assertCalledWithScopeInfo<T extends ScopeTypeInfo>(
 }
 
 export async function assertCalledWithoutScopeInfo<T extends ScopeTypeInfo>(
-  fake: sinon.SinonSpy<[scopeInfos: T[]], void>,
+  fake: SinonSpy<[scopeInfos: T[]], void>,
   ...scopeTypes: ScopeType[]
 ) {
-  await sleepWithBackoff(100);
-  sinon.assert.called(fake);
+  const scopeInfos = await waitForScopeInfos(fake, (scopeInfos) =>
+    scopeTypes.every(
+      (scopeType) =>
+        scopeInfos.find((scopeInfo) =>
+          isEqual(scopeInfo.scopeType, scopeType),
+        ) == null,
+    ),
+  );
 
   for (const scopeType of scopeTypes) {
     assert.isUndefined(
-      fake.lastCall.args[0].find((scopeInfo) =>
-        isEqual(scopeInfo.scopeType, scopeType),
-      ),
+      scopeInfos.find((scopeInfo) => isEqual(scopeInfo.scopeType, scopeType)),
     );
   }
 
   fake.resetHistory();
 }
+
+async function waitForScopeInfos<T extends ScopeTypeInfo>(
+  fake: SinonSpy<[scopeInfos: T[]], void>,
+  predicate: (scopeInfos: T[]) => boolean,
+): Promise<T[]> {
+  const success = await waitFor(
+    () => fake.called && predicate(fake.lastCall.args[0]),
+  );
+  if (success) {
+    return fake.lastCall.args[0];
+  }
+  assert.fail("Timed out waiting for expected scope info");
+}

packages/cursorless-vscode-e2e/src/suite/scopeProvider/runCustomRegexScopeInfoTest.ts

@@ -33,10 +33,6 @@ export async function runCustomRegexScopeInfoTest() {
     await assertCalledWithScopeInfo(fake, unsupported);
 
     await openNewEditor(contents);
-    // The scope provider relies on the open document event (among others) to
-    // update available scopes. Add a short sleep here to give it time to
-    // trigger.
-    await sleep(100);
     await assertCalledWithScopeInfo(fake, present);
 
     await unlink(cursorlessTalonStateJsonPath);

packages/cursorless-vscode-e2e/src/suite/scopeVisualizer/scopeVisualizer.vscode.test.ts

@@ -1,14 +1,20 @@
+import { asyncSafety } from "@cursorless/common";
+import { getCursorlessApi } from "@cursorless/vscode-common";
 import { commands } from "vscode";
 import { endToEndTestSetup } from "../../endToEndTestSetup";
 import { runBasicMultilineContentTest } from "./runBasicMultilineContentTest";
 import { runBasicRemovalTest } from "./runBasicRemovalTest";
 import { runNestedMultilineContentTest } from "./runNestedMultilineContentTest";
 import { runUpdateTest } from "./runUpdateTest";
-import { asyncSafety } from "@cursorless/common";
 
-suite("scope visualizer", async function () {
+suite("scope visualizer", function () {
   endToEndTestSetup(this);
 
+  suiteSetup(async () => {
+    const { ide } = (await getCursorlessApi()).testHelpers!;
+    ide.configuration.mockConfiguration("decorationDebounceDelayMs", 0);
+  });
+
   teardown(() => commands.executeCommand("cursorless.hideScopeVisualizer"));
 
   test(

packages/cursorless-vscode-e2e/src/suite/tutorial/tutorial.vscode.test.ts

@@ -3,19 +3,20 @@ import {
   LATEST_VERSION,
   asyncSafety,
   getSnapshotForComparison,
-  sleep,
 } from "@cursorless/common";
 import { getRecordedTestsDirPath, loadFixture } from "@cursorless/node-common";
 import {
   getCursorlessApi,
   runCursorlessCommand,
+  type SpyWebViewEvent,
 } from "@cursorless/vscode-common";
+import { isEqual } from "lodash-es";
 import assert from "node:assert";
 import path from "path";
 import sinon from "sinon";
 import { commands } from "vscode";
-import { endToEndTestSetup, sleepWithBackoff } from "../../endToEndTestSetup";
-import { isEqual } from "lodash-es";
+import { endToEndTestSetup } from "../../endToEndTestSetup";
+import { waitFor } from "../waitFor";
 
 suite("tutorial", async function () {
   const { getSpy } = endToEndTestSetup(this);
@@ -73,56 +74,42 @@ async function runBasicTutorialTest(spyIde: SpyIDE) {
   );
   await checkStepSetup(fixtures[0]);
 
-  // Allow for debounce
-  await sleep(100);
-
-  // Another sleep just in case
-  await sleepWithBackoff(50);
-
   // We allow duplicate messages because they're idempotent. Not sure why some
   // platforms get the init message twice but it doesn't matter.
-  const result = getTutorialWebviewEventLog();
+
   // This is the initial message that the webview sends to the extension.
   // Seeing this means that the javascript in the webview successfully loaded.
-  assert(
-    result.some((e) =>
-      isEqual(e, {
-        type: "messageReceived",
-        data: {
-          type: "getInitialState",
-        },
-      }),
-    ),
+  await waitForEvent(
+    getTutorialWebviewEventLog,
+    (e) => e.type === "messageReceived" && e.data.type === "getInitialState",
   );
 
   // This is the response from the extension to the webview's initial message.
-  assert(
-    result.some((e) =>
-      isEqual(e, {
-        type: "messageSent",
-        data: {
-          type: "doingTutorial",
-          hasErrors: false,
-          id: "tutorial-1-basics",
-          stepNumber: 0,
-          stepContent: [
-            [
-              {
-                type: "string",
-                value: "Say ",
-              },
-              {
-                type: "command",
-                value: "take cap",
-              },
-            ],
+  await waitForEvent(getTutorialWebviewEventLog, (e) =>
+    isEqual(e, {
+      type: "messageSent",
+      data: {
+        type: "doingTutorial",
+        hasErrors: false,
+        id: "tutorial-1-basics",
+        stepNumber: 0,
+        stepContent: [
+          [
+            {
+              type: "string",
+              value: "Say ",
+            },
+            {
+              type: "command",
+              value: "take cap",
+            },
           ],
-          stepCount: 11,
-          title: "Introduction",
-          preConditionsMet: true,
-        },
-      }),
-    ),
+        ],
+        stepCount: 11,
+        title: "Introduction",
+        preConditionsMet: true,
+      },
+    }),
   );
 
   // Check that we focus the tutorial webview when the user starts the tutorial
@@ -149,18 +136,10 @@ async function runBasicTutorialTest(spyIde: SpyIDE) {
     usePrePhraseSnapshot: false,
   });
 
-  // Allow for debounce
-  await sleep(100);
-
-  // Another sleep just in case
-  await sleepWithBackoff(50);
-
-  assert(
-    getTutorialWebviewEventLog().some(
-      (message) =>
-        message.type === "messageSent" &&
-        message.data.preConditionsMet === false,
-    ),
+  await waitForEvent(
+    getTutorialWebviewEventLog,
+    (event) =>
+      event.type === "messageSent" && event.data.preConditionsMet === false,
   );
 
   // Test resuming tutorial
@@ -234,3 +213,15 @@ const runNoOpCursorlessCommand = () =>
     },
     usePrePhraseSnapshot: false,
   });
+
+async function waitForEvent(
+  getTutorialWebviewEventLog: () => SpyWebViewEvent[],
+  predicate: (event: SpyWebViewEvent) => boolean,
+) {
+  const success = await waitFor(() =>
+    getTutorialWebviewEventLog().some(predicate),
+  );
+  if (!success) {
+    assert.fail("Timed out waiting for tutorial event log");
+  }
+}

packages/cursorless-vscode-e2e/src/suite/waitFor.ts

@@ -0,0 +1,23 @@
+import { sleepWithBackoff } from "../endToEndTestSetup";
+
+type Predicate = () => boolean | Promise<boolean>;
+
+/**
+ * Waits for a predicate to become true, checking periodically with an
+ * increasing delay between checks. Returns true if the predicate becomes true
+ * within the given number of iterations, and false otherwise.
+ */
+export async function waitFor(
+  predicate: Predicate,
+  iterations = 20,
+): Promise<boolean> {
+  for (let i = 0; i < iterations; i++) {
+    if (await Promise.resolve(predicate())) {
+      return true;
+    }
+
+    await sleepWithBackoff(25);
+  }
+
+  return false;
+}