From 1455168545442f28138b5869eb8ae30de9020c79 Mon Sep 17 00:00:00 2001 From: Charles Ouimet Date: Mon, 25 May 2026 14:14:07 -0400 Subject: [PATCH 1/2] [issues/558] Automate 3 assisted tests with `CMD_BIND_TO_CUSTOM_AI_BY_ID` + `dummyAi.getText()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Three [assisted] integration tests that required human setup (picker bind, keybinding press, PASS/FAIL verdict) are now fully automated using `CMD_BIND_TO_CUSTOM_AI_BY_ID` for programmatic binding, `CMD_COPY_LINK_RELATIVE` for programmatic send, and `dummyAi.getText()` for outcome verification. `clipboardPreservation.test.ts` now has zero `[assisted]` tests. ## Changes - `core-send-commands-r-l-003`: replaced human-in-the-loop with programmatic bind → send → `dummyAi.getText()` assertion. Removed `waitForHumanVerdict` import. - `clipboard-preservation-004`: same pattern. Programmatic bind to `rangelink.dummy-ai-extension`, programmatic send, `dummyAi.getText()` outcome check. - `clipboard-preservation-010`: same pattern. Bind to `rangelink.dummy-ai-extension-focus-fail`, programmatic send, log-based assertions for focus-failure path. - All three tests now assert exact padded values (` ${link} `) via strict equality — no `.trim()` in any assertion. - Removed `waitForHuman` and `CLIPBOARD_SENTINEL` from clipboardPreservation imports. `CLIPBOARD_SENTINEL` was only referenced in `waitForHuman` console steps (shown to the human driver); the sentinel mechanics still work through `writeClipboardSentinel`, `assertClipboardRestored`, and `assertClipboardChanged` which use the constant internally. - `assistedTestHelper.ts`: fixed `waitForHumanVerdict` overlap bug. When Mocha times out a verdict-blocked test and starts the next one, the new invocation now disposes the previous test's stale status bar items AND dismisses its `withProgress` notification toast before creating its own. Root cause: Mocha timeout fires on a `waitForHumanVerdict`-blocked test, Mocha fails it and starts the next test, but `settleWith` never ran so the first test's PASS/FAIL buttons and progress toast persisted. Fix: module-level `activeVerdictDisposables` + `activeVerdictReject` track the current invocation's state; a new call disposes stale items and rejects the prior promise to dismiss the toast. - `.vscode-test.mjs`: added `RANGELINK_MOCHA_TIMEOUT` env var override for debugging timeout behavior without code changes (defaults to `ASSISTED_TIMEOUT_MS` when unset). - QA YAML: all three TCs moved from `automated: assisted` to `automated: true`. Automated count: 156 → 159, assisted count: 112 → 108 (one other TC changed independently). ## Test Plan - [x] All 1979 unit tests pass - [x] `core-send-commands-r-l-003` passes fully automated (1.7s) - [x] `clipboard-preservation-004` passes fully automated (1.6s) - [x] `clipboard-preservation-010` passes fully automated (1.5s) - [x] QA coverage validator passes (159 automated, 108 assisted) - [x] Zero `.trim()` calls remain in integration test assertions ## Related - Closes https://github.com/couimet/rangeLink/issues/558 --- .../.vscode-test.mjs | 6 +- .../qa/qa-test-cases-v1.1.0.yaml | 7 +-- .../helpers/assistedTestHelper.ts | 26 ++++++++- .../suite/clipboardPreservation.test.ts | 58 ++++++++----------- .../suite/coreSendCommands.test.ts | 29 +++++----- 5 files changed, 72 insertions(+), 54 deletions(-) diff --git a/packages/rangelink-vscode-extension/.vscode-test.mjs b/packages/rangelink-vscode-extension/.vscode-test.mjs index faaf9f7e..82ab93d0 100644 --- a/packages/rangelink-vscode-extension/.vscode-test.mjs +++ b/packages/rangelink-vscode-extension/.vscode-test.mjs @@ -2,9 +2,13 @@ import { defineConfig } from '@vscode/test-cli'; import { ASSISTED_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs'; +const MOCHA_TIMEOUT = process.env.RANGELINK_MOCHA_TIMEOUT + ? Number(process.env.RANGELINK_MOCHA_TIMEOUT) + : ASSISTED_TIMEOUT_MS; + export default defineConfig([ { ...BASE_CONFIG, - mocha: { timeout: ASSISTED_TIMEOUT_MS, ...BASE_CONFIG.mocha }, + mocha: { timeout: MOCHA_TIMEOUT, ...BASE_CONFIG.mocha }, }, ]); diff --git a/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml b/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml index a0bee74a..d3a689b5 100644 --- a/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml +++ b/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml @@ -389,7 +389,7 @@ test_cases: feature: 'Clipboard Preservation' scenario: 'always mode: clipboard content before AI assistant paste is restored after' expected_result: 'Dummy AI tier1 contains the exact link (relPath#L1-L3). Clipboard contains the sentinel value. assertClipboardRestored passes — preserve=always silently restored the clipboard after AI delivery.' - automated: assisted + automated: true - id: clipboard-preservation-005 labels: @@ -437,8 +437,7 @@ test_cases: feature: 'Clipboard Preservation' scenario: 'AI assistant auto-paste fails (focus command throws): RangeLink stays in clipboard for manual paste' expected_result: 'Clipboard contains the generated RangeLink (not CLIPBOARD_SENTINEL). isClipboardRestorationApplicable returns false because getUserInstruction(Failure) is defined. The "Paste manually" warning toast was shown.' - command_to_run: 'pnpm test:release:grep "clipboard-preservation-010"' - automated: assisted + automated: true - id: clipboard-preservation-011 labels: @@ -1372,7 +1371,7 @@ test_cases: feature: 'Core Send Commands — R-L' scenario: 'R-L sends RangeLink to bound AI assistant destination' expected_result: 'RangeLink appears in AI chat input. Success toast confirms send.' - automated: assisted + automated: true - id: core-send-commands-r-c-001 labels: diff --git a/packages/rangelink-vscode-extension/src/__integration-tests__/helpers/assistedTestHelper.ts b/packages/rangelink-vscode-extension/src/__integration-tests__/helpers/assistedTestHelper.ts index 50a7ee38..3cbf9b3d 100644 --- a/packages/rangelink-vscode-extension/src/__integration-tests__/helpers/assistedTestHelper.ts +++ b/packages/rangelink-vscode-extension/src/__integration-tests__/helpers/assistedTestHelper.ts @@ -64,6 +64,12 @@ const VERDICT_FAIL_COMMAND_PREFIX = 'rangelink._test.verdict.fail'; // patterns would otherwise throw `command '...' already exists`. let verdictInvocationCounter = 0; +// Tracks the currently-active verdict state so a new invocation can clean up +// stale UI from a previous invocation that never settled (e.g., Mocha timed +// out the test before the human clicked a button). +let activeVerdictDisposables: vscode.Disposable[] | undefined; +let activeVerdictReject: ((reason: unknown) => void) | undefined; + /** * Pauses the test until the human clicks Pass or Fail in the status bar. * @@ -113,6 +119,19 @@ export const waitForHumanVerdict = async ( nodeConsole.log('Click the PASS or FAIL button in the status bar (bottom-left) when done.'); nodeConsole.log(SECTION_LINE); + // Dispose any stale UI from a previous invocation that never settled + // (e.g., Mocha timed out the test before the human clicked a button). + if (activeVerdictDisposables !== undefined) { + for (const d of activeVerdictDisposables) { + d.dispose(); + } + activeVerdictDisposables = undefined; + } + if (activeVerdictReject !== undefined) { + activeVerdictReject(new Error('Superseded by a new waitForHumanVerdict invocation')); + activeVerdictReject = undefined; + } + const invocationId = ++verdictInvocationCounter; const passCommand = `${VERDICT_PASS_COMMAND_PREFIX}.${invocationId}`; const failCommand = `${VERDICT_FAIL_COMMAND_PREFIX}.${invocationId}`; @@ -126,13 +145,16 @@ export const waitForHumanVerdict = async ( (progress) => { progress.report({ message: 'Click PASS or FAIL in the bottom-left status bar' }); - return new Promise((resolve) => { + return new Promise((resolve, reject) => { + activeVerdictReject = reject; const disposables: vscode.Disposable[] = []; let settled = false; const settleWith = (verdict: HumanVerdict): void => { if (settled) return; settled = true; + activeVerdictDisposables = undefined; + activeVerdictReject = undefined; for (const d of disposables) { d.dispose(); } @@ -159,6 +181,8 @@ export const waitForHumanVerdict = async ( failItem.color = new vscode.ThemeColor('testing.iconFailed'); failItem.show(); disposables.push(failItem); + + activeVerdictDisposables = disposables; }); }, ); diff --git a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/clipboardPreservation.test.ts b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/clipboardPreservation.test.ts index d6ee10a0..d3baeb31 100644 --- a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/clipboardPreservation.test.ts +++ b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/clipboardPreservation.test.ts @@ -3,6 +3,7 @@ import assert from 'node:assert'; import * as vscode from 'vscode'; import { + CMD_BIND_TO_CUSTOM_AI_BY_ID, CMD_BIND_TO_TEXT_EDITOR_HERE, CMD_COPY_LINK_RELATIVE, CMD_PASTE_CURRENT_FILE_PATH_RELATIVE, @@ -20,8 +21,6 @@ import { openAndDismiss, standardSuite, TERMINAL_READY_MS, - waitForHuman, - CLIPBOARD_SENTINEL, writeClipboardSentinel, } from '../helpers'; import type { CapturingTerminal } from '../helpers/capturingPtyHelpers'; @@ -206,7 +205,7 @@ standardSuite('Clipboard Preservation — Assisted', (ss) => { ss.log('✓ Clipboard restored to sentinel and phrase landed in destination file after R-V'); }); - test('[assisted] clipboard-preservation-004: always mode — AI assistant paste restores clipboard', async () => { + test('clipboard-preservation-004: always mode — AI assistant paste restores clipboard', async () => { const { uri: fileUri } = ss.createContentFile('cbp-004', 10, (i) => `line ${i + 1} content`); const relPath = vscode.workspace.asRelativePath(fileUri); @@ -218,30 +217,26 @@ standardSuite('Clipboard Preservation — Assisted', (ss) => { editor.selection = new vscode.Selection(0, 0, 2, 6); await ss.settle(); + await vscode.commands.executeCommand(CMD_BIND_TO_CUSTOM_AI_BY_ID, { + extensionId: 'rangelink.dummy-ai-extension', + }); + await ss.settle(); + const logCapture = getLogCapture(); logCapture.mark('before-004'); - await waitForHuman( - 'clipboard-preservation-004', - `Press Cmd+R Cmd+D → bind "Dummy AI (Tier 1)", click back into the editor, press Cmd+R Cmd+L`, - [ - '1. Press Cmd+R Cmd+D → select "Dummy AI (Tier 1)" from the picker', - '2. Click back into the editor (lines 1-3 are pre-selected)', - '3. Press Cmd+R Cmd+L — the link should appear in Dummy AI Tier 1 textarea', - '4. Press Cancel to continue (assertions happen automatically)', - ], - ); + await vscode.commands.executeCommand(CMD_COPY_LINK_RELATIVE); await ss.settle(); + assertClipboardPreservationRan(logCapture, 'before-004', 'R-L'); const dummyText = (await vscode.commands.executeCommand('dummyAi.getText')) as { tier1: string; tier2: string; }; - // trim() strips smart-padding spaces (pasteLink='both' adds leading/trailing space) assert.strictEqual( - dummyText.tier1.trim(), - expectedLink, - `Expected Dummy AI tier1="${expectedLink}", got: ${JSON.stringify(dummyText.tier1)}`, + dummyText.tier1, + ` ${expectedLink} `, + `Expected Dummy AI tier1=" ${expectedLink} ", got: ${JSON.stringify(dummyText.tier1)}`, ); await assertClipboardRestored('clipboard-preservation-004: always + AI paste'); ss.log('✓ Clipboard restored to sentinel and link landed in Dummy AI after R-L'); @@ -338,29 +333,23 @@ standardSuite('Clipboard Preservation — Assisted', (ss) => { ss.log('✓ Clipboard unchanged after picker dismissed (no operation performed)'); }); - test('[assisted] clipboard-preservation-010: focus command failure preserves link in clipboard for manual paste', async () => { + test('clipboard-preservation-010: focus command failure preserves link in clipboard for manual paste', async () => { const { uri: fileUri } = ss.createContentFile('cbp-010', 5, (i) => `line ${i + 1}`); - const relPath = vscode.workspace.asRelativePath(fileUri); - await ss.openEditor(fileUri); + const editor = await ss.openEditor(fileUri); + editor.selection = new vscode.Selection(0, 0, 3, 0); await ss.settle(); await writeClipboardSentinel(); + await vscode.commands.executeCommand(CMD_BIND_TO_CUSTOM_AI_BY_ID, { + extensionId: 'rangelink.dummy-ai-extension-focus-fail', + }); + await ss.settle(); + const logCapture = getLogCapture(); logCapture.mark('before-010'); - await waitForHuman( - 'clipboard-preservation-010', - `clipboard.preserve="always". Bind "Dummy AI (Focus-Fail)" via Cmd+R Cmd+D, then select lines 1-3 in the test file and press Cmd+R Cmd+L. The focus command throws — you should see a warning. Sentinel: "${CLIPBOARD_SENTINEL}".`, - [ - '1. Press Cmd+R Cmd+D → select "Dummy AI (Focus-Fail)" from the picker', - `2. Click back into the test file (${relPath}) and select lines 1-3`, - '3. Press Cmd+R Cmd+L — the focus command will throw an intentional error', - '4. Observe the warning message (manual paste instruction)', - '5. Press Cancel to continue (test verifies the link stayed in the clipboard)', - ], - ); - + await vscode.commands.executeCommand(CMD_COPY_LINK_RELATIVE); await ss.settle(); const lines010 = logCapture.getLinesSince('before-010'); @@ -384,10 +373,11 @@ standardSuite('Clipboard Preservation — Assisted', (ss) => { ); const generatedLink = extractGeneratedLink(lines010); assert.ok(generatedLink, 'Expected "Generated link:" log line'); + const PADDED_GENERATED_LINK = ` ${generatedLink} `; assert.strictEqual( clipboardContent, - generatedLink, - `Expected clipboard to equal generated link "${generatedLink}", got: ${clipboardContent}`, + PADDED_GENERATED_LINK, + `Expected clipboard to equal "${PADDED_GENERATED_LINK}", got: ${JSON.stringify(clipboardContent)}`, ); ss.log( '✓ Clipboard not restored after focus failure — link stays in clipboard for manual paste', diff --git a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts index bbb93b21..3a02abf1 100644 --- a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts +++ b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import * as vscode from 'vscode'; import { + CMD_BIND_TO_CUSTOM_AI_BY_ID, CMD_BIND_TO_TEXT_EDITOR_HERE, CMD_COPY_LINK_ONLY_RELATIVE, CMD_COPY_LINK_RELATIVE, @@ -85,7 +86,7 @@ standardSuite('Core Send Commands', (ss) => { ss.log('✓ R-L sent exact RangeLink to bound text editor destination'); }); - test('[assisted] core-send-commands-r-l-003: R-L sends RangeLink to bound AI assistant destination', async () => { + test('core-send-commands-r-l-003: R-L sends RangeLink to bound AI assistant destination', async () => { const CSC_R_L_003_LINE_COUNT = 10; const { uri: fileUri } = ss.createContentFile( 'csc-r-l-003', @@ -97,30 +98,30 @@ standardSuite('Core Send Commands', (ss) => { const expectedLink = `${relPath}#L1-L3`; await ss.openEditor(fileUri); + const doc = await vscode.workspace.openTextDocument(fileUri); + const editor = await vscode.window.showTextDocument(doc); + editor.selection = new vscode.Selection(0, 0, 3, 0); + await ss.settle(); + + await vscode.commands.executeCommand(CMD_BIND_TO_CUSTOM_AI_BY_ID, { + extensionId: 'rangelink.dummy-ai-extension', + }); await ss.settle(); const logCapture = getLogCapture(); logCapture.mark('before-r-l-003'); - await waitForHuman( - 'core-send-commands-r-l-003', - `Bind Dummy AI Tier 1 via Cmd+R Cmd+D, select lines 1-3 in ${relPath}, press Cmd+R Cmd+L.`, - [ - '1. Press Cmd+R Cmd+D → select "Dummy AI (Tier 1)"', - `2. Click into ${relPath}, select lines 1-3`, - '3. Press Cmd+R Cmd+L', - ], - ); - + await vscode.commands.executeCommand(CMD_COPY_LINK_RELATIVE); await ss.settle(); + const dummyText = (await vscode.commands.executeCommand('dummyAi.getText')) as { tier1: string; tier2: string; }; assert.strictEqual( - dummyText.tier1.trim(), - expectedLink, - `Expected Dummy AI tier1="${expectedLink}", got: ${JSON.stringify(dummyText.tier1)}`, + dummyText.tier1, + ` ${expectedLink} `, + `Expected Dummy AI tier1=" ${expectedLink} ", got: ${JSON.stringify(dummyText.tier1)}`, ); assert.ok( From 15a4ed3abc9363cd5845fce80d22252fc105fcba Mon Sep 17 00:00:00 2001 From: Charles Ouimet Date: Mon, 25 May 2026 14:32:37 -0400 Subject: [PATCH 2/2] [PR feedback] Address 3 CodeRabbit findings from PR #601 review Validate RANGELINK_MOCHA_TIMEOUT env var to prevent NaN/negative values reaching Mocha. Fix clipboard-preservation-004 expected_result to document the padded link contract. Remove redundant ss.openEditor call in core-send-commands-r-l-003. Ignored Feedback: - Extract waitForHumanVerdict into smaller helpers: Splitting tight UI-wiring code into single-caller helpers adds indirection without improving testability or readability. The function is self-contained and its concerns are inherently coupled. Ref: https://github.com/couimet/rangeLink/pull/601#pullrequestreview-4358236828 --- packages/rangelink-vscode-extension/.vscode-test.mjs | 5 ++--- .../rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml | 2 +- .../src/__integration-tests__/suite/coreSendCommands.test.ts | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/rangelink-vscode-extension/.vscode-test.mjs b/packages/rangelink-vscode-extension/.vscode-test.mjs index 82ab93d0..26996fe1 100644 --- a/packages/rangelink-vscode-extension/.vscode-test.mjs +++ b/packages/rangelink-vscode-extension/.vscode-test.mjs @@ -2,9 +2,8 @@ import { defineConfig } from '@vscode/test-cli'; import { ASSISTED_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs'; -const MOCHA_TIMEOUT = process.env.RANGELINK_MOCHA_TIMEOUT - ? Number(process.env.RANGELINK_MOCHA_TIMEOUT) - : ASSISTED_TIMEOUT_MS; +const parsed = Number(process.env.RANGELINK_MOCHA_TIMEOUT); +const MOCHA_TIMEOUT = Number.isFinite(parsed) && parsed >= 0 ? parsed : ASSISTED_TIMEOUT_MS; export default defineConfig([ { diff --git a/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml b/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml index d3a689b5..ff6cc533 100644 --- a/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml +++ b/packages/rangelink-vscode-extension/qa/qa-test-cases-v1.1.0.yaml @@ -388,7 +388,7 @@ test_cases: - clipboard feature: 'Clipboard Preservation' scenario: 'always mode: clipboard content before AI assistant paste is restored after' - expected_result: 'Dummy AI tier1 contains the exact link (relPath#L1-L3). Clipboard contains the sentinel value. assertClipboardRestored passes — preserve=always silently restored the clipboard after AI delivery.' + expected_result: 'Dummy AI tier1 contains the exact padded link ` relPath#L1C1-L3C7 ` (character-precise, with smart-padding spaces). Clipboard contains the sentinel value. assertClipboardRestored passes — preserve=always silently restored the clipboard after AI delivery.' automated: true - id: clipboard-preservation-005 diff --git a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts index 3a02abf1..ffbba074 100644 --- a/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts +++ b/packages/rangelink-vscode-extension/src/__integration-tests__/suite/coreSendCommands.test.ts @@ -97,7 +97,6 @@ standardSuite('Core Send Commands', (ss) => { const relPath = vscode.workspace.asRelativePath(fileUri); const expectedLink = `${relPath}#L1-L3`; - await ss.openEditor(fileUri); const doc = await vscode.workspace.openTextDocument(fileUri); const editor = await vscode.window.showTextDocument(doc); editor.selection = new vscode.Selection(0, 0, 3, 0);