E2E AutoTest #149
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E AutoTest | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| # Every weekday (Mon–Fri) at 13:00 Shanghai time (05:00 UTC) | |
| - cron: '0 5 * * 1-5' | |
| workflow_dispatch: | |
| inputs: | |
| test_plan: | |
| description: "Test plan to run (leave empty for all)" | |
| required: false | |
| default: "" | |
| type: string | |
| vsix_urls: | |
| description: "VSIX URLs, GitHub release tag URLs, GitHub Actions run URLs, or redhat-developer/vscode-java run IDs, comma-separated (e.g. https://github.com/redhat-developer/vscode-java/releases/tag/v1.54.0,https://github.com/redhat-developer/vscode-java/actions/runs/28102068371,https://host/vscode-java-debug-0.58.0.vsix). Release/run inputs auto-resolve to the platform-specific VSIX for the runner OS." | |
| required: false | |
| default: "" | |
| type: string | |
| pre_release: | |
| description: "Install pre-release versions of marketplace extensions" | |
| required: false | |
| default: true | |
| type: boolean | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| # ── Job 1a: Build vscode-java-pack VSIX from the PR/manual branch ─────── | |
| build-pack: | |
| if: ${{ github.event_name != 'schedule' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build extension | |
| run: npm run build | |
| - name: Package branch VSIX | |
| run: npx @vscode/vsce@latest package -o vscode-java-pack-pr.vsix | |
| - name: Upload branch VSIX | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pack-vsix | |
| path: vscode-java-pack-pr.vsix | |
| retention-days: 1 | |
| # ── Job 1b: Discover test plans ────────────────────────── | |
| discover: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.scan.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Resolve test plan matrix | |
| id: scan | |
| shell: bash | |
| run: | | |
| requested="${{ inputs.test_plan }}" | |
| if [ -n "$requested" ]; then | |
| # Strip optional .yaml suffix and emit a single-entry matrix | |
| plan="${requested%.yaml}" | |
| matrix=$(printf '%s' "$plan" | jq -R . | jq -sc .) | |
| else | |
| matrix=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .) | |
| fi | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| echo "Found plans: $matrix" | |
| # ── Job 2: Run each test plan in parallel ─────────────── | |
| e2e-test: | |
| needs: [discover, build-pack] | |
| # build-pack is skipped on schedule; PR/manual runs install this branch's pack VSIX first. | |
| if: ${{ always() && needs.discover.result == 'success' && (github.event_name == 'schedule' || needs.build-pack.result == 'success') }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [windows-latest, ubuntu-latest, macos-latest] | |
| plan: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clone vscode-java (test projects) | |
| run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java | |
| - name: Clone eclipse.jdt.ls (Gradle test projects) | |
| run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 (for java25 test plans) | |
| if: contains(matrix.plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Configure JDK 25 path for runner OS | |
| if: contains(matrix.plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| $planFile = "test-plans/${{ matrix.plan }}.yaml" | |
| if ($IsWindows) { | |
| $target = "C:\Program Files\Java\jdk-25" | |
| New-Item -ItemType Junction -Path $target -Target $env:JAVA_HOME -Force | Out-Null | |
| Write-Host "Created junction: $target -> $env:JAVA_HOME" | |
| } else { | |
| # Linux and macOS: symlink JAVA_HOME to a known location and | |
| # rewrite the Windows-style java.jdt.ls.java.home in the plan | |
| # so the language server resolves JDK 25 on this runner. | |
| $target = "/opt/jdk-25" | |
| sudo mkdir -p /opt | |
| sudo ln -sfn $env:JAVA_HOME $target | |
| Write-Host "Created symlink: $target -> $env:JAVA_HOME" | |
| (Get-Content $planFile -Raw) -replace 'C:\\\\Program Files\\\\Java\\\\jdk-25', $target | Set-Content $planFile | |
| Write-Host "Patched $planFile java.jdt.ls.java.home -> $target" | |
| } | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Setup virtual display (Linux) | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y xvfb | |
| Xvfb :99 -screen 0 1920x1080x24 & | |
| echo "DISPLAY=:99" >> "$GITHUB_ENV" | |
| # Give Xvfb a moment to start before the autotest CLI launches VS Code. | |
| sleep 2 | |
| - name: Download vscode-java-pack VSIX (from branch) | |
| if: ${{ github.event_name != 'schedule' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pack-vsix | |
| path: vsix | |
| - name: Download VSIX files | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| VSCODE_JAVA_ARTIFACT_TOKEN: ${{ secrets.VSCODE_JAVA_ARTIFACT_TOKEN }} | |
| run: | | |
| New-Item -ItemType Directory -Path vsix -Force | Out-Null | |
| $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } | |
| # Map runner OS/arch to vscode-java platform identifiers | |
| $platformMap = @{ "Windows" = "win32"; "Linux" = "linux"; "macOS" = "darwin" } | |
| $archMap = @{ "X64" = "x64"; "ARM64" = "arm64" } | |
| $platform = $platformMap["${{ runner.os }}"] | |
| $arch = $archMap["${{ runner.arch }}"] | |
| $platformId = "$platform-$arch" | |
| Write-Host "Runner platform: $platformId (${{ runner.os }}/${{ runner.arch }})" | |
| $githubHeaders = @{ | |
| Accept = "application/vnd.github+json" | |
| "X-GitHub-Api-Version" = "2022-11-28" | |
| "User-Agent" = "vscode-java-pack-autotest" | |
| } | |
| $artifactToken = if ($env:VSCODE_JAVA_ARTIFACT_TOKEN) { $env:VSCODE_JAVA_ARTIFACT_TOKEN } else { $env:GITHUB_TOKEN } | |
| $githubArtifactHeaders = $githubHeaders.Clone() | |
| if ($artifactToken) { | |
| $githubArtifactHeaders.Authorization = "Bearer $artifactToken" | |
| } | |
| function Select-VsixForPlatform { | |
| param( | |
| [Parameter(Mandatory = $true)] | |
| [System.IO.FileInfo[]] $VsixFiles, | |
| [Parameter(Mandatory = $true)] | |
| [string] $PlatformId | |
| ) | |
| $platformVsix = $VsixFiles | Where-Object { $_.Name -like "*-$PlatformId-*" } | Sort-Object Name | Select-Object -First 1 | |
| if ($platformVsix) { | |
| Write-Host " Found platform-specific VSIX: $($platformVsix.Name)" | |
| return $platformVsix | |
| } | |
| $universalVsix = $VsixFiles | Where-Object { $_.Name -notmatch '-(darwin|linux|win32)-' } | Sort-Object Name | Select-Object -First 1 | |
| if ($universalVsix) { | |
| Write-Host " No platform-specific VSIX found, using universal: $($universalVsix.Name)" | |
| return $universalVsix | |
| } | |
| throw "No matching VSIX found for platform $PlatformId. Available VSIX files: $($VsixFiles.Name -join ', ')" | |
| } | |
| function Download-GitHubActionsRunVsix { | |
| param( | |
| [Parameter(Mandatory = $true)] | |
| [string] $Owner, | |
| [Parameter(Mandatory = $true)] | |
| [string] $Repo, | |
| [Parameter(Mandatory = $true)] | |
| [string] $RunId | |
| ) | |
| Write-Host "Resolving GitHub Actions run: $Owner/$Repo#$RunId for platform $platformId" | |
| $artifactsUrl = "https://api.github.com/repos/$Owner/$Repo/actions/runs/$RunId/artifacts" | |
| $artifacts = @((Invoke-RestMethod -Uri $artifactsUrl -Headers $githubArtifactHeaders -UseBasicParsing).artifacts | Where-Object { -not $_.expired }) | |
| if (-not $artifacts) { | |
| throw "No non-expired artifacts found for GitHub Actions run $Owner/$Repo#$RunId" | |
| } | |
| $artifact = $artifacts | Where-Object { $_.name -eq "vscode-java" } | Select-Object -First 1 | |
| if (-not $artifact -and $artifacts.Count -eq 1) { | |
| $artifact = $artifacts | Select-Object -First 1 | |
| } | |
| if (-not $artifact) { | |
| throw "No vscode-java artifact found for GitHub Actions run $Owner/$Repo#$RunId. Available artifacts: $($artifacts.name -join ', ')" | |
| } | |
| $safeArtifactName = $artifact.name -replace '[^\w.-]', '_' | |
| $downloadDir = Join-Path "downloaded-artifacts" "$Owner-$Repo-$RunId-$safeArtifactName" | |
| $zipPath = "$downloadDir.zip" | |
| New-Item -ItemType Directory -Path $downloadDir -Force | Out-Null | |
| Write-Host " Downloading artifact: $($artifact.name)" | |
| try { | |
| Invoke-WebRequest -Uri $artifact.archive_download_url -OutFile $zipPath -Headers $githubArtifactHeaders -UseBasicParsing | |
| } catch { | |
| throw "Failed to download artifact $($artifact.name) from $Owner/$Repo#$RunId. Cross-repo artifact downloads may require a VSCODE_JAVA_ARTIFACT_TOKEN secret with actions:read access to the source repository. $($_.Exception.Message)" | |
| } | |
| Expand-Archive -LiteralPath $zipPath -DestinationPath $downloadDir -Force | |
| $vsixFiles = @(Get-ChildItem $downloadDir -Recurse -Filter "*.vsix") | |
| if ($vsixFiles.Count -eq 0) { | |
| throw "Artifact $($artifact.name) from $Owner/$Repo#$RunId does not contain VSIX files" | |
| } | |
| $selectedVsix = Select-VsixForPlatform -VsixFiles $vsixFiles -PlatformId $platformId | |
| Copy-Item -LiteralPath $selectedVsix.FullName -Destination (Join-Path "vsix" $selectedVsix.Name) -Force | |
| } | |
| $resolvedUrls = @() | |
| foreach ($url in $urls) { | |
| if ($url -match '^https://github\.com/([^/]+)/([^/]+)/releases/tag/(.+)$') { | |
| $owner = $Matches[1]; $repo = $Matches[2]; $tag = $Matches[3] | |
| Write-Host "Resolving GitHub release: $owner/$repo@$tag for platform $platformId" | |
| $apiUrl = "https://api.github.com/repos/$owner/$repo/releases/tags/$tag" | |
| $release = Invoke-RestMethod -Uri $apiUrl -Headers $githubHeaders -UseBasicParsing | |
| $platformAsset = $release.assets | Where-Object { $_.name -like "*-$platformId-*" -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($platformAsset) { | |
| Write-Host " Found platform-specific VSIX: $($platformAsset.name)" | |
| $resolvedUrls += $platformAsset.browser_download_url | |
| } else { | |
| $universalAsset = $release.assets | Where-Object { $_.name -notmatch '-(darwin|linux|win32)-' -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($universalAsset) { | |
| Write-Host " No platform-specific VSIX found, using universal: $($universalAsset.name)" | |
| $resolvedUrls += $universalAsset.browser_download_url | |
| } else { | |
| Write-Host "::warning::No matching VSIX found in release $owner/$repo@$tag for platform $platformId" | |
| } | |
| } | |
| } elseif ($url -match '^https://github\.com/([^/]+)/([^/]+)/actions/runs/(\d+)') { | |
| Download-GitHubActionsRunVsix -Owner $Matches[1] -Repo $Matches[2] -RunId $Matches[3] | |
| } elseif ($url -match '^\d+$') { | |
| Download-GitHubActionsRunVsix -Owner "redhat-developer" -Repo "vscode-java" -RunId $url | |
| } else { | |
| $resolvedUrls += $url | |
| } | |
| } | |
| foreach ($url in $resolvedUrls) { | |
| $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0]) | |
| Write-Host "Downloading: $url → vsix/$fileName" | |
| Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing | |
| } | |
| Write-Host "Downloaded VSIX files:" | |
| Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host " $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" } | |
| - name: Use vscode-java-pack as primary extension for VSIX override runs | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| run: | | |
| $planFile = "test-plans/${{ matrix.plan }}.yaml" | |
| $content = Get-Content $planFile -Raw | |
| $content = $content -replace '(?m)^(\s*)extension:\s*"?redhat\.java"?\s*$', '$1extension: "vscjava.vscode-java-pack"' | |
| $content = $content -replace '(?m)^(\s*)extensions:\s*\r?\n\1\s*-\s*"vscjava\.vscode-java-pack"\s*\r?\n', '' | |
| Set-Content $planFile $content | |
| Write-Host "Patched $planFile to install vscode-java through vscode-java-pack before applying VSIX overrides." | |
| - name: Run ${{ matrix.plan }} | |
| shell: pwsh | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: | | |
| $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml") | |
| if (Test-Path vsix) { | |
| $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | | |
| Sort-Object @{ Expression = { if ($_.Name -like "vscode-java-pack*") { 0 } else { 1 } } }, Name | | |
| ForEach-Object { $_.FullName }) -join "," | |
| if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) } | |
| } | |
| # PR/manual runs test the branch-built pack VSIX first, then any | |
| # supplied VSIX overrides such as a vscode-java Actions run artifact. | |
| # LLM verification activates automatically when AZURE_OPENAI_* secrets | |
| # are available (e.g. internal PRs); fork PRs without secret access | |
| # simply skip the LLM step (LLMClient.isConfigured() returns false). | |
| # Scheduled & manual runs default to --pre-release unless explicitly disabled. | |
| $isPR = "${{ github.event_name }}" -eq "pull_request" | |
| if (-not $isPR -and "${{ inputs.pre_release }}" -ne "false") { | |
| $autotestArgs += "--pre-release" | |
| } | |
| Write-Host "Running: autotest $($autotestArgs -join ' ')" | |
| & autotest @autotestArgs | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.plan }}-${{ matrix.os }} | |
| path: test-results/ | |
| retention-days: 30 | |
| # ── Job 3: Aggregate analysis ─────────────────────────── | |
| analyze: | |
| if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }} | |
| needs: e2e-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download all results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| path: all-results | |
| merge-multiple: false | |
| - name: Organize results | |
| run: | | |
| mkdir -p test-results | |
| # Each artifact is named results-<plan>-<os>; keep that suffix so | |
| # Windows and Linux runs of the same plan don't collide. | |
| for dir in all-results/results-*/; do | |
| artifact=$(basename "$dir") # results-<plan>-<os> | |
| suffix="${artifact#results-}" # <plan>-<os> | |
| find "$dir" -name "results.json" -exec dirname {} \; | while read d; do | |
| cp -r "$d" "test-results/$suffix" | |
| done | |
| done | |
| echo "Organized results:" | |
| ls test-results/ | |
| - name: Analyze results | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: autotest analyze test-results --output test-results | |
| - name: Write Job Summary | |
| if: always() | |
| run: | | |
| if [ -f test-results/summary.md ]; then | |
| cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload aggregate results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-aggregate-summary | |
| path: test-results/summary.md | |
| retention-days: 30 |