diff --git a/workspaces/orchestrator/.eslintignore b/workspaces/orchestrator/.eslintignore index d989225489..8da64e7158 100644 --- a/workspaces/orchestrator/.eslintignore +++ b/workspaces/orchestrator/.eslintignore @@ -1,4 +1,5 @@ playwright.config.ts +e2e-tests/ dist-dynamic dist-scalprum !.eslintrc.js diff --git a/workspaces/orchestrator/.gitignore b/workspaces/orchestrator/.gitignore index 4af66de389..49f77bc2c4 100644 --- a/workspaces/orchestrator/.gitignore +++ b/workspaces/orchestrator/.gitignore @@ -53,7 +53,8 @@ site *.session.sql # E2E test reports -e2e-test-report/ +e2e-test-report*/ +test-results/ # Sonataflow Dev Container Temp files packages/backend/.devModeTemp diff --git a/workspaces/orchestrator/e2e-tests/orchestrator.test.ts b/workspaces/orchestrator/e2e-tests/orchestrator.test.ts new file mode 100644 index 0000000000..85426e0ca4 --- /dev/null +++ b/workspaces/orchestrator/e2e-tests/orchestrator.test.ts @@ -0,0 +1,282 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, Page, type BrowserContext } from '@playwright/test'; +import { Orchestrator } from './pages/orchestrator'; +import { runAccessibilityTests } from './utils/accessibility'; +import { OrchestratorHelper } from './utils/helper'; +import { OrchestratorMessages, getTranslations } from './utils/translations'; + +const LOCALE_DISPLAY_NAMES: Record = { + en: 'English', + de: 'Deutsch', + es: 'Español', + fr: 'Français', + it: 'Italiano', + ja: '日本語', +}; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function countHeadingPattern(template: string): RegExp { + const regexBody = template.split('{{count}}').map(escapeRegExp).join('\\d+'); + return new RegExp(`^${regexBody}$`); +} + +/** + * Get the display name for a locale code + */ +function getLocaleDisplayName(locale: string): string { + const baseLocale = locale.split('-')[0]; + return LOCALE_DISPLAY_NAMES[baseLocale] || locale; +} + +test.describe('Orchestrator workflow runs', () => { + let orchestrator: Orchestrator; + let orchestratorHelper: OrchestratorHelper; + let translations: OrchestratorMessages; + let sharedPage!: Page; + let sharedContext!: BrowserContext; + + async function switchToLocale(page: Page, locale: string): Promise { + const baseLocale = locale.split('-')[0]; + if (baseLocale === 'en') return; + + const displayName = getLocaleDisplayName(locale); + const localeDisplayPattern = new RegExp( + `^(${Object.values(LOCALE_DISPLAY_NAMES).map(escapeRegExp).join('|')})$`, + ); + + await page.goto('/settings'); + await page.waitForLoadState('networkidle'); + const languageButton = page + .getByRole('button', { name: localeDisplayPattern }) + .first(); + await languageButton.waitFor({ state: 'visible', timeout: 30_000 }); + await languageButton.click(); + await page.getByRole('option', { name: displayName }).click(); + await page.goto('/'); + } + + test.beforeAll(async ({ browser }) => { + sharedContext = await browser.newContext(); + sharedPage = await sharedContext.newPage(); + const currentLocale = await sharedPage.evaluate( + () => globalThis.navigator.language.split('-')[0], + ); + await sharedPage.goto('/'); + await sharedPage.getByRole('button', { name: 'Enter' }).click(); + await switchToLocale(sharedPage, currentLocale); + translations = getTranslations(currentLocale); + orchestrator = new Orchestrator(sharedPage, translations, currentLocale); + orchestratorHelper = new OrchestratorHelper(sharedPage, translations); + }); + + test.beforeEach(async () => { + await orchestrator.navigateToOrchestrator('Orchestrator'); + }); + + test.afterAll(async () => { + if (sharedContext) { + await sharedContext.close(); + } + }); + + test.describe('Orchestrator > Workflow runs page', () => { + test.beforeEach(async () => { + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.workflows, + ); + }); + + test('Verify workflow runs table', async ({ + browser: _browser, + }, testInfo) => { + await runAccessibilityTests(sharedPage, testInfo); + await orchestratorHelper.verifyTableHeadingAndRows([ + translations.table.headers.name, + translations.table.headers.workflowStatus, + translations.table.headers.lastRun, + translations.table.headers.lastRunStatus, + translations.table.headers.description, + translations.table.headers.version, + 'Actions', + ]); + + await orchestratorHelper.searchInputPlaceholder('Hello World workflow'); + await expect( + sharedPage + .getByRole('row', { name: 'Hello World workflow' }) + .getByRole('button', { + name: translations.table.actions.run, + exact: true, + }) + .first(), + ).toBeVisible(); + await expect( + sharedPage.getByRole('row', { name: 'Hello World workflow' }), + ).toContainText(translations.workflow.status.available); + await sharedPage + .getByRole('row', { name: 'Hello World workflow' }) + .getByRole('button', { + name: translations.table.actions.viewInputSchema, + }) + .click(); + const workflowSchema = sharedPage.getByRole('dialog', { + name: /Hello World Workflow input schema/i, + }); + await expect(workflowSchema).toBeVisible(); + await expect( + workflowSchema.getByText(translations.messages.noInputSchemaWorkflow), + ).toBeVisible(); + await sharedPage + .getByRole('button', { name: 'close', exact: true }) + .click(); + }); + + test('Run Test Object Type Support in ui:props workflow', async ({ + browser: _browser, + }) => { + const workflowName = 'Test Object Type Support in ui:props'; + const workflowInputs = { + name: 'test-name', + email: 'test@test.com', + simpleText: 'sample testing', + objectExample: '{"kind":"demo","id":42,"tags":["a","b"]}', + }; + + await orchestrator.runUiPropsWorkflow(workflowName, workflowInputs); + + await expect(sharedPage).toHaveURL(/\/orchestrator\/instances\/.+/); + await orchestratorHelper.verifyHeading( + translations.run.pageTitle.replace('{{processName}}', workflowName), + ); + await orchestrator.verifyUiPropsWorkflowInstanceDetails(workflowName); + await orchestrator.verifyUiPropsWorkflowRunVariables(workflowInputs); + }); + + test('Greeting workflow execution and workflow tab validation', async ({ + browser: _browser, + }) => { + const workflowName = 'Greeting workflow'; + + await orchestrator.runGreetingWorkflow(workflowName); + await orchestrator.navigateToOrchestrator('Orchestrator'); + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.workflows, + ); + await orchestrator.searchWorkflow(workflowName); + await orchestrator.validateGreetingWorkflowTableRow(workflowName); + }); + + test('Greeting workflow run details validation', async ({ + browser: _browser, + }) => { + const workflowName = 'Greeting workflow'; + + await orchestrator.runGreetingWorkflow(workflowName); + await orchestrator.reRunGreetingWorkflow(); + await orchestrator.verifyWorkflowRunDetails(); + }); + + test('Sample Retry Test', async ({ browser: _browser }) => { + const workflowName = 'Sample Retry Test'; + + await orchestrator.runSampleRetryTest(workflowName); + await orchestrator.verifySampleRetryTest(); + }); + }); + + test.describe('Orchestrator > All runs page', () => { + test.beforeEach(async () => { + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.allRuns, + ); + }); + + test('Verify all runs tab', async ({ browser: _browser }, testInfo) => { + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.workflows, + ); + await orchestrator.searchWorkflow('Hello World workflow'); + await sharedPage + .getByRole('row', { name: 'Hello World workflow' }) + .getByRole('button', { + name: translations.table.actions.run, + exact: true, + }) + .click(); + await orchestrator.submitWorkflowRunFromReview(); + await orchestratorHelper.verifyHeading( + translations.run.pageTitle.replace( + '{{processName}}', + 'Hello World workflow', + ), + ); + await orchestrator.navigateToOrchestrator('Orchestrator'); + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.allRuns, + ); + await expect( + sharedPage.getByText( + countHeadingPattern(translations.table.title.allRuns), + ), + ).toBeVisible(); + await runAccessibilityTests(sharedPage, testInfo); + await orchestrator.verifyWorkflowRunTabDetails(); + }); + + // Remove the fixme with the fix of bug https://redhat.atlassian.net/browse/RHDHBUGS-3401 + test.fixme('All runs tab workflow details validation', async ({ + browser: _browser, + }) => { + await sharedPage + .getByRole('link', { name: 'Hello World workflow' }) + .first() + .click(); + await orchestratorHelper.verifyHeading('Hello World workflow'); + await orchestrator.verifyWorkflowDetails(); + await orchestrator.navigateToWorkflowRunTab( + translations.page.tabs.workflowRuns, + ); + await orchestrator.verifyWorkflowRunTab(); + const runLocator = await sharedPage + .getByText( + countHeadingPattern(translations.table.title.allWorkflowRuns), + ) + .textContent(); + const runCount = parseInt(runLocator?.match(/\d+/)?.[0] || '0'); + await sharedPage + .getByRole('button', { + name: translations.table.actions.run, + exact: true, + }) + .first() + .click(); + await orchestrator.submitWorkflowRunFromReview(); + await orchestratorHelper.verifyHeading( + translations.run.pageTitle.replace( + '{{processName}}', + 'Hello World workflow', + ), + ); + await sharedPage.goto(`/orchestrator/workflows/hello_world/runs`); + await orchestrator.verifyWorkflowRunTab(runCount + 1); + }); + }); +}); diff --git a/workspaces/orchestrator/e2e-tests/pages/orchestrator.ts b/workspaces/orchestrator/e2e-tests/pages/orchestrator.ts new file mode 100644 index 0000000000..be9a2582f3 --- /dev/null +++ b/workspaces/orchestrator/e2e-tests/pages/orchestrator.ts @@ -0,0 +1,542 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Page, expect, Locator } from '@playwright/test'; +import { OrchestratorHelper } from '../utils/helper'; +import type { OrchestratorMessages } from '../utils/translations'; + +export type CreateGithubBranchWorkflowInputs = { + owner: string; + repo: string; + baseBranch: string; + targetBranch: string; +}; + +export type UiPropsWorkflowInputs = { + name: string; + email: string; + simpleText: string; + objectExample: string; +}; + +type SampleRetryHits = { + allProps: number; + statusCodesNoMatch: number; + noRetry: number; +}; + +export class Orchestrator { + private page: Page; + private orchestratorHelper: OrchestratorHelper; + private translations: OrchestratorMessages; + private locale: string; + private sampleRetryHits: SampleRetryHits = { + allProps: 0, + statusCodesNoMatch: 0, + noRetry: 0, + }; + + constructor(page: Page, translations: OrchestratorMessages, locale = 'en') { + this.page = page; + this.translations = translations; + this.locale = locale.split('-')[0]; + this.orchestratorHelper = new OrchestratorHelper(page, translations); + } + + async navigateToOrchestrator(_navText: string) { + await this.page.goto('/orchestrator'); + await this.page + .getByRole('heading', { name: this.translations.page.title }) + .first() + .waitFor({ state: 'visible', timeout: 30_000 }); + } + + async navigateToWorkflowRunTab(navText: string) { + const navLink = this.page.getByRole('tab', { name: navText }).first(); + await navLink.waitFor({ state: 'visible', timeout: 8_000 }); + await navLink.click(); + await expect(navLink).toHaveAttribute('aria-selected', 'true'); + } + + async verifyKeyValueRowElements(rowTitle: string, rowValue: string) { + const rowLocator = this.page.locator('.v5-MuiTableRow-root'); + await expect(rowLocator.filter({ hasText: rowTitle })).toContainText( + rowValue, + ); + } + + async searchWorkflow(workflowName: string) { + await this.orchestratorHelper.searchInputPlaceholder(workflowName); + await expect( + this.page.getByRole('row', { name: workflowName }), + ).toBeVisible(); + } + + async openWorkflowFromTable(workflowName: string) { + await this.page + .getByRole('row', { name: workflowName }) + .getByRole('link', { name: workflowName }) + .click(); + await this.orchestratorHelper.verifyHeading(workflowName); + } + + async clickRunWorkflowFromDetails() { + await this.orchestratorHelper.clickButton( + this.translations.workflow.buttons.run, + ); + await expect( + this.page.getByText(this.translations.run.title), + ).toBeVisible(); + } + + // async fillCreateGithubBranchWorkflowInputs( + // inputs: CreateGithubBranchWorkflowInputs, + // ) { + // await this.orchestratorHelper.fillInputByLabel('owner', inputs.owner); + // await this.orchestratorHelper.fillInputByLabel('repo', inputs.repo); + // await this.orchestratorHelper.fillInputByLabel( + // 'baseBranch', + // inputs.baseBranch, + // ); + // await this.orchestratorHelper.fillInputByLabel( + // 'targetBranch', + // inputs.targetBranch, + // ); + // } + + async submitWorkflowRunForm() { + const nextButton = this.page.getByRole('button', { + name: this.translations.common.next, + }); + if (await nextButton.isVisible()) { + await nextButton.click(); + } + await this.orchestratorHelper.clickButton(this.translations.common.run); + } + + async fillUiPropsWorkflowInputs(inputs: UiPropsWorkflowInputs) { + await this.page.getByRole('textbox', { name: 'Name' }).fill(inputs.name); + await this.page.getByRole('textbox', { name: 'Email' }).fill(inputs.email); + await this.orchestratorHelper.clickButton(this.translations.common.next); + await this.page + .getByRole('textbox', { name: 'Simple Text Field' }) + .fill(inputs.simpleText); + await this.page + .getByRole('textbox', { name: 'Object Type Example' }) + .fill(inputs.objectExample); + await this.orchestratorHelper.clickButton(this.translations.common.next); + } + + async submitWorkflowRunFromReview() { + await expect( + this.page.getByText(this.translations.run.title).first(), + ).toBeVisible(); + await this.orchestratorHelper.clickButton(this.translations.common.run); + } + + async verifyUiPropsWorkflowInstanceDetails(workflowName: string) { + const displayWorkflowName = + workflowName.charAt(0).toUpperCase() + + workflowName.slice(1).toLocaleLowerCase('en-US'); + + await expect( + this.page.getByText( + `${this.translations.workflow.fields.runStatus} ${this.translations.table.status.completed}`, + ), + ).toBeVisible({ timeout: 60_000 }); + // TODO: Remove the if statement for japanese language tests with solving bug https://redhat.atlassian.net/browse/RHDHBUGS-3406 + if (this.locale === 'ja') { + await expect( + this.page.getByText( + `${this.translations.run.results}${this.translations.table.actions.run}`, + ), + ).toBeVisible(); + } else { + await expect( + this.page.getByText( + `${this.translations.run.results}${this.translations.run.status.completed}`, + ), + ).toBeVisible(); + } + await expect( + this.page.getByText( + `${this.translations.workflow.fields.workflow}${displayWorkflowName}`, + ), + ).toBeVisible(); + await expect( + this.page.getByText( + `${this.translations.workflow.fields.workflowStatus} ${this.translations.workflow.status.available}`, + ), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.workflowId, + }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.duration, + }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.started, + }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.description, + }), + ).toBeVisible(); + } + + private formatRunVariablesStringField( + fieldName: string, + value: string, + ): string { + const escapedValue = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + return `"${fieldName}": "${escapedValue}"`; + } + + async verifyUiPropsWorkflowRunVariables(inputs: UiPropsWorkflowInputs) { + await this.orchestratorHelper.clickLink( + this.translations.run.viewVariables, + ); + await expect( + this.page.getByText(`{ "name": "${inputs.name}"`), + ).toBeVisible(); + await expect( + this.page.getByText(`"email": "${inputs.email}"`), + ).toBeVisible(); + await expect( + this.page.getByText(`{ "simpleText": "${inputs.simpleText}"`), + ).toBeVisible(); + await expect( + this.page.getByText( + this.formatRunVariablesStringField( + 'objectExample', + inputs.objectExample, + ), + ), + ).toBeVisible(); + await this.orchestratorHelper.closeBar(this.translations.common.close); + } + + async fillGreetingWorkflowForm(language = 'English', name = 'John') { + await this.page.getByLabel('Language', { exact: true }).click(); + await this.page.getByRole('option', { name: language }).click(); + const nameField = this.page.getByLabel('Name', { exact: true }); + if (await nameField.isVisible()) { + await nameField.fill(name); + } + const nextButton = this.page.getByRole('button', { + name: this.translations.common.next, + }); + if (await nextButton.isVisible()) { + await nextButton.click(); + } + } + + async runGreetingWorkflowFromExecuteForm() { + await this.orchestratorHelper.clickButton(this.translations.common.run); + await expect( + this.page.getByText(this.translations.table.status.completed, { + exact: true, + }), + ).toBeVisible({ timeout: 120_000 }); + } + + async runGreetingWorkflow(workflowName: string, language = 'English') { + await this.searchWorkflow(workflowName); + await this.openWorkflowFromTable(workflowName); + await this.clickRunWorkflowFromDetails(); + await this.fillGreetingWorkflowForm(language); + await this.runGreetingWorkflowFromExecuteForm(); + } + + async reRunGreetingWorkflow(language = 'english') { + await expect( + this.page.getByText(this.translations.workflow.buttons.runAgain), + ).toBeVisible(); + // Uncomment the below lines and remove the line 254 with bug fix https://redhat.atlassian.net/browse/RHDHBUGS-3400 + // await this.orchestratorHelper.clickButton( + // this.translations.workflow.buttons.runAgain, + // ); + await this.page + .locator('div') + .filter({ hasText: this.translations.workflow.buttons.runAgain }) + .last() + .getByRole('button') + .click(); + await this.fillGreetingWorkflowForm(language); + await this.runGreetingWorkflowFromExecuteForm(); + } + + async validateGreetingWorkflowTableRow(workflowName: string) { + const workflowRow = this.page.getByRole('row', { name: workflowName }); + + await expect(workflowRow).toContainText( + this.translations.workflow.status.available, + ); + await expect(workflowRow).toContainText( + this.translations.table.status.completed, + ); + await expect(workflowRow).toContainText('YAML based greeting workflow'); + await expect( + workflowRow + .getByRole('button', { + name: this.translations.table.actions.run, + exact: true, + }) + .first(), + ).toBeVisible(); + await expect( + workflowRow + .getByRole('button', { name: this.translations.table.actions.viewRuns }) + .first(), + ).toBeVisible(); + await expect( + workflowRow + .getByRole('button', { + name: this.translations.table.actions.viewInputSchema, + }) + .first(), + ).toBeVisible(); + } + + async verifyWorkflowRunDetails() { + await expect( + this.page.getByText(this.translations.workflow.details, { exact: true }), + ).toBeVisible(); + await expect( + this.page.getByText(this.translations.run.results, { exact: true }), + ).toBeVisible(); + await expect( + this.page.getByText(this.translations.workflow.progress, { exact: true }), + ).toBeVisible(); + await expect( + this.page + .locator('div') + .filter({ hasText: this.translations.table.status.completed }) + .first(), + ).toBeVisible(); + } + + async verifyWorkflowDetails() { + await expect( + this.page.getByText(this.translations.workflow.details, { exact: true }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.workflowStatus, + }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.description, + }), + ).toBeVisible(); + await expect( + this.page.getByRole('heading', { + name: this.translations.workflow.fields.version, + }), + ).toBeVisible(); + await expect( + this.page.getByText(this.translations.workflow.status.available), + ).toBeVisible(); + await expect( + this.page.getByText(this.translations.workflow.definition, { + exact: true, + }), + ).toBeVisible(); + await expect(this.page.locator('pre').first()).toBeVisible(); + await expect( + this.page.getByRole('button', { name: 'Copy text' }), + ).toBeVisible(); + } + + async verifyWorkflowRunTab(runsCount?: number) { + const escapedRunsCount = this.translations.table.title.allWorkflowRuns + .split('{{count}}') + .map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('\\d+'); + await expect( + this.page.getByText( + runsCount + ? this.translations.table.title.allWorkflowRuns.replace( + '{{count}}', + runsCount.toString(), + ) + : new RegExp(`^${escapedRunsCount}$`), + ), + ).toBeVisible({ timeout: 5000 }); + await this.orchestratorHelper.verifyTableHeadingAndRows([ + 'ID', + this.translations.table.headers.runStatus, + this.translations.table.headers.started, + this.translations.table.headers.duration, + ]); + await expect( + this.page + .getByRole('button', { + name: this.translations.table.actions.run, + exact: true, + }) + .first(), + ).toBeVisible(); + await expect( + this.page + .getByLabel(this.translations.table.filters.status) + .getByRole('button', { name: 'All' }), + ).toBeVisible(); + await expect( + this.page + .getByLabel(this.translations.table.filters.started) + .getByRole('button', { name: 'All' }), + ).toBeVisible(); + } + + async runUiPropsWorkflow( + workflowName: string, + inputs: UiPropsWorkflowInputs, + ) { + await this.searchWorkflow(workflowName); + await this.openWorkflowFromTable(workflowName); + await this.clickRunWorkflowFromDetails(); + await this.fillUiPropsWorkflowInputs(inputs); + await this.submitWorkflowRunFromReview(); + } + + async verifyWorkflowRunTabDetails() { + await this.orchestratorHelper.verifyTableHeadingAndRows([ + 'ID', + this.translations.table.headers.workflowName, + this.translations.table.headers.runStatus, + this.translations.table.headers.started, + this.translations.table.headers.duration, + ]); + const statuses = [ + this.translations.table.status.running, + this.translations.table.status.failed, + this.translations.table.status.completed, + this.translations.table.status.aborted, + this.translations.table.status.pending, + ]; + await this.page + .getByLabel(this.translations.table.filters.status) + .getByRole('button', { name: 'All' }) + .click(); + for (const status of statuses) { + await expect(this.page.getByRole('option', { name: status })).toHaveText( + status, + ); + await this.page.getByRole('option', { name: status }).click(); + await this.page.getByRole('button', { name: status }).click(); + } + await this.page.getByRole('option', { name: 'All', exact: true }).click(); + const startTimings = [ + this.translations.table.filters.startedOptions.today, + this.translations.table.filters.startedOptions.yesterday, + this.translations.table.filters.startedOptions.last7days, + this.translations.table.filters.startedOptions.thisMonth, + ]; + await this.page + .getByLabel(this.translations.table.filters.started) + .getByRole('button', { name: 'All' }) + .click(); + for (const startTime of startTimings) { + await expect( + this.page.getByRole('option', { name: startTime }), + ).toHaveText(startTime); + await this.page.getByRole('option', { name: startTime }).click(); + await this.page.getByRole('button', { name: startTime }).click(); + } + await this.page.getByRole('option', { name: 'All' }).click(); + } + + private retryTestJsonOk(value: string) { + return { + status: 200, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ value }), + }; + } + + async setupSampleRetryTestRoutes() { + this.sampleRetryHits = { + allProps: 0, + statusCodesNoMatch: 0, + noRetry: 0, + }; + + await this.page.route('**/api/retry-test/**', async route => { + const url = route.request().url(); + + if (url.includes('all-props')) { + this.sampleRetryHits.allProps += 1; + if (this.sampleRetryHits.allProps <= 3) { + await route.fulfill({ status: 404, body: 'unavailable' }); + } else { + await route.fulfill(this.retryTestJsonOk('ok')); + } + return; + } + + if (url.includes('status-codes-no-404')) { + this.sampleRetryHits.statusCodesNoMatch += 1; + await route.fulfill({ status: 404, body: 'not found' }); + return; + } + + if (url.includes('no-retry-props')) { + this.sampleRetryHits.noRetry += 1; + await route.fulfill({ status: 503, body: 'no retry' }); + return; + } + + await route.continue(); + }); + } + + async cleanupSampleRetryTestRoutes() { + await this.page.unroute('**/api/retry-test/**'); + } + + async runSampleRetryTest(workflowName: string) { + await this.setupSampleRetryTestRoutes(); + await this.searchWorkflow(workflowName); + await this.openWorkflowFromTable(workflowName); + await this.clickRunWorkflowFromDetails(); + } + + async verifySampleRetryTest() { + await expect( + this.page.getByRole('textbox', { name: 'Retry Test (all props)' }), + ).toHaveValue('ok', { timeout: 150_000 }); + await expect( + this.page.getByTestId('root_retryStatusCodesNoMatch-error-text'), + ).toBeVisible({ timeout: 60_000 }); + await expect( + this.page.getByTestId('root_retryNoProps-error-text'), + ).toBeVisible({ timeout: 60_000 }); + + expect(this.sampleRetryHits.allProps).toBe(4); + expect(this.sampleRetryHits.statusCodesNoMatch).toBe(1); + expect(this.sampleRetryHits.noRetry).toBe(1); + + await this.cleanupSampleRetryTestRoutes(); + } +} diff --git a/workspaces/orchestrator/e2e-tests/utils/accessibility.ts b/workspaces/orchestrator/e2e-tests/utils/accessibility.ts new file mode 100644 index 0000000000..586d0c10c3 --- /dev/null +++ b/workspaces/orchestrator/e2e-tests/utils/accessibility.ts @@ -0,0 +1,41 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AxeBuilder from '@axe-core/playwright'; +import { Page, expect, TestInfo } from '@playwright/test'; + +export async function runAccessibilityTests( + page: Page, + testInfo: TestInfo, + attachName = 'accessibility-scan-results.json', + options: { skipFailures?: boolean } = { skipFailures: true }, +) { + const accessibilityScanResults = await new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']) + .analyze(); + + await testInfo.attach(attachName, { + body: JSON.stringify(accessibilityScanResults.violations, null, 2), + contentType: 'application/json', + }); + + if (!options?.skipFailures) { + expect( + accessibilityScanResults.violations, + 'Accessibility violations found', + ).toEqual([]); + } +} diff --git a/workspaces/orchestrator/e2e-tests/utils/helper.ts b/workspaces/orchestrator/e2e-tests/utils/helper.ts new file mode 100644 index 0000000000..ba5d3288d8 --- /dev/null +++ b/workspaces/orchestrator/e2e-tests/utils/helper.ts @@ -0,0 +1,98 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Page, expect, Locator } from '@playwright/test'; +import type { OrchestratorMessages } from './translations'; + +export class OrchestratorHelper { + private page: Page; + private translations: OrchestratorMessages; + + constructor(page: Page, translations: OrchestratorMessages) { + this.page = page; + this.translations = translations; + } + + async verifyTableHeadings(texts: string[]) { + // Wait for the table to load by checking for the presence of table rows + await this.page.waitForSelector('table tbody tr', { state: 'visible' }); + for (const column of texts) { + const columnSelector = `table th:has-text("${column}")`; + // check if columnSelector has at least one element or more + const columnCount = await this.page.locator(columnSelector).count(); + expect(columnCount).toBeGreaterThan(0); + } + } + + async verifyTableHeadingAndRows(texts: string[]) { + await this.verifyTableHeadings(texts); + // Checks if the table has at least one row with data + // Excludes rows that have cells spanning multiple columns, such as "No data available" messages + const rowSelector = `table tbody tr:not(:has(td[colspan]))`; + const rowCount = await this.page.locator(rowSelector).count(); + expect(rowCount).toBeGreaterThan(0); + } + + async searchInputPlaceholder(searchTerm: string) { + await this.page.fill(`input[placeholder="Filter"]`, searchTerm); + } + + async verifyHeading(heading: string | RegExp, timeout: number = 20000) { + await this.page + .getByRole('heading', { name: heading }) + .first() + .waitFor({ state: 'visible', timeout: timeout }); + } + + async closeBar(buttonLocator: string | RegExp, timeout: number = 20000) { + const barButton = this.page + .getByRole('button', { name: buttonLocator }) + .first(); + await barButton.waitFor({ state: 'visible', timeout: timeout }); + await barButton.click(); + } + + async clickLink(options: string | { href: string } | { ariaLabel: string }) { + let linkLocator: Locator; + + if (typeof options === 'string') { + linkLocator = this.page.locator('a').filter({ hasText: options }).first(); + } else if ('href' in options) { + linkLocator = this.page.locator(`a[href*="${options.href}"]`).first(); + } else { + linkLocator = this.page + .locator(`div[aria-label='${options.ariaLabel}'] a`) + .first(); + } + + await linkLocator.waitFor({ state: 'visible' }); + await linkLocator.click(); + } + + async clickTab(tabName: string) { + const tabLocator = this.page.getByRole('tab', { name: tabName }); + await tabLocator.waitFor({ state: 'visible' }); + await tabLocator.click(); + } + + async clickButton(buttonName: string) { + const buttonLocator = this.page + .getByRole('button', { name: buttonName }) + .first(); + await buttonLocator.waitFor({ state: 'visible' }); + await buttonLocator.click(); + } +} diff --git a/workspaces/orchestrator/e2e-tests/utils/translations.ts b/workspaces/orchestrator/e2e-tests/utils/translations.ts new file mode 100644 index 0000000000..66efdea84e --- /dev/null +++ b/workspaces/orchestrator/e2e-tests/utils/translations.ts @@ -0,0 +1,78 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These translation files are not exported by the package, so relative imports are necessary for e2e tests +/* eslint-disable @backstage/no-relative-monorepo-imports */ +import { orchestratorMessages } from '../../plugins/orchestrator/src/translations/ref.js'; +import orchestratorTranslationDe from '../../plugins/orchestrator/src/translations/de.js'; +import orchestratorTranslationEs from '../../plugins/orchestrator/src/translations/es.js'; +import orchestratorTranslationFr from '../../plugins/orchestrator/src/translations/fr.js'; +import orchestratorTranslationIt from '../../plugins/orchestrator/src/translations/it.js'; +import orchestratorTranslationJa from '../../plugins/orchestrator/src/translations/ja.js'; +/* eslint-enable @backstage/no-relative-monorepo-imports */ + +export type OrchestratorMessages = typeof orchestratorMessages; + +function transform(messages: typeof orchestratorTranslationDe.messages) { + const result = Object.keys(messages).reduce((res, key) => { + const path = key.split('.'); + const lastIndex = path.length - 1; + path.reduce((acc, currentPath, i) => { + acc[currentPath] = + lastIndex === i ? messages[key] : acc[currentPath] || {}; + return acc[currentPath]; + }, res); + return res; + }, {}); + + return result as OrchestratorMessages; +} + +export function getTranslations(locale: string) { + switch (locale) { + case 'en': + return orchestratorMessages; + case 'de': + return transform(orchestratorTranslationDe.messages); + case 'es': + return transform(orchestratorTranslationEs.messages); + case 'fr': + return transform(orchestratorTranslationFr.messages); + case 'it': + return transform(orchestratorTranslationIt.messages); + case 'ja': + return transform(orchestratorTranslationJa.messages); + default: + return orchestratorMessages; + } +} + +/** + * Replace multiple placeholders in a template string + * @param template - Template string with placeholders like {{key}} + * @param replacements - Object with key-value pairs for replacement + * @returns String with all placeholders replaced + */ +export function replaceTemplate( + template: string, + replacements: Record, +): string { + let result = template; + for (const [key, value] of Object.entries(replacements)) { + result = result.replaceAll(`{{${key}}}`, String(value)); + } + return result; +} diff --git a/workspaces/orchestrator/package.json b/workspaces/orchestrator/package.json index 953497679d..f1e78630db 100644 --- a/workspaces/orchestrator/package.json +++ b/workspaces/orchestrator/package.json @@ -17,8 +17,12 @@ "build:api-reports:only": "backstage-repo-tools api-reports --allow-all-warnings -o ae-wrong-input-file-type --validate-release-tags --exclude 'plugins/orchestrator-swf-editor-envelope'", "build:knip-reports": "backstage-repo-tools knip-reports", "clean": "backstage-cli repo clean", - "test": "backstage-cli repo test", - "test:all": "backstage-cli repo test --coverage", + "test": "backstage-cli repo test --detectOpenHandles", + "test:all": "backstage-cli repo test --coverage --detectOpenHandles", + "test:legacy": "APP_MODE=legacy playwright test", + "test:nfs": "APP_MODE=nfs playwright test", + "test:e2e:ci": "yarn test:legacy && yarn test:nfs", + "playwright": "sh -c 'if [ \"$1\" = test ] && [ $# -eq 1 ]; then yarn test:e2e:ci; else exec playwright \"$@\"; fi' _", "fix": "backstage-cli repo fix", "lint": "backstage-cli repo lint --since origin/main", "lint:all": "backstage-cli repo lint", @@ -46,6 +50,7 @@ "@changesets/cli": "^2.27.1", "@ianvs/prettier-plugin-sort-imports": "^4.4.0", "@jest/environment-jsdom-abstract": "^30.3.0", + "@playwright/test": "1.60.0", "@types/jest": "^30.0.0", "@types/jsdom": "^27.0.0", "jest": "^30.3.0", diff --git a/workspaces/orchestrator/packages/app-legacy/package.json b/workspaces/orchestrator/packages/app-legacy/package.json index a53607f43a..61ebc2f379 100644 --- a/workspaces/orchestrator/packages/app-legacy/package.json +++ b/workspaces/orchestrator/packages/app-legacy/package.json @@ -26,6 +26,7 @@ "@backstage/core-app-api": "^1.20.1", "@backstage/core-components": "^0.18.10", "@backstage/core-plugin-api": "^1.12.6", + "@backstage/frontend-plugin-api": "^0.17.1", "@backstage/integration-react": "^1.2.18", "@backstage/plugin-api-docs": "^0.14.1", "@backstage/plugin-catalog": "^2.0.5", @@ -62,8 +63,9 @@ "react-use": "^17.4.0" }, "devDependencies": { + "@axe-core/playwright": "^4.10.0", "@backstage/test-utils": "^1.7.18", - "@playwright/test": "1.58.2", + "@playwright/test": "1.60.0", "@testing-library/dom": "^9.0.0", "@testing-library/jest-dom": "^6.0.0", "@testing-library/react": "^14.0.0", diff --git a/workspaces/orchestrator/packages/app-legacy/src/apis.ts b/workspaces/orchestrator/packages/app-legacy/src/apis.ts index b43dcbd3bc..8375494242 100644 --- a/workspaces/orchestrator/packages/app-legacy/src/apis.ts +++ b/workspaces/orchestrator/packages/app-legacy/src/apis.ts @@ -19,6 +19,7 @@ import { configApiRef, createApiFactory, } from '@backstage/core-plugin-api'; +import { toastApiRef } from '@backstage/frontend-plugin-api'; import { ScmAuth, ScmIntegrationsApi, @@ -32,4 +33,11 @@ export const apis: AnyApiFactory[] = [ factory: ({ configApi }) => ScmIntegrationsApi.fromConfig(configApi), }), ScmAuth.createDefaultApiFactory(), + createApiFactory({ + api: toastApiRef, + deps: {}, + factory: () => ({ + post: () => ({ close: () => {} }), + }), + }), ]; diff --git a/workspaces/orchestrator/packages/app/package.json b/workspaces/orchestrator/packages/app/package.json index 33f232c13b..69e863a5e6 100644 --- a/workspaces/orchestrator/packages/app/package.json +++ b/workspaces/orchestrator/packages/app/package.json @@ -41,7 +41,9 @@ "react-router-dom": "^6.3.0" }, "devDependencies": { + "@axe-core/playwright": "^4.10.0", "@backstage/test-utils": "^1.7.18", + "@playwright/test": "1.60.0", "@testing-library/dom": "^9.0.0", "@testing-library/jest-dom": "^6.0.0", "@testing-library/react": "^14.0.0", diff --git a/workspaces/orchestrator/playwright.config.ts b/workspaces/orchestrator/playwright.config.ts new file mode 100644 index 0000000000..1a9f696e3d --- /dev/null +++ b/workspaces/orchestrator/playwright.config.ts @@ -0,0 +1,72 @@ +/* + * Copyright The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineConfig } from '@playwright/test'; + +const LOCALES = ['en', 'de', 'es', 'fr', 'it', 'ja'] as const; +// APP_MODE: 'legacy' (packages/app-legacy) or 'nfs' (packages/app with new frontend system) +const appMode = process.env.APP_MODE || 'legacy'; +const startCommand = appMode === 'legacy' ? 'yarn start:legacy' : 'yarn start'; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + timeout: 90 * 1000, + + expect: { + timeout: 10 * 1000, // Global expect timeout + }, + + // Run your local dev server before starting the tests + webServer: process.env.PLAYWRIGHT_URL + ? [] + : [ + { + command: startCommand, + port: 3000, + reuseExistingServer: false, + cwd: __dirname, + }, + ], + + forbidOnly: !!process.env.CI, + + retries: process.env.CI ? 2 : 0, + + reporter: [ + ['html', { open: 'never', outputFolder: `e2e-test-report-${appMode}` }], + ], + + use: { + actionTimeout: 0, + baseURL: process.env.PLAYWRIGHT_URL ?? 'http://localhost:3000', + screenshot: 'only-on-failure', + trace: 'on-first-retry', + }, + + outputDir: `node_modules/.cache/e2e-test-results-${appMode}`, + + testDir: 'e2e-tests', + + projects: LOCALES.map(locale => ({ + name: locale, + use: { + channel: 'chrome' as const, + locale, + }, + })), +}); diff --git a/workspaces/orchestrator/yarn.lock b/workspaces/orchestrator/yarn.lock index 0bfd608017..325e4a5d6a 100644 --- a/workspaces/orchestrator/yarn.lock +++ b/workspaces/orchestrator/yarn.lock @@ -1034,6 +1034,17 @@ __metadata: languageName: node linkType: hard +"@axe-core/playwright@npm:^4.10.0": + version: 4.11.3 + resolution: "@axe-core/playwright@npm:4.11.3" + dependencies: + axe-core: "npm:~4.11.4" + peerDependencies: + playwright-core: ">= 1.0.0" + checksum: 10c0/da1854726dbc461a71ac25e0435f5dd9b7d143dc9142f53b1aeb4a8d7edcb4533eddb59949e8a07c4f4e3dce85ae43b7f249b3801e8b255f605fc974b94616fe + languageName: node + linkType: hard + "@azure/abort-controller@npm:^2.0.0, @azure/abort-controller@npm:^2.1.2": version: 2.1.2 resolution: "@azure/abort-controller@npm:2.1.2" @@ -7852,6 +7863,7 @@ __metadata: "@changesets/cli": "npm:^2.27.1" "@ianvs/prettier-plugin-sort-imports": "npm:^4.4.0" "@jest/environment-jsdom-abstract": "npm:^30.3.0" + "@playwright/test": "npm:1.60.0" "@types/jest": "npm:^30.0.0" "@types/jsdom": "npm:^27.0.0" enquirer: "npm:^2.4.1" @@ -10813,14 +10825,14 @@ __metadata: languageName: node linkType: hard -"@playwright/test@npm:1.58.2": - version: 1.58.2 - resolution: "@playwright/test@npm:1.58.2" +"@playwright/test@npm:1.60.0": + version: 1.60.0 + resolution: "@playwright/test@npm:1.60.0" dependencies: - playwright: "npm:1.58.2" + playwright: "npm:1.60.0" bin: playwright: cli.js - checksum: 10c0/2164c03ad97c3653ff02e8818a71f3b2bbc344ac07924c9d8e31cd57505d6d37596015a41f51396b3ed8de6840f59143eaa9c21bf65515963da20740119811da + checksum: 10c0/86b06e6437933e741c7cd43f362024e857e7bc28a55fcbb0553ef55e01a2a403c64f4786868de8af86a6e303fe99e98a18a42ba19489f43ae122e457f9e2d189 languageName: node linkType: hard @@ -16930,6 +16942,7 @@ __metadata: version: 0.0.0-use.local resolution: "app-legacy@workspace:packages/app-legacy" dependencies: + "@axe-core/playwright": "npm:^4.10.0" "@backstage-community/plugin-rbac": "npm:^1.33.2" "@backstage/app-defaults": "npm:^1.7.8" "@backstage/catalog-model": "npm:^1.9.0" @@ -16937,6 +16950,7 @@ __metadata: "@backstage/core-app-api": "npm:^1.20.1" "@backstage/core-components": "npm:^0.18.10" "@backstage/core-plugin-api": "npm:^1.12.6" + "@backstage/frontend-plugin-api": "npm:^0.17.1" "@backstage/integration-react": "npm:^1.2.18" "@backstage/plugin-api-docs": "npm:^0.14.1" "@backstage/plugin-catalog": "npm:^2.0.5" @@ -16962,7 +16976,7 @@ __metadata: "@mui/lab": "npm:5.0.0-alpha.177" "@mui/material": "npm:5.18.0" "@mui/styles": "npm:5.18.0" - "@playwright/test": "npm:1.58.2" + "@playwright/test": "npm:1.60.0" "@red-hat-developer-hub/backstage-plugin-orchestrator": "workspace:^" "@red-hat-developer-hub/backstage-plugin-orchestrator-form-widgets": "workspace:^" "@red-hat-developer-hub/backstage-plugin-theme": "npm:^0.14.0" @@ -16999,6 +17013,7 @@ __metadata: version: 0.0.0-use.local resolution: "app@workspace:packages/app" dependencies: + "@axe-core/playwright": "npm:^4.10.0" "@backstage/cli": "npm:^0.36.2" "@backstage/core-compat-api": "npm:^0.5.11" "@backstage/core-components": "npm:^0.18.10" @@ -17013,6 +17028,7 @@ __metadata: "@backstage/ui": "npm:^0.15.0" "@mui/icons-material": "npm:^5.17.1" "@mui/material": "npm:^5.17.1" + "@playwright/test": "npm:1.60.0" "@red-hat-developer-hub/backstage-plugin-orchestrator": "workspace:^" "@red-hat-developer-hub/backstage-plugin-orchestrator-form-widgets": "workspace:^" "@red-hat-developer-hub/backstage-plugin-theme": "npm:^0.14.0" @@ -17520,10 +17536,10 @@ __metadata: languageName: node linkType: hard -"axe-core@npm:^4.10.0": - version: 4.10.1 - resolution: "axe-core@npm:4.10.1" - checksum: 10c0/53d865efb7284fd69bc95ced1a1709fd603ea07f06e272da06942e7cfeca1c823e09bde28f57178e3a1a4c9a089fe4b5d274c871e3e6522a3b1bffec8eaa7dd8 +"axe-core@npm:^4.10.0, axe-core@npm:~4.11.4": + version: 4.11.4 + resolution: "axe-core@npm:4.11.4" + checksum: 10c0/c4aa83fc3eac5f7a0d0cb1a28f9d073acf0c06ce8daacc38608faa278c57ce084c028c850746b98817ae4c101c30c1a32e95ea34748c4b4c7419b9b81221ef84 languageName: node linkType: hard @@ -31185,27 +31201,27 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.58.2": - version: 1.58.2 - resolution: "playwright-core@npm:1.58.2" +"playwright-core@npm:1.60.0": + version: 1.60.0 + resolution: "playwright-core@npm:1.60.0" bin: playwright-core: cli.js - checksum: 10c0/5aa15b2b764e6ffe738293a09081a6f7023847a0dbf4cd05fe10eed2e25450d321baf7482f938f2d2eb330291e197fa23e57b29a5b552b89927ceb791266225b + checksum: 10c0/99ccd43923b6e9355e0723b7fe221e6326efd4687f8dafff951313662aea11db51f542a9c2122c704c445fb9baae1c9ec9fa6f895126bbddd9fe92313f6942c9 languageName: node linkType: hard -"playwright@npm:1.58.2": - version: 1.58.2 - resolution: "playwright@npm:1.58.2" +"playwright@npm:1.60.0": + version: 1.60.0 + resolution: "playwright@npm:1.60.0" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.58.2" + playwright-core: "npm:1.60.0" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/d060d9b7cc124bd8b5dffebaab5e84f6b34654a553758fe7b19cc598dfbee93f6ecfbdc1832b40a6380ae04eade86ef3285ba03aa0b136799e83402246dc0727 + checksum: 10c0/714ad76d85b4865d7e43c0012f9039800c1485373388973ed39d79339cee5ad467052d1e2f1eaeca107a1cb6e65342186a8578a4c3504853d84c3a691250d5db languageName: node linkType: hard