feat(e2e): add plugin sanity check test (RHIDP-13508)

.ci/pipelines/env_variables.sh

@@ -56,9 +56,10 @@ IMAGE_REPO="${IMAGE_REPO:-${QUAY_REPO:-rhdh-community/rhdh}}"
 QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility
 
 # Catalog index image reference.
+# Defaults to the nightly catalog index for the current release branch.
 # Override via Gangway for RC (e.g., --catalog-index-image quay.io/rhdh/plugin-catalog-index:1.9-60) or
 # GA verification (e.g., --catalog-index-image registry.access.redhat.com/rhdh/plugin-catalog-index:1.9.4).
-CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}"
+CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-quay.io/rhdh/plugin-catalog-index:${RELEASE_VERSION}}"
 if [[ -n "${CATALOG_INDEX_IMAGE}" ]]; then
   # Derived components for Helm chart (requires separate registry/repository/tag)
   CATALOG_INDEX_TAG="${CATALOG_INDEX_IMAGE##*:}"

e2e-tests/package.json

@@ -3,13 +3,17 @@
   "version": "1.11.0",
   "private": true,
   "type": "module",
+  "engines": {
+    "node": "24"
+  },
   "scripts": {
     "showcase": "playwright test --project=showcase",
     "showcase-rbac": "playwright test --project=showcase-rbac",
     "showcase-k8s": "playwright test --project=showcase-k8s",
     "showcase-rbac-k8s": "playwright test --project=showcase-rbac-k8s",
     "showcase-operator": "playwright test --project=showcase-operator",
     "showcase-operator-rbac": "playwright test --project=showcase-operator-rbac",
+    "showcase-runtime-db": "playwright test --project=showcase-runtime-db",
     "showcase-runtime": "playwright test --project=showcase-runtime",
     "showcase-upgrade": "playwright test --project=showcase-upgrade",
     "showcase-auth-providers": "playwright test --project=showcase-auth-providers",
@@ -19,13 +23,39 @@
     "showcase-localization-fr": "LOCALE=fr playwright test --project=showcase-localization-fr",
     "showcase-localization-it": "LOCALE=it playwright test --project=showcase-localization-it",
     "showcase-localization-ja": "LOCALE=ja playwright test --project=showcase-localization-ja",
-    "lint": "oxlint .",
-    "lint:fix": "oxlint --fix .",
-    "test:list": "playwright test --list",
-    "fmt": "oxfmt .",
-    "fmt:check": "oxfmt --check .",
+    "lint:check": "eslint . --ext .js,.ts",
+    "lint:fix": "eslint . \"playwright/**/*.{ts,js}\" --fix",
     "postinstall": "playwright install chromium",
-    "shellcheck": "git ls-files -z '*.sh' | xargs -0 shellcheck --severity=warning --color=always"
+    "tsc": "tsc",
+    "tsc:check": "tsc -p tsconfig.json",
+    "shellcheck": "git ls-files -z '*.sh' | xargs -0 shellcheck --severity=warning --color=always",
+    "prettier:check": "prettier --ignore-unknown --check .",
+    "prettier:fix": "prettier --ignore-unknown --write ."
+  },
+  "devDependencies": {
+    "@axe-core/playwright": "4.11.2",
+    "@backstage/backend-test-utils": "^1.11.4",
+    "@backstage/plugin-catalog-backend": "3.5.0",
+    "@backstage/plugin-scaffolder-backend": "3.3.0",
+    "@eslint/js": "9.39.4",
+    "@microsoft/microsoft-graph-types": "2.43.1",
+    "@playwright/test": "1.59.1",
+    "@red-hat-developer-hub/cli-module-install-dynamic-plugins": "^0.2.0",
+    "@types/node": "24.12.2",
+    "@types/pg": "8.20.0",
+    "@typescript-eslint/eslint-plugin": "8.59.4",
+    "@typescript-eslint/parser": "8.59.4",
+    "eslint": "9.39.4",
+    "eslint-plugin-check-file": "3.3.1",
+    "eslint-plugin-playwright": "2.10.4",
+    "ioredis": "5.10.1",
+    "monocart-coverage-reports": "2.12.11",
+    "otplib": "12.0.1",
+    "prettier": "3.8.3",
+    "prettier-plugin-sh": "0.18.1",
+    "shellcheck": "4.1.0",
+    "typescript": "5.9.3",
+    "typescript-eslint": "8.59.4"
   },
   "dependencies": {
     "@azure/arm-network": "34.2.0",
@@ -39,29 +69,8 @@
     "octokit": "4.1.4",
     "pg": "8.22.0",
     "uuid": "14.0.0",
-    "winston": "3.14.2"
-  },
-  "devDependencies": {
-    "@axe-core/playwright": "4.11.3",
-    "@microsoft/microsoft-graph-types": "2.43.1",
-    "@playwright/test": "1.61.0",
-    "@types/js-yaml": "4.0.9",
-    "@types/node": "24.13.2",
-    "@types/node-fetch": "2.6.13",
-    "@types/pg": "8.20.0",
-    "eslint-plugin-check-file": "3.3.1",
-    "eslint-plugin-playwright": "2.10.4",
-    "ioredis": "5.11.1",
-    "monocart-coverage-reports": "2.12.12",
-    "otplib": "12.0.1",
-    "oxfmt": "0.56.0",
-    "oxlint": "1.71.0",
-    "oxlint-tsgolint": "0.23.0",
-    "shellcheck": "4.1.0",
-    "typescript": "6.0.3"
-  },
-  "engines": {
-    "node": "24"
+    "winston": "3.14.2",
+    "yaml": "2.9.0"
   },
   "packageManager": "yarn@4.12.0"
 }

e2e-tests/playwright.config.ts

@@ -98,6 +98,7 @@ export default defineConfig({
         "**/playwright/e2e/external-database/verify-tls-config-with-external-azure-db.spec.ts",
         "**/playwright/e2e/plugin-division-mode-schema/*.spec.ts",
         "**/playwright/e2e/configuration-test/config-map.spec.ts",
+        "**/playwright/e2e/plugin-dynamic-loading.spec.ts",
       ],
     },
     {
@@ -195,6 +196,7 @@ export default defineConfig({
         "**/playwright/e2e/home-page-customization.spec.ts",
         "**/playwright/e2e/plugins/frontend/sidebar.spec.ts",
         "**/playwright/e2e/instance-health-check.spec.ts",
+        "**/playwright/e2e/plugin-dynamic-loading.spec.ts",
       ],
     },
     {

e2e-tests/playwright/e2e/plugin-dynamic-loading.spec.ts

@@ -0,0 +1,250 @@
+/**
+ * Plugin Dynamic Loading Test (Comprehensive Loading Validation)
+ *
+ * This is a COMPREHENSIVE test that actually downloads and loads plugins from the
+ * catalog index, validating they work with a real Backstage backend. This is the
+ * "full validation" counterpart to plugin-sanity-check.spec.ts.
+ *
+ * Test Strategy:
+ * 1. Download plugins from catalog index using @red-hat-developer-hub/cli-module-install-dynamic-plugins
+ * 2. Load backend plugins and verify they have valid default exports
+ * 3. Start test backend with @backstage/backend-test-utils (validates plugins actually work)
+ * 4. Validate frontend plugins have required bundle artifacts
+ *
+ * Runtime: ~3 minutes for extraction + ~2 seconds for backend startup validation.
+ *
+ * IMPORTANT: This test provides comprehensive validation that complements
+ * plugin-sanity-check.spec.ts:
+ * - plugin-sanity-check.spec.ts: Fast format validation (~seconds)
+ * - This test: Full loading validation (~3 minutes)
+ *
+ * Both tests run in nightly CI and catch different types of issues:
+ * - Format/structure errors → caught by plugin-sanity-check.spec.ts
+ * - Loading/runtime errors → caught by this test
+ *
+ * Based on POC from PR #4523 but modernized to use @red-hat-developer-hub/cli-module-install-dynamic-plugins
+ * instead of the Python script.
+ */
+
+import { test, expect } from "@support/coverage/test";
+import { startTestBackend, mockServices } from "@backstage/backend-test-utils";
+import catalogPlugin from "@backstage/plugin-catalog-backend";
+import scaffolderPlugin from "@backstage/plugin-scaffolder-backend";
+import { mkdtemp, rm, writeFile, mkdir } from "fs/promises";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { tmpdir } from "os";
+import { execSync } from "child_process";
+import {
+  loadManifest,
+  loadBackendPlugins,
+  validateFrontendBundle,
+} from "../utils/plugin-loader";
+import { buildMergedConfig, KNOWN_FAILURES } from "../utils/plugin-config";
+import type { PluginError } from "../utils/plugin-types";
+import {
+  reportCatalogIndex,
+  reportDownloadStarted,
+  reportDownloadCompleted,
+  reportCliVerification,
+  reportCliCommand,
+  reportCliFailure,
+  reportManifestLoaded,
+  reportBackendLoadingStarted,
+  reportLoadErrors,
+  reportBackendStartupStarted,
+  reportBackendSuccess,
+  reportStartupFailure,
+  reportFrontendValidationStarted,
+  reportFrontendValidation,
+  reportSummary,
+} from "../utils/plugin-reporter";
+import { patchModuleResolution } from "../utils/module-resolution-patch";
+
+// eslint-disable-next-line @typescript-eslint/naming-convention -- ESM compatibility requires __filename/__dirname
+const __filename = fileURLToPath(import.meta.url);
+// eslint-disable-next-line @typescript-eslint/naming-convention -- ESM compatibility requires __filename/__dirname
+const __dirname = dirname(__filename);
+
+// Patch module resolution once before all tests.
+// NOTE: Safe because showcase-sanity-plugins runs serially (no parallel workers).
+patchModuleResolution(join(__dirname, "..", "..", "node_modules"));
+
+const coreFeatures = [catalogPlugin, scaffolderPlugin];
+
+test.describe("Plugin Dynamic Loading", () => {
+  test.beforeAll(async () => {
+    test.info().annotations.push({
+      type: "component",
+      description: "plugins",
+    });
+  });
+
+  test(
+    "All plugins from catalog index load and backend starts",
+    { tag: "@sanity" },
+    async ({}, testInfo) => {
+      // Skip test if CATALOG_INDEX_IMAGE is not set
+      // This test requires the catalog index to download plugins from.
+      // In nightly CI (showcase-sanity-plugins), this env var is always set.
+      // In PR checks (showcase), this test is excluded via testIgnore.
+      if (!process.env.CATALOG_INDEX_IMAGE) {
+        testInfo.skip(
+          true,
+          "CATALOG_INDEX_IMAGE not set - skipping external catalog download. " +
+            "This test only runs in nightly jobs where CATALOG_INDEX_IMAGE is configured.",
+        );
+        return;
+      }
+
+      // 5 minutes timeout: ~3 min plugin download + ~2s backend startup + 2 min buffer
+      test.setTimeout(300_000);
+
+      // Get catalog index image from environment (now guaranteed to exist)
+      const catalogIndexImage = process.env.CATALOG_INDEX_IMAGE;
+
+      reportCatalogIndex(catalogIndexImage);
+
+      // Create temporary directories
+      const tempDir = await mkdtemp(join(tmpdir(), "rhdh-plugin-test-"));
+      const dynamicPluginsRoot = join(tempDir, "dynamic-plugins-root");
+
+      try {
+        // Step 1: Create minimal dynamic-plugins.yaml to trigger catalog index extraction
+        await mkdir(dynamicPluginsRoot, { recursive: true });
+
+        // Empty plugins list triggers catalog index extraction via CATALOG_INDEX_IMAGE env var
+        const dynamicPluginsConfig = `plugins: []`;
+        await writeFile(
+          join(dynamicPluginsRoot, "dynamic-plugins.yaml"),
+          dynamicPluginsConfig,
+        );
+
+        reportDownloadStarted();
+
+        // Step 2: Verify CLI package is available
+        try {
+          const cliVersion = execSync(
+            "npx @red-hat-developer-hub/cli-module-install-dynamic-plugins --version",
+            { encoding: "utf-8", stdio: "pipe" },
+          ).trim();
+          reportCliVerification(cliVersion);
+        } catch (versionError) {
+          console.error("❌ CLI not available:", versionError.message);
+          throw new Error(
+            `CLI @red-hat-developer-hub/cli-module-install-dynamic-plugins not available`,
+          );
+        }
+
+        // Step 3: Run install-dynamic-plugins
+        const installCmd = `npx @red-hat-developer-hub/cli-module-install-dynamic-plugins install ${dynamicPluginsRoot}`;
+
+        reportCliCommand(installCmd, catalogIndexImage);
+
+        try {
+          execSync(installCmd, {
+            env: {
+              ...process.env,
+              CATALOG_INDEX_IMAGE: catalogIndexImage,
+            },
+            stdio: "inherit",
+          });
+        } catch (error) {
+          const exitCode = error.status || error.code || "unknown";
+          reportCliFailure(exitCode);
+          throw new Error(
+            `install-dynamic-plugins failed (exit ${exitCode}). Check logs above for details.`,
+          );
+        }
+
+        reportDownloadCompleted();
+
+        // Step 4: Load manifest
+        const manifest = loadManifest(dynamicPluginsRoot);
+        reportManifestLoaded(manifest.backend.length, manifest.frontend.length);
+
+        // Filter out known failures
+        const backendPlugins = manifest.backend.filter(
+          (p) => !KNOWN_FAILURES.has(p.dirName),
+        );
+        const frontendPlugins = manifest.frontend.filter(
+          (p) => !KNOWN_FAILURES.has(p.dirName),
+        );
+
+        // Step 5: Load backend plugins
+        reportBackendLoadingStarted(backendPlugins.length);
+        const { loaded, errors: loadErrors } =
+          loadBackendPlugins(backendPlugins);
+
+        reportLoadErrors(loadErrors);
+        expect(loaded.length).toBeGreaterThan(0);
+
+        // Step 6: Build config and start test backend
+        reportBackendStartupStarted();
+        const config = buildMergedConfig(loaded);
+        const features = [
+          ...coreFeatures,
+          ...loaded.map((p) => p.feature),
+          mockServices.rootConfig.factory({ data: config }),
+        ];
+
+        let backend;
+        try {
+          backend = await startTestBackend({
+            features,
+          });
+
+          reportBackendSuccess(loaded);
+
+          // Stop backend
+          await backend.stop();
+        } catch (err) {
+          reportStartupFailure(err, loaded, config);
+          throw err;
+        }
+
+        // Fail test if there were load errors
+        if (loadErrors.length > 0) {
+          throw new Error(
+            `${loadErrors.length} plugin(s) failed to load:\n` +
+              loadErrors
+                .map((e) => `  - ${e.plugin.name}: ${e.error}`)
+                .join("\n"),
+          );
+        }
+
+        // Step 7: Validate frontend plugins
+        reportFrontendValidationStarted(frontendPlugins.length);
+        const frontendErrors: PluginError[] = [];
+        const validFrontend: { name: string; version: string }[] = [];
+
+        for (const plugin of frontendPlugins) {
+          const error = validateFrontendBundle(plugin);
+          if (error) {
+            frontendErrors.push({ plugin, error });
+          } else {
+            validFrontend.push({ name: plugin.name, version: plugin.version });
+          }
+        }
+
+        reportFrontendValidation(
+          frontendPlugins.length,
+          frontendErrors,
+          validFrontend,
+        );
+
+        expect(frontendErrors).toEqual([]);
+
+        // Step 8: Report summary
+        reportSummary(manifest, loaded.length, validFrontend.length);
+
+        expect(
+          manifest.backend.length + manifest.frontend.length,
+        ).toBeGreaterThan(0);
+      } finally {
+        // Cleanup
+        await rm(tempDir, { recursive: true, force: true });
+      }
+    },
+  );
+});