diff --git a/.gitignore b/.gitignore index 0e1605fad4..efd94139b2 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,5 @@ backups/ /playwright/.cache/ .claude + +load_testing/data/ diff --git a/load_testing/README.md b/load_testing/README.md index 48f993b2d1..eca8f0c80a 100644 --- a/load_testing/README.md +++ b/load_testing/README.md @@ -3,7 +3,7 @@ ### Usage (Docker) ```shell -./scripts/k6.sh -e BACKEND_BASE_URL=#### -e FRONTEND_BASE_URL=#### +./scripts/k6.sh run /app/learn.smoke.ts -e BACKEND_BASE_URL=#### -e FRONTEND_BASE_URL=#### ``` ### Usage (local k6) @@ -11,5 +11,25 @@ - Install [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) ```shell -k6 run learn.ts -e BACKEND_BASE_URL=#### -e FRONTEND_BASE_URL=#### +k6 run learn.smoke.ts -e BACKEND_BASE_URL=#### -e FRONTEND_BASE_URL=#### ``` + +### Available tests + +**Note: the numbers for average-load/stress should be updated over time** + +- `learn.smoke.ts` - for lightweight smoke testing of deployments (4 users) +- `learn.average-load.ts` - simulates an average amount of user load (100 users) +- `learn.stress.ts` - simulates a stressful amount of user load (200 users) + +### Environment Variables + +**Note: for base urls, if you access the services via a port, include the port number** + +| Name | Decription | Example value | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| `BACKEND_BASE_URL` | The base url to the backend API service. | `https://api.learn.odl.local` | +| `FRONTEND_BASE_URL` | The base url to the frontend service. | `https://learn.odl.local` | +| `SSO_BASE_URL` | The base url to the keycloak service. | `https://keycloak.odl.local` | +| `IGNORE_HTTPS_ERRORS` | Ignore https certificate errors. Only recommemded for local test certificates. | `true` | +| `USERS_JSON_FILE` | Data file for users auth info. Expected to be in the format `[{"email": "", "password": ""}, ...]`. Put this is the `data/` subdirectory as files in there are gitignored. | `data/users.json` | diff --git a/load_testing/auth.ts b/load_testing/auth.ts index 46b66ffc54..8c09b6a846 100644 --- a/load_testing/auth.ts +++ b/load_testing/auth.ts @@ -1,13 +1,45 @@ import { SharedArray } from "k6/data" -export function getAccessToken(): String { +export type AuthCredential = { + email: string + password: string +} + +export function getAccessToken(): string | null { return __ENV.AUTH_ACCESS_TOKEN } -export const users = new SharedArray("users", function () { - if (!__ENV.USERS_JSON_FILE) { - return [] +export function hasAccessToken(): boolean { + return !getAccessToken() +} + +function _validate_credentials(credentials) { + if (!Array.isArray(credentials)) { + throw Error("Expected an array of credentials") + } + + for (let index = 0; index < credentials.length; index++) { + const credential = credentials[index] + if (!Object.hasOwn(credential, "email")) { + throw Error(`User entry is missing 'email' at index ${index}`) + } + if (!Object.hasOwn(credential, "password")) { + throw Error(`User entry is missing 'password' at index ${index}`) + } } +} + +export const credentials: AuthCredential[] = new SharedArray( + "credentials", + function () { + if (!__ENV.USERS_JSON_FILE) { + return [] + } + + const parsed = JSON.parse(open(__ENV.USERS_JSON_FILE)) + + _validate_credentials(parsed) - return JSON.parse(open(__ENV.USERS_JSON_FILE)) -}) + return parsed + }, +) diff --git a/load_testing/backend/test.ts b/load_testing/backend/test.ts index 105ce4c166..35f533151f 100644 --- a/load_testing/backend/test.ts +++ b/load_testing/backend/test.ts @@ -8,6 +8,7 @@ import { import { createV0Client, createV1Client } from "./client/client.ts" import { NewsEventsListParams } from "./client/v0/api.schemas.ts" import { FeaturedListParams } from "./client/v1/api.schemas.ts" +import { hasAccessToken } from "../auth.ts" export function testBackend() { group("api", function () { @@ -15,23 +16,32 @@ export function testBackend() { exec.vu.metrics.tags.apiVersion = "v0" const client = createV0Client() - let res = client.videoShortsList({ limit: 50 }) + group("video shorts", () => { + const res = client.videoShortsList({ limit: 50 }) - check(res, { - "is status 200": (r) => r.response.status === 200, - "has results": (r) => r.response.json("results").length > 0, + group + check(res, { + "is status 200": (r) => r.response.status === 200, + "has results": (r) => r.response.json("results").length > 0, + }) }) - res = client.usersMeRetrieve() - check(res, { - "is status 200": (r) => r.response.status === 200, + group("users/me", () => { + const res = client.usersMeRetrieve() + check(res, { + "is status 200": (r) => r.response.status === 200, + "expected auth state": (r) => + r.response.json("is_authenticated") === hasAccessToken(), + }) }) - res = client.testimonialsList({ position: 1 }) + group("testimonials", () => { + const res = client.testimonialsList({ position: 1 }) - check(res, { - "is status 200": (r) => r.response.status === 200, - "has results": (r) => r.response.json("results").length > 0, + check(res, { + "is status 200": (r) => r.response.status === 200, + "has results": (r) => r.response.json("results").length > 0, + }) }) group("news", function () { diff --git a/load_testing/config.ts b/load_testing/config.ts index d000f4bcf5..d4188b7124 100644 --- a/load_testing/config.ts +++ b/load_testing/config.ts @@ -1,10 +1,18 @@ import { NewBrowserContextOptions } from "k6/browser" -export const BACKEND_BASE_URL: string = __ENV.BACKEND_BASE_URL -export const FRONTEND_BASE_URL: string = __ENV.FRONTEND_BASE_URL +export const BACKEND_BASE_URL: string = __ENV.BACKEND_BASE_URL?.replace( + /\/$/, + "", +) +export const FRONTEND_BASE_URL: string = __ENV.FRONTEND_BASE_URL?.replace( + /\/$/, + "", +) +export const SSO_BASE_URL: string = __ENV.SSO_BASE_URL?.replace(/\/$/, "") + +export const IGNORE_HTTPS_ERRORS: boolean = + (__ENV.IGNORE_HTTPS_ERRORS || "false").toLowerCase() == "true" export const BROWSER_CONTEXT_OPTIONS: NewBrowserContextOptions = { - ignoreHTTPSErrors: Object.hasOwn(__ENV, "BROWSER_IGNORE_HTTPS_ERRORS") - ? __ENV.BROWSER_IGNORE_HTTPS_ERRORS - : false, + ignoreHTTPSErrors: IGNORE_HTTPS_ERRORS, } diff --git a/load_testing/data/.keep b/load_testing/data/.keep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/load_testing/frontend/test.ts b/load_testing/frontend/test.ts index 9d6aa2e5bd..479ea19b7e 100644 --- a/load_testing/frontend/test.ts +++ b/load_testing/frontend/test.ts @@ -1,15 +1,29 @@ import { check } from "k6" import { browser, Page } from "k6/browser" -import { randomIntBetween } from "https://jslib.k6.io/k6-utils/1.2.0/index.js" -import { BROWSER_CONTEXT_OPTIONS, FRONTEND_BASE_URL } from "../config.ts" +import { + randomIntBetween, + randomItem, +} from "https://jslib.k6.io/k6-utils/1.2.0/index.js" +import { + BROWSER_CONTEXT_OPTIONS, + FRONTEND_BASE_URL, + SSO_BASE_URL, +} from "../config.ts" +import { AuthCredential, credentials } from "../auth.ts" +import { escapeRegex } from "../utils.ts" -async function home(page: Page) { +type Context = { + loggedIn: boolean + credential: AuthCredential | null +} + +async function home(page: Page, context: Context) { await page.goto(FRONTEND_BASE_URL) const carousel = await page.getByTestId("resource-carousel") const articlesCount = await carousel.locator("article").count() await check(carousel, { - "has 12 items": () => articlesCount === 12, + "home page carousel has 12 items": () => articlesCount === 12, }) await carousel @@ -18,7 +32,7 @@ async function home(page: Page) { .click() } -async function search(page: Page) { +async function search(page: Page, context: Context) { await page.goto(`${FRONTEND_BASE_URL}/search?sortby=-views`) await page.goto(`${FRONTEND_BASE_URL}/search?resource_type_group=program`) await page.goto( @@ -26,27 +40,114 @@ async function search(page: Page) { ) } -async function topics(page: Page) { +async function topics(page: Page, context: Context) { await page.goto(`${FRONTEND_BASE_URL}/topics`) } -async function departments(page: Page) { +async function departments(page: Page, context: Context) { await page.goto(`${FRONTEND_BASE_URL}/departments`) } -async function units(page: Page) { +async function units(page: Page, context: Context) { await page.goto(`${FRONTEND_BASE_URL}/units`) } +async function login(page: Page, context: Context) { + const credential: AuthCredential = randomItem(credentials) + + if (credential == null) { + console.log("Login > skipping because no credentials provided") + } + if (page.url() == "about:blank") { + await page.goto(FRONTEND_BASE_URL) + } + + console.log(`Login > using '${credential.email}'`) + + const loginButton = page.getByTestId("login-button-desktop") + await loginButton.first().click() + + await loginKeycloak(page, credential, context) + + await page.waitForURL(/.*\/dashboard.*/) + + console.log("Login > reached dashboard") +} + +const ESCAPED_SSO_URL = escapeRegex(SSO_BASE_URL) + +const KEYCLOAK_USERNAME_URL_RE = new RegExp( + `${ESCAPED_SSO_URL}\/realms\/[a-z-]+\/protocol\/openid-connect\/auth.*`, +) +const KEYCLOAK_PASSWORD_URL_RE = new RegExp( + `${ESCAPED_SSO_URL}\/realms\/[a-z-]+\/login-actions\/authenticate.*`, +) + +async function loginKeycloak( + page: Page, + credential: AuthCredential, + context: Context, +) { + await page.waitForURL(KEYCLOAK_USERNAME_URL_RE, { + waitUntil: "load", + }) + + console.log("Login > Keycloak > on login email page") + + const credentialnameInput = await page.locator("input[name=username]") + console.log("Login > Keycloak > found username input") + await credentialnameInput.focus() + await credentialnameInput.fill(credential.email) + console.log("Login > Keycloak > entered email address") + + await page.locator("button[type=submit]").click() + console.log("Login > Keycloak > submitting email address") + + await page.waitForNavigation(KEYCLOAK_PASSWORD_URL_RE) + console.log("Login > Keycloak > on login password page") + + const passwordInput = await page.locator("input[name=password]") + console.log("Login > Keycloak > found password input") + await passwordInput.focus() + await passwordInput.fill(credential.password, { + force: true, + }) + await page.locator("button[type=submit]").click() + console.log("Login > Keycloak > submitting password") + await page.waitForNavigation() + + context.loggedIn = true + context.credential = credential +} + +async function dashboard(page: Page, context: Context) { + if (!context.loggedIn) { + return + } + await page.goto(`${FRONTEND_BASE_URL}/dashboard`) +} + export async function testFrontend() { const page = await browser.newPage(BROWSER_CONTEXT_OPTIONS) + const context: Context = { + loggedIn: false, + credential: null, + } try { - await home(page) - await search(page) - await topics(page) - await departments(page) - await units(page) + // always home page first + await home(page, context) + await search(page, context) + await topics(page, context) + await departments(page, context) + await units(page, context) + await login(page, context) + await dashboard(page, context) + await search(page, context) + await topics(page, context) + await departments(page, context) + await units(page, context) + await home(page, context) } finally { await page.close() } diff --git a/load_testing/learn.average-load.ts b/load_testing/learn.average-load.ts new file mode 100644 index 0000000000..36905de3ab --- /dev/null +++ b/load_testing/learn.average-load.ts @@ -0,0 +1,44 @@ +import { IGNORE_HTTPS_ERRORS } from "./config.ts" + +export { testBackend } from "./backend/test.ts" +export { testFrontend } from "./frontend/test.ts" + +const MAX_VUS = 100 +const BROWSER_VU_SHARE = 0.5 +const BACKEND_VU_SHARE = 1 - BROWSER_VU_SHARE + +const BROWSER_VUS = Math.floor(MAX_VUS * BROWSER_VU_SHARE) +const BACKEND_VUS = Math.floor(MAX_VUS * BACKEND_VU_SHARE) + +export const options = { + scenarios: { + browser: { + exec: "testFrontend", + executor: "ramping-vus", + options: { + browser: { + type: "chromium", + }, + }, + stages: [ + { duration: "5m", target: BROWSER_VUS }, + { duration: "30m", target: BROWSER_VUS }, + { duration: "5m", target: 0 }, + ], + }, + backend: { + exec: "testBackend", + executor: "ramping-vus", + stages: [ + { duration: "5m", target: BACKEND_VUS }, + { duration: "30m", target: BACKEND_VUS }, + { duration: "5m", target: 0 }, + ], + }, + }, + thresholds: { + // the rate of successful checks should be higher than 90% + checks: ["rate>0.9"], + }, + insecureSkipTLSVerify: IGNORE_HTTPS_ERRORS, +} diff --git a/load_testing/learn.ts b/load_testing/learn.smoke.ts similarity index 59% rename from load_testing/learn.ts rename to load_testing/learn.smoke.ts index bc8f78a81e..649a4d0577 100644 --- a/load_testing/learn.ts +++ b/load_testing/learn.smoke.ts @@ -1,13 +1,19 @@ +import { IGNORE_HTTPS_ERRORS } from "./config.ts" + export { testBackend } from "./backend/test.ts" export { testFrontend } from "./frontend/test.ts" +const MAX_VUS = 4 +const BROWSER_VU_SHARE = 0.5 +const BACKEND_VU_SHARE = 1 - BROWSER_VU_SHARE + export const options = { scenarios: { browser: { exec: "testFrontend", executor: "constant-vus", - vus: 10, - duration: "30s", + vus: Math.floor(MAX_VUS * BROWSER_VU_SHARE), + duration: "1m", options: { browser: { type: "chromium", @@ -17,12 +23,13 @@ export const options = { backend: { exec: "testBackend", executor: "constant-vus", - vus: 10, - duration: "30s", + vus: Math.floor(MAX_VUS * BACKEND_VU_SHARE), + duration: "1m", }, }, thresholds: { // the rate of successful checks should be higher than 90% checks: ["rate>0.9"], }, + insecureSkipTLSVerify: IGNORE_HTTPS_ERRORS, } diff --git a/load_testing/learn.stress.ts b/load_testing/learn.stress.ts new file mode 100644 index 0000000000..1f648c76c9 --- /dev/null +++ b/load_testing/learn.stress.ts @@ -0,0 +1,44 @@ +import { IGNORE_HTTPS_ERRORS } from "./config.ts" + +export { testBackend } from "./backend/test.ts" +export { testFrontend } from "./frontend/test.ts" + +const MAX_VUS = 200 +const BROWSER_VU_SHARE = 0.5 +const BACKEND_VU_SHARE = 1 - BROWSER_VU_SHARE + +const BROWSER_VUS = Math.floor(MAX_VUS * BROWSER_VU_SHARE) +const BACKEND_VUS = Math.floor(MAX_VUS * BACKEND_VU_SHARE) + +export const options = { + scenarios: { + browser: { + exec: "testFrontend", + executor: "ramping-vus", + options: { + browser: { + type: "chromium", + }, + }, + stages: [ + { duration: "5m", target: BROWSER_VUS }, + { duration: "30m", target: BROWSER_VUS }, + { duration: "5m", target: 0 }, + ], + }, + backend: { + exec: "testBackend", + executor: "ramping-vus", + stages: [ + { duration: "5m", target: BACKEND_VUS }, + { duration: "30m", target: BACKEND_VUS }, + { duration: "5m", target: 0 }, + ], + }, + }, + thresholds: { + // the rate of successful checks should be higher than 90% + checks: ["rate>0.9"], + }, + insecureSkipTLSVerify: IGNORE_HTTPS_ERRORS, +} diff --git a/load_testing/utils.ts b/load_testing/utils.ts new file mode 100644 index 0000000000..199065a5db --- /dev/null +++ b/load_testing/utils.ts @@ -0,0 +1,3 @@ +export function escapeRegex(str: string): string { + return str.replace(/[\.\+\*\$\?\^\(\)\{\}\\\[\]\|]/g, "\\$&") +} diff --git a/scripts/k6.sh b/scripts/k6.sh index 80c01807b6..5095e0b288 100755 --- a/scripts/k6.sh +++ b/scripts/k6.sh @@ -11,4 +11,4 @@ docker run --rm -ti \ -v $ROOT_DIR/load_testing:/app \ --add-host learn.odl.local:host-gateway \ grafana/k6:master-with-browser \ - run /app/learn.ts "$@" + run "$@"