diff --git a/scalability/2_curiosity.js b/scalability/2_curiosity.js new file mode 100644 index 0000000..9309d07 --- /dev/null +++ b/scalability/2_curiosity.js @@ -0,0 +1,66 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { queryUsersList, queryUserById } from "./workflows/querier.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + submitters: { + executor: "constant-vus", + vus: 10, + duration: "2m", + exec: "submitRequests", + }, + auditors: { + executor: "ramping-arrival-rate", + startTime: "2m", + startRate: 100, + timeUnit: "1s", + preAllocatedVUs: 200, + maxVUs: 500, + stages: [ + // generate ~60k requests in the 2-minute sustained burst (500 req/s × 120s) + { duration: "30s", target: 500 }, // ramp to 500 req/s + { duration: "2m", target: 500 }, // sustain ~60k requests + { duration: "30s", target: 10 }, // cool down + ], + exec: "auditorRead", + }, + }, +}; + +export function setup() { + let customers = []; + let userIds = []; + for (let i = 0; i < 5; i++) customers.push(crypto.randomUUID()); + for (let i = 0; i < 50; i++) userIds.push(crypto.randomUUID()); + return { + customers: customers, + userIds: userIds, + tag: "curiosity", + generator: "normal", + urgentRatio: 0.05, + }; +} + +export function submitRequests(data) { + submitter(data); + sleep(5); +} + +export function auditorRead(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let customer = data.customers[0]; + + // Query list of users for the customer + let res = queryUsersList(host, customer); + sleep(1); + + // Query a random user discovered from the list endpoint + if (res.users && res.users.length > 0) { + let user = res.users[Math.floor(Math.random() * res.users.length)]; + queryUserById(host, customer, user.id); + } + sleep(1); +} diff --git a/scalability/3_compliance_early.js b/scalability/3_compliance_early.js index bd977b3..cedceea 100644 --- a/scalability/3_compliance_early.js +++ b/scalability/3_compliance_early.js @@ -55,7 +55,6 @@ export function setup() { tag: "compliance_early", generator: "normal", urgentRatio: 0.05, - photoComplexity: 4, }; } diff --git a/scalability/4_compliance_mid.js b/scalability/4_compliance_mid.js new file mode 100644 index 0000000..c83e32f --- /dev/null +++ b/scalability/4_compliance_mid.js @@ -0,0 +1,81 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { + queryRequestsList, + queryStats, + queryBatchUserResults, +} from "./workflows/querier.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + submitters: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "2m", target: 5 }, // ramp up to 5 VUs, at ~10s/iteration -> 6 iterations per minute per VU, so 6 * 5 * 2 = 60 iterations (POST + poll GET) in the first 2 minute + { duration: "3m", target: 50 }, // ramp up to 50 VUs, so 6 * 50 * 3 = 900 iterations in the next 3 minutes + { duration: "2m", target: 10 }, // ramp down to 10 VUs, so 6 * 10 * 2 = 120 iterations in the next 2 minutes + { duration: "3m", target: 2 }, // ramp down to 2 VUs, so 6 * 2 * 3 = 36 iterations in the last 3 minutes + ], + exec: "submitRequests", + }, + readers: { + executor: "ramping-vus", + startVUs: 0, + startTime: "1m", + stages: [ + { duration: "2m", target: 3 }, + { duration: "3m", target: 15 }, + { duration: "2m", target: 15 }, + { duration: "3m", target: 3 }, + ], + exec: "readResults", + }, + auditors: { + executor: "constant-vus", + startTime: "1m", + vus: 5, + duration: "10m", + exec: "auditBatch", + }, + }, +}; + +export function setup() { + let customers = []; + let userIds = []; + for (let i = 0; i < 15; i++) customers.push(crypto.randomUUID()); + for (let i = 0; i < 60; i++) userIds.push(crypto.randomUUID()); + return { + customers: customers, + userIds: userIds, + tag: "compliance_mid", + generator: "normal", + urgentRatio: 0.15, + }; +} + +export function submitRequests(data) { + submitter(data); + sleep(5); +} + +export function readResults(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + + queryRequestsList(host, customer); + sleep(1); + queryStats(host, customer); + sleep(5); +} + +export function auditBatch(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let batch = data.customers.slice(0, 5); + queryBatchUserResults(host, batch); + sleep(5); +} diff --git a/scalability/5_compliance_peak.js b/scalability/5_compliance_peak.js new file mode 100644 index 0000000..d0e0228 --- /dev/null +++ b/scalability/5_compliance_peak.js @@ -0,0 +1,97 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { + pollPendingRequests, + queryBatchUserResults, + queryUserResults, + queryRequestsList, +} from "./workflows/querier.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + submitters: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "1m", target: 5 }, // ramp up to 5 VUs, at ~10s/iteration -> 6 iterations per minute per VU, so 6 * 5 * 1 = 30 iterations (POST + poll GET) in the first minute + { duration: "2m", target: 40 }, // ramp up to 40 VUs, so 6 * 40 * 2 = 480 iterations in the next 2 minutes + { duration: "2m", target: 40 }, // stay at 40 VUs, so 6 * 40 * 2 = 480 iterations in the next 2 minutes + { duration: "1m", target: 2 }, // ramp down to 2 VUs, so 6 * 2 * 1 = 12 iterations in the last minute + ], + exec: "submitRequests", + }, + readers: { + executor: "ramping-vus", + startVUs: 0, + startTime: "1m", + stages: [ + { duration: "1m", target: 5 }, + { duration: "4m", target: 60 }, + { duration: "4m", target: 60 }, + { duration: "1m", target: 10 }, + ], + exec: "readResults", + }, + auditors: { + executor: "ramping-vus", + startVUs: 0, + startTime: "1m", + stages: [ + { duration: "2m", target: 5 }, + { duration: "4m", target: 15 }, + { duration: "3m", target: 15 }, + { duration: "1m", target: 1 }, + ], + exec: "auditBatch", + }, + }, +}; + +export function setup() { + let customers = []; + let userIds = []; + for (let i = 0; i < 20; i++) customers.push(crypto.randomUUID()); + for (let i = 0; i < 90; i++) userIds.push(crypto.randomUUID()); + return { + customers: customers, + userIds: userIds, + tag: "compliance_peak", + generator: "heavy", + urgentRatio: 0.3, + }; +} + +export function submitRequests(data) { + submitter(data); + sleep(10); +} + +export function readResults(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + + // List requests, then re-query any that are still pending + queryRequestsList(host, customer); + sleep(1); + pollPendingRequests(host, customer); + sleep(5); +} + +export function auditBatch(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + + // Batch queries across all clients + queryBatchUserResults(host, data.customers); + sleep(1); + + // Per-user queries across all customers + for (let i = 0; i < data.customers.length; i++) { + const customer = data.customers[i]; + queryUserResults(host, customer); + sleep(1); + } + sleep(1); +} diff --git a/scalability/6_compliance_tail.js b/scalability/6_compliance_tail.js new file mode 100644 index 0000000..2514dc4 --- /dev/null +++ b/scalability/6_compliance_tail.js @@ -0,0 +1,88 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { + pollPendingRequests, + queryBatchUserResults, + queryUserResults, +} from "./workflows/querier.js"; +import { randomIntBetween } from "https://jslib.k6.io/k6-utils/1.2.0/index.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + submitters: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "2m", target: 10 }, + { duration: "2m", target: 40 }, + { duration: "2m", target: 20 }, + { duration: "2m", target: 35 }, + { duration: "2m", target: 5 }, + ], + exec: "submitRequests", + }, + readers: { + executor: "constant-vus", + vus: 20, + startTime: "1m", + duration: "10m", + exec: "readResults", + }, + auditors: { + executor: "constant-vus", + vus: 8, + startTime: "1m", + duration: "10m", + exec: "auditBatch", + }, + }, +}; + +export function setup() { + let customers = []; + let userIds = []; + for (let i = 0; i < 20; i++) customers.push(crypto.randomUUID()); + for (let i = 0; i < 80; i++) userIds.push(crypto.randomUUID()); + return { + customers: customers, + userIds: userIds, + tag: "compliance_tail", + generator: "heavy", + urgentRatio: 0.1, + }; +} + +export function submitRequests(data) { + submitter(data); + sleep(5); +} + +export function readResults(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + + pollPendingRequests(host, customer); + sleep(5); +} + +export function auditBatch(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + + // Moderate batch queries + let batchSize = randomIntBetween(4, data.customers.length); + let startIdx = Math.floor( + Math.random() * Math.max(1, data.customers.length - batchSize), + ); + let batch = data.customers.slice(startIdx, startIdx + batchSize); + queryBatchUserResults(host, batch); + sleep(1); + + // Per-user queries max 50 users + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + queryUserResults(host, customer, 50); + sleep(5); +} diff --git a/scalability/7_auditing_bau.js b/scalability/7_auditing_bau.js new file mode 100644 index 0000000..5e9b28d --- /dev/null +++ b/scalability/7_auditing_bau.js @@ -0,0 +1,122 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { + queryRequestsList, + queryStats, + queryBatchUserResults, + queryUserResults, +} from "./workflows/querier.js"; +import { randomIntBetween } from "https://jslib.k6.io/k6-utils/1.2.0/index.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + normal_submitters: { + executor: "constant-vus", + vus: 10, + duration: "10m", + exec: "submitNormal", + }, + backlog_submitters: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "2m", target: 5 }, + { duration: "3m", target: 25 }, + { duration: "3m", target: 25 }, + { duration: "2m", target: 5 }, + ], + exec: "submitBacklog", + }, + moderate_readers: { + executor: "constant-vus", + vus: 10, + startTime: "1m", + duration: "10m", + exec: "readResults", + }, + heavy_auditors: { + executor: "ramping-vus", + startVUs: 0, + startTime: "1m", + stages: [ + { duration: "2m", target: 5 }, + { duration: "3m", target: 25 }, + { duration: "3m", target: 25 }, + { duration: "2m", target: 5 }, + ], + exec: "heavyAudit", + }, + }, +}; + +export function setup() { + let normalCustomers = []; + let backlogCustomers = []; + let userIds = []; + for (let i = 0; i < 16; i++) normalCustomers.push(crypto.randomUUID()); + for (let i = 0; i < 4; i++) backlogCustomers.push(crypto.randomUUID()); + for (let i = 0; i < 100; i++) userIds.push(crypto.randomUUID()); + return { + customers: normalCustomers.concat(backlogCustomers), + normalCustomers: normalCustomers, + backlogCustomers: backlogCustomers, + userIds: userIds, + tag: "auditing_bau", + generator: "normal", + urgentRatio: 0.02, + }; +} + +export function submitNormal(data) { + let normalData = Object.assign({}, data, { + customers: data.normalCustomers, + tag: "auditing_bau_normal", + generator: "normal", + urgentRatio: 0.02, + }); + submitter(normalData); + sleep(5); +} + +export function submitBacklog(data) { + let backlogData = Object.assign({}, data, { + customers: data.backlogCustomers, + tag: "auditing_bau_backlog", + generator: "heavy", + urgentRatio: 0.05, + }); + submitter(backlogData); + sleep(5); +} + +export function readResults(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + + queryRequestsList(host, customer); + sleep(1); + queryStats(host, customer); + sleep(5); +} + +export function heavyAudit(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + + // Large batch queries across all clients + let batchSize = randomIntBetween(8, data.customers.length); + let startIdx = Math.floor( + Math.random() * Math.max(1, data.customers.length - batchSize), + ); + let batch = data.customers.slice(startIdx, startIdx + batchSize); + queryBatchUserResults(host, batch); + sleep(1); + + // Large per-user queries max 50 users + let customer = + data.customers[Math.floor(Math.random() * data.customers.length)]; + queryUserResults(host, customer, 50); + sleep(5); +} diff --git a/scalability/8_research.js b/scalability/8_research.js new file mode 100644 index 0000000..84d5ffe --- /dev/null +++ b/scalability/8_research.js @@ -0,0 +1,73 @@ +import { sleep } from "k6"; +import { submitter } from "./workflows/submitter.js"; +import { + queryRequestsList, + queryStats, + queryRequestById, +} from "./workflows/querier.js"; + +const BASE_URL = __ENV.TEST_HOST; + +export let options = { + scenarios: { + normal_submitters: { + executor: "constant-vus", + vus: 10, + duration: "5m", + exec: "submitRequests", + }, + researcher: { + executor: "constant-vus", + vus: 4, + startTime: "1m", + duration: "5m", + exec: "researchScan", + }, + }, +}; + +export function setup() { + let customers = []; + let userIds = []; + for (let i = 0; i < 15; i++) customers.push(crypto.randomUUID()); + for (let i = 0; i < 100; i++) userIds.push(crypto.randomUUID()); + return { + customers: customers, + userIds: userIds, + tag: "research", + generator: "normal", + urgentRatio: 0.05, + }; +} + +export function submitRequests(data) { + submitter(data); + sleep(5); +} + +export function researchScan(data) { + let host = BASE_URL.endsWith("/") ? BASE_URL.slice(0, -1) : BASE_URL; + + for (let i = 0; i < data.customers.length; i++) { + let customer = data.customers[i]; + + // Get all requests for this customer + let res = queryRequestsList(host, customer); + sleep(1); + + // Get stats + queryStats(host, customer); + sleep(1); + + // Query each request result for this customer + if (res.items) { + for (let j = 0; j < res.items.length; j++) { + queryRequestById(host, customer, res.items[j].id); + sleep(0.5); + } + } + + sleep(1); + } + sleep(5); +} diff --git a/scalability/bin/runk6.sh b/scalability/bin/runk6.sh index 32ca7a4..d82748a 100755 --- a/scalability/bin/runk6.sh +++ b/scalability/bin/runk6.sh @@ -12,4 +12,10 @@ mkdir -p logs # k6run 0_debug k6run 1_normal +k6run 2_curiosity k6run 3_compliance_early +k6run 4_compliance_mid +k6run 5_compliance_peak +k6run 6_compliance_tail +k6run 7_auditing_bau +k6run 8_research diff --git a/scalability/data/photo.js b/scalability/data/photo.js index fafbe3b..deaca72 100644 --- a/scalability/data/photo.js +++ b/scalability/data/photo.js @@ -75,12 +75,12 @@ export function getPhotos(generator) { count = 1; break; case "heavy": - complexity = randomIntBetween(13, 16); + complexity = randomIntBetween(13, 20); count = 1; break; case "normal": default: - complexity = randomIntBetween(8, 16); + complexity = randomIntBetween(8, 20); count = randomIntBetween(1, 2); break; } diff --git a/scalability/workflows/querier.js b/scalability/workflows/querier.js index 515931a..6ed7b0e 100644 --- a/scalability/workflows/querier.js +++ b/scalability/workflows/querier.js @@ -13,7 +13,13 @@ const queriesTotal = new Counter("queries_total"); const queryDelay = new Trend("query_delay"); const errors = new Counter("errors"); -function timedGet(url, tags) { +function timedGet(url, tags, params) { + if (params) { + let query = Object.entries(params) + .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`) + .join("&"); + url = url + "?" + query; + } let start = Date.now(); let res = http.get(url, { headers: { Accept: "application/json" } }); let elapsed = Date.now() - start; @@ -26,29 +32,49 @@ export function queryUsersList(hostUrl, customer) { let url = hostUrl + "/analysis/" + customer + "/users"; let res = timedGet(url, { endpoint: "/users", type: "list" }); + let users = null; try { let success = check(res, checkUserList); - if (!success) { + if (success) { + users = res.json(); + } else { errors.add(1, { endpoint: "/users" }); } } catch (e) { errors.add(1, { endpoint: "/users" }); } + return { res: res, users: users }; +} + +export function queryUserById(hostUrl, customer, userId) { + let url = hostUrl + "/analysis/" + customer + "/users/" + userId; + let res = timedGet(url, { endpoint: "/users", type: "detail" }); + try { + let success = check(res, checkUser); + if (!success) { + errors.add(1, { endpoint: "/users/" + userId }); + } + } catch (e) { + errors.add(1, { endpoint: "/users/" + userId }); + } return res; } export function queryRequestsList(hostUrl, customer, params) { let url = hostUrl + "/analysis/" + customer + "/requests"; - let res = timedGet(url, { endpoint: "/requests", type: "list" }); + let res = timedGet(url, { endpoint: "/requests", type: "list" }, params); + let items = null; try { let success = check(res, checkAnalysisList); - if (!success) { + if (success) { + items = res.json(); + } else { errors.add(1, { endpoint: "/requests" }); } } catch (e) { errors.add(1, { endpoint: "/requests" }); } - return res; + return { res: res, items: items }; } export function queryRequestById(hostUrl, customer, requestId) { @@ -79,6 +105,16 @@ export function queryStats(hostUrl, customer) { return res; } +export function pollPendingRequests(hostUrl, customer) { + let result = queryRequestsList(hostUrl, customer, { status: "pending" }); + if (result.items) { + for (let i = 0; i < result.items.length; i++) { + queryRequestById(hostUrl, customer, result.items[i].id); + sleep(0.5); + } + } +} + export function queryBatchUserResults(hostUrl, customers) { for (let i = 0; i < customers.length; i++) { let customer = customers[i]; @@ -86,13 +122,21 @@ export function queryBatchUserResults(hostUrl, customers) { sleep(0.5); - let res = queryRequestsList(hostUrl, customer, { status: "pending" }); - if (res.status === 200 && Array.isArray(res.json())) { - let pending = res.json(); - for (let i = 0; i < pending.length; i++) { - queryRequestById(hostUrl, customer, pending[i].id); - sleep(0.5); - } - } + pollPendingRequests(hostUrl, customer); + + sleep(0.5); + } +} + +export function queryUserResults(hostUrl, customer, maxUsers) { + let res = queryUsersList(hostUrl, customer); + let users = res.users; + if (!users) { + return; + } + let limit = maxUsers || users.length; + for (let i = 0; i < Math.min(users.length, limit); i++) { + queryUserById(hostUrl, customer, users[i].id); + sleep(0.5); } } diff --git a/scalability/workflows/submitter.js b/scalability/workflows/submitter.js index 78f3b74..065d54f 100644 --- a/scalability/workflows/submitter.js +++ b/scalability/workflows/submitter.js @@ -56,6 +56,10 @@ export function submitter(data, timeout = 60) { let requestId = res.json().id; + // Wait for expected processing time before polling + sleep(5); + timeout -= 5; + // Check if completed while (timeout > 0) { let analysis = http.get(url + "/" + requestId, {