From 33bfcfabed0925a1bf3419cc066e616b42315339 Mon Sep 17 00:00:00 2001 From: kusuma chalasani Date: Wed, 20 May 2026 13:30:10 +0530 Subject: [PATCH] Include spring-quarkus-perf-comparison benchmark Signed-off-by: kusuma chalasani --- spring-quarkus-perf-comparison/README.md | 135 +++ spring-quarkus-perf-comparison/STRUCTURE.md | 132 +++ .../VARIABLE_LOAD_SESSION_GUIDE.md | 621 +++++++++++++ spring-quarkus-perf-comparison/config.env | 225 +++++ .../image-build/README.md | 0 .../image-build/build-all-images.sh | 698 ++++++++++++++ .../image-build/config.env | 56 ++ .../dockerfiles/apps/Dockerfile.quarkus3-jvm | 23 + .../apps/Dockerfile.quarkus3-spring-compat | 27 + .../apps/Dockerfile.quarkus3-virtual | 24 + .../dockerfiles/apps/Dockerfile.spring3-jvm | 21 + .../dockerfiles/apps/Dockerfile.spring4-jvm | 21 + .../postgres/Dockerfile.postgres-fruits | 5 + .../postgres/create-postgres-data.sql | 111 +++ .../manifests/app-template.yaml | 143 +++ .../manifests/fixed-load-hf.yml | 112 +++ .../manifests/postgresql.yaml | 72 ++ .../manifests/variable-load-24h.hf.yml | 242 +++++ .../manifests/variable-load-4h.hf.yml | 132 +++ .../run-benchmark.sh | 858 ++++++++++++++++++ .../scripts/common-utils.sh | 217 +++++ .../scripts/deploy-app.sh | 445 +++++++++ .../scripts/measure-memory.sh | 109 +++ .../scripts/measure-startup.sh | 99 ++ .../scripts/perf/perf-test-ootb-vs-tuned.sh | 444 +++++++++ .../results-tools/aggregate-session.sh | 278 ++++++ .../scripts/results-tools/analyze-session.sh | 468 ++++++++++ .../scripts/results-tools/combine-sessions.sh | 146 +++ .../scripts/results-tools/compare-all.sh | 312 +++++++ .../scripts/results-tools/compare-sessions.sh | 544 +++++++++++ .../results-tools/consolidate-session.sh | 162 ++++ .../results-tools/generate-final-reports.sh | 183 ++++ .../scripts/results-tools/generate-report.sh | 440 +++++++++ .../results-tools/generate-session-report.sh | 523 +++++++++++ .../results-tools/regenerate-metrics.sh | 333 +++++++ .../scripts/run-load-test.sh | 149 +++ .../scripts/run-variable-load-multi-phase.sh | 733 +++++++++++++++ 37 files changed, 9243 insertions(+) create mode 100644 spring-quarkus-perf-comparison/README.md create mode 100644 spring-quarkus-perf-comparison/STRUCTURE.md create mode 100644 spring-quarkus-perf-comparison/VARIABLE_LOAD_SESSION_GUIDE.md create mode 100644 spring-quarkus-perf-comparison/config.env create mode 100644 spring-quarkus-perf-comparison/image-build/README.md create mode 100755 spring-quarkus-perf-comparison/image-build/build-all-images.sh create mode 100644 spring-quarkus-perf-comparison/image-build/config.env create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-jvm create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-spring-compat create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-virtual create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring3-jvm create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring4-jvm create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/Dockerfile.postgres-fruits create mode 100644 spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/create-postgres-data.sql create mode 100644 spring-quarkus-perf-comparison/manifests/app-template.yaml create mode 100644 spring-quarkus-perf-comparison/manifests/fixed-load-hf.yml create mode 100644 spring-quarkus-perf-comparison/manifests/postgresql.yaml create mode 100644 spring-quarkus-perf-comparison/manifests/variable-load-24h.hf.yml create mode 100644 spring-quarkus-perf-comparison/manifests/variable-load-4h.hf.yml create mode 100755 spring-quarkus-perf-comparison/run-benchmark.sh create mode 100644 spring-quarkus-perf-comparison/scripts/common-utils.sh create mode 100755 spring-quarkus-perf-comparison/scripts/deploy-app.sh create mode 100755 spring-quarkus-perf-comparison/scripts/measure-memory.sh create mode 100755 spring-quarkus-perf-comparison/scripts/measure-startup.sh create mode 100755 spring-quarkus-perf-comparison/scripts/perf/perf-test-ootb-vs-tuned.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/aggregate-session.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/analyze-session.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/combine-sessions.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/compare-all.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/compare-sessions.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/consolidate-session.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/generate-final-reports.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/generate-report.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/generate-session-report.sh create mode 100755 spring-quarkus-perf-comparison/scripts/results-tools/regenerate-metrics.sh create mode 100755 spring-quarkus-perf-comparison/scripts/run-load-test.sh create mode 100755 spring-quarkus-perf-comparison/scripts/run-variable-load-multi-phase.sh diff --git a/spring-quarkus-perf-comparison/README.md b/spring-quarkus-perf-comparison/README.md new file mode 100644 index 00000000..e1494098 --- /dev/null +++ b/spring-quarkus-perf-comparison/README.md @@ -0,0 +1,135 @@ +# Spring-Quarkus-Perf-Comparison Benchmark + +This repository contains benchmark tools and configurations for testing Kruize with various application frameworks. + +> **Note:** For detailed repository structure and file organization, see [STRUCTURE.md](STRUCTURE.md) + + +## Prerequisites + +- Kubernetes or OpenShift cluster +- `kubectl` or `oc` CLI tools +- JBang (for running load tests) - **auto-installed if not present** +- `jq` for JSON processing + + +### 1. Main Benchmark Script (`run-benchmark.sh`) + +The main orchestration script that runs complete end-to-end benchmarks. It handles everything automatically: +- Sets up OpenShift project +- Deploys PostgreSQL database +- Deploys applications with different configurations (JAVA_OPTS based on scenario) +- Runs multiple test types (startup, memory, load) +- Runs the benchmark at a fixed load for 30s. +- Supports multiple iterations for statistical significance +- Generates comprehensive metrics and reports +- Compares performance across runtimes and scenarios +- Optionally cleans up deployments after tests + +**Note:** JAVA_OPTS are configured in `config.env` based on the selected scenario: +- `ootb` scenario uses `JAVA_OPTS_OOTB` from config.env +- `tuned` scenario uses `JAVA_OPTS_TUNED` from config.env (can be replaced with Kruize recommendations) + +**Usage:** +```bash +./run-benchmark.sh [OPTIONS] +``` + +**Options:** +``` +--scenario Scenario: ootb (default) or tuned +--runtimes Comma-separated list of runtimes +--tests Comma-separated list of tests +--iterations Number of iterations (default: 3) +--output-dir Output directory for results +--registry Container registry +--image-tag Image tag +--project OpenShift project name +--cleanup Cleanup after tests +--no-cleanup Don't cleanup after tests +--help Show help message +``` + +**Available Runtimes:** +- `quarkus3-jvm` - Quarkus 3 with standard JVM +- `quarkus3-virtual` - Quarkus 3 with Virtual Threads +- `spring3-jvm` - Spring Boot 3 with standard JVM +- `spring4-jvm` - Spring Boot 4 with standard JVM + +**Available Tests:** +- `measure-startup` - Measure application startup time +- `measure-memory` - Measure memory usage (RSS) +- `run-load-test` - Run load test using JBang + +**Available Scenarios:** +- `ootb` - Out-of-the-box (default JVM settings) +- `tuned` - Tuned (optimized JVM settings) + +**Examples:** +```bash +# Run default configuration +./run-benchmark.sh + +# Run specific scenario and runtimes +./run-benchmark.sh --scenario tuned --runtimes quarkus3-jvm,spring4-jvm + +# Run with custom iterations +./run-benchmark.sh --iterations 5 --tests run-load-test + +# Run without cleanup +./run-benchmark.sh --no-cleanup +``` + +### 2. Multi-Phase Load Performance Test (`scripts/perf`) + +Run the performance comparison for multiple scenarios for a runtime. +```bash +# Run default configuration ((Compares OOTB vs Tuned for quarkus3-jvm)) +perf-test-ootb-vs-tuned.sh +``` + +This workflow simulates fluid, real-world patterns with a multi-phase variable load testing approach featuring: + +- **Dynamic Workloads**: Multiple phases with shifting load intensities, thread counts, and durations. +- **Session-Based Iteration Tracking**: Run tests repeatedly with auto-incrementing iteration tracking. +- **Advanced Analytics**: Statistical aggregation calculates mean, median, standard deviation, and consistency metrics (Coefficient of Variation) to detect test drift. +- **Phase-Level Cross-Session Insights**: Directly compare OOTB vs Tuned configurations phase-by-phase. + +**For detailed documentation on running variable load, see [VARIABLE_LOAD_SESSION_GUIDE.md](VARIABLE_LOAD_SESSION_GUIDE.md)** + + +### 3. Image Build (`image-build/`) + +Contains Dockerfiles and scripts to build benchmark application images with Micrometer Prometheus metrics enabled for Kruize analysis. + +> **Note:** For detailed image building instructions, see [image-build/README.md](image-build/README.md) + + +## Components + +### Manifests (`manifests/`) + +Kubernetes/OpenShift deployment manifests: +- `app-template.yaml` - Application deployment template (includes ServiceMonitor) +- `postgresql.yaml` - PostgreSQL database deployment + +### Scripts (`scripts/`) + +Deployment, testing, and analysis scripts: + +**Deployment:** +- `deploy-app.sh` - Deploy benchmark applications + +**Testing:** +- `run-variable-load-multi-phase.sh` - Run multi-phase variable load tests with session-based iteration tracking +- `measure-startup.sh` - Measure application startup time +- `measure-memory.sh` - Measure memory usage (RSS) +- `run-load-test.sh` - Run load tests +- `common-utils.sh` - Shared utility functions for all scripts + +**Analysis:** +- `results-tools/compare-all.sh` - Compare all runtimes and scenarios (text output) +- `results-tools/generate-report.sh` - Generate interactive HTML reports with scenario and runtime comparisons +- `results-tools/aggregate-session.sh` - Aggregate metrics across multiple iterations within a session +- `results-tools/compare-sessions.sh` - Compare two sessions phase-by-phase (e.g., OOTB vs Tuned) +- `results-tools/analyze-session.sh` - Analyze within-session variability and consistency diff --git a/spring-quarkus-perf-comparison/STRUCTURE.md b/spring-quarkus-perf-comparison/STRUCTURE.md new file mode 100644 index 00000000..e4618395 --- /dev/null +++ b/spring-quarkus-perf-comparison/STRUCTURE.md @@ -0,0 +1,132 @@ +# Benchmarks Git Repository Structure + +This document describes the structure and contents of the benchmarks_git repository. + +## Directory Structure + +``` +benchmarks_git/ +├── README.md # Main documentation +├── STRUCTURE.md # This file +├── run-benchmark.sh # Main benchmark orchestration script +├── config.env # Main configuration file +├── image-build/ # Container image building +│ ├── build-all-images.sh # Build script for all images +│ ├── config.env # Configuration for image builds +│ └── dockerfiles/ # Dockerfiles directory +│ ├── apps/ # Application Dockerfiles +│ │ ├── Dockerfile.quarkus3-jvm +│ │ ├── Dockerfile.quarkus3-spring-compat +│ │ ├── Dockerfile.quarkus3-virtual +│ │ ├── Dockerfile.spring3-jvm +│ │ └── Dockerfile.spring4-jvm +│ └── postgres/ # PostgreSQL Dockerfiles +│ ├── Dockerfile.postgres-fruits +│ └── create-postgres-data.sql +├── manifests/ # Kubernetes/OpenShift manifests +│ ├── app-template.yaml # Application deployment template (includes ServiceMonitor) +│ ├── postgresql.yaml # PostgreSQL database +│ ├── fixed-load-hf.yml # Hyperfoil fixed load test benchmark +│ ├── variable-load-4h.hf.yml # 4-hour variable load test benchmark +│ └── variable-load-24h.hf.yml # 24-hour variable load test benchmark +└── scripts/ # Deployment, testing, and analysis scripts + ├── deploy-app.sh # Deploy applications + ├── measure-startup.sh # Measure application startup time + ├── measure-memory.sh # Measure memory usage (RSS) + ├── run-load-test.sh # Run load tests + ├── run-variable-load-multi-phase.sh # Multi-phase variable load test + └── results-tools/ # Results analysis tools + ├── compare-all.sh # Compare all runtimes and scenarios + ├── compare-results.sh # Compare specific results + └── generate-report.sh # Generate HTML reports +``` + +## Source Mapping + +### From `docs/benchmark-images/` +- `build-all-images.sh` → `image-build/build-all-images.sh` +- `config.env` → `image-build/config.env` +- `dockerfiles/Dockerfile.quarkus*` → `image-build/dockerfiles/apps/` +- `dockerfiles/Dockerfile.spring*` → `image-build/dockerfiles/apps/` +- `dockerfiles/Dockerfile.postgres-fruits` → `image-build/dockerfiles/postgres/` +- `dockerfiles/create-postgres-data.sql` → `image-build/dockerfiles/postgres/` + +### From `docs/openshift-benchmark/` +- `config.env` → `config.env` (top level) +- `manifests/app-template.yaml` → `manifests/app-template.yaml` +- `manifests/postgresql.yaml` → `manifests/postgresql.yaml` +- `scripts/deploy-app.sh` → `scripts/deploy-app.sh` +- `scripts/measure-*.sh` → `scripts/measure-*.sh` +- `scripts/run-load-test.sh` → `scripts/run-load-test.sh` +- `scripts/run-variable-load-multi-phase.sh` → `scripts/run-variable-load-multi-phase.sh` +- `run-benchmark.sh` → `run-benchmark.sh` (top level) +- `results-tools/*` → `scripts/results-tools/*` + +## File Counts + +- **Top Level:** + - 1 main benchmark script + - 1 main config file + - 2 documentation files + +- **Image Build:** + - 1 build script + - 1 config file + - 5 application Dockerfiles + - 1 PostgreSQL Dockerfile + - 1 SQL initialization script + +- **Manifests:** + - 2 manifest files + +- **Scripts:** + - 5 deployment/testing scripts + - 3 results analysis tools + +**Total:** 24 files organized for the kruize/benchmarks repository + +## Key Features + +### Organized Dockerfiles +Dockerfiles are organized by purpose: +- `dockerfiles/apps/` - Application images (Quarkus, Spring Boot variants) +- `dockerfiles/postgres/` - PostgreSQL database with initialization + +### End-to-End Benchmarking +The `run-benchmark.sh` script orchestrates complete benchmark workflows: +- Deploys applications with different configurations +- Runs multiple test types (startup, memory, load) +- Supports multiple iterations for statistical significance +- Generates comprehensive metrics and reports +- Compares performance across runtimes and scenarios + +### Multi-Phase Variable Load Testing +Uses `run-variable-load-multi-phase.sh` for realistic workload simulation: +- Multiple phases with different load intensities +- Configurable duration and thread counts +- Supports both read and write operations +- Generates metrics for Kruize analysis + +### PostgreSQL Database +Standard PostgreSQL deployment used by benchmark applications: +- `postgresql.yaml` - PostgreSQL database for data persistence + +### Results Analysis +Comprehensive analysis tools in `scripts/results-tools/`: +- Compare all runtimes and scenarios +- Generate detailed comparison reports +- Create HTML reports for visualization + +## Dependencies + +The `run-benchmark.sh` script requires: +- OpenShift CLI (`oc`) +- `jq` for JSON processing +- JBang (for load testing) +- All manifests in `manifests/` +- All scripts in `scripts/` +- Results analysis tools in `scripts/results-tools/` + +## Usage + +See [README.md](README.md) for detailed usage instructions. \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/VARIABLE_LOAD_SESSION_GUIDE.md b/spring-quarkus-perf-comparison/VARIABLE_LOAD_SESSION_GUIDE.md new file mode 100644 index 00000000..6f144fa6 --- /dev/null +++ b/spring-quarkus-perf-comparison/VARIABLE_LOAD_SESSION_GUIDE.md @@ -0,0 +1,621 @@ +# Variable Load Testing with Session-Based Analysis + +This guide explains how to use the session-based iteration tracking and analysis features for multi-phase variable load testing. + +## Table of Contents + +1. [Overview](#overview) +2. [Quick Start](#quick-start) +3. [Session Concepts](#session-concepts) +4. [Complete Workflow](#complete-workflow) +5. [Analysis Tools](#analysis-tools) +6. [Use Cases](#use-cases) +7. [Best Practices](#best-practices) + +## Overview + +The session-based testing framework allows you to: + +- **Run multiple iterations** of the same test configuration +- **Track iterations automatically** within a session +- **Compare different configurations** (e.g., OOTB vs Tuned) +- **Analyze variability** within a session to measure consistency +- **Aggregate metrics** across iterations for statistical analysis + +### Key Features + +- ✅ Automatic iteration numbering +- ✅ Phase-level statistical aggregation (mean, median, stddev) +- ✅ Cross-session comparison (A vs B) +- ✅ Within-session variability analysis +- ✅ Coefficient of Variation (CV) for consistency measurement +- ✅ Compatible with existing comparison and reporting tools + +## Quick Start + +### 1. Run Tests with Session Tracking + +```bash +# Run baseline configuration (iteration 1) +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario ootb \ + --session-id baseline \ + --duration custom \ + --phase-duration 3m + +# Run again (iteration 2 - auto-detected) +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario ootb \ + --session-id baseline \ + --duration custom \ + --phase-duration 3m + +# Run optimized configuration +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario tuned \ + --session-id optimized \ + --duration custom \ + --phase-duration 3m +``` + +### 2. Aggregate Session Results + +```bash +# Aggregate baseline session +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-baseline + +# Aggregate optimized session +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-optimized +``` + +### 3. Compare Sessions + +```bash +# Compare OOTB vs Tuned +./scripts/results-tools/compare-sessions.sh \ + --session-a ./variable-load-results/session-baseline \ + --session-b ./variable-load-results/session-optimized +``` + +### 4. Analyze Variability + +```bash +# Check consistency within baseline session +./scripts/results-tools/analyze-session.sh \ + --session-dir ./variable-load-results/session-baseline +``` + +## Session Concepts + +### What is a Session? + +A **session** is a group of related test runs with the same configuration. Each run within a session is called an **iteration**. + +``` +session-baseline/ +├── quarkus3-jvm-ootb-iter1.json # First run +├── quarkus3-jvm-ootb-iter2.json # Second run +├── quarkus3-jvm-ootb-iter3.json # Third run +└── aggregated.json # Statistical summary +``` + +### Session ID + +The `--session-id` parameter groups related test runs: + +- **Same session-id** = iterations of the same configuration +- **Different session-ids** = different configurations to compare + +### Iteration Numbers + +Iteration numbers are **automatically detected** by counting existing files in the session directory. You can also specify `--iteration` explicitly if needed. + +### Output Structure + +``` +variable-load-results/ +├── session-baseline/ +│ ├── quarkus3-jvm-ootb-iter1.json +│ ├── quarkus3-jvm-ootb-iter2.json +│ ├── quarkus3-jvm-ootb-iter3.json +│ ├── aggregated.json +│ └── [phase logs and metadata] +└── session-optimized/ + ├── quarkus3-jvm-tuned-iter1.json + ├── quarkus3-jvm-tuned-iter2.json + ├── aggregated.json + └── [phase logs and metadata] +``` + +## Complete Workflow + +### Step 1: Run Multiple Iterations + +Run the same test configuration multiple times (3-5 iterations recommended): + +```bash +# Iteration 1 +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario ootb \ + --session-id baseline \ + --duration 4h + +# Iteration 2 (auto-detected) +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario ootb \ + --session-id baseline \ + --duration 4h + +# Iteration 3 (auto-detected) +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario ootb \ + --session-id baseline \ + --duration 4h +``` + +**Output:** +``` +variable-load-results/session-baseline/ +├── quarkus3-jvm-ootb-iter1.json +├── quarkus3-jvm-ootb-iter2.json +└── quarkus3-jvm-ootb-iter3.json +``` + +### Step 2: Aggregate Session Data + +Calculate statistics across all iterations: + +```bash +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-baseline +``` + +**Output:** `session-baseline/aggregated.json` + +This file contains mean, median, stddev, min, max for each metric in each phase. + +### Step 3: Analyze Variability + +Check how consistent the performance is: + +```bash +./scripts/results-tools/analyze-session.sh \ + --session-dir ./variable-load-results/session-baseline +``` + +**Output:** +``` +PHASE: low +Configuration: 2 threads, 50 connections +Overall Stability Score: 95.23/100 + +THROUGHPUT: + Mean: 1234.56 req/s + Std Dev: 45.67 req/s + CV: 3.70% + Range: 123.45 req/s + Stability: 96.30/100 +``` + +### Step 4: Run Alternative Configuration + +Test a different configuration (e.g., tuned): + +```bash +# Run tuned configuration (3 iterations) +for i in {1..3}; do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm \ + --scenario tuned \ + --session-id optimized \ + --duration 4h +done +``` + +### Step 5: Aggregate Alternative Session + +```bash +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-optimized +``` + +### Step 6: Compare Sessions + +Compare the two configurations: + +```bash +./scripts/results-tools/compare-sessions.sh \ + --session-a ./variable-load-results/session-baseline \ + --session-b ./variable-load-results/session-optimized +``` + +**Output:** +``` +PHASE: peak +Configuration: 6 threads, 500 connections + +Metric Session A Session B Improvement +-------------------------------------------------------------------------------- +Throughput (req/s) 1234.56 1456.78 +18.00% ✓ +Mean Latency (ms) 45.67 38.90 -14.82% ✓ +P99 Latency (ms) 123.45 98.76 -20.00% ✓ +``` + +## Analysis Tools + +### 1. aggregate-session.sh + +**Purpose:** Calculate statistics across iterations + +**Usage:** +```bash +./scripts/results-tools/aggregate-session.sh \ + --session-dir \ + [--output ] +``` + +**Output:** JSON file with mean, median, stddev, min, max for each metric + +**When to use:** +- After running multiple iterations +- Before comparing sessions +- Before analyzing variability + +### 2. compare-sessions.sh + +**Purpose:** Compare two sessions phase-by-phase + +**Usage:** +```bash +./scripts/results-tools/compare-sessions.sh \ + --session-a \ + --session-b \ + [--format text|json] \ + [--output ] +``` + +**Output:** Phase-level comparison showing improvements/regressions + +**When to use:** +- Comparing OOTB vs Tuned +- Comparing different JVM versions +- Comparing different configurations + +### 3. analyze-session.sh + +**Purpose:** Analyze within-session variability + +**Usage:** +```bash +./scripts/results-tools/analyze-session.sh \ + --session-dir \ + [--format text|json] \ + [--output ] +``` + +**Output:** Variability metrics (CV, stability scores) + +**When to use:** +- Checking test consistency +- Determining if more iterations are needed +- Validating test reliability +### 4. generate-final-reports.sh + +**Purpose:** Generate all final output files in one command + +**Usage:** +```bash +./scripts/results-tools/generate-final-reports.sh +``` + +**Example:** +```bash +./scripts/results-tools/generate-final-reports.sh ./variable-load-results +``` + +**What it does:** +1. Consolidates each session into a single JSON file +2. Combines all sessions into one JSON file +3. Generates comparison text file +4. Generates comparison HTML report + +**Output Files:** +``` +variable-load-results/ +├── session-ootb.json # All ootb iterations with full metrics +├── session-tuned.json # All tuned iterations with full metrics +├── combined.json # Both sessions combined +├── session-ootb-vs-session-tuned-comparison.txt # Text comparison table +└── comparison-report.html # HTML comparison report +``` + +**File Descriptions:** + +- **session-*.json**: Contains all iterations for a session with complete metrics from each run + - Preserves ALL original metrics from iteration files + - Useful for detailed analysis and custom processing + - Structure: `{ session_name, runtime, scenario, total_iterations, iterations: [...] }` + +- **combined.json**: All sessions in one file + - Top-level structure: `{ total_sessions, sessions: { "session-name": {...}, ... } }` + - Useful for programmatic access to all data + +- **session-*-vs-*-comparison.txt**: Human-readable comparison table + - Phase-by-phase performance comparison + - Shows actual values for both sessions + - Includes improvement percentages with ✓/✗ indicators + +- **comparison-report.html**: Interactive HTML report + - Visual comparison with charts + - Includes variability analysis + - Easy to share with stakeholders + +**When to use:** +- After completing all test runs +- When you need a complete set of output files +- For final reporting and analysis + + +## Use Cases + +### Use Case 1: OOTB vs Tuned Comparison + +**Goal:** Compare out-of-the-box vs tuned configuration + +```bash +# 1. Run OOTB (3 iterations) +for i in {1..3}; do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm --scenario ootb \ + --session-id baseline --duration 4h +done + +# 2. Run Tuned (3 iterations) +for i in {1..3}; do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm --scenario tuned \ + --session-id optimized --duration 4h +done + +# 3. Aggregate both +./scripts/results-tools/aggregate-session.sh --session-dir ./variable-load-results/session-baseline +./scripts/results-tools/aggregate-session.sh --session-dir ./variable-load-results/session-optimized + +# 4. Compare +./scripts/results-tools/compare-sessions.sh \ + --session-a ./variable-load-results/session-baseline \ + --session-b ./variable-load-results/session-optimized +``` + +### Use Case 2: Quick Testing with Short Phases + +**Goal:** Fast iteration for development/debugging + +```bash +# Run 3 quick iterations (3 minutes per phase) +for i in {1..3}; do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm --scenario ootb \ + --session-id quick-test \ + --duration custom --phase-duration 3m +done + +# Analyze consistency +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-quick-test + +./scripts/results-tools/analyze-session.sh \ + --session-dir ./variable-load-results/session-quick-test +``` + +### Use Case 3: Long-Term Stability Testing + +**Goal:** 24-hour test with multiple iterations + +```bash +# Run 5 iterations of 24-hour test +for i in {1..5}; do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm --scenario ootb \ + --session-id stability-test \ + --duration 24h +done + +# Analyze variability +./scripts/results-tools/analyze-session.sh \ + --session-dir ./variable-load-results/session-stability-test +``` + +## Best Practices + +### Number of Iterations + +- **Quick tests (3m phases):** 3-5 iterations +- **Standard tests (4h):** 3 iterations minimum +- **Long tests (24h):** 2-3 iterations +- **Production validation:** 5+ iterations + +### Session Naming + +Use descriptive session IDs: + +```bash +# Recommended format +--session-id baseline-jdk17 +--session-id tuned-jdk21 +--session-id production-config + +# Avoid +--session-id test1 +--session-id run2 +``` + +### Consistency Checks + +Always check variability before comparing: + +```bash +# 1. Run iterations +# 2. Aggregate +./scripts/results-tools/aggregate-session.sh --session-dir + +# 3. Check consistency +./scripts/results-tools/analyze-session.sh --session-dir + +# 4. If CV > 20%, consider running more iterations +# 5. Only then compare sessions +``` + +### Understanding CV (Coefficient of Variation) + +CV% represents the ratio of standard deviation to mean, expressed as a percentage: +- **Lower CV%** = More consistent results across iterations +- **Higher CV%** = More variability between iterations + +**General Guidelines:** +- **CV < 5%:** Very low variability - results are highly consistent +- **CV 5-10%:** Low variability - results show good repeatability +- **CV 10-20%:** Moderate variability - consider running more iterations +- **CV > 20%:** High variability - investigate causes or run more iterations + +**Note:** These are general reference points. The acceptable CV% depends on your specific use case, metric type, and testing goals. + +### Phase Duration Selection + +| Duration Mode | Use Case | Total Time | Iterations Recommended | +|--------------|----------|------------|----------------------| +| `custom --phase-duration 3m` | Development/Debug | ~15 min | 3-5 | +| `1h` | Quick validation | ~1 hour | 3 | +| `4h` | Standard testing | ~4 hours | 3 | +| `24h` | Production simulation | ~24 hours | 2-3 | + +### Disk Space Considerations + +Each iteration generates: +- Phase logs (~10-50 MB per phase) +- JSON results (~1-5 MB) +- Metadata files (~1 KB) + +**Example:** 5 phases × 3 iterations × 20 MB = ~300 MB per session + +## Troubleshooting + +### Issue: Iteration not auto-detected + +**Symptom:** Script creates iteration 1 again instead of incrementing + +**Solution:** Check that you're using the same `--session-id` and that files match the pattern `*-iter*.json` + +### Issue: Aggregation fails + +**Symptom:** `aggregate-session.sh` reports no iteration files found + +**Solution:** +1. Check that JSON files exist in session directory +2. Verify files match pattern: `--iter.json` +3. Ensure `run-variable-load-multi-phase.sh` completed successfully + +### Issue: High variability (CV > 20%) + +**Symptom:** `analyze-session.sh` shows poor consistency + +**Solutions:** +1. Run more iterations (5-10) +2. Check for external factors (network issues, resource contention) +3. Increase phase duration for more stable measurements +4. Verify system is idle during tests + +### Issue: Comparison shows unexpected results + +**Symptom:** Session B shows regression instead of improvement + +**Solutions:** +1. Check that both sessions have similar iteration counts +2. Verify both used same duration mode and phase settings +3. Review individual iteration files for anomalies +4. Check variability analysis for both sessions + +## Advanced Usage + +### Custom Iteration Numbers + +```bash +# Explicitly specify iteration +./scripts/run-variable-load-multi-phase.sh \ + --runtime quarkus3-jvm --scenario ootb \ + --session-id baseline --iteration 5 +``` + +### JSON Output for Automation + +```bash +# Generate JSON comparison +./scripts/results-tools/compare-sessions.sh \ + --session-a ./variable-load-results/session-baseline \ + --session-b ./variable-load-results/session-optimized \ + --format json --output comparison.json + +# Generate JSON variability analysis +./scripts/results-tools/analyze-session.sh \ + --session-dir ./variable-load-results/session-baseline \ + --format json --output variability.json +``` + +### Scripted Workflow + +```bash +#!/bin/bash +# automated-comparison.sh + +RUNTIME="quarkus3-jvm" +ITERATIONS=3 + +# Run baseline +for i in $(seq 1 $ITERATIONS); do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime $RUNTIME --scenario ootb \ + --session-id baseline --duration 4h +done + +# Run optimized +for i in $(seq 1 $ITERATIONS); do + ./scripts/run-variable-load-multi-phase.sh \ + --runtime $RUNTIME --scenario tuned \ + --session-id optimized --duration 4h +done + +# Aggregate +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-baseline + +./scripts/results-tools/aggregate-session.sh \ + --session-dir ./variable-load-results/session-optimized + +# Compare +./scripts/results-tools/compare-sessions.sh \ + --session-a ./variable-load-results/session-baseline \ + --session-b ./variable-load-results/session-optimized \ + --output comparison-report.txt + +echo "Comparison complete! See comparison-report.txt" +``` + +## Summary + +The session-based testing framework provides: + +1. **Automatic iteration tracking** - No manual numbering needed +2. **Statistical aggregation** - Mean, median, stddev across iterations +3. **Cross-session comparison** - Compare different configurations +4. **Variability analysis** - Measure test consistency +5. **Phase-level insights** - Understand performance at each load level + +This enables reliable, reproducible performance testing with confidence in the results. \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/config.env b/spring-quarkus-perf-comparison/config.env new file mode 100644 index 00000000..6ec923ac --- /dev/null +++ b/spring-quarkus-perf-comparison/config.env @@ -0,0 +1,225 @@ +#!/bin/bash +# Configuration file for OpenShift Benchmark Suite + +# ============================================================================ +# Container Registry Configuration +# ============================================================================ +export REGISTRY="quay.io" +export REGISTRY_USER="your-username" +export IMAGE_TAG="v1.0" + +# ============================================================================ +# Runtime Version Information +# ============================================================================ +# These versions reflect what's in the container images +export BASE_IMAGE="registry.access.redhat.com/ubi9/openjdk-21:latest" +export JAVA_VERSION="21" +export JVM_VENDOR="Red Hat OpenJDK" +export QUARKUS_VERSION="3.34.3" +export SPRING_BOOT_VERSION="4.0.0" + +# ============================================================================ +# OpenShift Configuration +# ============================================================================ +export OPENSHIFT_PROJECT="quarkus-perf-benchmark" +export OPENSHIFT_CLUSTER_URL="https://api.your-cluster.com:6443" + +# ============================================================================ +# Container Configuration +# ============================================================================ +# Default container name for deployments +export CONTAINER_NAME="app" + +# ============================================================================ +# Benchmark Configuration +# ============================================================================ +# Scenarios: Comma-separated list of scenarios to run +# Options: ootb (Out-of-the-Box), tuned (Optimized) +# Examples: "ootb" for single scenario, "ootb,tuned" for both +export SCENARIOS="ootb,tuned" + +# Runtimes to test (comma-separated) +# Options: quarkus3-jvm, quarkus3-virtual, spring4-jvm, spring4-virtual +export RUNTIMES="quarkus3-jvm,spring4-jvm" + +# Tests to run (comma-separated) +# Options: measure-startup, measure-memory, run-load-test +export TESTS="measure-startup,measure-memory,run-load-test" + +# Number of iterations per test +export ITERATIONS="1" + +# ============================================================================ +# JVM Configuration - OOTB Scenario +# ============================================================================ +export JAVA_OPTS_OOTB=" " + +# ============================================================================ +# JVM Configuration - Tuned Scenario +# ============================================================================ +# These can be replaced with Kruize recommendations +export JAVA_OPTS_TUNED="-XX:MaxRAMPercentage=80 -XX:+UseG1GC " + +# ============================================================================ +# Resource Limits - Default (used if scenario-specific not defined) +# ============================================================================ +export CPU_REQUEST="500m" +export CPU_LIMIT="1000m" +export MEMORY_REQUEST="512Mi" +export MEMORY_LIMIT="1Gi" + +# ============================================================================ +# Resource Limits - OOTB Scenario +# ============================================================================ +export CPU_REQUEST_OOTB="${CPU_REQUEST_OOTB:-500m}" +export CPU_LIMIT_OOTB="${CPU_LIMIT_OOTB:-1000m}" +export MEMORY_REQUEST_OOTB="${MEMORY_REQUEST_OOTB:-512Mi}" +export MEMORY_LIMIT_OOTB="${MEMORY_LIMIT_OOTB:-1Gi}" + +# ============================================================================ +# Resource Limits - Tuned Scenario +# ============================================================================ +# These can be replaced with Kruize recommendations +export CPU_REQUEST_TUNED="${CPU_REQUEST_TUNED:-750m}" +export CPU_LIMIT_TUNED="${CPU_LIMIT_TUNED:-2000m}" +export MEMORY_REQUEST_TUNED="${MEMORY_REQUEST_TUNED:-768Mi}" +export MEMORY_LIMIT_TUNED="${MEMORY_LIMIT_TUNED:-1536Mi}" + +# ============================================================================ +# PostgreSQL Configuration +# ============================================================================ +export POSTGRES_IMAGE="postgres:15" +export POSTGRES_USER="quarkus" +export POSTGRES_PASSWORD="quarkus" +export POSTGRES_DB="quarkus" +export POSTGRES_CPU_REQUEST="500m" +export POSTGRES_CPU_LIMIT="1000m" +export POSTGRES_MEMORY_REQUEST="256Mi" +export POSTGRES_MEMORY_LIMIT="512Mi" + +# ============================================================================ +# Load Test Configuration +# ============================================================================ +# Note: Using Hyperfoil 0.25 for Java 17 compatibility +# If you have Java 21+, you can upgrade to 0.29+ in scripts/run-load-test.sh +export HYPERFOIL_VERSION="0.25" +export LOAD_TEST_DURATION="30s" +export LOAD_TEST_CONNECTIONS="50" +export LOAD_TEST_RATE="5000" +export LOAD_TEST_WARMUP="10s" + +# ============================================================================ +# Results Configuration +# ============================================================================ +export RESULTS_DIR="./results" +export RESULTS_FORMAT="json" # json, csv, or both + +# ============================================================================ +# Timeout Configuration +# ============================================================================ +export DEPLOYMENT_TIMEOUT="300" # seconds +export STARTUP_TIMEOUT="120" # seconds +export TEST_TIMEOUT="600" # seconds + +# ============================================================================ +# Monitoring Configuration +# ============================================================================ +export ENABLE_PROMETHEUS="true" +export PROMETHEUS_NAMESPACE="openshift-monitoring" +export METRICS_COLLECTION_INTERVAL="5s" + +# ============================================================================ +# Cleanup Configuration +# ============================================================================ +export CLEANUP_AFTER_TEST="true" +export KEEP_FAILED_DEPLOYMENTS="true" + +# ============================================================================ +# Logging Configuration +# ============================================================================ +export LOG_LEVEL="INFO" # DEBUG, INFO, WARN, ERROR +export LOG_FILE="benchmark.log" + +# ============================================================================ +# Advanced Configuration +# ============================================================================ +# Enable detailed metrics collection +export COLLECT_JVM_METRICS="true" +export COLLECT_GC_METRICS="true" +export COLLECT_THREAD_METRICS="true" + +# Enable profiling (may impact performance) +export ENABLE_PROFILING="false" +export PROFILING_TOOL="async-profiler" + +# ============================================================================ +# Kruize Integration +# ============================================================================ +export KRUIZE_ENABLED="false" +export KRUIZE_URL="http://kruize.kruize.svc.cluster.local:8080" +export KRUIZE_EXPERIMENT_NAME="quarkus-perf-benchmark" + +# ============================================================================ +# Notification Configuration +# ============================================================================ +export ENABLE_NOTIFICATIONS="false" +export NOTIFICATION_EMAIL="" +export NOTIFICATION_SLACK_WEBHOOK="" + +# ============================================================================ +# Helper Functions +# ============================================================================ + +# Get full image name +get_image_name() { + local runtime=$1 + echo "${REGISTRY}/${REGISTRY_USER}/spring-quarkus-perf-${runtime}:${IMAGE_TAG}" +} + +# Get JAVA_OPTS for scenario +get_java_opts() { + local scenario=$1 + if [ "$scenario" = "ootb" ]; then + echo "$JAVA_OPTS_OOTB" + else + echo "$JAVA_OPTS_TUNED" + fi +} + +# Validate configuration +validate_config() { + local errors=0 + + if [ -z "$REGISTRY_USER" ] || [ "$REGISTRY_USER" = "your-username" ]; then + echo "ERROR: REGISTRY_USER not configured" + errors=$((errors + 1)) + fi + + if [ -z "$OPENSHIFT_PROJECT" ]; then + echo "ERROR: OPENSHIFT_PROJECT not configured" + errors=$((errors + 1)) + fi + + if [ "$errors" -gt 0 ]; then + echo "Configuration validation failed with $errors error(s)" + return 1 + fi + + return 0 +} + +# Print configuration summary +print_config() { + echo "============================================" + echo "Benchmark Configuration Summary" + echo "============================================" + echo "Registry: ${REGISTRY}/${REGISTRY_USER}" + echo "Image Tag: ${IMAGE_TAG}" + echo "Project: ${OPENSHIFT_PROJECT}" + echo "Scenarios: ${SCENARIOS}" + echo "Runtimes: ${RUNTIMES}" + echo "Tests: ${TESTS}" + echo "Iterations: ${ITERATIONS}" + echo "Results Dir: ${RESULTS_DIR}" + echo "============================================" +} \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/image-build/README.md b/spring-quarkus-perf-comparison/image-build/README.md new file mode 100644 index 00000000..e69de29b diff --git a/spring-quarkus-perf-comparison/image-build/build-all-images.sh b/spring-quarkus-perf-comparison/image-build/build-all-images.sh new file mode 100755 index 00000000..cdf7b083 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/build-all-images.sh @@ -0,0 +1,698 @@ +#!/bin/bash +################################################################################ +# Main script to build all benchmark container images +# This script: +# 1. Clones the spring-quarkus-perf-comparison repository +# 2. Builds all 4 application variants (Quarkus JVM/Virtual, Spring JVM/Virtual) +# 3. Creates container images for each variant +# 4. Optionally pushes images to container registry +################################################################################ + +# Note: We don't use 'set -e' here because we want to continue building other images +# even if one fails. Each critical operation has its own error handling. + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source configuration +if [ -f "${SCRIPT_DIR}/config.env" ]; then + source "${SCRIPT_DIR}/config.env" +else + echo "ERROR: config.env not found!" + exit 1 +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored output +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_step() { + echo "" + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}" +} + +# Function to check prerequisites +check_prerequisites() { + print_step "Checking Prerequisites" + + local missing_tools=() + + # Check for git + if ! command -v git &> /dev/null; then + missing_tools+=("git") + fi + + # Check for maven + if ! command -v mvn &> /dev/null; then + missing_tools+=("maven") + fi + + # Check for podman or docker + if command -v podman &> /dev/null; then + CONTAINER_CMD="podman" + print_info "Using podman for container builds" + elif command -v docker &> /dev/null; then + CONTAINER_CMD="docker" + print_info "Using docker for container builds" + else + missing_tools+=("podman or docker") + fi + + # Check Java version + if ! command -v java &> /dev/null; then + missing_tools+=("java") + else + print_info "Java version: $(java -version 2>&1 | head -n 1)" + fi + + if [ ${#missing_tools[@]} -gt 0 ]; then + print_error "Missing required tools: ${missing_tools[*]}" + echo "" + echo "Please install the missing tools:" + echo " - git: https://git-scm.com/downloads" + echo " - maven: https://maven.apache.org/download.cgi" + echo " - podman: https://podman.io/getting-started/installation" + echo " - java: Use SDKMAN (https://sdkman.io/)" + exit 1 + fi + + print_success "All prerequisites met" +} + +# Function to clone repository if needed +clone_repository() { + print_step "Cloning Repository" + + if [ -d "${REPO_DIR}" ]; then + print_info "Repository already exists at ${REPO_DIR}" + read -p "Do you want to pull latest changes? (y/n): " -n 1 -r + echo "" + if [[ $REPLY =~ ^[Yy]$ ]]; then + cd "${REPO_DIR}" + git pull + cd "${SCRIPT_DIR}" + print_success "Repository updated" + fi + else + print_info "Cloning repository from ${REPO_URL}..." + git clone "${REPO_URL}" "${REPO_DIR}" + print_success "Repository cloned to ${REPO_DIR}" + fi +} + +# Function to add Prometheus Micrometer dependencies to Quarkus pom.xml +add_quarkus_micrometer_deps() { + local pom_file=$1 + local app_name=$2 + + print_info "Adding Prometheus Micrometer dependencies to ${app_name}..." + + # Check if dependencies already exist + if grep -q "quarkus-micrometer-registry-prometheus" "${pom_file}"; then + print_info "Prometheus Micrometer dependencies already present in ${app_name}" + return 0 + fi + + # Create backup + cp "${pom_file}" "${pom_file}.backup" + + # Find the tag within main section (not in ) + # and add our dependencies BEFORE it + # Use awk to ensure we're adding to the right section + awk ' + // { in_dep_mgmt=1 } + /<\/dependencyManagement>/ { in_dep_mgmt=0 } + // && !in_dep_mgmt { in_deps=1 } + { + if (/<\/dependencies>/ && in_deps && !in_dep_mgmt) { + # Insert BEFORE closing tag + print " " + print " " + print " io.quarkus" + print " quarkus-micrometer" + print " " + print " " + print " io.quarkus" + print " quarkus-micrometer-registry-prometheus" + print " " + in_deps=0 + } + print + } + ' "${pom_file}.backup" > "${pom_file}" + + if [ $? -eq 0 ]; then + print_success "Added Prometheus Micrometer dependencies to ${app_name}" + rm "${pom_file}.backup" + else + print_error "Failed to add dependencies to ${app_name}" + mv "${pom_file}.backup" "${pom_file}" + return 1 + fi +} + +# Function to add Prometheus Micrometer configuration to Quarkus application config +add_quarkus_micrometer_config() { + local config_dir=$1 + local app_name=$2 + + print_info "Adding Prometheus Micrometer configuration to ${app_name}..." + + # Check for YAML or properties file + local config_file="" + if [ -f "${config_dir}/application.yml" ]; then + config_file="${config_dir}/application.yml" + elif [ -f "${config_dir}/application.yaml" ]; then + config_file="${config_dir}/application.yaml" + elif [ -f "${config_dir}/application.properties" ]; then + config_file="${config_dir}/application.properties" + else + print_warning "No configuration file found in ${config_dir}, skipping Micrometer config" + return 0 + fi + + # Check if configuration already exists + if grep -q "micrometer" "${config_file}"; then + print_info "Micrometer configuration already present in ${app_name}" + return 0 + fi + + # Create backup + cp "${config_file}" "${config_file}.backup" + + # Add configuration based on file type + if [[ "${config_file}" == *.yml ]] || [[ "${config_file}" == *.yaml ]]; then + # Add YAML configuration + cat >> "${config_file}" << 'EOF' + + # Micrometer Prometheus Configuration + micrometer: + enabled: true + export: + prometheus: + enabled: true + path: /q/metrics + binder: + jvm: true + http-server: + enabled: true + http-client: + enabled: true + system: true +EOF + else + # Add properties configuration + cat >> "${config_file}" << 'EOF' + +# Micrometer Prometheus Configuration +quarkus.micrometer.enabled=true +quarkus.micrometer.export.prometheus.enabled=true +quarkus.micrometer.export.prometheus.path=/q/metrics + +# Enable metric binders +quarkus.micrometer.binder.jvm=true +quarkus.micrometer.binder.http-server.enabled=true +quarkus.micrometer.binder.http-client.enabled=true +quarkus.micrometer.binder.system=true +EOF + fi + + if [ $? -eq 0 ]; then + print_success "Added Prometheus Micrometer configuration to ${app_name}" + else + print_error "Failed to add configuration to ${app_name}" + mv "${config_file}.backup" "${config_file}" + return 1 + fi +} + +# Function to add Spring Boot Actuator and Prometheus dependencies +add_spring_micrometer_deps() { + local pom_file=$1 + local app_name=$2 + + print_info "Adding Prometheus Micrometer dependencies to ${app_name}..." + + # Check if dependencies already exist + if grep -q "micrometer-registry-prometheus" "${pom_file}"; then + print_info "Prometheus Micrometer dependencies already present in ${app_name}" + return 0 + fi + + # Create backup + cp "${pom_file}" "${pom_file}.backup" + + # Add dependencies before closing tag + sed -i '/<\/dependencies>/i\ + \ + \ + org.springframework.boot\ + spring-boot-starter-actuator\ + \ + \ + io.micrometer\ + micrometer-registry-prometheus\ + ' "${pom_file}" + + if [ $? -eq 0 ]; then + print_success "Added Prometheus Micrometer dependencies to ${app_name}" + else + print_error "Failed to add dependencies to ${app_name}" + mv "${pom_file}.backup" "${pom_file}" + return 1 + fi +} + +# Function to add Spring Boot Actuator configuration +add_spring_micrometer_config() { + local props_file=$1 + local app_name=$2 + + print_info "Adding Prometheus Micrometer configuration to ${app_name}..." + + # Check if configuration already exists + if grep -q "management.endpoints.web.exposure.include" "${props_file}"; then + print_info "Prometheus Micrometer configuration already present in ${app_name}" + return 0 + fi + + # Create backup + cp "${props_file}" "${props_file}.backup" + + # Add configuration + cat >> "${props_file}" << 'EOF' + +# Spring Boot Actuator Configuration +management.endpoints.web.exposure.include=health,info,metrics,prometheus +management.endpoints.web.base-path=/actuator +management.endpoint.prometheus.enabled=true +management.metrics.enable.jvm=true +management.metrics.enable.process=true +management.metrics.enable.system=true +management.metrics.enable.http=true +EOF + + if [ $? -eq 0 ]; then + print_success "Added Prometheus Micrometer configuration to ${app_name}" + else + print_error "Failed to add configuration to ${app_name}" + mv "${props_file}.backup" "${props_file}" + return 1 + fi +} + +# Function to build application JAR +build_application() { + local app_dir=$1 + local app_name=$2 + local app_type=$3 # "quarkus" or "spring" + + print_info "Building ${app_name}..." + + # Check if directory exists + if [ ! -d "${REPO_DIR}/${app_dir}" ]; then + print_warning "Directory ${REPO_DIR}/${app_dir} not found, skipping ${app_name}" + return 1 + fi + + cd "${REPO_DIR}/${app_dir}" || { + print_error "Failed to change to directory ${REPO_DIR}/${app_dir}" + return 1 + } + + # Add Prometheus Micrometer support + if [ "${app_type}" = "quarkus" ]; then + add_quarkus_micrometer_deps "pom.xml" "${app_name}" + add_quarkus_micrometer_config "src/main/resources" "${app_name}" + elif [ "${app_type}" = "spring" ]; then + add_spring_micrometer_deps "pom.xml" "${app_name}" + if [ -f "src/main/resources/application.properties" ]; then + add_spring_micrometer_config "src/main/resources/application.properties" "${app_name}" + fi + fi + + # Build the application + if [ "${SKIP_TESTS}" = "true" ]; then + mvn clean package -DskipTests + else + mvn clean package + fi + + if [ $? -eq 0 ]; then + print_success "${app_name} built successfully" + cd "${SCRIPT_DIR}" + return 0 + else + print_error "Failed to build ${app_name}" + cd "${SCRIPT_DIR}" + return 1 + fi +} + +# Function to build container image +build_image() { + local dockerfile=$1 + local context_dir=$2 + local image_name=$3 + local app_name=$4 + + print_info "Building container image for ${app_name}..." + + # Check if context directory exists + if [ ! -d "${context_dir}" ]; then + print_warning "Context directory ${context_dir} not found, skipping ${app_name} image build" + return 1 + fi + + # Check if Dockerfile exists + if [ ! -f "${SCRIPT_DIR}/dockerfiles/${dockerfile}" ]; then + print_warning "Dockerfile ${SCRIPT_DIR}/dockerfiles/${dockerfile} not found, skipping ${app_name} image build" + return 1 + fi + + ${CONTAINER_CMD} build \ + -f "${SCRIPT_DIR}/dockerfiles/${dockerfile}" \ + -t "${image_name}" \ + "${context_dir}" + + if [ $? -eq 0 ]; then + print_success "Image ${image_name} built successfully" + return 0 + else + print_error "Failed to build image ${image_name}" + return 1 + fi +} + +# Function to push image to registry +push_image() { + local image_name=$1 + + print_info "Pushing ${image_name} to registry..." + + ${CONTAINER_CMD} push "${image_name}" + + if [ $? -eq 0 ]; then + print_success "Image ${image_name} pushed successfully" + else + print_error "Failed to push image ${image_name}" + return 1 + fi +} + +# Function to list built images +list_images() { + print_step "Built Images" + ${CONTAINER_CMD} images | head -n 1 + ${CONTAINER_CMD} images | grep -E "(quarkus3|spring3|spring4)" | grep "${REGISTRY_USER}" || echo "No images found" +} + +# Main execution +main() { + echo "========================================================================" + echo " Spring-Quarkus Performance Benchmark - Image Builder" + echo "========================================================================" + echo "" + + # Parse command line arguments + SELECTED_RUNTIMES=("$@") + + # Show usage if --help is specified + if [[ " ${SELECTED_RUNTIMES[@]} " =~ " --help " ]] || [[ " ${SELECTED_RUNTIMES[@]} " =~ " -h " ]]; then + cat << EOF +Usage: $0 [RUNTIMES...] + +Build container images for Spring-Quarkus performance benchmarks. +All images include Prometheus Micrometer support for metrics collection. + +Arguments: + RUNTIMES Optional. Specify which runtimes to build (space-separated). + If not specified, builds all runtimes. + +Available runtimes: + quarkus3-jvm Quarkus 3 with standard JVM + quarkus3-virtual Quarkus 3 with Virtual Threads + quarkus3-spring-compat Quarkus 3 with Spring API compatibility + spring3-jvm Spring Boot 3 with standard JVM + spring4-jvm Spring Boot 4 with standard JVM + all Build all runtimes (default) + +Examples: + # Build all runtimes + $0 + $0 all + + # Build only Quarkus JVM + $0 quarkus3-jvm + + # Build Quarkus with Spring compatibility + $0 quarkus3-spring-compat + + # Build multiple specific runtimes + $0 quarkus3-jvm spring4-jvm + + # Build all Quarkus variants + $0 quarkus3-jvm quarkus3-virtual quarkus3-spring-compat + + # Build all Spring Boot variants + $0 spring3-jvm spring4-jvm + +EOF + exit 0 + fi + + # Display selected runtimes + if [ ${#SELECTED_RUNTIMES[@]} -eq 0 ]; then + print_info "Building all runtimes (default)" + SELECTED_RUNTIMES=("all") + else + print_info "Selected runtimes: ${SELECTED_RUNTIMES[*]}" + fi + + # Check prerequisites + check_prerequisites + + # Clone repository + clone_repository + + # Build applications + print_step "Step 1: Building Applications (with Prometheus Micrometer)" + + local build_quarkus3=false + local build_quarkus3_virtual=false + local build_quarkus3_spring_compat=false + local build_spring3=false + local build_spring4=false + + # Determine what needs to be built + for runtime in "${SELECTED_RUNTIMES[@]}"; do + case "$runtime" in + all) + build_quarkus3=true + build_quarkus3_virtual=true + build_quarkus3_spring_compat=true + build_spring3=true + build_spring4=true + ;; + quarkus3-jvm) + build_quarkus3=true + ;; + quarkus3-virtual) + build_quarkus3_virtual=true + ;; + quarkus3-spring-compat) + build_quarkus3_spring_compat=true + ;; + spring3-jvm) + build_spring3=true + ;; + spring4-jvm) + build_spring4=true + ;; + *) + print_warning "Unknown runtime: $runtime" + ;; + esac + done + + # Build applications + if [ "$build_quarkus3" = true ]; then + build_application "quarkus3" "Quarkus 3 JVM" "quarkus" + fi + + if [ "$build_quarkus3_virtual" = true ]; then + build_application "quarkus3-virtual" "Quarkus 3 Virtual Threads" "quarkus" + fi + + if [ "$build_quarkus3_spring_compat" = true ]; then + build_application "quarkus3-spring-compatibility" "Quarkus 3 Spring Compatibility" "quarkus" + fi + + if [ "$build_spring3" = true ]; then + build_application "springboot3" "Spring Boot 3" "spring" + fi + + if [ "$build_spring4" = true ]; then + build_application "springboot4" "Spring Boot 4" "spring" + fi + + # Build container images + print_step "Step 2: Building Container Images" + + local images_built=0 + + for runtime in "${SELECTED_RUNTIMES[@]}"; do + case "$runtime" in + all) + if build_image "apps/Dockerfile.quarkus3-jvm" "${REPO_DIR}/quarkus3" "${QUARKUS3_JVM_IMAGE}" "Quarkus 3 JVM"; then + ((images_built++)) + fi + if build_image "apps/Dockerfile.quarkus3-virtual" "${REPO_DIR}/quarkus3-virtual" "${QUARKUS3_VIRTUAL_IMAGE}" "Quarkus 3 Virtual"; then + ((images_built++)) + fi + if build_image "apps/Dockerfile.quarkus3-spring-compat" "${REPO_DIR}/quarkus3-spring-compatibility" "${QUARKUS3_SPRING_COMPAT_IMAGE}" "Quarkus 3 Spring Compat"; then + ((images_built++)) + fi + if build_image "apps/Dockerfile.spring3-jvm" "${REPO_DIR}/springboot3" "${SPRING3_JVM_IMAGE}" "Spring Boot 3 JVM"; then + ((images_built++)) + fi + if build_image "apps/Dockerfile.spring4-jvm" "${REPO_DIR}/springboot4" "${SPRING4_JVM_IMAGE}" "Spring Boot 4 JVM"; then + ((images_built++)) + fi + ;; + quarkus3-jvm) + if build_image "apps/Dockerfile.quarkus3-jvm" "${REPO_DIR}/quarkus3" "${QUARKUS3_JVM_IMAGE}" "Quarkus 3 JVM"; then + ((images_built++)) + fi + ;; + quarkus3-virtual) + if build_image "apps/Dockerfile.quarkus3-virtual" "${REPO_DIR}/quarkus3-virtual" "${QUARKUS3_VIRTUAL_IMAGE}" "Quarkus 3 Virtual"; then + ((images_built++)) + fi + ;; + quarkus3-spring-compat) + if build_image "apps/Dockerfile.quarkus3-spring-compat" "${REPO_DIR}/quarkus3-spring-compatibility" "${QUARKUS3_SPRING_COMPAT_IMAGE}" "Quarkus 3 Spring Compat"; then + ((images_built++)) + fi + ;; + spring3-jvm) + if build_image "apps/Dockerfile.spring3-jvm" "${REPO_DIR}/springboot3" "${SPRING3_JVM_IMAGE}" "Spring Boot 3 JVM"; then + ((images_built++)) + fi + ;; + spring4-jvm) + if build_image "apps/Dockerfile.spring4-jvm" "${REPO_DIR}/springboot4" "${SPRING4_JVM_IMAGE}" "Spring Boot 4 JVM"; then + ((images_built++)) + fi + ;; + esac + done + + if [ $images_built -eq 0 ]; then + print_error "No valid runtimes specified!" + echo "" + exit 1 + fi + + print_success "${images_built} image(s) built successfully!" + + # List images + list_images + + # Ask if user wants to push images + echo "" + read -p "Do you want to push images to ${REGISTRY}/${REGISTRY_USER}? (y/n): " -n 1 -r + echo "" + + if [[ $REPLY =~ ^[Yy]$ ]]; then + print_step "Step 3: Pushing Images to Registry" + + # Login to registry + print_info "Logging in to ${REGISTRY}..." + ${CONTAINER_CMD} login ${REGISTRY} + + if [ $? -ne 0 ]; then + print_error "Failed to login to registry" + exit 1 + fi + + # Push selected images + for runtime in "${SELECTED_RUNTIMES[@]}"; do + case "$runtime" in + all) + push_image "${QUARKUS3_JVM_IMAGE}" + push_image "${QUARKUS3_VIRTUAL_IMAGE}" + push_image "${QUARKUS3_SPRING_COMPAT_IMAGE}" + push_image "${SPRING3_JVM_IMAGE}" + push_image "${SPRING3_VIRTUAL_IMAGE}" + push_image "${SPRING4_JVM_IMAGE}" + push_image "${SPRING4_VIRTUAL_IMAGE}" + ;; + quarkus3-jvm) + push_image "${QUARKUS3_JVM_IMAGE}" + ;; + quarkus3-virtual) + push_image "${QUARKUS3_VIRTUAL_IMAGE}" + ;; + quarkus3-spring-compat) + push_image "${QUARKUS3_SPRING_COMPAT_IMAGE}" + ;; + spring3-jvm) + push_image "${SPRING3_JVM_IMAGE}" + ;; + spring3-virtual) + push_image "${SPRING3_VIRTUAL_IMAGE}" + ;; + spring4-jvm) + push_image "${SPRING4_JVM_IMAGE}" + ;; + spring4-virtual) + push_image "${SPRING4_VIRTUAL_IMAGE}" + ;; + esac + done + + print_success "Images pushed successfully!" + fi + + echo "" + echo "========================================================================" + print_success "Build Process Completed!" + echo "========================================================================" + echo "" + echo "Images created: ${images_built}" + echo "" + echo "Next steps:" + echo " 1. Verify images: ${CONTAINER_CMD} images | grep ${REGISTRY_USER}" + echo " 2. Test locally: ${CONTAINER_CMD} run -p 8080:8080 " + echo " 3. Check metrics: curl http://localhost:8080/q/metrics (Quarkus)" + echo " curl http://localhost:8080/actuator/prometheus (Spring)" + echo "" +} + +# Run main function +main "$@" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/config.env b/spring-quarkus-perf-comparison/image-build/config.env new file mode 100644 index 00000000..27ce039a --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/config.env @@ -0,0 +1,56 @@ +#!/bin/bash +################################################################################ +# Configuration for Spring-Quarkus Performance Benchmark Image Builder +################################################################################ + +# Version Configuration +export JAVA_VERSION="25.0.2-tem" +export GRAALVM_VERSION="25.0.2-graalce" +export QUARKUS_VERSION="3.34.3" +export SPRING_BOOT3_VERSION="3.4.5" +export SPRING_BOOT4_VERSION="4.0.5" + +# Container Registry Configuration +# Change these to match your registry +export REGISTRY="quay.io" # Options: quay.io, docker.io, or your registry +export REGISTRY_USER="your-username" # Your registry username +export IMAGE_TAG="v1.0" # Image tag/version + +# Repository Configuration +export REPO_URL="https://github.com/quarkusio/spring-quarkus-perf-comparison.git" +export REPO_DIR="/tmp/spring-quarkus-perf-comparison" + +# Image Names (automatically constructed) +# Quarkus 3 variants +export QUARKUS3_JVM_IMAGE="${REGISTRY}/${REGISTRY_USER}/quarkus3-jvm:${IMAGE_TAG}" +export QUARKUS3_VIRTUAL_IMAGE="${REGISTRY}/${REGISTRY_USER}/quarkus3-virtual:${IMAGE_TAG}" +export QUARKUS3_SPRING_COMPAT_IMAGE="${REGISTRY}/${REGISTRY_USER}/quarkus3-spring-compat:${IMAGE_TAG}" + +# Spring Boot 3 variants +export SPRING3_JVM_IMAGE="${REGISTRY}/${REGISTRY_USER}/spring3-jvm:${IMAGE_TAG}" +export SPRING3_VIRTUAL_IMAGE="${REGISTRY}/${REGISTRY_USER}/spring3-virtual:${IMAGE_TAG}" + +# Spring Boot 4 variants +export SPRING4_JVM_IMAGE="${REGISTRY}/${REGISTRY_USER}/spring4-jvm:${IMAGE_TAG}" +export SPRING4_VIRTUAL_IMAGE="${REGISTRY}/${REGISTRY_USER}/spring4-virtual:${IMAGE_TAG}" + +# Build Configuration +export SKIP_TESTS="true" +export MAVEN_OPTS="-Xmx2g" + +# Base Image +export BASE_IMAGE="registry.access.redhat.com/ubi9/openjdk-21:latest" + +# Print configuration +echo "==========================================" +echo "Benchmark Image Builder Configuration" +echo "==========================================" +echo "Java Version: ${JAVA_VERSION}" +echo "GraalVM Version: ${GRAALVM_VERSION}" +echo "Quarkus Version: ${QUARKUS_VERSION}" +echo "Spring Boot 3 Version: ${SPRING_BOOT3_VERSION}" +echo "Spring Boot 4 Version: ${SPRING_BOOT4_VERSION}" +echo "Registry: ${REGISTRY}" +echo "Registry User: ${REGISTRY_USER}" +echo "Image Tag: ${IMAGE_TAG}" +echo "==========================================" \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-jvm b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-jvm new file mode 100644 index 00000000..72d55e4d --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-jvm @@ -0,0 +1,23 @@ +# Dockerfile for Quarkus 3 with standard JVM +FROM registry.access.redhat.com/ubi9/openjdk-21:latest + +LABEL maintainer="Kruize Performance Team" +LABEL description="Quarkus 3.34.3 JVM mode for performance benchmarking" + +WORKDIR /deployments + +# Copy Quarkus application +COPY target/quarkus-app/lib/ /deployments/lib/ +COPY target/quarkus-app/*.jar /deployments/ +COPY target/quarkus-app/app/ /deployments/app/ +COPY target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 + +# Default JVM options (OOTB scenario) +ENV JAVA_OPTS="-Xms256m -Xmx512m -XX:+UseG1GC" + +CMD ["sh", "-c", "java $JAVA_OPTS -jar /deployments/quarkus-run.jar"] + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-spring-compat b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-spring-compat new file mode 100644 index 00000000..b77c1593 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-spring-compat @@ -0,0 +1,27 @@ +# Dockerfile for Quarkus 3 with Spring API Compatibility +FROM registry.access.redhat.com/ubi9/openjdk-21:latest + +# Set working directory +WORKDIR /deployments + +# Copy the application JAR +COPY target/quarkus-app/lib/ /deployments/lib/ +COPY target/quarkus-app/*.jar /deployments/ +COPY target/quarkus-app/app/ /deployments/app/ +COPY target/quarkus-app/quarkus/ /deployments/quarkus/ + +# Set environment variables +ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +# Expose port +EXPOSE 8080 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8080/q/health/ready || exit 1 + +# Run the application +ENTRYPOINT ["java", "-jar", "/deployments/quarkus-run.jar"] + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-virtual b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-virtual new file mode 100644 index 00000000..22db9363 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.quarkus3-virtual @@ -0,0 +1,24 @@ +# Dockerfile for Quarkus 3 with Virtual Threads +FROM registry.access.redhat.com/ubi9/openjdk-21:latest + +LABEL maintainer="Kruize Performance Team" +LABEL description="Quarkus 3.34.3 with Virtual Threads for performance benchmarking" + +WORKDIR /deployments + +# Copy Quarkus application +COPY target/quarkus-app/lib/ /deployments/lib/ +COPY target/quarkus-app/*.jar /deployments/ +COPY target/quarkus-app/app/ /deployments/app/ +COPY target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 + +# Default JVM options with Virtual Threads enabled +ENV JAVA_OPTS="-Xms256m -Xmx512m -XX:+UseG1GC" +ENV QUARKUS_VIRTUAL_THREADS_ENABLED="true" + +CMD ["sh", "-c", "java $JAVA_OPTS -jar /deployments/quarkus-run.jar"] + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring3-jvm b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring3-jvm new file mode 100644 index 00000000..c5435387 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring3-jvm @@ -0,0 +1,21 @@ +# Dockerfile for Spring Boot 3 with standard JVM +FROM registry.access.redhat.com/ubi9/openjdk-21:latest + +# Set working directory +WORKDIR /deployments + +# Copy the application JAR +COPY target/*.jar /deployments/app.jar + +# Expose port +EXPOSE 8080 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8080/actuator/health || exit 1 + +# Run the application with JAVA_OPTS support +# JAVA_OPTS can be passed via environment variable at runtime +ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /deployments/app.jar"] + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring4-jvm b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring4-jvm new file mode 100644 index 00000000..49a68004 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/apps/Dockerfile.spring4-jvm @@ -0,0 +1,21 @@ +# Dockerfile for Spring Boot 4 with standard JVM +FROM registry.access.redhat.com/ubi9/openjdk-21:latest + +# Set working directory +WORKDIR /deployments + +# Copy the application JAR +COPY target/*.jar /deployments/app.jar + +# Expose port +EXPOSE 8080 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8080/actuator/health || exit 1 + +# Run the application with JAVA_OPTS support +# JAVA_OPTS can be passed via environment variable at runtime +ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /deployments/app.jar"] + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/Dockerfile.postgres-fruits b/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/Dockerfile.postgres-fruits new file mode 100644 index 00000000..a5365d41 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/Dockerfile.postgres-fruits @@ -0,0 +1,5 @@ +FROM quay.io/centos7/postgresql-13-centos7:latest + +ADD create-postgres-data.sql /tmp/create-postgres-data.sql + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/create-postgres-data.sql b/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/create-postgres-data.sql new file mode 100644 index 00000000..52d21b17 --- /dev/null +++ b/spring-quarkus-perf-comparison/image-build/dockerfiles/postgres/create-postgres-data.sql @@ -0,0 +1,111 @@ +BEGIN; + +-- Database schema from spring-quarkus-perf-comparison +-- Source: https://github.com/quarkusio/spring-quarkus-perf-comparison/blob/main/scripts/dbdata/db.sql + +-- Create tables + +CREATE TABLE fruits ( + id bigint NOT NULL, + description VARCHAR(255), + name VARCHAR(255) NOT NULL UNIQUE, + PRIMARY KEY (id) +); +GRANT ALL PRIVILEGES ON fruits TO fruits; + +CREATE TABLE store_fruit_prices ( + price numeric(12,2) NOT NULL, + fruit_id bigint NOT NULL, + store_id bigint NOT NULL, + PRIMARY KEY (fruit_id, store_id) +); +GRANT ALL PRIVILEGES ON store_fruit_prices TO fruits; + +CREATE TABLE stores ( + id bigint NOT NULL, + address VARCHAR(255) NOT NULL, + city VARCHAR(255) NOT NULL, + country VARCHAR(255) NOT NULL, + currency VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + PRIMARY KEY (id) +); +GRANT ALL PRIVILEGES ON stores TO fruits; + +ALTER TABLE IF EXISTS store_fruit_prices + ADD CONSTRAINT fruit_id_fk + FOREIGN KEY (fruit_id) + REFERENCES fruits; + +ALTER TABLE IF EXISTS store_fruit_prices + ADD CONSTRAINT store_id_fk + FOREIGN KEY (store_id) + REFERENCES stores; + +-- Fruits +INSERT INTO fruits(id, name, description) VALUES (1, 'Apple', 'Hearty fruit'); +INSERT INTO fruits(id, name, description) VALUES (2, 'Pear', 'Juicy fruit'); +INSERT INTO fruits(id, name, description) VALUES (3, 'Banana', 'Tropical yellow fruit'); +INSERT INTO fruits(id, name, description) VALUES (4, 'Orange', 'Citrus fruit rich in vitamin C'); +INSERT INTO fruits(id, name, description) VALUES (5, 'Strawberry', 'Sweet red berry'); +INSERT INTO fruits(id, name, description) VALUES (6, 'Mango', 'Exotic tropical fruit'); +INSERT INTO fruits(id, name, description) VALUES (7, 'Grape', 'Small purple or green fruit'); +INSERT INTO fruits(id, name, description) VALUES (8, 'Pineapple', 'Large tropical fruit'); +INSERT INTO fruits(id, name, description) VALUES (9, 'Watermelon', 'Large refreshing summer fruit'); +INSERT INTO fruits(id, name, description) VALUES (10, 'Kiwi', 'Small fuzzy green fruit'); + +CREATE SEQUENCE fruits_seq START WITH 11 INCREMENT BY 1; +GRANT ALL PRIVILEGES ON SEQUENCE fruits_seq TO fruits; + +-- Stores +INSERT INTO stores(id, name, address, city, country, currency) VALUES (1, 'Store 1', '123 Main St', 'Anytown', 'USA', 'USD'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (2, 'Store 2', '456 Main St', 'Paris', 'France', 'EUR'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (3, 'Store 3', '789 Oak Ave', 'London', 'UK', 'GBP'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (4, 'Store 4', '321 Cherry Ln', 'Tokyo', 'Japan', 'JPY'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (5, 'Store 5', '555 Maple Dr', 'Toronto', 'Canada', 'CAD'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (6, 'Store 6', '888 Pine St', 'Sydney', 'Australia', 'AUD'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (7, 'Store 7', '999 Elm Rd', 'Berlin', 'Germany', 'EUR'); +INSERT INTO stores(id, name, address, city, country, currency) VALUES (8, 'Store 8', '147 Birch Blvd', 'Mexico City', 'Mexico', 'MXN'); + +CREATE SEQUENCE stores_seq START WITH 9 INCREMENT BY 1; +GRANT ALL PRIVILEGES ON SEQUENCE stores_seq TO fruits; + +-- Prices +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (1, 1, 1.29); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (1, 2, 0.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (1, 3, 0.59); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (1, 4, 1.19); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (1, 5, 3.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (2, 1, 2.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (2, 2, 1.19); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (2, 3, 0.89); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (2, 4, 1.79); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (2, 6, 2.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (3, 1, 1.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (3, 2, 1.29); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (3, 5, 3.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (3, 7, 2.79); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (4, 1, 189.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (4, 3, 99.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (4, 4, 149.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (4, 8, 599.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (5, 1, 1.79); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (5, 2, 1.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (5, 5, 4.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (5, 9, 6.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (6, 1, 2.19); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (6, 6, 3.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (6, 8, 5.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (6, 10, 1.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (7, 2, 1.39); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (7, 4, 1.89); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (7, 7, 2.49); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (7, 9, 4.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (8, 1, 25.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (8, 3, 12.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (8, 6, 39.99); +INSERT INTO store_fruit_prices(store_id, fruit_id, price) VALUES (8, 8, 49.99); + +COMMIT; + +-- Made with Bob diff --git a/spring-quarkus-perf-comparison/manifests/app-template.yaml b/spring-quarkus-perf-comparison/manifests/app-template.yaml new file mode 100644 index 00000000..c58fda1e --- /dev/null +++ b/spring-quarkus-perf-comparison/manifests/app-template.yaml @@ -0,0 +1,143 @@ +--- +# Template for application deployment +# Variables to replace: +# {{RUNTIME}} - Runtime name (e.g., quarkus3-jvm, spring4-jvm) +# {{SCENARIO}} - Scenario name (ootb or tuned) +# {{IMAGE}} - Full container image path +# {{CONTAINER_NAME}} - Container name (default: app) +# {{JAVA_OPTS}} - JVM options +# {{CPU_REQ}} - CPU request +# {{CPU_LIM}} - CPU limit +# {{MEM_REQ}} - Memory request +# {{MEM_LIM}} - Memory limit +# {{METRICS_PATH}} - Metrics endpoint path (/q/metrics for Quarkus, /actuator/prometheus for Spring) + +apiVersion: v1 +kind: Service +metadata: + name: {{RUNTIME}}-{{SCENARIO}} + labels: + app: {{RUNTIME}}-{{SCENARIO}} + runtime: {{RUNTIME}} + scenario: {{SCENARIO}} +spec: + ports: + - port: 8080 + targetPort: 8080 + name: http + selector: + app: {{RUNTIME}}-{{SCENARIO}} +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{RUNTIME}}-{{SCENARIO}} + labels: + app: {{RUNTIME}}-{{SCENARIO}} + runtime: {{RUNTIME}} + scenario: {{SCENARIO}} +spec: + to: + kind: Service + name: {{RUNTIME}}-{{SCENARIO}} + port: + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{RUNTIME}}-{{SCENARIO}} + labels: + app: {{RUNTIME}}-{{SCENARIO}} + runtime: {{RUNTIME}} + scenario: {{SCENARIO}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{RUNTIME}}-{{SCENARIO}} + template: + metadata: + labels: + app: {{RUNTIME}}-{{SCENARIO}} + runtime: {{RUNTIME}} + scenario: {{SCENARIO}} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "{{METRICS_PATH}}" + spec: + containers: + - name: {{CONTAINER_NAME}} + image: {{IMAGE}} + imagePullPolicy: Always + ports: + - containerPort: 8080 + name: http + protocol: TCP + env: + - name: JAVA_OPTS + value: "{{JAVA_OPTS}}" + - name: QUARKUS_PROFILE + value: "prod" + - name: QUARKUS_DATASOURCE_JDBC_URL + value: "jdbc:postgresql://postgresql:5432/fruits" + - name: QUARKUS_DATASOURCE_USERNAME + value: "fruits" + - name: QUARKUS_DATASOURCE_PASSWORD + value: "fruits" + - name: SPRING_DATASOURCE_URL + value: "jdbc:postgresql://postgresql:5432/fruits" + - name: SPRING_DATASOURCE_USERNAME + value: "fruits" + - name: SPRING_DATASOURCE_PASSWORD + value: "fruits" + resources: + requests: + memory: "{{MEM_REQ}}" + cpu: "{{CPU_REQ}}" + limits: + memory: "{{MEM_LIM}}" + cpu: "{{CPU_LIM}}" + livenessProbe: + httpGet: + path: /q/health/live + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /q/health/ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + startupProbe: + httpGet: + path: /q/health/started + port: 8080 + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 30 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{RUNTIME}}-{{SCENARIO}} + labels: + app: {{RUNTIME}}-{{SCENARIO}} + runtime: {{RUNTIME}} + scenario: {{SCENARIO}} +spec: + selector: + matchLabels: + app: {{RUNTIME}}-{{SCENARIO}} + endpoints: + - port: http + path: {{METRICS_PATH}} + interval: 30s + scheme: http diff --git a/spring-quarkus-perf-comparison/manifests/fixed-load-hf.yml b/spring-quarkus-perf-comparison/manifests/fixed-load-hf.yml new file mode 100644 index 00000000..f3d6aa4e --- /dev/null +++ b/spring-quarkus-perf-comparison/manifests/fixed-load-hf.yml @@ -0,0 +1,112 @@ +# ============================================================================= +# Hyperfoil Load Test Configuration +# ============================================================================= +# +# This configuration file defines a load test scenario for the application's read-all +# endpoint. It includes a warmup phase followed by a delay, then the main load test phase. +# +# All phases target the same HTTP endpoint. +# +# This is to replace how we did things exactly as before: +# warmup +# wrk -t 2 -c 100 -d 2m --timeout 2m http://localhost:8080/fruits +# +# cooldown +# sleep 30s +# +# loadTest +# wrk -t 2 -c 100 -d 30s --timeout 30s http://localhost:8080/fruits +# +# Configurable Parameters: +# ---------------------------------- +# PROTOCOL - HTTP protocol to use (default: http) +# Options: http, https +# HOST - Target server hostname (default: localhost) +# PORT - Target server port (default: 8080) +# CONNECTIONS - Number of HTTP connections (-c wrk param) (default: 100) +# HIDE_WARMUP_PHASE - Hide the warmup phase details (default: false) +# PATH - HTTP endpoint path to test (default: /fruits) +# WARMUP_DURATION - Duration of the warmup phase (-d wrk param) (default: 2m) +# WARMUP_MAX_DURATION - Maximum duration of the warmup phase (default: 4m) +# WARMUP_PAUSE_DURATION - Duration of the pause between warmup and main load test (default: 30s) +# WARMUP_REQUEST_TIMEOUT - Timeout for warmup requests (--timeout wrk param) (default: 2m) +# LOAD_REQUEST_TIMEOUT - Timeout for main load test requests (--timeout wrk param) (default: 30s) +# LOAD_DURATION - Duration of the main load test phase (-d wrk param) (default: 30s) +# LOAD_MAX_DURATION - Maximum duration of the main load test phase (default: 1m) +# THREADS - Number of threads to use (-t wrk param) (default: 2) +# +# Usage: +# ------ +# Override parameters using: -P PARAM_NAME=value +# Example: jbang run@hyperfoil -PHOST=example.com -PPORT=443 -PPROTOCOL=https load-test-fixed-threads-read-all.hf.yml +# +# ============================================================================= + +name: load-test-fixed-threads-read-all +threads: !param THREADS 2 +http: + - protocol: !param PROTOCOL http + host: !param HOST localhost + port: !param PORT 8080 + sharedConnections: !param CONNECTIONS 100 + allowHttp2: false + useHttpCache: false +ergonomics: + repeatCookies: false + userAgentFromSession: false + +phases: + - warmup: + always: + users: !param CONNECTIONS 100 + duration: !param WARMUP_DURATION 2m + maxDuration: !param WARMUP_MAX_DURATION 4m + isWarmup: !param HIDE_WARMUP_PHASE false + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getAllFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: !param WARMUP_REQUEST_TIMEOUT 2m + headers: + accept: application/json + + - cooldown: + noop: + startAfter: warmup + duration: !param WARMUP_PAUSE_DURATION 30s + + - logStart: + atOnce: + users: 1 + startAfterStrict: + - warmup + - cooldown + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - log: + - timestamp: + toVar: now + pattern: "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" + - log: "loadTest started at ${now}" + + - loadTest: + always: + users: !param CONNECTIONS 100 + duration: !param LOAD_DURATION 30s + maxDuration: !param LOAD_MAX_DURATION 1m + startAfterStrict: logStart + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getAllFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: !param LOAD_REQUEST_TIMEOUT 30s + headers: + accept: application/json diff --git a/spring-quarkus-perf-comparison/manifests/postgresql.yaml b/spring-quarkus-perf-comparison/manifests/postgresql.yaml new file mode 100644 index 00000000..d0584414 --- /dev/null +++ b/spring-quarkus-perf-comparison/manifests/postgresql.yaml @@ -0,0 +1,72 @@ +--- +# PostgreSQL deployment for OpenShift - Quarkus Benchmark +# Based on kruize/benchmarks TechEmpower setup +# Uses custom image with spring-quarkus-perf-comparison schema +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql + labels: + app: postgresql +spec: + replicas: 1 + selector: + matchLabels: + app: postgresql + template: + metadata: + labels: + name: postgresql + app: postgresql + app.kubernetes.io/name: "postgresql" + version: v1 + spec: + containers: + - name: postgresql + image: quay.io/kruizehub/quarkus-postgres:openshift + imagePullPolicy: IfNotPresent + env: + - name: POSTGRESQL_USER + value: fruits + - name: POSTGRESQL_PASSWORD + value: fruits + - name: POSTGRESQL_DATABASE + value: fruits + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "sleep 10 && psql -a fruits < /tmp/create-postgres-data.sql"] + ports: + - containerPort: 5432 + resources: + requests: + cpu: 0.5 + memory: "512Mi" + limits: + cpu: 2 + memory: "1024Mi" + volumeMounts: + - name: postgresql-data + mountPath: /var/lib/pgsql/data + volumes: + - name: postgresql-data + emptyDir: {} + +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql + labels: + name: postgresql +spec: + type: ClusterIP + selector: + app: postgresql + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/manifests/variable-load-24h.hf.yml b/spring-quarkus-perf-comparison/manifests/variable-load-24h.hf.yml new file mode 100644 index 00000000..5b4841dc --- /dev/null +++ b/spring-quarkus-perf-comparison/manifests/variable-load-24h.hf.yml @@ -0,0 +1,242 @@ +# ============================================================================= +# Hyperfoil 24-Hour Variable Load Test Configuration +# ============================================================================= +# +# This configuration creates a realistic variable load pattern over 24 hours +# using the same approach as run-benchmark.sh (concurrent users/connections) +# +# Key Parameters: +# - THREADS: Global thread count (applies to all phases) +# - *_CONNECTIONS: Per-phase concurrent user count (varies load) +# +# Configurable Parameters: +# ---------------------------------- +# PROTOCOL - HTTP protocol (default: http) +# HOST - Target hostname (default: localhost) +# PORT - Target port (default: 8080) +# PATH - Endpoint path (default: /fruits) +# THREADS - Global thread count (default: 4) +# +# Per-Phase Connection Parameters: +# NIGHT_CONNECTIONS - 0-6h: Low traffic (default: 50) +# MORNING_RAMP_1_CONNECTIONS - 6-7h: Ramp up (default: 100) +# MORNING_RAMP_2_CONNECTIONS - 7-8h: Ramp up (default: 200) +# MORNING_RAMP_3_CONNECTIONS - 8-9h: Ramp up (default: 300) +# MORNING_PEAK_CONNECTIONS - 9-12h: High load (default: 400) +# LUNCH_CONNECTIONS - 12-14h: Moderate (default: 250) +# AFTERNOON_PEAK_CONNECTIONS - 14-18h: Very high (default: 500) +# EVENING_1_CONNECTIONS - 18-19h: Decline (default: 400) +# EVENING_2_CONNECTIONS - 19-20h: Decline (default: 300) +# EVENING_3_CONNECTIONS - 20-21h: Decline (default: 200) +# LATE_NIGHT_CONNECTIONS - 21-24h: Low (default: 100) +# +# ============================================================================= + +name: variable-load-24h +threads: !param THREADS 4 +http: + - protocol: !param PROTOCOL http + host: !param HOST localhost + port: !param PORT 8080 + sharedConnections: 600 + allowHttp2: false + useHttpCache: false +ergonomics: + repeatCookies: false + userAgentFromSession: false + +phases: + # Night: 0-6h (6 hours) - Low traffic + - night: + always: + users: !param NIGHT_CONNECTIONS 50 + duration: 6h + maxDuration: 6h10m + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Morning ramp: 6-9h (3 hours) - Gradual increase + - morning-ramp-1: + always: + users: !param MORNING_RAMP_1_CONNECTIONS 100 + duration: 1h + maxDuration: 1h10m + startAfter: night + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + - morning-ramp-2: + always: + users: !param MORNING_RAMP_2_CONNECTIONS 200 + duration: 1h + maxDuration: 1h10m + startAfter: morning-ramp-1 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + - morning-ramp-3: + always: + users: !param MORNING_RAMP_3_CONNECTIONS 300 + duration: 1h + maxDuration: 1h10m + startAfter: morning-ramp-2 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Morning peak: 9-12h (3 hours) - High sustained load + - morning-peak: + always: + users: !param MORNING_PEAK_CONNECTIONS 400 + duration: 3h + maxDuration: 3h10m + startAfter: morning-ramp-3 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Lunch: 12-14h (2 hours) - Moderate sustained + - lunch: + always: + users: !param LUNCH_CONNECTIONS 250 + duration: 2h + maxDuration: 2h10m + startAfter: morning-peak + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Afternoon peak: 14-18h (4 hours) - Very high sustained + - afternoon-peak: + always: + users: !param AFTERNOON_PEAK_CONNECTIONS 500 + duration: 4h + maxDuration: 4h10m + startAfter: lunch + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Evening: 18-21h (3 hours) - Declining + - evening-decline-1: + always: + users: !param EVENING_1_CONNECTIONS 400 + duration: 1h + maxDuration: 1h10m + startAfter: afternoon-peak + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + - evening-decline-2: + always: + users: !param EVENING_2_CONNECTIONS 300 + duration: 1h + maxDuration: 1h10m + startAfter: evening-decline-1 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + - evening-decline-3: + always: + users: !param EVENING_3_CONNECTIONS 200 + duration: 1h + maxDuration: 1h10m + startAfter: evening-decline-2 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Late night: 21-24h (3 hours) - Low traffic + - late-night: + always: + users: !param LATE_NIGHT_CONNECTIONS 100 + duration: 3h + maxDuration: 3h10m + startAfter: evening-decline-3 + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/manifests/variable-load-4h.hf.yml b/spring-quarkus-perf-comparison/manifests/variable-load-4h.hf.yml new file mode 100644 index 00000000..5343f31e --- /dev/null +++ b/spring-quarkus-perf-comparison/manifests/variable-load-4h.hf.yml @@ -0,0 +1,132 @@ +# ============================================================================= +# Hyperfoil 4-Hour Variable Load Test Configuration +# ============================================================================= +# +# This configuration creates a compressed variable load pattern over 4 hours +# using the same approach as run-benchmark.sh (concurrent users/connections) +# +# Key Parameters: +# - THREADS: Global thread count (applies to all phases) +# - *_CONNECTIONS: Per-phase concurrent user count (varies load) +# +# Configurable Parameters: +# ---------------------------------- +# PROTOCOL - HTTP protocol (default: http) +# HOST - Target hostname (default: localhost) +# PORT - Target port (default: 8080) +# PATH - Endpoint path (default: /fruits) +# THREADS - Global thread count (default: 4) +# +# Per-Phase Connection Parameters (4-hour cycle): +# LOW_CONNECTIONS - 0-1h: Low traffic (default: 50) +# RAMP_UP_CONNECTIONS - 1-1.5h: Ramp up (default: 200) +# PEAK_CONNECTIONS - 1.5-3h: High load (default: 500) +# RAMP_DOWN_CONNECTIONS - 3-3.5h: Ramp down (default: 200) +# END_LOW_CONNECTIONS - 3.5-4h: Low traffic (default: 100) +# +# ============================================================================= + +name: variable-load-4h +threads: !param THREADS 4 +http: + - protocol: !param PROTOCOL http + host: !param HOST localhost + port: !param PORT 8080 + sharedConnections: 600 + allowHttp2: false + useHttpCache: false +ergonomics: + repeatCookies: false + userAgentFromSession: false + +phases: + # Low: 0-1h - Low traffic + - low: + always: + users: !param LOW_CONNECTIONS 50 + duration: 1h + maxDuration: 1h10m + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Ramp up: 1-1.5h - Gradual increase + - ramp-up: + always: + users: !param RAMP_UP_CONNECTIONS 200 + duration: 30m + maxDuration: 40m + startAfter: low + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Peak: 1.5-3h - High sustained load + - peak: + always: + users: !param PEAK_CONNECTIONS 500 + duration: 1h30m + maxDuration: 1h40m + startAfter: ramp-up + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # Ramp down: 3-3.5h - Gradual decrease + - ramp-down: + always: + users: !param RAMP_DOWN_CONNECTIONS 200 + duration: 30m + maxDuration: 40m + startAfter: peak + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + + # End low: 3.5-4h - Low traffic + - end-low: + always: + users: !param END_LOW_CONNECTIONS 100 + duration: 30m + maxDuration: 40m + startAfter: ramp-down + scenario: + maxRequests: 1 + maxSequences: 1 + orderedSequences: + - getFruits: + - httpRequest: + GET: !param PATH /fruits + timeout: 30s + headers: + accept: application/json + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/run-benchmark.sh b/spring-quarkus-perf-comparison/run-benchmark.sh new file mode 100755 index 00000000..95915ea4 --- /dev/null +++ b/spring-quarkus-perf-comparison/run-benchmark.sh @@ -0,0 +1,858 @@ +#!/bin/bash + +# ============================================================================ +# OpenShift Benchmark Runner +# ============================================================================ +# Main script to run performance benchmarks on OpenShift +# Supports multiple runtimes, scenarios, and test types + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Load configuration +if [ -f "config.env" ]; then + source config.env +else + echo "ERROR: config.env not found" + exit 1 +fi + +# ============================================================================ +# Global Variables +# ============================================================================ +TIMESTAMP=$(date +%Y-%m-%d-%H-%M-%S) +RUN_DIR="${RESULTS_DIR}/${TIMESTAMP}" +LOG_FILE="${RUN_DIR}/benchmark.log" + +# ============================================================================ +# Helper Functions +# ============================================================================ + +log() { + local level=$1 + shift + local message="$@" + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$level] $message" | tee -a "$LOG_FILE" +} + +log_info() { + log "INFO" "$@" +} + +log_error() { + log "ERROR" "$@" +} + +log_debug() { + if [ "$LOG_LEVEL" = "DEBUG" ]; then + log "DEBUG" "$@" + fi +} + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Run performance benchmarks on OpenShift cluster + +Options: + --scenarios Comma-separated scenarios: ootb,tuned (default: ${SCENARIOS}) + --runtimes Comma-separated list of runtimes (default: ${RUNTIMES}) + --tests Comma-separated list of tests (default: ${TESTS}) + --iterations Number of iterations (default: ${ITERATIONS}) + --output-dir Output directory (default: ${RESULTS_DIR}) + --registry Container registry (default: ${REGISTRY}) + --image-tag Image tag (default: ${IMAGE_TAG}) + --project OpenShift project (default: ${OPENSHIFT_PROJECT}) + --cleanup Cleanup after tests (default: ${CLEANUP_AFTER_TEST}) + --no-cleanup Don't cleanup after tests + --help Show this help message + +Examples: + # Run default configuration + $0 + + # Run specific scenarios and runtimes + $0 --scenarios ootb,tuned --runtimes quarkus3-jvm,spring4-jvm + + # Run with custom iterations + $0 --iterations 5 --tests run-load-test + + # Run without cleanup + $0 --no-cleanup + +Available Runtimes: + - quarkus3-jvm Quarkus 3 with standard JVM + - quarkus3-virtual Quarkus 3 with Virtual Threads + - spring4-jvm Spring Boot 4 with standard JVM + - spring4-virtual Spring Boot 4 with Virtual Threads + +Available Tests: + - measure-startup Measure startup time + - measure-memory Measure memory usage (RSS) + - run-load-test Run Hyperfoil load test + +Available Scenarios: + - ootb Out-of-the-box (default JVM settings) + - tuned Tuned (optimized JVM settings) + +EOF + exit 0 +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --scenario|--scenarios) + SCENARIOS="$2" + shift 2 + ;; + --runtimes) + RUNTIMES="$2" + shift 2 + ;; + --tests) + TESTS="$2" + shift 2 + ;; + --iterations) + ITERATIONS="$2" + shift 2 + ;; + --output-dir) + RESULTS_DIR="$2" + RUN_DIR="${RESULTS_DIR}/${TIMESTAMP}" + shift 2 + ;; + --registry) + REGISTRY="$2" + shift 2 + ;; + --image-tag) + IMAGE_TAG="$2" + shift 2 + ;; + --project) + OPENSHIFT_PROJECT="$2" + shift 2 + ;; + --cleanup) + CLEANUP_AFTER_TEST="true" + shift + ;; + --no-cleanup) + CLEANUP_AFTER_TEST="false" + shift + ;; + --help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac + done +} + +# Check prerequisites +check_prerequisites() { + log_info "Checking prerequisites..." + + # Check oc CLI + if ! command -v oc &> /dev/null; then + log_error "oc CLI not found. Please install OpenShift CLI." + exit 1 + fi + + # Check jq + if ! command -v jq &> /dev/null; then + log_error "jq not found. Please install jq for JSON processing." + exit 1 + fi + + # Check JBang (required for load testing) + if [[ "$TESTS" == *"run-load-test"* ]]; then + if ! command -v jbang &> /dev/null; then + log_info "JBang not found. Installing JBang..." + if curl -Ls https://sh.jbang.dev | bash -s - app setup; then + # Source the jbang environment + export PATH="$HOME/.jbang/bin:$PATH" + log_info "JBang installed successfully" + else + log_error "Failed to install JBang automatically" + log_error "Please install manually from: https://www.jbang.dev/" + exit 1 + fi + fi + log_info "JBang found: $(jbang version)" + fi + + # Check oc login + if ! oc whoami &> /dev/null; then + log_error "Not logged in to OpenShift. Please run 'oc login' first." + exit 1 + fi + + # Validate configuration + if ! validate_config; then + log_error "Configuration validation failed" + exit 1 + fi + + log_info "Prerequisites check passed" +} + +# Setup OpenShift project +setup_project() { + log_info "Setting up OpenShift project: ${OPENSHIFT_PROJECT}" + + if oc get project "${OPENSHIFT_PROJECT}" &> /dev/null; then + log_info "Project ${OPENSHIFT_PROJECT} already exists" + else + log_info "Creating project ${OPENSHIFT_PROJECT}" + oc new-project "${OPENSHIFT_PROJECT}" + fi + + oc project "${OPENSHIFT_PROJECT}" +} + +# Deploy PostgreSQL +deploy_postgresql() { + log_info "Deploying PostgreSQL..." + + if oc get deployment postgresql &> /dev/null; then + log_info "PostgreSQL already deployed" + return 0 + fi + + oc apply -f manifests/postgresql.yaml + + log_info "Waiting for PostgreSQL to be ready..." + oc wait --for=condition=available --timeout=300s deployment/postgresql + + log_info "PostgreSQL deployed successfully" +} + +# Note: Hyperfoil deployment removed - using curl-based load testing instead +# This matches the original repository's approach (they use qDup/JBang CLI) + +# Generate manifest from template +generate_manifest() { + local runtime=$1 + local scenario=$2 + local output_file=$3 + + local image=$(get_image_name "$runtime") + local java_opts=$(get_java_opts "$scenario") + local container_name="${CONTAINER_NAME:-app}" + + # Get scenario-specific resources + local cpu_req cpu_lim mem_req mem_lim + case "$scenario" in + ootb) + cpu_req="${CPU_REQUEST_OOTB:-${CPU_REQUEST}}" + cpu_lim="${CPU_LIMIT_OOTB:-${CPU_LIMIT}}" + mem_req="${MEMORY_REQUEST_OOTB:-${MEMORY_REQUEST}}" + mem_lim="${MEMORY_LIMIT_OOTB:-${MEMORY_LIMIT}}" + ;; + tuned) + cpu_req="${CPU_REQUEST_TUNED:-${CPU_REQUEST}}" + cpu_lim="${CPU_LIMIT_TUNED:-${CPU_LIMIT}}" + mem_req="${MEMORY_REQUEST_TUNED:-${MEMORY_REQUEST}}" + mem_lim="${MEMORY_LIMIT_TUNED:-${MEMORY_LIMIT}}" + ;; + *) + # For custom scenarios, use defaults + cpu_req="${CPU_REQUEST}" + cpu_lim="${CPU_LIMIT}" + mem_req="${MEMORY_REQUEST}" + mem_lim="${MEMORY_LIMIT}" + ;; + esac + + # Determine metrics path based on runtime + local metrics_path + case "$runtime" in + quarkus*|quarkus3-spring-compat) + metrics_path="/q/metrics" + ;; + spring*) + metrics_path="/actuator/prometheus" + ;; + *) + metrics_path="/metrics" + ;; + esac + + log_debug "Generating manifest for ${runtime}-${scenario}" + log_debug "Image: ${image}" + log_debug "Container Name: ${container_name}" + log_debug "JAVA_OPTS: ${java_opts}" + log_debug "Resources: CPU ${cpu_req}/${cpu_lim}, Memory ${mem_req}/${mem_lim}" + log_debug "Metrics path: ${metrics_path}" + + sed -e "s|{{RUNTIME}}|${runtime}|g" \ + -e "s|{{SCENARIO}}|${scenario}|g" \ + -e "s|{{IMAGE}}|${image}|g" \ + -e "s|{{CONTAINER_NAME}}|${container_name}|g" \ + -e "s|{{JAVA_OPTS}}|${java_opts}|g" \ + -e "s|{{CPU_REQ}}|${cpu_req}|g" \ + -e "s|{{CPU_LIM}}|${cpu_lim}|g" \ + -e "s|{{MEM_REQ}}|${mem_req}|g" \ + -e "s|{{MEM_LIM}}|${mem_lim}|g" \ + -e "s|{{METRICS_PATH}}|${metrics_path}|g" \ + manifests/app-template.yaml > "$output_file" +} + +# Deploy application +deploy_app() { + local runtime=$1 + local scenario=$2 + + log_info "Deploying ${runtime} with ${scenario} scenario..." + + local manifest_file="${RUN_DIR}/manifests/${runtime}-${scenario}.yaml" + mkdir -p "$(dirname "$manifest_file")" + + generate_manifest "$runtime" "$scenario" "$manifest_file" + + oc apply -f "$manifest_file" + + log_info "Waiting for ${runtime}-${scenario} to be ready..." + if ! oc wait --for=condition=available --timeout=${DEPLOYMENT_TIMEOUT}s deployment/${runtime}-${scenario}; then + log_error "Deployment ${runtime}-${scenario} failed to become ready" + return 1 + fi + + log_info "${runtime}-${scenario} deployed successfully" + return 0 +} + +# Run benchmark for a runtime/scenario combination +run_benchmark() { + local runtime=$1 + local scenario=$2 + local iteration=$3 + + log_info "Running benchmark: ${runtime}-${scenario} (iteration ${iteration}/${ITERATIONS})" + + local result_file="${RUN_DIR}/results/${runtime}-${scenario}-${iteration}.json" + mkdir -p "$(dirname "$result_file")" + + # Initialize result JSON + cat > "$result_file" << EOF +{ + "runtime": "${runtime}", + "scenario": "${scenario}", + "iteration": ${iteration}, + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "tests": {} +} +EOF + + # Run each test + IFS=',' read -ra TEST_ARRAY <<< "$TESTS" + for test in "${TEST_ARRAY[@]}"; do + log_info "Running test: ${test}" + + case $test in + measure-startup) + ./scripts/measure-startup.sh "$runtime" "$scenario" "$result_file" + ;; + measure-memory) + ./scripts/measure-memory.sh "$runtime" "$scenario" "$result_file" + ;; + run-load-test) + ./scripts/run-load-test.sh "$runtime" "$scenario" "$result_file" + ;; + *) + log_error "Unknown test: ${test}" + ;; + esac + done + + log_info "Benchmark completed: ${runtime}-${scenario} (iteration ${iteration})" +} + +# Cleanup deployment +cleanup_deployment() { + local runtime=$1 + local scenario=$2 + + if [ "$CLEANUP_AFTER_TEST" = "true" ]; then + log_info "Cleaning up ${runtime}-${scenario}..." + oc delete deployment,service,route ${runtime}-${scenario} --ignore-not-found=true + else + log_info "Skipping cleanup for ${runtime}-${scenario}" + fi +} +# Print configuration +print_config() { + log_info "==========================================" + log_info "Configuration:" + log_info " Scenarios: ${SCENARIOS}" + log_info " Runtimes: ${RUNTIMES}" + log_info " Tests: ${TESTS}" + log_info " Iterations: ${ITERATIONS}" + log_info " OpenShift Project: ${OPENSHIFT_PROJECT}" + log_info " Results Directory: ${RUN_DIR}" + log_info " Cleanup After Test: ${CLEANUP_AFTER_TEST}" + log_info "==========================================" +} + + +# Main benchmark execution +run_benchmarks() { + log_info "Starting benchmark run: ${TIMESTAMP}" + print_config + + # Setup + setup_project + deploy_postgresql + + # Note: No need to deploy Hyperfoil - using curl-based load testing + + # Parse scenarios and runtimes + IFS=',' read -ra SCENARIO_ARRAY <<< "$SCENARIOS" + IFS=',' read -ra RUNTIME_ARRAY <<< "$RUNTIMES" + + log_info "Will run ${#SCENARIO_ARRAY[@]} scenario(s) x ${#RUNTIME_ARRAY[@]} runtime(s) x ${ITERATIONS} iteration(s)" + + # Run benchmarks for each scenario + for scenario in "${SCENARIO_ARRAY[@]}"; do + log_info "==========================================" + log_info "Starting scenario: ${scenario}" + log_info "==========================================" + + # Run benchmarks for each runtime + for runtime in "${RUNTIME_ARRAY[@]}"; do + log_info "Processing runtime: ${runtime} (scenario: ${scenario})" + + # Run multiple iterations + for ((i=1; i<=ITERATIONS; i++)); do + log_info "Iteration ${i}/${ITERATIONS} for ${runtime}-${scenario}" + + # Deploy application + if ! deploy_app "$runtime" "$scenario"; then + log_error "Failed to deploy ${runtime}-${scenario}" + continue + fi + + # Wait for application to stabilize + log_info "Waiting for application to stabilize..." + sleep 10 + + # Run benchmark + run_benchmark "$runtime" "$scenario" "$i" + + # Cleanup + cleanup_deployment "$runtime" "$scenario" + + # Wait between iterations + if [ $i -lt $ITERATIONS ]; then + log_info "Waiting before next iteration..." + sleep 5 + fi + done + + # Wait between runtimes + log_info "Completed ${runtime}-${scenario}" + sleep 5 + done + + log_info "Completed scenario: ${scenario}" + done + + log_info "All benchmarks completed" +} + +# Generate comprehensive metrics JSON (similar to reference repo format) +generate_metrics_json() { + log_info "Generating comprehensive metrics JSON..." + + local metrics_file="${RUN_DIR}/metrics.json" + local start_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local stop_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + # Start building JSON structure + cat > "$metrics_file" << 'EOF_START' +{ + "timing": { + "start": "START_TIME_PLACEHOLDER", + "stop": "STOP_TIME_PLACEHOLDER" + }, + "results": { +EOF_START + + # Replace placeholders + sed -i "s/START_TIME_PLACEHOLDER/${start_time}/" "$metrics_file" + sed -i "s/STOP_TIME_PLACEHOLDER/${stop_time}/" "$metrics_file" + + # Parse scenarios and runtimes + IFS=',' read -ra SCENARIO_ARRAY <<< "$SCENARIOS" + IFS=',' read -ra RUNTIME_ARRAY <<< "$RUNTIMES" + local first_entry=true + + log_info "Processing scenarios: ${SCENARIOS}" + log_info "Processing runtimes: ${RUNTIMES}" + log_info "Number of combinations: $((${#SCENARIO_ARRAY[@]} * ${#RUNTIME_ARRAY[@]}))" + + # Check if we have any runtimes + if [ ${#RUNTIME_ARRAY[@]} -eq 0 ] || [ -z "${RUNTIMES}" ]; then + log_error "No runtimes found. RUNTIMES variable is empty." + return 1 + fi + + # Add results for each scenario-runtime combination + for scenario in "${SCENARIO_ARRAY[@]}"; do + for runtime in "${RUNTIME_ARRAY[@]}"; do + local runtime_scenario_key="${runtime}-${scenario}" + log_info "Processing: ${runtime_scenario_key}" + if [ "$first_entry" = false ]; then + echo "," >> "$metrics_file" + fi + first_entry=false + + # Aggregate results across iterations + local total_throughput=0 + local total_memory=0 + local total_startup=0 + local total_errors=0 + local count=0 + + # Parse all iteration results for this runtime-scenario combination + for ((i=1; i<=ITERATIONS; i++)); do + local result_file="${RUN_DIR}/results/${runtime}-${scenario}-${i}.json" + log_debug "Checking for result file: ${result_file}" + if [ -f "$result_file" ]; then + log_debug "Found result file: ${result_file}" + # Extract metrics using jq - support both old and new field names + local throughput=$(jq -r '.tests.load_test.requests_per_sec // .tests."run-load-test".throughput // 0' "$result_file" 2>/dev/null || echo "0") + local memory=$(jq -r '.tests.memory.rss_mb // .tests."measure-memory".rss_mib // 0' "$result_file" 2>/dev/null || echo "0") + local startup=$(jq -r '.tests.startup.startup_time_ms // .tests."measure-startup".startup_ms // 0' "$result_file" 2>/dev/null || echo "0") + local errors=$(jq -r '.tests.load_test.errors // .tests."run-load-test".errors // 0' "$result_file" 2>/dev/null || echo "0") + + log_debug "Extracted metrics - throughput: ${throughput}, memory: ${memory}, startup: ${startup}" + + total_throughput=$(awk "BEGIN {print $total_throughput + $throughput}" 2>/dev/null || echo "0") + total_memory=$(awk "BEGIN {print $total_memory + $memory}" 2>/dev/null || echo "0") + total_startup=$(awk "BEGIN {print $total_startup + $startup}" 2>/dev/null || echo "0") + total_errors=$(awk "BEGIN {print $total_errors + $errors}" 2>/dev/null || echo "0") + ((count++)) || true + else + log_debug "Result file not found: ${result_file}" + fi + done + + # Calculate averages + local av_throughput=0 + local av_memory=0 + local av_startup=0 + local av_errors=0 + local throughput_density=0 + + if [ $count -gt 0 ]; then + av_throughput=$(awk "BEGIN {printf \"%.2f\", $total_throughput / $count}") + av_memory=$(awk "BEGIN {printf \"%.2f\", $total_memory / $count}") + av_startup=$(awk "BEGIN {printf \"%.2f\", $total_startup / $count}") + av_errors=$(awk "BEGIN {printf \"%.0f\", $total_errors / $count}") + + # Calculate throughput density (throughput per MiB) + if [ "$(awk "BEGIN {print ($av_memory > 0)}")" -eq 1 ]; then + throughput_density=$(awk "BEGIN {printf \"%.6f\", $av_throughput / $av_memory}") + fi + fi + + # Write runtime-scenario results + cat >> "$metrics_file" << EOF + "${runtime_scenario_key}": { + "load": { + "throughput": [${av_throughput}], + "connectionErrors": [${av_errors}], + "requestTimeouts": [0], + "appErrors": [0], + "app4xxErrors": [0], + "app5xxErrors": [0], + "rss": [${av_memory}], + "throughputDensity": [${throughput_density}], + "avThroughput": ${av_throughput}, + "avMaxRss": ${av_memory}, + "maxThroughputDensity": ${throughput_density}, + "avConnectionErrors": ${av_errors}, + "avRequestTimeouts": 0, + "avAppErrors": 0, + "avApp4xxErrors": 0, + "avApp5xxErrors": 0 + }, + "startup": { + "timings": [${av_startup}], + "avStartupTime": ${av_startup} + } + } +EOF + done + done + + # Add configuration section + cat >> "$metrics_file" << EOF + }, + "config": { + "units": { + "timings": { + "startup": "ms" + }, + "rss": { + "startup": "MiB", + "firstRequest": "MiB", + "load": "MiB" + }, + "load": { + "throughput": "req/s", + "throughputDensity": "req/s per MiB", + "errors": { + "connectionErrors": "count", + "requestTimeouts": "count" + } + } + }, + "jvm": { + "vendor": "${JVM_VENDOR}", + "version": "${JAVA_VERSION}", + "base_image": "${BASE_IMAGE}", + "java_opts": { + "ootb": "${JAVA_OPTS_OOTB:-}", + "tuned": "${JAVA_OPTS_TUNED:-}" + } + }, + "frameworks": { + "quarkus": { + "version": "${QUARKUS_VERSION}" + }, + "spring_boot": { + "version": "${SPRING_BOOT_VERSION}" + } + }, + "run": { + "description": "Spring and Quarkus Performance Comparison on OpenShift", + "identifier": "openshift-benchmark-${TIMESTAMP}", + "scenarios": "${SCENARIOS}", + "iterations": ${ITERATIONS} + }, + "resources": { + "application": { + "ootb": { + "cpu": { + "request": "${CPU_REQUEST_OOTB:-${CPU_REQUEST}}", + "limit": "${CPU_LIMIT_OOTB:-${CPU_LIMIT}}" + }, + "memory": { + "request": "${MEMORY_REQUEST_OOTB:-${MEMORY_REQUEST}}", + "limit": "${MEMORY_LIMIT_OOTB:-${MEMORY_LIMIT}}" + } + }, + "tuned": { + "cpu": { + "request": "${CPU_REQUEST_TUNED:-${CPU_REQUEST}}", + "limit": "${CPU_LIMIT_TUNED:-${CPU_LIMIT}}" + }, + "memory": { + "request": "${MEMORY_REQUEST_TUNED:-${MEMORY_REQUEST}}", + "limit": "${MEMORY_LIMIT_TUNED:-${MEMORY_LIMIT}}" + } + } + }, + "database": { + "cpu": { + "request": "${POSTGRES_CPU_REQUEST}", + "limit": "${POSTGRES_CPU_LIMIT}" + }, + "memory": { + "request": "${POSTGRES_MEMORY_REQUEST}", + "limit": "${POSTGRES_MEMORY_LIMIT}" + } + } + }, + "load_test": { + "tool": "hyperfoil-jbang", + "version": "${HYPERFOIL_VERSION}", + "duration": "${LOAD_TEST_DURATION}", + "connections": ${LOAD_TEST_CONNECTIONS}, + "rate": ${LOAD_TEST_RATE}, + "warmup": "${LOAD_TEST_WARMUP}" + }, + "profiler": { + "name": "none", + "events": "cpu" + }, + "repository": { + "url": "https://github.com/quarkusio/spring-quarkus-perf-comparison.git", + "branch": "main" + }, + "openshift": { + "project": "${OPENSHIFT_PROJECT}", + "cluster_url": "${OPENSHIFT_CLUSTER_URL}" + } + }, + "env": { + "host": { + "os": "$(cat /etc/os-release 2>/dev/null | grep PRETTY_NAME | cut -d'"' -f2 || echo 'Unknown')", + "type": "$(dmidecode -s system-product-name 2>/dev/null || echo 'Unknown')", + "kernel": "$(uname -r)", + "cpu": "$(lscpu | grep 'Model name' | cut -d':' -f2 | xargs || echo 'Unknown')", + "memory": "$(free -h | grep Mem | awk '{print $2}')" + }, + "run": { + "host": { + "user": "$(whoami)", + "name": "$(hostname)", + "target": "$(whoami)@$(hostname)" + } + } + } +} +EOF + + log_info "Comprehensive metrics JSON generated: ${metrics_file}" + + # Output to stdout for logging (similar to reference repo) + echo "" + log_info "set-state: RUN.output.config $(jq -c '.config' "$metrics_file")" + log_info "set-state: RUN.output.env $(jq -c '.env' "$metrics_file")" + log_info "echo '$(cat "$metrics_file")' > ${metrics_file}" +} + +# Generate summary +generate_summary() { + log_info "Generating summary..." + + local summary_file="${RUN_DIR}/summary.json" + + cat > "$summary_file" << EOF +{ + "timestamp": "${TIMESTAMP}", + "scenarios": "${SCENARIOS}", + "runtimes": "${RUNTIMES}", + "tests": "${TESTS}", + "iterations": ${ITERATIONS}, + "results_directory": "${RUN_DIR}" +} +EOF + + log_info "Summary saved to: ${summary_file}" + log_info "Results directory: ${RUN_DIR}" + + # Generate comprehensive metrics JSON + if ! generate_metrics_json; then + log_error "Failed to generate metrics.json" + log_error "You can regenerate it later using: ./scripts/results-tools/regenerate-metrics.sh ${RUN_DIR}/results" + return 1 + fi +} + +# Generate comprehensive comparison report +generate_comparison() { + log_info "Generating comprehensive comparison report..." + + # Check if comparison script exists + if [ ! -f "${SCRIPT_DIR}/scripts/results-tools/compare-all.sh" ]; then + log_error "Comparison script not found: ${SCRIPT_DIR}/scripts/results-tools/compare-all.sh" + return 1 + fi + + # Run comprehensive comparison (compares all runtimes and scenarios) + "${SCRIPT_DIR}/scripts/results-tools/compare-all.sh" "${RUN_DIR}" > tee "${RUN_DIR}/comparison.txt" + + log_info "Comparison report saved to: ${RUN_DIR}/comparison.txt" + return 0 +} + +# Generate HTML report +generate_html_report() { + log_info "Generating HTML report..." + + # Check if report generation script exists + if [ ! -f "${SCRIPT_DIR}/scripts/results-tools/generate-report.sh" ]; then + log_error "Report generation script not found: ${SCRIPT_DIR}/scripts/results-tools/generate-report.sh" + return 1 + fi + + # Generate HTML report + "${SCRIPT_DIR}/scripts/results-tools/generate-report.sh" "${RUN_DIR}" + + if [ -f "${RUN_DIR}/report.html" ]; then + log_info "HTML report generated: ${RUN_DIR}/report.html" + log_info "Open in browser: file://${RUN_DIR}/report.html" + else + log_error "Failed to generate HTML report" + return 1 + fi +} + +# ============================================================================ +# Main Execution +# ============================================================================ + +main() { + # Parse arguments + parse_args "$@" + + # Create results directory + mkdir -p "$RUN_DIR" + + # Start logging + log_info "==========================================" + log_info "OpenShift Benchmark Runner" + log_info "==========================================" + + # Check prerequisites + check_prerequisites + + # Run benchmarks + run_benchmarks + + # Generate summary + if ! generate_summary; then + log_error "Failed to generate summary and metrics" + log_error "Benchmark results are in: ${RUN_DIR}/results/" + log_error "You can regenerate metrics.json using: ./scripts/results-tools/regenerate-metrics.sh ${RUN_DIR}/results" + fi + + # Generate analysis reports + log_info "" + log_info "==========================================" + log_info "Generating Analysis Reports" + log_info "==========================================" + + # Generate comprehensive comparison (all runtimes x all scenarios) + if generate_comparison; then + log_info "✓ Comprehensive comparison report generated" + else + log_info "⚠ Comparison report generation failed" + fi + + # Generate HTML report + if generate_html_report; then + log_info "✓ HTML report generated" + else + log_info "⚠ HTML report generation failed" + fi + + log_info "" + log_info "==========================================" + log_info "Benchmark Run Completed Successfully" + log_info "==========================================" + log_info "Results directory: ${RUN_DIR}" + log_info "Summary: ${RUN_DIR}/summary.json" + log_info "Comparison: ${RUN_DIR}/comparison.txt" + log_info "HTML Report: file://${RUN_DIR}/report.html" + log_info "==========================================" +} + +# Run main function +main "$@" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/common-utils.sh b/spring-quarkus-perf-comparison/scripts/common-utils.sh new file mode 100644 index 00000000..4279fa67 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/common-utils.sh @@ -0,0 +1,217 @@ +#!/bin/bash + +# ============================================================================ +# Common Utilities for Benchmark Scripts +# ============================================================================ +# Shared functions used across multiple benchmark scripts + +# Color codes for terminal output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Basic logging functions (no color) +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +log_error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR] $@" >&2 +} + +log_warn() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [WARN] $@" >&2 +} + +# Colored logging functions (for interactive use) +log_info_color() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error_color() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" >&2 +} + +log_warn_color() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] [WARN]${NC} $@" >&2 +} + +log_debug() { + if [ "${DEBUG:-false}" = "true" ]; then + echo -e "${CYAN}[$(date +'%Y-%m-%d %H:%M:%S')] [DEBUG]${NC} $@" >&2 + fi +} + +# Validate that a directory exists and is readable +# Usage: validate_directory +# Returns: 0 on success, 1 on failure +validate_directory() { + local dir_path=$1 + local description=${2:-"Directory"} + + if [ -z "$dir_path" ]; then + log_error_color "$description path is empty" + return 1 + fi + + if [ ! -d "$dir_path" ]; then + log_error_color "$description not found: $dir_path" + return 1 + fi + + if [ ! -r "$dir_path" ]; then + log_error_color "$description is not readable: $dir_path" + return 1 + fi + + return 0 +} + +# Validate that a file exists and is readable +# Usage: validate_file +# Returns: 0 on success, 1 on failure +validate_file() { + local file_path=$1 + local description=${2:-"File"} + + if [ -z "$file_path" ]; then + log_error_color "$description path is empty" + return 1 + fi + + if [ ! -f "$file_path" ]; then + log_error_color "$description not found: $file_path" + return 1 + fi + + if [ ! -r "$file_path" ]; then + log_error_color "$description is not readable: $file_path" + return 1 + fi + + return 0 +} + +# Check if jbang is installed +# Returns: 0 if installed, 1 if not +check_jbang() { + if ! command -v jbang >/dev/null 2>&1; then + log_error_color "JBang is not installed" + log_error_color "Please install JBang from https://www.jbang.dev/" + log_error_color "" + log_error_color "Quick install:" + log_error_color " curl -Ls https://sh.jbang.dev | bash -s - app setup" + return 1 + fi + log_info_color "JBang found: $(jbang version)" + return 0 +} + +# Check if required command exists +# Usage: require_command +# Returns: 0 if exists, 1 if not +require_command() { + local cmd=$1 + local pkg=${2:-$1} + + if ! command -v "$cmd" >/dev/null 2>&1; then + log_error_color "Required command not found: $cmd" + log_error_color "Please install: $pkg" + return 1 + fi + return 0 +} + +# Parse Hyperfoil results and generate JSON +# Usage: parse_hyperfoil_results +parse_hyperfoil_results() { + local log_file=$1 + local phase_name=$2 + local threads=$3 + local connections=$4 + local duration=$5 + + if [ ! -f "$log_file" ]; then + log_error "Log file not found: $log_file" + echo "{}" + return 1 + fi + + # Extract the phase line from Hyperfoil output + # Try different phase names: "main", "loadTest", "warmup" + local phase_line=$(grep -E "^(main|loadTest|warmup)" "$log_file" | head -1) + + if [ -z "$phase_line" ]; then + log_error "Could not find phase results in log: $log_file" + echo "{}" + return 1 + fi + + # Parse values from Hyperfoil output + # Format: "phase scenario 578.76 req/s 17379 86.28 ms 60.92 ms 599.79 ms 98.04 ms 124.26 ms 297.80 ms 400.56 ms 599.79 ms 0 0 0 ns 17379 0 0 0 0" + local throughput=$(echo "$phase_line" | grep -oP '\d+\.\d+\s+req/s' | grep -oP '\d+\.\d+') + local total_requests=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="req/s") print $(i+1)}') + local latency_mean=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="req/s") print $(i+2)}') + local latency_p50=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="req/s") print $(i+8)}') + local latency_p90=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="req/s") print $(i+10)}') + local latency_p99=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="req/s") print $(i+12)}') + local errors=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="ns") print $(i-1)}') + local success_2xx=$(echo "$phase_line" | awk '{for(i=1;i<=NF;i++) if($i=="ns") print $(i+1)}') + + # Set defaults if parsing failed + throughput=${throughput:-0} + total_requests=${total_requests:-0} + latency_mean=${latency_mean:-0} + latency_p50=${latency_p50:-0} + latency_p90=${latency_p90:-0} + latency_p99=${latency_p99:-0} + errors=${errors:-0} + success_2xx=${success_2xx:-0} + + # Convert duration to seconds (handle formats like "1h", "30m", "1h30m", "60s") + local duration_sec=$(echo "$duration" | awk ' + { + total = 0 + if (match($0, /([0-9]+)h/, arr)) total += arr[1] * 3600 + if (match($0, /([0-9]+)m/, arr)) total += arr[1] * 60 + if (match($0, /([0-9]+)s/, arr)) total += arr[1] + print total + } + ') + + # If duration_sec is 0, try simple conversion (remove 's' suffix) + if [ "$duration_sec" = "0" ]; then + duration_sec=$(echo "$duration" | sed 's/s$//') + fi + + # Generate JSON + cat << EOF +{ + "phase_name": "${phase_name}", + "configuration": { + "threads": ${threads}, + "connections": ${connections}, + "duration": "${duration}", + "duration_seconds": ${duration_sec} + }, + "results": { + "tool": "hyperfoil-jbang", + "requests_total": ${total_requests}, + "requests_per_sec": ${throughput}, + "latency_mean_ms": ${latency_mean}, + "latency_p50_ms": ${latency_p50}, + "latency_p90_ms": ${latency_p90}, + "latency_p99_ms": ${latency_p99}, + "errors": ${errors}, + "success_2xx": ${success_2xx} + } +} +EOF + + return 0 +} + +# Made with Bob \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/scripts/deploy-app.sh b/spring-quarkus-perf-comparison/scripts/deploy-app.sh new file mode 100755 index 00000000..42d6d3d0 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/deploy-app.sh @@ -0,0 +1,445 @@ +#!/bin/bash +################################################################################ +# Application Deployment Script +# +# Deploys application and PostgreSQL database to OpenShift +# Outputs the application endpoint URL for use with load testing +################################################################################ + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} [INFO] $*" +} + +log_success() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} [SUCCESS] $*" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} [ERROR] $*" +} + +# Default configuration +RUNTIME="quarkus3-jvm" +SCENARIO="ootb" +NAMESPACE="quarkus-perf-benchmark" +IMAGE="" +CONTAINER_NAME="app" +JAVA_OPTS="" + +# Default resource requests and limits +CPU_REQUEST="500m" +CPU_LIMIT="3000m" +MEMORY_REQUEST="512Mi" +MEMORY_LIMIT="2Gi" + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Deploy application and database to OpenShift for benchmarking. + +Options: + --runtime RUNTIME Runtime to deploy (default: ${RUNTIME}) + Options: quarkus3-jvm, quarkus3-virtual, quarkus3-spring-compat, + spring3-jvm, spring3-virtual, spring4-jvm, spring4-virtual + --scenario SCENARIO Deployment scenario (default: ${SCENARIO}) + Common options: ootb (out-of-the-box), tuned (optimized) + Can be any custom name when using --java-opts + --namespace NS OpenShift namespace (default: ${NAMESPACE}) + --image IMAGE Container image (optional, uses config.env if not specified) + --container-name NAME Container name (default: ${CONTAINER_NAME}) + --java-opts OPTS Custom JAVA_OPTS (optional, uses scenario defaults) + + Resource Configuration: + --cpu-request CPU CPU request (default: ${CPU_REQUEST}) + --cpu-limit CPU CPU limit (default: ${CPU_LIMIT}) + --memory-request MEM Memory request (default: ${MEMORY_REQUEST}) + --memory-limit MEM Memory limit (default: ${MEMORY_LIMIT}) + + --help, -h Show this help message + +Examples: + # Deploy quarkus3-jvm with OOTB settings + $0 --runtime quarkus3-jvm --scenario ootb + + # Deploy spring4-virtual with tuned settings + $0 --runtime spring4-virtual --scenario tuned + + # Deploy with custom image + $0 --runtime quarkus3-jvm --image quay.io/myuser/quarkus3-jvm:v1.0 + + # Deploy with custom JAVA_OPTS and scenario name + $0 --runtime quarkus3-jvm --scenario custom --java-opts "-Xmx1g -Xms512m -XX:+UseG1GC" + + # Deploy with custom resource limits + $0 --runtime quarkus3-jvm --cpu-limit 4000m --memory-limit 4Gi + + # Deploy with Kruize recommendations + $0 --runtime quarkus3-jvm --scenario tuned --cpu-request 750m --cpu-limit 2500m \\ + --memory-request 768Mi --memory-limit 1536Mi --java-opts "-Xmx1200m -Xms768m" + +EOF + exit 0 +} + +# Parse arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --runtime) + RUNTIME="$2" + shift 2 + ;; + --scenario) + SCENARIO="$2" + shift 2 + ;; + --namespace) + NAMESPACE="$2" + shift 2 + ;; + --image) + IMAGE="$2" + shift 2 + ;; + --container-name) + CONTAINER_NAME="$2" + shift 2 + ;; + --java-opts) + JAVA_OPTS="$2" + shift 2 + ;; + --cpu-request) + CPU_REQUEST="$2" + shift 2 + ;; + --cpu-limit) + CPU_LIMIT="$2" + shift 2 + ;; + --memory-request) + MEMORY_REQUEST="$2" + shift 2 + ;; + --memory-limit) + MEMORY_LIMIT="$2" + shift 2 + ;; + --help|-h) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac + done +} + +# Get image name from config +get_image_name_from_config() { + if [ -n "$IMAGE" ]; then + echo "$IMAGE" + return + fi + + # Try to load from config.env + local config_file="${SCRIPT_DIR}/../config.env" + if [ -f "$config_file" ]; then + source "$config_file" + + # Use the helper function from config.env + if type get_image_name &>/dev/null; then + get_image_name "$RUNTIME" + else + log_error "get_image_name function not found in config.env" + echo "" + fi + else + log_error "Config file not found: $config_file" + echo "" + fi +} + +# Get JAVA_OPTS based on scenario +get_java_opts_from_config() { + if [ -n "$JAVA_OPTS" ]; then + echo "$JAVA_OPTS" + return + fi + + # Try to load from config.env + local config_file="${SCRIPT_DIR}/../config.env" + if [ -f "$config_file" ]; then + source "$config_file" + + # Use the helper function from config.env + if type get_java_opts &>/dev/null; then + local opts=$(get_java_opts "$SCENARIO") + # If function returns empty for custom scenario, that's OK + echo "$opts" + else + # Fallback to defaults for known scenarios + case "$SCENARIO" in + ootb) + echo "$JAVA_OPTS_OOTB" + ;; + tuned) + echo "$JAVA_OPTS_TUNED" + ;; + *) + # For custom scenarios, return empty (user should provide --java-opts) + echo "" + ;; + esac + fi + else + # Fallback defaults if config not found + case "$SCENARIO" in + ootb) + echo "" + ;; + tuned) + echo "-Xmx512m -Xms512m -XX:+UseParallelGC -XX:+UseNUMA" + ;; + *) + echo "" + ;; + esac + fi +} + +# Get resource configuration based on scenario +get_resources_from_config() { + local scenario_upper=$(echo "$SCENARIO" | tr '[:lower:]' '[:upper:]') + + # Try to load from config.env + local config_file="${SCRIPT_DIR}/../config.env" + if [ -f "$config_file" ]; then + source "$config_file" + fi + + # Set resources based on scenario, with fallback to defaults + case "$SCENARIO" in + ootb) + CPU_REQUEST="${CPU_REQUEST_OOTB:-${CPU_REQUEST}}" + CPU_LIMIT="${CPU_LIMIT_OOTB:-${CPU_LIMIT}}" + MEMORY_REQUEST="${MEMORY_REQUEST_OOTB:-${MEMORY_REQUEST}}" + MEMORY_LIMIT="${MEMORY_LIMIT_OOTB:-${MEMORY_LIMIT}}" + ;; + tuned) + CPU_REQUEST="${CPU_REQUEST_TUNED:-${CPU_REQUEST}}" + CPU_LIMIT="${CPU_LIMIT_TUNED:-${CPU_LIMIT}}" + MEMORY_REQUEST="${MEMORY_REQUEST_TUNED:-${MEMORY_REQUEST}}" + MEMORY_LIMIT="${MEMORY_LIMIT_TUNED:-${MEMORY_LIMIT}}" + ;; + *) + # For custom scenarios, use defaults or command-line provided values + # (already set in global variables) + ;; + esac +} + +# Deploy PostgreSQL +deploy_postgresql() { + log_info "Deploying PostgreSQL database..." + + if oc get deployment postgresql -n "${NAMESPACE}" &> /dev/null; then + log_info "PostgreSQL already deployed" + return 0 + fi + + local manifest_file="${SCRIPT_DIR}/../manifests/postgresql.yaml" + if [ ! -f "$manifest_file" ]; then + log_error "PostgreSQL manifest not found: $manifest_file" + return 1 + fi + + oc apply -f "$manifest_file" -n "${NAMESPACE}" + + log_info "Waiting for PostgreSQL to be ready..." + if ! oc wait --for=condition=available --timeout=300s deployment/postgresql -n "${NAMESPACE}"; then + log_error "PostgreSQL deployment failed" + return 1 + fi + + log_success "PostgreSQL deployed successfully" +} + +# Deploy application +deploy_application() { + local image=$(get_image_name_from_config) + local java_opts=$(get_java_opts_from_config) + + # Get scenario-specific resources (updates global variables) + get_resources_from_config + + if [ -z "$image" ]; then + log_error "Could not determine image for runtime: $RUNTIME" + log_error "Please specify --image or ensure config.env is properly configured" + return 1 + fi + + # Determine metrics path based on runtime + local metrics_path + case "$RUNTIME" in + quarkus*|quarkus3-spring-compat) + metrics_path="/q/metrics" + ;; + spring*) + metrics_path="/actuator/prometheus" + ;; + *) + metrics_path="/q/metrics" # Default to Quarkus + ;; + esac + + log_info "Deploying ${RUNTIME} with ${SCENARIO} scenario..." + log_info "Image: ${image}" + log_info "Container Name: ${CONTAINER_NAME}" + if [ -n "$java_opts" ]; then + log_info "JAVA_OPTS: ${java_opts}" + else + log_info "JAVA_OPTS: (none - using JVM defaults)" + fi + log_info "Metrics Path: ${metrics_path}" + log_info "Resources:" + log_info " CPU Request: ${CPU_REQUEST}, Limit: ${CPU_LIMIT}" + log_info " Memory Request: ${MEMORY_REQUEST}, Limit: ${MEMORY_LIMIT}" + + local app_name="${RUNTIME}-${SCENARIO}" + local manifest_file="${SCRIPT_DIR}/../manifests/app-template.yaml" + + if [ ! -f "$manifest_file" ]; then + log_error "Application manifest not found: $manifest_file" + return 1 + fi + + # Generate deployment manifest + sed -e "s|{{RUNTIME}}|${RUNTIME}|g" \ + -e "s|{{SCENARIO}}|${SCENARIO}|g" \ + -e "s|{{IMAGE}}|${image}|g" \ + -e "s|{{CONTAINER_NAME}}|${CONTAINER_NAME}|g" \ + -e "s|{{JAVA_OPTS}}|${java_opts}|g" \ + -e "s|{{CPU_REQ}}|${CPU_REQUEST}|g" \ + -e "s|{{CPU_LIM}}|${CPU_LIMIT}|g" \ + -e "s|{{MEM_REQ}}|${MEMORY_REQUEST}|g" \ + -e "s|{{MEM_LIM}}|${MEMORY_LIMIT}|g" \ + -e "s|{{METRICS_PATH}}|${metrics_path}|g" \ + "$manifest_file" | oc apply -n "${NAMESPACE}" -f - + + log_info "Waiting for ${app_name} to be ready..." + if ! oc wait --for=condition=available --timeout=300s deployment/${app_name} -n "${NAMESPACE}"; then + log_error "Application deployment failed" + return 1 + fi + + log_success "Application deployed successfully" +} + +# Get application endpoint +get_endpoint() { + local app_name="${RUNTIME}-${SCENARIO}" + + log_info "Getting application endpoint..." + + # Try to get route first (OpenShift) + local route_host=$(oc get route ${app_name} -n ${NAMESPACE} -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + + if [ -n "$route_host" ]; then + echo "http://${route_host}" + return 0 + fi + + # Fallback to service ClusterIP + local service_ip=$(oc get svc ${app_name} -n ${NAMESPACE} -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "") + + if [ -n "$service_ip" ]; then + echo "http://${service_ip}:8080" + return 0 + fi + + log_error "Could not determine application endpoint" + return 1 +} + +# Main execution +main() { + parse_args "$@" + + log_info "==========================================" + log_info "Application Deployment" + log_info "==========================================" + log_info "Runtime: ${RUNTIME}" + log_info "Scenario: ${SCENARIO}" + log_info "Namespace: ${NAMESPACE}" + log_info "Resources: CPU ${CPU_REQUEST}/${CPU_LIMIT}, Memory ${MEMORY_REQUEST}/${MEMORY_LIMIT}" + log_info "==========================================" + + # Check if logged in to OpenShift + if ! oc whoami &> /dev/null; then + log_error "Not logged in to OpenShift. Please run 'oc login' first." + exit 1 + fi + + # Create/use namespace + if ! oc get project "${NAMESPACE}" &> /dev/null; then + log_info "Creating namespace: ${NAMESPACE}" + oc new-project "${NAMESPACE}" + else + log_info "Using existing namespace: ${NAMESPACE}" + oc project "${NAMESPACE}" + fi + + # Deploy PostgreSQL + if ! deploy_postgresql; then + log_error "PostgreSQL deployment failed" + exit 1 + fi + + # Deploy application + if ! deploy_application; then + log_error "Application deployment failed" + exit 1 + fi + + # Get endpoint + local endpoint=$(get_endpoint) + if [ $? -ne 0 ]; then + log_error "Failed to get application endpoint" + exit 1 + fi + + log_success "==========================================" + log_success "Deployment Complete!" + log_success "==========================================" + log_success "Application: ${RUNTIME}-${SCENARIO}" + log_success "Namespace: ${NAMESPACE}" + log_success "Endpoint: ${endpoint}" + log_success "==========================================" + log_info "" + log_info "To run variable load test, use:" + log_info "./run-variable-load.sh --url ${endpoint} --runtime ${RUNTIME}" + log_info "" + log_info "To check application status:" + log_info "oc get pods -n ${NAMESPACE}" + log_info "" + log_info "To view logs:" + log_info "oc logs -f deployment/${RUNTIME}-${SCENARIO} -n ${NAMESPACE}" +} + +main "$@" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/measure-memory.sh b/spring-quarkus-perf-comparison/scripts/measure-memory.sh new file mode 100755 index 00000000..84d3b564 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/measure-memory.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# ============================================================================ +# Measure Memory Usage +# ============================================================================ +# Measures RSS (Resident Set Size) memory usage after application startup + +set -e + +# Capture parameters BEFORE sourcing config.env to avoid overwriting +RUNTIME_PARAM=$1 +SCENARIO_PARAM=$2 +RESULT_FILE=$3 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../config.env" + +# Use the parameters passed to the script, not the defaults from config.env +RUNTIME="$RUNTIME_PARAM" +SCENARIO="$SCENARIO_PARAM" + +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +log_error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR] $@" +} + +measure_memory() { + local app_name="${RUNTIME}-${SCENARIO}" + + log_info "Measuring memory usage for ${app_name}..." + + # Get pod name + local pod_name=$(oc get pods -l app=${app_name} -o jsonpath='{.items[0].metadata.name}') + if [ -z "$pod_name" ]; then + log_error "Pod not found for ${app_name}" + return 1 + fi + + log_info "Pod: ${pod_name}" + + # Wait for application to stabilize + log_info "Waiting for application to stabilize (30 seconds)..." + sleep 30 + + # Get memory metrics from oc adm top (redirect stderr to suppress errors in variable) + local memory_usage=$(oc adm top pod ${pod_name} --no-headers 2>/dev/null | awk '{print $3}') + + if [ -z "$memory_usage" ]; then + log_error "Failed to get memory metrics from 'oc adm top'" + log_info "Attempting to get memory from metrics API..." + # Try alternative method using metrics API + memory_usage=$(get_memory_from_metrics_api "$pod_name" 2>&1) + fi + + # Convert to MB (handle Mi suffix) + local memory_mb=$(echo "$memory_usage" | sed 's/Mi//' | grep -oP '^\d+' || echo "0") + + # Validate memory_mb is a number + if ! [[ "$memory_mb" =~ ^[0-9]+$ ]]; then + log_error "Invalid memory value: $memory_mb, using 0" + memory_mb=0 + fi + + log_info "Memory usage (RSS): ${memory_mb}MB" + + # Get additional memory metrics from container stats + local memory_limit=$(oc get pod ${pod_name} -o jsonpath='{.spec.containers[0].resources.limits.memory}') + local memory_request=$(oc get pod ${pod_name} -o jsonpath='{.spec.containers[0].resources.requests.memory}') + + # Update result file + local temp_file=$(mktemp) + jq ".tests.memory = { + \"rss_mb\": ${memory_mb}, + \"memory_limit\": \"${memory_limit}\", + \"memory_request\": \"${memory_request}\", + \"measurement_time\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" + }" "$RESULT_FILE" > "$temp_file" + mv "$temp_file" "$RESULT_FILE" + + log_info "Memory measurement completed" + return 0 +} + +get_memory_from_metrics_api() { + local pod_name=$1 + + # Suppress log output to stderr so it doesn't get captured in variable + >&2 echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] Attempting to get memory from metrics API..." + + # Try to get memory from pod resource usage + local memory_bytes=$(oc get --raw /apis/metrics.k8s.io/v1beta1/namespaces/${OPENSHIFT_PROJECT}/pods/${pod_name} 2>/dev/null | jq -r '.containers[0].usage.memory' 2>/dev/null) + + if [ -n "$memory_bytes" ] && [ "$memory_bytes" != "null" ]; then + # Convert from Ki to Mi + local memory_mi=$(echo "$memory_bytes" | sed 's/Ki$//' | awk '{print int($1/1024)}') + echo "${memory_mi}" + else + >&2 echo "[$(date +'%Y-%m-%d %H:%M:%S')] [WARN] Could not get memory from metrics API, using 0" + echo "0" + fi +} + +# Main execution +measure_memory + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/measure-startup.sh b/spring-quarkus-perf-comparison/scripts/measure-startup.sh new file mode 100755 index 00000000..f17fc648 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/measure-startup.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# ============================================================================ +# Measure Startup Time +# ============================================================================ +# Measures the time from pod creation to first successful HTTP request + +set -e + +# Capture parameters BEFORE sourcing config.env to avoid overwriting +RUNTIME_PARAM=$1 +SCENARIO_PARAM=$2 +RESULT_FILE=$3 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../config.env" + +# Use the parameters passed to the script, not the defaults from config.env +RUNTIME="$RUNTIME_PARAM" +SCENARIO="$SCENARIO_PARAM" + +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +log_error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR] $@" +} + +measure_startup() { + local app_name="${RUNTIME}-${SCENARIO}" + + log_info "Measuring startup time for ${app_name}..." + + # Get pod name + local pod_name=$(oc get pods -l app=${app_name} -o jsonpath='{.items[0].metadata.name}') + if [ -z "$pod_name" ]; then + log_error "Pod not found for ${app_name}" + return 1 + fi + + # Get pod creation time + local pod_created=$(oc get pod ${pod_name} -o jsonpath='{.metadata.creationTimestamp}') + local pod_created_epoch=$(date -d "${pod_created}" +%s%3N) + + log_info "Pod created at: ${pod_created}" + + # Get route URL + local route_url=$(oc get route ${app_name} -o jsonpath='{.spec.host}') + if [ -z "$route_url" ]; then + log_error "Route not found for ${app_name}" + return 1 + fi + + local url="http://${route_url}/fruits" + log_info "Testing URL: ${url}" + + # Wait for first successful request + local max_attempts=60 + local attempt=0 + local first_success_time="" + + while [ $attempt -lt $max_attempts ]; do + if curl -s -f -m 5 "${url}" > /dev/null 2>&1; then + first_success_time=$(date +%s%3N) + log_info "First successful request at attempt ${attempt}" + break + fi + attempt=$((attempt + 1)) + sleep 1 + done + + if [ -z "$first_success_time" ]; then + log_error "Application did not respond within ${max_attempts} seconds" + return 1 + fi + + # Calculate startup time + local startup_time=$((first_success_time - pod_created_epoch)) + + log_info "Startup time: ${startup_time}ms" + + # Update result file + local temp_file=$(mktemp) + jq ".tests.startup = { + \"startup_time_ms\": ${startup_time}, + \"pod_created\": \"${pod_created}\", + \"first_request_success\": \"$(date -d @$((first_success_time / 1000)) -u +%Y-%m-%dT%H:%M:%SZ)\" + }" "$RESULT_FILE" > "$temp_file" + mv "$temp_file" "$RESULT_FILE" + + log_info "Startup measurement completed" + return 0 +} + +# Main execution +measure_startup + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/perf/perf-test-ootb-vs-tuned.sh b/spring-quarkus-perf-comparison/scripts/perf/perf-test-ootb-vs-tuned.sh new file mode 100755 index 00000000..2e37aa6b --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/perf/perf-test-ootb-vs-tuned.sh @@ -0,0 +1,444 @@ +#!/bin/bash + +# ============================================================================ +# Performance Test: OOTB vs Tuned Comparison for Quarkus 3 JVM +# ============================================================================ +# This performance test compares Out-Of-The-Box (OOTB) vs Tuned configurations +# by alternating between them across iterations: +# Iteration 1: OOTB -> Tuned +# Iteration 2: OOTB -> Tuned +# +# Each configuration is deployed, load tested, then the next is deployed. +# The URL is automatically discovered based on runtime and scenario. +# +# Test Configuration: +# - Runtime: Quarkus 3 JVM +# - Configurations: OOTB and Tuned (alternating) +# - Iterations: 2 (each config tested twice) +# - Phase Duration: 10 seconds per phase (configurable) +# - Load Pattern: Alternating (low -> medium -> peak) +# - Wait Times: Configurable (stabilization, scenario switch, iteration) +# ============================================================================ + +set -e + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_SCRIPT="${SCRIPT_DIR}/../deploy-app.sh" +BENCHMARK_SCRIPT="${SCRIPT_DIR}/../run-variable-load-multi-phase.sh" +RESULTS_DIR="./variable-load-results" + +# Test Parameters +RUNTIME="quarkus3-jvm" +NAMESPACE="quarkus-perf-benchmark" +PHASE_DURATION="10s" +TOTAL_ITERATIONS=2 + +# Wait Times (in seconds) +STABILIZATION_WAIT=30 # Wait after deployment for app to stabilize +SCENARIO_SWITCH_WAIT=60 # Wait between switching scenarios (OOTB -> Tuned) +ITERATION_WAIT=60 # Wait between iterations + +# Alternating Load Configuration +# Low Load Phase +LOW_THREADS=1 +LOW_CONNECTIONS=10 + +# Medium Load Phase +MEDIUM_THREADS=2 +MEDIUM_CONNECTIONS=20 + +# Peak Load Phase +PEAK_THREADS=3 +PEAK_CONNECTIONS=30 + +# Color codes for output +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_step() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] [STEP]${NC} $@" +} + +log_perf() { + echo -e "${CYAN}[$(date +'%Y-%m-%d %H:%M:%S')] [PERF]${NC} $@" +} + +log_warn() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] [WARN]${NC} $@" +} + +# ============================================================================ +# Capture Startup Metrics +# ============================================================================ +capture_startup_metrics() { + local scenario=$1 + local iteration=$2 + local output_file=$3 + + log_perf "Capturing startup metrics for $scenario..." + + local pod_name=$(oc get pods -n "$NAMESPACE" -l app="${RUNTIME}-${scenario}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$pod_name" ]; then + log_warn "Could not find pod for ${RUNTIME}-${scenario}" + return 1 + fi + + # Get pod creation time and current time to calculate startup duration + local pod_start_time=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.startTime}') + local pod_ready_time=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].lastTransitionTime}') + + # Get memory usage from pod metrics + local memory_usage=$(oc top pod "$pod_name" -n "$NAMESPACE" --no-headers 2>/dev/null | awk '{print $3}') + local cpu_usage=$(oc top pod "$pod_name" -n "$NAMESPACE" --no-headers 2>/dev/null | awk '{print $2}') + + # Get memory limits and requests + local memory_limit=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].resources.limits.memory}') + local memory_request=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].resources.requests.memory}') + local cpu_limit=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].resources.limits.cpu}') + local cpu_request=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].resources.requests.cpu}') + + # Calculate startup time in seconds + local startup_seconds="" + if [ -n "$pod_start_time" ] && [ -n "$pod_ready_time" ]; then + local start_epoch=$(date -d "$pod_start_time" +%s 2>/dev/null || echo "") + local ready_epoch=$(date -d "$pod_ready_time" +%s 2>/dev/null || echo "") + if [ -n "$start_epoch" ] && [ -n "$ready_epoch" ]; then + startup_seconds=$((ready_epoch - start_epoch)) + fi + fi + + # Create startup metrics JSON + cat > "$output_file" << EOF +{ + "scenario": "$scenario", + "iteration": $iteration, + "pod_name": "$pod_name", + "startup_time_seconds": ${startup_seconds:-null}, + "pod_start_time": "$pod_start_time", + "pod_ready_time": "$pod_ready_time", + "memory": { + "current_usage": "$memory_usage", + "limit": "$memory_limit", + "request": "$memory_request" + }, + "cpu": { + "current_usage": "$cpu_usage", + "limit": "$cpu_limit", + "request": "$cpu_request" + }, + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} +EOF + + log_info "Startup metrics captured:" + log_info " Pod: $pod_name" + log_info " Startup Time: ${startup_seconds:-N/A} seconds" + log_info " Memory Usage: $memory_usage (Limit: $memory_limit, Request: $memory_request)" + log_info " CPU Usage: $cpu_usage (Limit: $cpu_limit, Request: $cpu_request)" + + return 0 +} + +# ============================================================================ +# Deploy and Run Performance Test for a Configuration +# ============================================================================ +deploy_and_test() { + local scenario=$1 + local iteration=$2 + local session_id=$3 + + log_step "=========================================" + log_step "Performance Test: ${scenario^^} - Iteration $iteration" + log_step "=========================================" + + # Step 1: Deploy the application + log_perf "Deploying $RUNTIME with $scenario configuration..." + $DEPLOY_SCRIPT \ + --runtime "$RUNTIME" \ + --scenario "$scenario" \ + --namespace "$NAMESPACE" + + log_info "Deployment complete. Waiting ${STABILIZATION_WAIT} seconds for application to stabilize..." + sleep $STABILIZATION_WAIT + + # Step 2: Capture startup metrics + local startup_metrics_file="${RESULTS_DIR}/session-${session_id}/${RUNTIME}-${scenario}-iter${iteration}-startup.json" + mkdir -p "$(dirname "$startup_metrics_file")" + capture_startup_metrics "$scenario" "$iteration" "$startup_metrics_file" + + # Step 3: Run variable load performance test + # The --scenario parameter will auto-discover the URL based on runtime and scenario + log_perf "Running variable load performance test for $scenario..." + log_info "URL will be auto-discovered from deployed service: $RUNTIME-$scenario" + + $BENCHMARK_SCRIPT \ + --runtime "$RUNTIME" \ + --scenario "$scenario" \ + --session-id "$session_id" \ + --namespace "$NAMESPACE" \ + --duration custom \ + --phase-duration "$PHASE_DURATION" \ + --low-threads $LOW_THREADS \ + --low-connections $LOW_CONNECTIONS \ + --med-threads $MEDIUM_THREADS \ + --med-connections $MEDIUM_CONNECTIONS \ + --peak-threads $PEAK_THREADS \ + --peak-connections $PEAK_CONNECTIONS + + log_info "Performance test complete for $scenario - Iteration $iteration" +} + +# ============================================================================ +# Main Performance Test Execution - Alternating OOTB and Tuned +# ============================================================================ +run_alternating_performance_tests() { + log_step "=========================================" + log_step "Starting Alternating Performance Tests" + log_step "Pattern: OOTB -> Tuned -> OOTB -> Tuned" + log_step "=========================================" + + for iteration in $(seq 1 $TOTAL_ITERATIONS); do + log_step "=========================================" + log_step "ITERATION $iteration of $TOTAL_ITERATIONS" + log_step "=========================================" + + # Run OOTB + deploy_and_test "ootb" "$iteration" "ootb" + + log_info "Waiting ${SCENARIO_SWITCH_WAIT} seconds before switching to Tuned configuration..." + sleep $SCENARIO_SWITCH_WAIT + + # Run Tuned + deploy_and_test "tuned" "$iteration" "tuned" + + # Pause between iterations (except after last iteration) + if [ $iteration -lt $TOTAL_ITERATIONS ]; then + log_info "Waiting ${ITERATION_WAIT} seconds before next iteration..." + sleep $ITERATION_WAIT + fi + done + + log_step "All alternating performance tests completed!" +} + +# ============================================================================ +# Analysis Phase 1: Aggregate Performance Results +# ============================================================================ +aggregate_performance_results() { + log_step "=========================================" + log_step "Aggregating Performance Test Results" + log_step "=========================================" + + AGGREGATE_SCRIPT="${SCRIPT_DIR}/../results-tools/aggregate-session.sh" + + # Aggregate OOTB performance results + log_info "Aggregating OOTB performance results..." + if [ -d "${RESULTS_DIR}/session-ootb" ]; then + $AGGREGATE_SCRIPT --session-dir "${RESULTS_DIR}/session-ootb" + else + log_warn "OOTB session directory not found, skipping aggregation" + fi + + # Aggregate Tuned performance results + log_info "Aggregating Tuned performance results..." + if [ -d "${RESULTS_DIR}/session-tuned" ]; then + $AGGREGATE_SCRIPT --session-dir "${RESULTS_DIR}/session-tuned" + else + log_warn "Tuned session directory not found, skipping aggregation" + fi + + log_step "Performance results aggregation completed!" +} + +# ============================================================================ +# Analysis Phase 2: Analyze Performance Variability +# ============================================================================ +analyze_performance_variability() { + log_step "=========================================" + log_step "Analyzing Performance Variability" + log_step "=========================================" + + ANALYZE_SCRIPT="${SCRIPT_DIR}/../results-tools/analyze-session.sh" + + # Analyze OOTB performance variability + log_info "Analyzing OOTB performance variability..." + if [ -d "${RESULTS_DIR}/session-ootb" ]; then + $ANALYZE_SCRIPT \ + --session-dir "${RESULTS_DIR}/session-ootb" \ + --output "${RESULTS_DIR}/session-ootb/performance-variability-analysis.txt" + fi + + # Analyze Tuned performance variability + log_info "Analyzing Tuned performance variability..." + if [ -d "${RESULTS_DIR}/session-tuned" ]; then + $ANALYZE_SCRIPT \ + --session-dir "${RESULTS_DIR}/session-tuned" \ + --output "${RESULTS_DIR}/session-tuned/performance-variability-analysis.txt" + fi + + log_step "Performance variability analysis completed!" +} + +# ============================================================================ +# Analysis Phase 3: Compare Performance Between Configurations +# ============================================================================ +compare_performance() { + log_step "=========================================" + log_step "Comparing Performance: OOTB vs Tuned" + log_step "=========================================" + + COMPARE_SCRIPT="${SCRIPT_DIR}/../results-tools/compare-sessions.sh" + + if [ -d "${RESULTS_DIR}/session-ootb" ] && [ -d "${RESULTS_DIR}/session-tuned" ]; then + log_info "Generating performance comparison report..." + $COMPARE_SCRIPT \ + --session-a "${RESULTS_DIR}/session-ootb" \ + --session-b "${RESULTS_DIR}/session-tuned" \ + --session-a-name "OOTB" \ + --session-b-name "Tuned" \ + --output "${RESULTS_DIR}/performance-comparison-ootb-vs-tuned.txt" + + # Also generate JSON format for programmatic access + $COMPARE_SCRIPT \ + --session-a "${RESULTS_DIR}/session-ootb" \ + --session-b "${RESULTS_DIR}/session-tuned" \ + --session-a-name "OOTB" \ + --session-b-name "Tuned" \ + --format json \ + --output "${RESULTS_DIR}/performance-comparison-ootb-vs-tuned.json" + + log_step "Performance comparison completed!" + else + log_warn "One or both session directories not found, skipping comparison" + fi +} + +# ============================================================================ +# Reporting Phase: Generate Performance Reports +# ============================================================================ +generate_performance_reports() { + log_step "=========================================" + log_step "Generating Performance Test Reports" + log_step "=========================================" + + REPORT_SCRIPT="${SCRIPT_DIR}/../results-tools/generate-session-report.sh" + + # Generate OOTB performance report + if [ -d "${RESULTS_DIR}/session-ootb" ]; then + log_info "Generating OOTB performance report..." + $REPORT_SCRIPT \ + --session-dir "${RESULTS_DIR}/session-ootb" \ + --output "${RESULTS_DIR}/session-ootb/performance-report.html" + fi + + # Generate Tuned performance report + if [ -d "${RESULTS_DIR}/session-tuned" ]; then + log_info "Generating Tuned performance report..." + $REPORT_SCRIPT \ + --session-dir "${RESULTS_DIR}/session-tuned" \ + --output "${RESULTS_DIR}/session-tuned/performance-report.html" + fi + + log_step "Performance reports generated!" +} + +# ============================================================================ +# Main Performance Test Execution +# ============================================================================ +main() { + log_step "=========================================" + log_step "Performance Test: OOTB vs Tuned" + log_step "Quarkus 3 JVM - Alternating Configuration" + log_step "=========================================" + log_info "Runtime: $RUNTIME" + log_info "Namespace: $NAMESPACE" + log_info "Iterations: $TOTAL_ITERATIONS (alternating OOTB and Tuned)" + log_info "Phase Duration: $PHASE_DURATION" + log_info "Load Pattern: Alternating (Low -> Medium -> Peak)" + log_info " - Low: $LOW_THREADS threads, $LOW_CONNECTIONS connections" + log_info " - Medium: $MEDIUM_THREADS threads, $MEDIUM_CONNECTIONS connections" + log_info " - Peak: $PEAK_THREADS threads, $PEAK_CONNECTIONS connections" + log_info "Results Directory: $RESULTS_DIR" + log_info "" + log_info "Test Sequence:" + log_info " Iteration 1: Deploy OOTB -> Run Load -> Deploy Tuned -> Run Load" + log_info " Iteration 2: Deploy OOTB -> Run Load -> Deploy Tuned -> Run Load" + log_info "" + log_info "Note: Application URLs are auto-discovered from deployed services" + log_info "" + + # Check if required scripts exist + if [ ! -f "$DEPLOY_SCRIPT" ]; then + echo "ERROR: Deployment script not found: $DEPLOY_SCRIPT" + exit 1 + fi + + if [ ! -f "$BENCHMARK_SCRIPT" ]; then + echo "ERROR: Benchmark script not found: $BENCHMARK_SCRIPT" + exit 1 + fi + + # Create results directory + mkdir -p "$RESULTS_DIR" + + # Execute performance test workflow + run_alternating_performance_tests + echo "" + + aggregate_performance_results + echo "" + + analyze_performance_variability + echo "" + + compare_performance + echo "" + + generate_performance_reports + echo "" + + # Performance Test Summary + log_step "=========================================" + log_step "Performance Test Execution Complete!" + log_step "=========================================" + log_info "Results Location: $RESULTS_DIR" + log_info "" + log_info "Performance Test Artifacts:" + log_info " OOTB Configuration:" + log_info " - ${RESULTS_DIR}/session-ootb/aggregated.json" + log_info " - ${RESULTS_DIR}/session-ootb/performance-variability-analysis.txt" + log_info " - ${RESULTS_DIR}/session-ootb/performance-report.html" + log_info " - ${RESULTS_DIR}/session-ootb/*-startup.json (startup metrics per iteration)" + log_info "" + log_info " Tuned Configuration:" + log_info " - ${RESULTS_DIR}/session-tuned/aggregated.json" + log_info " - ${RESULTS_DIR}/session-tuned/performance-variability-analysis.txt" + log_info " - ${RESULTS_DIR}/session-tuned/performance-report.html" + log_info " - ${RESULTS_DIR}/session-tuned/*-startup.json (startup metrics per iteration)" + log_info "" + log_info " Performance Comparison:" + log_info " - ${RESULTS_DIR}/performance-comparison-ootb-vs-tuned.txt" + log_info " - ${RESULTS_DIR}/performance-comparison-ootb-vs-tuned.json" + log_info "" + log_info "Next Steps:" + log_info " 1. Review startup metrics to compare initialization time and memory footprint" + log_info " 2. Review variability analysis to ensure CV% < 10% for reliable results" + log_info " 3. Check performance comparison report for throughput/latency differences" + log_info " 4. Open HTML reports in browser for detailed performance metrics" + log_info " 5. Analyze phase-by-phase performance under different load conditions" + log_step "=========================================" +} + +# Run main performance test +main "$@" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/aggregate-session.sh b/spring-quarkus-perf-comparison/scripts/results-tools/aggregate-session.sh new file mode 100755 index 00000000..d866d7df --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/aggregate-session.sh @@ -0,0 +1,278 @@ +#!/bin/bash + +# ============================================================================ +# Aggregate Session Results - Phase-Level Statistics +# ============================================================================ +# This script aggregates metrics across multiple iterations within a session, +# calculating mean, median, and standard deviation for each phase. +# +# Output: Creates aggregated JSON files compatible with existing comparison tools + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common utilities +source "${SCRIPT_DIR}/../common-utils.sh" + +# Default values +SESSION_DIR="" +OUTPUT_FILE="" + +usage() { + cat << EOF +Usage: $0 --session-dir [options] + +Aggregate metrics across multiple iterations within a session. +Calculates mean, median, and standard deviation for each phase. + +Required: + --session-dir Path to session directory (e.g., ./variable-load-results/session-baseline) + +Optional: + --output Output aggregated JSON file (default: /aggregated.json) + --help Show this help message + +Examples: + # Aggregate all iterations in a session + $0 --session-dir ./variable-load-results/session-baseline + + # Specify custom output file + $0 --session-dir ./variable-load-results/session-baseline --output baseline-stats.json + +Output Format: + The aggregated JSON contains mean, median, stddev, min, max for each metric + across all iterations, organized by phase. This format is compatible with + existing comparison and reporting tools. + +EOF + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --session-dir) + SESSION_DIR="$2" + shift 2 + ;; + --output) + OUTPUT_FILE="$2" + shift 2 + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate required parameters +if [ -z "$SESSION_DIR" ]; then + log_error_color "--session-dir is required" + usage +fi + +if ! validate_directory "$SESSION_DIR" "Session directory"; then + exit 1 +fi + +# Set default output file +if [ -z "$OUTPUT_FILE" ]; then + OUTPUT_FILE="${SESSION_DIR}/aggregated.json" +fi + +log_info_color "=========================================" +log_info_color "Aggregating Session Results" +log_info_color "=========================================" +log_info_color "Session Dir: $SESSION_DIR" +log_info_color "Output File: $OUTPUT_FILE" +log_info_color "" + +# Find consolidated iteration JSON files (not phase-specific files) +# Pattern: *-{scenario}-iter{N}.json (e.g., quarkus3-jvm-ootb-iter1.json) +ITERATION_FILES=($(find "$SESSION_DIR" -maxdepth 1 -name "*-iter[0-9]*.json" ! -name "*_phase*" | sort)) + +if [ ${#ITERATION_FILES[@]} -eq 0 ]; then + log_error_color "No iteration files found in $SESSION_DIR" + log_error_color "Expected files matching pattern: *-iter*.json" + exit 1 +fi + +log_info_color "Found ${#ITERATION_FILES[@]} iteration files:" +for file in "${ITERATION_FILES[@]}"; do + log_info_color " - $(basename $file)" +done +log_info_color "" + +# Use Python for statistical calculations +log_info_color "Calculating statistics..." + +python3 - "$SESSION_DIR" "$OUTPUT_FILE" << 'PYTHON_SCRIPT' +import json +import sys +import statistics +from pathlib import Path + +def safe_float(value): + """Convert value to float, handling None and invalid values""" + if value is None: + return None + try: + return float(value) + except (ValueError, TypeError): + return None + +def calculate_stats(values): + """Calculate mean, median, stddev, min, max for a list of values""" + # Filter out None values + valid_values = [v for v in values if v is not None] + + if not valid_values: + return { + "mean": None, + "median": None, + "stddev": None, + "min": None, + "max": None, + "count": 0 + } + + return { + "mean": round(statistics.mean(valid_values), 2), + "median": round(statistics.median(valid_values), 2), + "stddev": round(statistics.stdev(valid_values), 2) if len(valid_values) > 1 else 0, + "min": round(min(valid_values), 2), + "max": round(max(valid_values), 2), + "count": len(valid_values) + } + +# Read arguments from command line +if len(sys.argv) < 3: + print("ERROR: Missing required arguments", file=sys.stderr) + sys.exit(1) + +session_dir = sys.argv[1] +output_file = sys.argv[2] + +# Find consolidated iteration files (exclude phase-specific files) +all_iter_files = sorted(Path(session_dir).glob('*-iter[0-9]*.json')) +iteration_files = [f for f in all_iter_files if '_phase' not in f.name] + +if not iteration_files: + print(f"ERROR: No iteration files found", file=sys.stderr) + sys.exit(1) + +# Load all iterations +iterations_data = [] +for file_path in iteration_files: + try: + with open(file_path, 'r') as f: + data = json.load(f) + iterations_data.append(data) + except Exception as e: + print(f"WARNING: Failed to load {file_path}: {e}", file=sys.stderr) + +if not iterations_data: + print("ERROR: No valid iteration data loaded", file=sys.stderr) + sys.exit(1) + +# Extract metadata from first iteration +first_iter = iterations_data[0] +runtime = first_iter.get('runtime', 'unknown') +scenario = first_iter.get('scenario', 'unknown') +test_type = first_iter.get('test_type', 'multi-phase-variable-load') +duration_mode = first_iter.get('duration_mode', 'unknown') +session_id = first_iter.get('session_id', 'unknown') + +# Aggregate phase data +phase_aggregates = {} + +for iteration in iterations_data: + phases = iteration.get('phases', []) + + for phase in phases: + phase_name = phase.get('phase_name', 'unknown') + config = phase.get('configuration', {}) + results = phase.get('results', {}) + + if phase_name not in phase_aggregates: + phase_aggregates[phase_name] = { + 'throughput': [], + 'mean_latency': [], + 'p50_latency': [], + 'p90_latency': [], + 'p99_latency': [], + 'errors': [], + 'threads': config.get('threads'), + 'connections': config.get('connections'), + 'duration': config.get('duration') + } + + # Collect values from results section + phase_aggregates[phase_name]['throughput'].append(safe_float(results.get('requests_per_sec'))) + phase_aggregates[phase_name]['mean_latency'].append(safe_float(results.get('latency_mean_ms'))) + phase_aggregates[phase_name]['p50_latency'].append(safe_float(results.get('latency_p50_ms'))) + phase_aggregates[phase_name]['p90_latency'].append(safe_float(results.get('latency_p90_ms'))) + phase_aggregates[phase_name]['p99_latency'].append(safe_float(results.get('latency_p99_ms'))) + phase_aggregates[phase_name]['errors'].append(safe_float(results.get('errors', 0))) + +# Calculate statistics for each phase +aggregated_phases = [] + +for phase_name, data in phase_aggregates.items(): + phase_stats = { + 'phase_name': phase_name, + 'threads': data['threads'], + 'connections': data['connections'], + 'duration': data['duration'], + 'iterations': len(iterations_data), + 'requests_per_sec': calculate_stats(data['throughput']), + 'latency_mean_ms': calculate_stats(data['mean_latency']), + 'latency_p50_ms': calculate_stats(data['p50_latency']), + 'latency_p90_ms': calculate_stats(data['p90_latency']), + 'latency_p99_ms': calculate_stats(data['p99_latency']), + 'errors': calculate_stats(data['errors']) + } + aggregated_phases.append(phase_stats) + +# Create output JSON +output = { + 'runtime': runtime, + 'scenario': scenario, + 'test_type': 'aggregated-multi-phase', + 'duration_mode': duration_mode, + 'session_id': session_id, + 'total_iterations': len(iterations_data), + 'aggregation_timestamp': first_iter.get('timestamp', 'unknown'), + 'phases': aggregated_phases +} + +# Write output +with open(output_file, 'w') as f: + json.dump(output, f, indent=2) + +print(f"SUCCESS: Aggregated data written to {output_file}") + +PYTHON_SCRIPT + +# Check if Python script succeeded +if [ $? -eq 0 ]; then + log_info_color "" + log_info_color "=========================================" + log_info_color "Aggregation Complete!" + log_info_color "=========================================" + log_info_color "Output: $OUTPUT_FILE" + log_info_color "" + log_info_color "You can now use this aggregated file with:" + log_info_color " - compare-all.sh for comparison" + log_info_color " - generate-report.sh for HTML reports" +else + log_error_color "Aggregation failed" + exit 1 +fi + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/analyze-session.sh b/spring-quarkus-perf-comparison/scripts/results-tools/analyze-session.sh new file mode 100755 index 00000000..f6d5321f --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/analyze-session.sh @@ -0,0 +1,468 @@ +#!/bin/bash + +# ============================================================================ +# Analyze Session - Within-Session Variability Analysis +# ============================================================================ +# This script analyzes variability and consistency within a single session, +# measuring how stable performance is across multiple iterations. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +SESSION_DIR="" +OUTPUT_FILE="" +FORMAT="text" # text or json + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +log_warn() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] [WARN]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 --session-dir [options] + +Analyze within-session variability to measure performance consistency +across multiple iterations of the same test configuration. + +Required: + --session-dir Path to session directory + +Optional: + --output Output file for analysis results + --format Output format: text or json (default: text) + --help Show this help message + +Examples: + # Analyze variability in a session + $0 --session-dir ./variable-load-results/session-baseline + + # Generate JSON output + $0 --session-dir ./variable-load-results/session-baseline \\ + --format json --output variability-analysis.json + +Metrics Analyzed: + - Coefficient of Variation (CV%): (stddev/mean) × 100 (lower = more consistent) + - Range: max - min (absolute variability) + - Stability Score: 100 - CV (higher = more stable, 0-100 scale) + +EOF + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --session-dir) + SESSION_DIR="$2" + shift 2 + ;; + --output) + OUTPUT_FILE="$2" + shift 2 + ;; + --format) + FORMAT="$2" + shift 2 + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate required parameters +if [ -z "$SESSION_DIR" ]; then + log_error "--session-dir is required" + usage +fi + +if [ ! -d "$SESSION_DIR" ]; then + log_error "Session directory not found: $SESSION_DIR" + exit 1 +fi + +# Check for aggregated file +AGG_FILE="${SESSION_DIR}/aggregated.json" + +if [ ! -f "$AGG_FILE" ]; then + log_error "Aggregated file not found: $AGG_FILE" + log_error "Run aggregate-session.sh first: ./aggregate-session.sh --session-dir $SESSION_DIR" + exit 1 +fi + +log_info "=========================================" +log_info "Analyzing Session Variability" +log_info "=========================================" +log_info "Session Dir: $SESSION_DIR" +log_info "Format: $FORMAT" +if [ -n "$OUTPUT_FILE" ]; then + log_info "Output: $OUTPUT_FILE" +fi +log_info "" + +# Use Python for analysis +export SESSION_DIR +export OUTPUT_FILE +export FORMAT + +python3 - "$SESSION_DIR" "$OUTPUT_FILE" "$FORMAT" << 'PYTHON_SCRIPT' +import json +import sys +import os +import glob +import statistics + +def calculate_cv(mean, stddev): + """Calculate Coefficient of Variation (CV)""" + if mean is None or stddev is None or mean == 0: + return None + return round((stddev / mean) * 100, 2) + +def calculate_stability_score(cv): + """Calculate stability score (0-100, higher is better)""" + if cv is None: + return None + return max(0, round(100 - cv, 2)) + +def parse_memory_value(mem_str): + """Convert memory string (e.g., '512Mi', '1Gi') to MB""" + if not mem_str: + return None + mem_str = str(mem_str).strip() + if mem_str.endswith('Mi'): + return float(mem_str[:-2]) + elif mem_str.endswith('Gi'): + return float(mem_str[:-2]) * 1024 + elif mem_str.endswith('Ki'): + return float(mem_str[:-2]) / 1024 + else: + try: + return float(mem_str) / (1024 * 1024) + except: + return None + +def analyze_startup_metrics(session_dir): + """Analyze startup metrics variability""" + startup_files = glob.glob(os.path.join(session_dir, '*-startup.json')) + + if not startup_files: + return None + + startup_times = [] + memory_usages = [] + cpu_usages = [] + + for file_path in startup_files: + try: + with open(file_path, 'r') as f: + data = json.load(f) + + if data.get('startup_time_seconds'): + startup_times.append(data['startup_time_seconds']) + + mem_usage = data.get('memory', {}).get('current_usage', '') + if mem_usage: + mem_mb = parse_memory_value(mem_usage) + if mem_mb: + memory_usages.append(mem_mb) + + cpu_usage = data.get('cpu', {}).get('current_usage', '') + if cpu_usage and cpu_usage.endswith('m'): + try: + cpu_usages.append(float(cpu_usage[:-1])) + except: + pass + except Exception as e: + print(f"Warning: Could not parse {file_path}: {e}", file=sys.stderr) + continue + + result = { + 'iterations': len(startup_files) + } + + if startup_times and len(startup_times) > 1: + mean_time = statistics.mean(startup_times) + stddev_time = statistics.stdev(startup_times) + cv_time = calculate_cv(mean_time, stddev_time) + result['startup_time'] = { + 'mean': round(mean_time, 2), + 'stddev': round(stddev_time, 2), + 'cv': cv_time, + 'min': min(startup_times), + 'max': max(startup_times), + 'range': round(max(startup_times) - min(startup_times), 2), + 'stability_score': calculate_stability_score(cv_time) + } + + if memory_usages and len(memory_usages) > 1: + mean_mem = statistics.mean(memory_usages) + stddev_mem = statistics.stdev(memory_usages) + cv_mem = calculate_cv(mean_mem, stddev_mem) + result['memory_mb'] = { + 'mean': round(mean_mem, 2), + 'stddev': round(stddev_mem, 2), + 'cv': cv_mem, + 'min': round(min(memory_usages), 2), + 'max': round(max(memory_usages), 2), + 'range': round(max(memory_usages) - min(memory_usages), 2), + 'stability_score': calculate_stability_score(cv_mem) + } + + if cpu_usages and len(cpu_usages) > 1: + mean_cpu = statistics.mean(cpu_usages) + stddev_cpu = statistics.stdev(cpu_usages) + cv_cpu = calculate_cv(mean_cpu, stddev_cpu) + result['cpu_millicores'] = { + 'mean': round(mean_cpu, 2), + 'stddev': round(stddev_cpu, 2), + 'cv': cv_cpu, + 'min': round(min(cpu_usages), 2), + 'max': round(max(cpu_usages), 2), + 'range': round(max(cpu_usages) - min(cpu_usages), 2), + 'stability_score': calculate_stability_score(cv_cpu) + } + + return result if len(result) > 1 else None + +# Read command line arguments +if len(sys.argv) < 4: + print("ERROR: Missing required arguments", file=sys.stderr) + sys.exit(1) + +session_dir = sys.argv[1] +output_file = sys.argv[2] if sys.argv[2] else '' +output_format = sys.argv[3] if len(sys.argv) > 3 else 'text' + +agg_file = os.path.join(session_dir, 'aggregated.json') + +# Load aggregated data +with open(agg_file, 'r') as f: + data = json.load(f) + +# Extract metadata +session_id = data.get('session_id', 'unknown') +runtime = data.get('runtime', 'unknown') +total_iterations = data.get('total_iterations', 0) + +# Analyze startup metrics +startup_analysis = analyze_startup_metrics(session_dir) + +# Analyze each phase +phase_analyses = [] + +for phase in data.get('phases', []): + phase_name = phase.get('phase_name') + + # Calculate CVs for key metrics (using actual field names from aggregated.json) + throughput_cv = calculate_cv( + phase['requests_per_sec']['mean'], + phase['requests_per_sec']['stddev'] + ) + mean_latency_cv = calculate_cv( + phase['latency_mean_ms']['mean'], + phase['latency_mean_ms']['stddev'] + ) + p99_latency_cv = calculate_cv( + phase['latency_p99_ms']['mean'], + phase['latency_p99_ms']['stddev'] + ) + + # Calculate stability scores + throughput_stability = calculate_stability_score(throughput_cv) + latency_stability = calculate_stability_score(mean_latency_cv) + + # Overall stability (average of throughput and latency) + if throughput_stability is not None and latency_stability is not None: + overall_stability = round((throughput_stability + latency_stability) / 2, 2) + else: + overall_stability = None + + analysis = { + 'phase_name': phase_name, + 'threads': phase.get('threads'), + 'connections': phase.get('connections'), + 'iterations': phase.get('iterations', total_iterations), + 'requests_per_sec': { + 'mean': phase['requests_per_sec']['mean'], + 'stddev': phase['requests_per_sec']['stddev'], + 'cv': throughput_cv, + 'range': round(phase['requests_per_sec']['max'] - phase['requests_per_sec']['min'], 2), + 'stability_score': throughput_stability + }, + 'latency_mean_ms': { + 'mean': phase['latency_mean_ms']['mean'], + 'stddev': phase['latency_mean_ms']['stddev'], + 'cv': mean_latency_cv, + 'range': round(phase['latency_mean_ms']['max'] - phase['latency_mean_ms']['min'], 2), + 'stability_score': latency_stability + }, + 'latency_p99_ms': { + 'mean': phase['latency_p99_ms']['mean'], + 'stddev': phase['latency_p99_ms']['stddev'], + 'cv': p99_latency_cv, + 'range': round(phase['latency_p99_ms']['max'] - phase['latency_p99_ms']['min'], 2) + }, + 'overall_stability_score': overall_stability + } + + phase_analyses.append(analysis) + +# Generate output +if output_format == 'json': + output = { + 'session_id': session_id, + 'runtime': runtime, + 'total_iterations': total_iterations, + 'startup_analysis': startup_analysis, + 'phase_analyses': phase_analyses + } + + if output_file: + with open(output_file, 'w') as f: + json.dump(output, f, indent=2) + print(f"JSON analysis written to {output_file}") + else: + print(json.dumps(output, indent=2)) + +else: # text format + output_lines = [] + output_lines.append("=" * 100) + output_lines.append(f"SESSION VARIABILITY ANALYSIS: {session_id}") + output_lines.append("=" * 100) + output_lines.append(f"Runtime: {runtime}") + output_lines.append(f"Total Iterations: {total_iterations}") + output_lines.append("") + + # Add startup metrics analysis if available + if startup_analysis and startup_analysis.get('iterations', 0) > 0: + output_lines.append("=" * 100) + output_lines.append("STARTUP METRICS VARIABILITY") + output_lines.append("=" * 100) + output_lines.append(f"Iterations Analyzed: {startup_analysis['iterations']}") + output_lines.append("") + output_lines.append(f"{'Metric':<25} {'Mean':<12} {'StdDev':<12} {'CV%':<10} {'Range':<12} {'Stability':<10}") + output_lines.append("-" * 85) + + if 'startup_time' in startup_analysis: + st = startup_analysis['startup_time'] + output_lines.append(f"{'Startup Time (sec)':<25} " + f"{st['mean']:<12.2f} " + f"{st['stddev']:<12.2f} " + f"{st['cv']:<10.2f} " + f"{st['range']:<12.2f} " + f"{st['stability_score']:<10.2f}") + + if 'memory_mb' in startup_analysis: + mem = startup_analysis['memory_mb'] + output_lines.append(f"{'Memory (MB)':<25} " + f"{mem['mean']:<12.2f} " + f"{mem['stddev']:<12.2f} " + f"{mem['cv']:<10.2f} " + f"{mem['range']:<12.2f} " + f"{mem['stability_score']:<10.2f}") + + if 'cpu_millicores' in startup_analysis: + cpu = startup_analysis['cpu_millicores'] + output_lines.append(f"{'CPU (millicores)':<25} " + f"{cpu['mean']:<12.2f} " + f"{cpu['stddev']:<12.2f} " + f"{cpu['cv']:<10.2f} " + f"{cpu['range']:<12.2f} " + f"{cpu['stability_score']:<10.2f}") + + output_lines.append("") + output_lines.append("Startup Metrics Interpretation:") + output_lines.append(" - CV% < 5%: Very consistent startup behavior") + output_lines.append(" - CV% 5-10%: Acceptable variability") + output_lines.append(" - CV% > 10%: High variability, investigate environmental factors") + output_lines.append("") + + for analysis in phase_analyses: + output_lines.append("-" * 100) + output_lines.append(f"PHASE: {analysis['phase_name']}") + output_lines.append(f"Configuration: {analysis['threads']} threads, {analysis['connections']} connections") + output_lines.append(f"Iterations: {analysis['iterations']}") + output_lines.append(f"Overall Stability Score: {analysis['overall_stability_score']:.2f}/100") + output_lines.append("-" * 100) + output_lines.append("") + + # Throughput + output_lines.append("THROUGHPUT (requests/sec):") + output_lines.append(f" Mean: {analysis['requests_per_sec']['mean']:.2f} req/s") + output_lines.append(f" Std Dev: {analysis['requests_per_sec']['stddev']:.2f} req/s") + output_lines.append(f" CV: {analysis['requests_per_sec']['cv']:.2f}%") + output_lines.append(f" Range: {analysis['requests_per_sec']['range']:.2f} req/s") + output_lines.append(f" Stability: {analysis['requests_per_sec']['stability_score']:.2f}/100") + output_lines.append("") + + # Mean Latency + output_lines.append("MEAN LATENCY (ms):") + output_lines.append(f" Mean: {analysis['latency_mean_ms']['mean']:.2f} ms") + output_lines.append(f" Std Dev: {analysis['latency_mean_ms']['stddev']:.2f} ms") + output_lines.append(f" CV: {analysis['latency_mean_ms']['cv']:.2f}%") + output_lines.append(f" Range: {analysis['latency_mean_ms']['range']:.2f} ms") + output_lines.append(f" Stability: {analysis['latency_mean_ms']['stability_score']:.2f}/100") + output_lines.append("") + + # P99 Latency + output_lines.append("P99 LATENCY (ms):") + output_lines.append(f" Mean: {analysis['latency_p99_ms']['mean']:.2f} ms") + output_lines.append(f" Std Dev: {analysis['latency_p99_ms']['stddev']:.2f} ms") + output_lines.append(f" CV: {analysis['latency_p99_ms']['cv']:.2f}%") + output_lines.append(f" Range: {analysis['latency_p99_ms']['range']:.2f} ms") + output_lines.append("") + + output_lines.append("=" * 100) + output_lines.append("SUMMARY") + output_lines.append("=" * 100) + + # Calculate overall session stability + stability_scores = [a['overall_stability_score'] for a in phase_analyses if a['overall_stability_score'] is not None] + if stability_scores: + avg_stability = sum(stability_scores) / len(stability_scores) + output_lines.append(f"Average Session Stability Score: {avg_stability:.2f}/100") + + output_lines.append("=" * 100) + + output_text = "\n".join(output_lines) + + if output_file: + with open(output_file, 'w') as f: + f.write(output_text) + print(f"Text analysis written to {output_file}") + else: + print(output_text) + +PYTHON_SCRIPT + +if [ $? -eq 0 ]; then + log_info "" + log_info "Analysis complete!" +else + log_error "Analysis failed" + exit 1 +fi + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/combine-sessions.sh b/spring-quarkus-perf-comparison/scripts/results-tools/combine-sessions.sh new file mode 100755 index 00000000..90176669 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/combine-sessions.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# ============================================================================ +# Combine Multiple Sessions +# ============================================================================ +# Combines multiple session JSON files into a single combined JSON file + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 [session-file3...] --output + +Combines multiple session JSON files into a single combined JSON file. + +Arguments: + session-file1, session-file2, ... Session JSON files to combine + --output Output combined JSON file + +Example: + $0 ./variable-load-results/ootb/ootb.json \\ + ./variable-load-results/tuned/tuned.json \\ + --output ./variable-load-results/combined.json + +EOF + exit 1 +} + +# Parse arguments +SESSION_FILES=() +OUTPUT_FILE="" + +while [[ $# -gt 0 ]]; do + case $1 in + --output) + OUTPUT_FILE="$2" + shift 2 + ;; + *) + SESSION_FILES+=("$1") + shift + ;; + esac +done + +# Validate inputs +if [ ${#SESSION_FILES[@]} -lt 2 ]; then + log_error "At least 2 session files are required" + usage +fi + +if [ -z "$OUTPUT_FILE" ]; then + log_error "Output file is required (--output)" + usage +fi + +# Validate all session files exist +for file in "${SESSION_FILES[@]}"; do + if [ ! -f "$file" ]; then + log_error "Session file not found: $file" + exit 1 + fi +done + +log_info "Combining ${#SESSION_FILES[@]} session files" +log_info "Output file: $OUTPUT_FILE" + +# Combine using Python +python3 - "${SESSION_FILES[@]}" "$OUTPUT_FILE" << 'PYTHON_SCRIPT' +import sys +import json +from pathlib import Path + +def combine_sessions(session_files, output_file): + """Combine multiple session JSON files into a single file.""" + + sessions = {} + + for session_file in session_files: + try: + with open(session_file, 'r') as f: + data = json.load(f) + + session_name = data.get('session_name', Path(session_file).stem) + sessions[session_name] = data + + print(f"Loaded session: {session_name} ({data.get('total_iterations', 0)} iterations)") + + except Exception as e: + print(f"Warning: Failed to process {session_file}: {e}", file=sys.stderr) + continue + + if not sessions: + print("Error: No valid session data found", file=sys.stderr) + sys.exit(1) + + # Create combined structure + combined = { + 'total_sessions': len(sessions), + 'sessions': sessions + } + + # Write output + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, 'w') as f: + json.dump(combined, f, indent=2) + + print(f"Combined {len(sessions)} sessions into {output_file}") + +if __name__ == '__main__': + if len(sys.argv) < 3: + print("Usage: script ... ", file=sys.stderr) + sys.exit(1) + + session_files = sys.argv[1:-1] + output_file = sys.argv[-1] + + combine_sessions(session_files, output_file) +PYTHON_SCRIPT + +if [ $? -eq 0 ]; then + log_info "✓ Combination complete: $OUTPUT_FILE" +else + log_error "✗ Combination failed" + exit 1 +fi + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/compare-all.sh b/spring-quarkus-perf-comparison/scripts/results-tools/compare-all.sh new file mode 100755 index 00000000..dbea1780 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/compare-all.sh @@ -0,0 +1,312 @@ +#!/bin/bash + +# ============================================================================ +# Comprehensive Comparison Script +# ============================================================================ +# Compares ALL runtimes across ALL scenarios in a matrix format + +set -e + +RESULTS_DIR=$1 + +if [ -z "$RESULTS_DIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Remove trailing slash if present +RESULTS_DIR="${RESULTS_DIR%/}" + +# If path ends with /results, use it as is, otherwise append /results +if [[ "$RESULTS_DIR" == */results ]]; then + RESULTS_FILES_DIR="$RESULTS_DIR" +else + RESULTS_FILES_DIR="$RESULTS_DIR/results" +fi + +if [ ! -d "$RESULTS_FILES_DIR" ]; then + echo "ERROR: Results directory not found: $RESULTS_FILES_DIR" + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +# Calculate statistics for a metric +calculate_stats() { + local values=("$@") + local count=${#values[@]} + + if [ $count -eq 0 ]; then + echo "0 0 0 0 0" + return + fi + + # Calculate sum + local sum=0 + for val in "${values[@]}"; do + sum=$(awk "BEGIN {print $sum + $val}") + done + + # Calculate mean + local mean=$(awk "BEGIN {printf \"%.2f\", $sum / $count}") + + # Sort values for median + IFS=$'\n' sorted=($(sort -n <<<"${values[*]}")) + unset IFS + + # Calculate median + local median + if [ $((count % 2)) -eq 0 ]; then + local mid=$((count / 2)) + median=$(awk "BEGIN {printf \"%.2f\", (${sorted[$mid-1]} + ${sorted[$mid]}) / 2}") + else + local mid=$((count / 2)) + median=${sorted[$mid]} + fi + + # Calculate min and max + local min=${sorted[0]} + local max=${sorted[$count-1]} + + # Calculate standard deviation + local variance=0 + for val in "${values[@]}"; do + local diff=$(awk "BEGIN {print $val - $mean}") + local sq=$(awk "BEGIN {print $diff * $diff}") + variance=$(awk "BEGIN {print $variance + $sq}") + done + variance=$(awk "BEGIN {printf \"%.2f\", $variance / $count}") + local stddev=$(awk "BEGIN {printf \"%.2f\", sqrt($variance)}") + + echo "$mean $median $stddev $min $max" +} + +# Extract metric from result files +extract_metric() { + local runtime=$1 + local scenario=$2 + local metric_path=$3 + + local values=() + + for file in "$RESULTS_DIR"/results/${runtime}-${scenario}-*.json; do + if [ -f "$file" ]; then + local value=$(jq -r "$metric_path" "$file" 2>/dev/null) + if [ "$value" != "null" ] && [ -n "$value" ]; then + values+=($value) + fi + fi + done + + echo "${values[@]}" +} + +# Get mean value for a metric +get_mean() { + local runtime=$1 + local scenario=$2 + local metric_path=$3 + + local values=($(extract_metric "$runtime" "$scenario" "$metric_path")) + if [ ${#values[@]} -eq 0 ]; then + echo "N/A" + return + fi + + local stats=($(calculate_stats "${values[@]}")) + echo "${stats[0]}" +} + +# Calculate improvement percentage +calculate_improvement() { + local baseline=$1 + local optimized=$2 + + if [ "$baseline" = "0" ] || [ -z "$baseline" ] || [ "$baseline" = "0.00" ] || [ "$baseline" = "N/A" ] || [ "$optimized" = "N/A" ]; then + echo "N/A" + return + fi + + local improvement=$(awk "BEGIN {printf \"%.2f\", (($baseline - $optimized) / $baseline) * 100}") + printf "%+.2f%%" "$improvement" +} + +# Print comparison matrix for a metric +print_metric_matrix() { + local metric_name=$1 + local metric_path=$2 + local lower_is_better=$3 + + echo "" + echo "================================================================================" + echo "$metric_name" + echo "================================================================================" + + # Print header + printf "%-25s" "Runtime" + for scenario in "${SCENARIOS[@]}"; do + printf " | %-15s" "$scenario" + done + echo "" + printf "%s\n" "$(printf '=%.0s' {1..100})" + + # Print each runtime's values across scenarios + for runtime in "${RUNTIMES[@]}"; do + printf "%-25s" "$runtime" + for scenario in "${SCENARIOS[@]}"; do + local value=$(get_mean "$runtime" "$scenario" "$metric_path") + printf " | %-15s" "$value" + done + echo "" + done + + # If we have multiple scenarios, show improvement comparisons + if [ ${#SCENARIOS[@]} -gt 1 ]; then + echo "" + echo "Improvements (comparing scenarios):" + + # Compare first scenario vs others + local base_scenario="${SCENARIOS[0]}" + for ((i=1; i<${#SCENARIOS[@]}; i++)); do + local compare_scenario="${SCENARIOS[$i]}" + echo "" + echo " ${base_scenario} → ${compare_scenario}:" + + for runtime in "${RUNTIMES[@]}"; do + local base_val=$(get_mean "$runtime" "$base_scenario" "$metric_path") + local comp_val=$(get_mean "$runtime" "$compare_scenario" "$metric_path") + + if [ "$base_val" != "N/A" ] && [ "$comp_val" != "N/A" ]; then + local improvement + if [ "$lower_is_better" = "true" ]; then + improvement=$(calculate_improvement "$base_val" "$comp_val") + else + improvement=$(calculate_improvement "$comp_val" "$base_val") + fi + printf " %-25s: %s\n" "$runtime" "$improvement" + fi + done + done + fi + + # If we have multiple runtimes, show runtime comparisons + if [ ${#RUNTIMES[@]} -gt 1 ]; then + echo "" + echo "Runtime Comparisons (for each scenario):" + + for scenario in "${SCENARIOS[@]}"; do + echo "" + echo " Scenario: ${scenario}" + + # Find best and worst + local best_runtime="" + local best_value="" + local worst_runtime="" + local worst_value="" + + for runtime in "${RUNTIMES[@]}"; do + local value=$(get_mean "$runtime" "$scenario" "$metric_path") + if [ "$value" != "N/A" ]; then + if [ -z "$best_value" ]; then + best_value="$value" + best_runtime="$runtime" + worst_value="$value" + worst_runtime="$runtime" + else + if [ "$lower_is_better" = "true" ]; then + if (( $(awk "BEGIN {print ($value < $best_value)}") )); then + best_value="$value" + best_runtime="$runtime" + fi + if (( $(awk "BEGIN {print ($value > $worst_value)}") )); then + worst_value="$value" + worst_runtime="$runtime" + fi + else + if (( $(awk "BEGIN {print ($value > $best_value)}") )); then + best_value="$value" + best_runtime="$runtime" + fi + if (( $(awk "BEGIN {print ($value < $worst_value)}") )); then + worst_value="$value" + worst_runtime="$runtime" + fi + fi + fi + fi + done + + if [ -n "$best_runtime" ]; then + echo " Best: $best_runtime = $best_value" + echo " Worst: $worst_runtime = $worst_value" + + if [ "$best_runtime" != "$worst_runtime" ]; then + local diff + if [ "$lower_is_better" = "true" ]; then + diff=$(calculate_improvement "$worst_value" "$best_value") + else + diff=$(calculate_improvement "$best_value" "$worst_value") + fi + echo " Difference: $diff" + fi + fi + done + fi +} + +# Main execution +log_info "Analyzing results from: $RESULTS_DIR" + +# Find all runtimes and scenarios +RUNTIMES=() +SCENARIOS=() + +for file in "$RESULTS_FILES_DIR"/*.json; do + if [ -f "$file" ]; then + basename=$(basename "$file" .json) + # Extract runtime and scenario from filename: runtime-scenario-iteration.json + # Remove the iteration number (last part after last dash) + base_without_iteration=$(echo "$basename" | sed 's/-[^-]*$//') + # Now extract runtime (everything except last part) and scenario (last part) + runtime=$(echo "$base_without_iteration" | sed 's/-[^-]*$//') + scenario=$(echo "$base_without_iteration" | sed 's/^.*-//') + + if [[ ! " ${RUNTIMES[@]} " =~ " ${runtime} " ]]; then + RUNTIMES+=("$runtime") + fi + if [[ ! " ${SCENARIOS[@]} " =~ " ${scenario} " ]]; then + SCENARIOS+=("$scenario") + fi + fi +done + +log_info "Found runtimes: ${RUNTIMES[@]}" +log_info "Found scenarios: ${SCENARIOS[@]}" + +echo "" +echo "================================================================================" +echo "COMPREHENSIVE PERFORMANCE COMPARISON" +echo "================================================================================" +echo "Runtimes: ${RUNTIMES[@]}" +echo "Scenarios: ${SCENARIOS[@]}" +echo "================================================================================" + +# Print comparison matrices for each metric +print_metric_matrix "Startup Time (ms)" ".tests.startup.startup_time_ms" "true" +print_metric_matrix "Memory Usage (MB)" ".tests.memory.rss_mb" "true" +print_metric_matrix "Throughput (req/s)" ".tests.load_test.requests_per_sec" "false" +print_metric_matrix "Latency Mean (ms)" ".tests.load_test.latency_mean_ms" "true" +print_metric_matrix "Latency P50 (ms)" ".tests.load_test.latency_p50_ms" "true" +print_metric_matrix "Latency P90 (ms)" ".tests.load_test.latency_p90_ms" "true" +print_metric_matrix "Latency P99 (ms)" ".tests.load_test.latency_p99_ms" "true" + +echo "" +echo "================================================================================" +log_info "Comparison completed" +echo "================================================================================" + +# Made with Bob \ No newline at end of file diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/compare-sessions.sh b/spring-quarkus-perf-comparison/scripts/results-tools/compare-sessions.sh new file mode 100755 index 00000000..d9212bb2 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/compare-sessions.sh @@ -0,0 +1,544 @@ +#!/bin/bash + +# ============================================================================ +# Compare Sessions - Cross-Session Phase-Level Comparison +# ============================================================================ +# This script compares two sessions (e.g., OOTB vs Tuned) at the phase level, +# showing performance differences for each phase. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +SESSION_A="" +SESSION_B="" +SESSION_A_NAME="" +SESSION_B_NAME="" +OUTPUT_FILE="" +FORMAT="text" # text or json + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +log_warn() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] [WARN]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 --session-a --session-b [options] + +Compare two sessions at the phase level, showing performance differences. +Sessions should contain aggregated.json files (use aggregate-session.sh first). + +Required: + --session-a First session directory (e.g., baseline/OOTB) + --session-b Second session directory (e.g., optimized/Tuned) + +Optional: + --session-a-name Display name for session A (default: from session_id in aggregated.json) + --session-b-name Display name for session B (default: from session_id in aggregated.json) + --output Output file for comparison results + --format Output format: text or json (default: text) + --help Show this help message + +Examples: + # Compare OOTB vs Tuned sessions + $0 --session-a ./variable-load-results/session-baseline \\ + --session-b ./variable-load-results/session-optimized + + # Generate JSON output + $0 --session-a ./variable-load-results/session-baseline \\ + --session-b ./variable-load-results/session-optimized \\ + --format json --output comparison.json + +Workflow: + 1. Run tests with session-id for both configurations: + ./run-variable-load-multi-phase.sh --runtime quarkus3-jvm --scenario ootb --session-id baseline + ./run-variable-load-multi-phase.sh --runtime quarkus3-jvm --scenario tuned --session-id optimized + + 2. Aggregate each session: + ./aggregate-session.sh --session-dir ./variable-load-results/session-baseline + ./aggregate-session.sh --session-dir ./variable-load-results/session-optimized + + 3. Compare sessions: + $0 --session-a ./variable-load-results/session-baseline \\ + --session-b ./variable-load-results/session-optimized + +EOF + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --session-a) + SESSION_A="$2" + shift 2 + ;; + --session-b) + SESSION_B="$2" + shift 2 + ;; + --session-a-name) + SESSION_A_NAME="$2" + shift 2 + ;; + --session-b-name) + SESSION_B_NAME="$2" + shift 2 + ;; + --output) + OUTPUT_FILE="$2" + shift 2 + ;; + --format) + FORMAT="$2" + shift 2 + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate required parameters +if [ -z "$SESSION_A" ] || [ -z "$SESSION_B" ]; then + log_error "Both --session-a and --session-b are required" + usage +fi + +if [ ! -d "$SESSION_A" ]; then + log_error "Session A directory not found: $SESSION_A" + exit 1 +fi + +if [ ! -d "$SESSION_B" ]; then + log_error "Session B directory not found: $SESSION_B" + exit 1 +fi + +# Check for aggregated files +AGG_A="${SESSION_A}/aggregated.json" +AGG_B="${SESSION_B}/aggregated.json" + +if [ ! -f "$AGG_A" ]; then + log_error "Aggregated file not found: $AGG_A" + log_error "Run aggregate-session.sh first: ./aggregate-session.sh --session-dir $SESSION_A" + exit 1 +fi + +if [ ! -f "$AGG_B" ]; then + log_error "Aggregated file not found: $AGG_B" + log_error "Run aggregate-session.sh first: ./aggregate-session.sh --session-dir $SESSION_B" + exit 1 +fi + +log_info "=========================================" +log_info "Comparing Sessions" +log_info "=========================================" +log_info "Session A: $SESSION_A" +log_info "Session B: $SESSION_B" +log_info "Format: $FORMAT" +if [ -n "$OUTPUT_FILE" ]; then + log_info "Output: $OUTPUT_FILE" +fi +log_info "" + +# Use Python for comparison +export SESSION_A +export SESSION_B +export SESSION_A_NAME +export SESSION_B_NAME +export OUTPUT_FILE +export FORMAT + +python3 - "$SESSION_A" "$SESSION_B" "$SESSION_A_NAME" "$SESSION_B_NAME" "$OUTPUT_FILE" "$FORMAT" << 'PYTHON_SCRIPT' +import json +import sys +import os +import glob + +def calculate_improvement(baseline, optimized): + """Calculate percentage improvement (positive = better)""" + if baseline is None or optimized is None or baseline == 0: + return None + return round(((optimized - baseline) / baseline) * 100, 2) + +def parse_memory_value(mem_str): + """Convert memory string (e.g., '512Mi', '1Gi') to MB""" + if not mem_str: + return None + mem_str = str(mem_str).strip() + if mem_str.endswith('Mi'): + return float(mem_str[:-2]) + elif mem_str.endswith('Gi'): + return float(mem_str[:-2]) * 1024 + elif mem_str.endswith('Ki'): + return float(mem_str[:-2]) / 1024 + else: + # Assume bytes + try: + return float(mem_str) / (1024 * 1024) + except: + return None + +def load_startup_metrics(session_dir): + """Load and aggregate startup metrics from all iterations""" + startup_files = glob.glob(os.path.join(session_dir, '*-startup.json')) + + if not startup_files: + return None + + startup_times = [] + memory_usages = [] + cpu_usages = [] + + for file_path in startup_files: + try: + with open(file_path, 'r') as f: + data = json.load(f) + + if data.get('startup_time_seconds'): + startup_times.append(data['startup_time_seconds']) + + mem_usage = data.get('memory', {}).get('current_usage', '') + if mem_usage: + mem_mb = parse_memory_value(mem_usage) + if mem_mb: + memory_usages.append(mem_mb) + + # CPU usage is typically like "10m" (millicores) + cpu_usage = data.get('cpu', {}).get('current_usage', '') + if cpu_usage and cpu_usage.endswith('m'): + try: + cpu_usages.append(float(cpu_usage[:-1])) + except: + pass + except Exception as e: + print(f"Warning: Could not parse {file_path}: {e}", file=sys.stderr) + continue + + if not startup_times and not memory_usages: + return None + + result = { + 'iterations': len(startup_files) + } + + if startup_times: + result['startup_time'] = { + 'mean': round(sum(startup_times) / len(startup_times), 2), + 'min': min(startup_times), + 'max': max(startup_times) + } + + if memory_usages: + result['memory_mb'] = { + 'mean': round(sum(memory_usages) / len(memory_usages), 2), + 'min': round(min(memory_usages), 2), + 'max': round(max(memory_usages), 2) + } + + if cpu_usages: + result['cpu_millicores'] = { + 'mean': round(sum(cpu_usages) / len(cpu_usages), 2), + 'min': round(min(cpu_usages), 2), + 'max': round(max(cpu_usages), 2) + } + + return result + +def format_improvement(value, metric_type='throughput'): + """Format improvement with color coding""" + if value is None: + return "N/A" + + # For throughput: higher is better + # For latency/errors: lower is better + if metric_type in ['throughput']: + if value > 0: + return f"+{value}% ✓" + elif value < 0: + return f"{value}% ✗" + else: # latency, errors + if value < 0: + return f"{value}% ✓" + elif value > 0: + return f"+{value}% ✗" + + return f"{value}%" + +# Read command line arguments +if len(sys.argv) < 7: + print("ERROR: Missing required arguments", file=sys.stderr) + sys.exit(1) + +session_a_dir = sys.argv[1] +session_b_dir = sys.argv[2] +session_a_name_override = sys.argv[3] if sys.argv[3] else '' +session_b_name_override = sys.argv[4] if sys.argv[4] else '' +output_file = sys.argv[5] if sys.argv[5] else '' +output_format = sys.argv[6] if len(sys.argv) > 6 else 'text' + +agg_a_path = os.path.join(session_a_dir, 'aggregated.json') +agg_b_path = os.path.join(session_b_dir, 'aggregated.json') + +# Load aggregated data +with open(agg_a_path, 'r') as f: + data_a = json.load(f) + +with open(agg_b_path, 'r') as f: + data_b = json.load(f) + +# Extract metadata +session_a_id = session_a_name_override if session_a_name_override else data_a.get('session_id', 'A') +session_b_id = session_b_name_override if session_b_name_override else data_b.get('session_id', 'B') +runtime = data_a.get('runtime', 'unknown') + +# Load startup metrics +startup_a = load_startup_metrics(session_a_dir) +startup_b = load_startup_metrics(session_b_dir) + +# Create phase comparison +phase_comparisons = [] + +phases_a = {p['phase_name']: p for p in data_a.get('phases', [])} +phases_b = {p['phase_name']: p for p in data_b.get('phases', [])} + +for phase_name in phases_a.keys(): + if phase_name not in phases_b: + continue + + phase_a = phases_a[phase_name] + phase_b = phases_b[phase_name] + + comparison = { + 'phase_name': phase_name, + 'threads': phase_a.get('threads'), + 'connections': phase_a.get('connections'), + session_a_id: { + 'requests_per_sec': phase_a['requests_per_sec']['mean'], + 'latency_mean_ms': phase_a['latency_mean_ms']['mean'], + 'latency_p99_ms': phase_a['latency_p99_ms']['mean'], + 'errors': phase_a['errors']['mean'] + }, + session_b_id: { + 'requests_per_sec': phase_b['requests_per_sec']['mean'], + 'latency_mean_ms': phase_b['latency_mean_ms']['mean'], + 'latency_p99_ms': phase_b['latency_p99_ms']['mean'], + 'errors': phase_b['errors']['mean'] + }, + 'improvement': { + 'requests_per_sec': calculate_improvement( + phase_a['requests_per_sec']['mean'], + phase_b['requests_per_sec']['mean'] + ), + 'latency_mean_ms': calculate_improvement( + phase_a['latency_mean_ms']['mean'], + phase_b['latency_mean_ms']['mean'] + ), + 'latency_p99_ms': calculate_improvement( + phase_a['latency_p99_ms']['mean'], + phase_b['latency_p99_ms']['mean'] + ), + 'errors': calculate_improvement( + phase_a['errors']['mean'], + phase_b['errors']['mean'] + ) + } + } + + phase_comparisons.append(comparison) + +# Generate output +if output_format == 'json': + output = { + 'runtime': runtime, + session_a_id: { + 'iterations': data_a.get('total_iterations'), + 'startup_metrics': startup_a + }, + session_b_id: { + 'iterations': data_b.get('total_iterations'), + 'startup_metrics': startup_b + }, + 'phase_comparisons': phase_comparisons + } + + # Add startup comparison if both sessions have startup metrics + if startup_a and startup_b: + startup_comparison = {} + + if 'startup_time' in startup_a and 'startup_time' in startup_b: + startup_comparison['startup_time'] = { + session_a_id: startup_a['startup_time']['mean'], + session_b_id: startup_b['startup_time']['mean'], + 'improvement_pct': calculate_improvement( + startup_a['startup_time']['mean'], + startup_b['startup_time']['mean'] + ) + } + + if 'memory_mb' in startup_a and 'memory_mb' in startup_b: + startup_comparison['memory_mb'] = { + session_a_id: startup_a['memory_mb']['mean'], + session_b_id: startup_b['memory_mb']['mean'], + 'improvement_pct': calculate_improvement( + startup_a['memory_mb']['mean'], + startup_b['memory_mb']['mean'] + ) + } + + if 'cpu_millicores' in startup_a and 'cpu_millicores' in startup_b: + startup_comparison['cpu_millicores'] = { + session_a_id: startup_a['cpu_millicores']['mean'], + session_b_id: startup_b['cpu_millicores']['mean'], + 'improvement_pct': calculate_improvement( + startup_a['cpu_millicores']['mean'], + startup_b['cpu_millicores']['mean'] + ) + } + + output['startup_comparison'] = startup_comparison + + if output_file: + with open(output_file, 'w') as f: + json.dump(output, f, indent=2) + print(f"JSON comparison written to {output_file}") + else: + print(json.dumps(output, indent=2)) + +else: # text format + output_lines = [] + output_lines.append("=" * 100) + output_lines.append(f"SESSION COMPARISON: {session_a_id} vs {session_b_id}") + output_lines.append("=" * 100) + output_lines.append(f"Runtime: {runtime}") + output_lines.append(f"Session A ({session_a_id}): {data_a.get('total_iterations')} iterations") + output_lines.append(f"Session B ({session_b_id}): {data_b.get('total_iterations')} iterations") + output_lines.append("") + + # Add startup metrics comparison if available + if startup_a and startup_b: + output_lines.append("=" * 100) + output_lines.append("STARTUP METRICS COMPARISON") + output_lines.append("=" * 100) + output_lines.append("") + output_lines.append(f"{'Metric':<25} {session_a_id:<20} {session_b_id:<20} {'Improvement':<20}") + output_lines.append("-" * 85) + + # Startup Time + if 'startup_time' in startup_a and 'startup_time' in startup_b: + time_a = startup_a['startup_time']['mean'] + time_b = startup_b['startup_time']['mean'] + improvement = calculate_improvement(time_a, time_b) + output_lines.append(f"{'Startup Time (seconds)':<25} " + f"{time_a:<20.2f} " + f"{time_b:<20.2f} " + f"{format_improvement(improvement, 'latency'):<20}") + + # Memory Usage + if 'memory_mb' in startup_a and 'memory_mb' in startup_b: + mem_a = startup_a['memory_mb']['mean'] + mem_b = startup_b['memory_mb']['mean'] + improvement = calculate_improvement(mem_a, mem_b) + output_lines.append(f"{'Memory Usage (MB)':<25} " + f"{mem_a:<20.2f} " + f"{mem_b:<20.2f} " + f"{format_improvement(improvement, 'latency'):<20}") + + # CPU Usage + if 'cpu_millicores' in startup_a and 'cpu_millicores' in startup_b: + cpu_a = startup_a['cpu_millicores']['mean'] + cpu_b = startup_b['cpu_millicores']['mean'] + improvement = calculate_improvement(cpu_a, cpu_b) + output_lines.append(f"{'CPU Usage (millicores)':<25} " + f"{cpu_a:<20.2f} " + f"{cpu_b:<20.2f} " + f"{format_improvement(improvement, 'latency'):<20}") + + output_lines.append("") + output_lines.append("Note: For startup metrics, lower values are better") + output_lines.append("") + + for comp in phase_comparisons: + output_lines.append("-" * 100) + output_lines.append(f"PHASE: {comp['phase_name']}") + output_lines.append(f"Configuration: {comp['threads']} threads, {comp['connections']} connections") + output_lines.append("-" * 100) + output_lines.append("") + + output_lines.append(f"{'Metric':<20} {session_a_id:<20} {session_b_id:<20} {'Improvement':<20}") + output_lines.append("-" * 80) + + # Throughput + output_lines.append(f"{'Throughput (req/s)':<20} " + f"{comp[session_a_id]['requests_per_sec']:<20.2f} " + f"{comp[session_b_id]['requests_per_sec']:<20.2f} " + f"{format_improvement(comp['improvement']['requests_per_sec'], 'throughput'):<20}") + + # Mean Latency + output_lines.append(f"{'Mean Latency (ms)':<20} " + f"{comp[session_a_id]['latency_mean_ms']:<20.2f} " + f"{comp[session_b_id]['latency_mean_ms']:<20.2f} " + f"{format_improvement(comp['improvement']['latency_mean_ms'], 'latency'):<20}") + + # P99 Latency + output_lines.append(f"{'P99 Latency (ms)':<20} " + f"{comp[session_a_id]['latency_p99_ms']:<20.2f} " + f"{comp[session_b_id]['latency_p99_ms']:<20.2f} " + f"{format_improvement(comp['improvement']['latency_p99_ms'], 'latency'):<20}") + + # Errors + output_lines.append(f"{'Errors':<20} " + f"{comp[session_a_id]['errors']:<20.2f} " + f"{comp[session_b_id]['errors']:<20.2f} " + f"{format_improvement(comp['improvement']['errors'], 'errors'):<20}") + + output_lines.append("") + + output_lines.append("=" * 100) + output_lines.append("Legend: ✓ = Improvement, ✗ = Regression") + output_lines.append(" For throughput: higher is better") + output_lines.append(" For latency/errors: lower is better") + output_lines.append("=" * 100) + + output_text = "\n".join(output_lines) + + if output_file: + with open(output_file, 'w') as f: + f.write(output_text) + print(f"Text comparison written to {output_file}") + else: + print(output_text) + +PYTHON_SCRIPT + +if [ $? -eq 0 ]; then + log_info "" + log_info "Comparison complete!" +else + log_error "Comparison failed" + exit 1 +fi + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/consolidate-session.sh b/spring-quarkus-perf-comparison/scripts/results-tools/consolidate-session.sh new file mode 100755 index 00000000..4e959f4f --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/consolidate-session.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# ============================================================================ +# Consolidate Session Data +# ============================================================================ +# Combines all iteration JSON files for a session into a single JSON file +# preserving ALL metrics from the original files + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 [output-file] + +Consolidates all iteration JSON files from a session into a single JSON file, +preserving ALL metrics from the original iteration files. + +Arguments: + session-directory Directory containing iteration JSON files + output-file Output JSON file (default: /.json) + +Example: + $0 ./variable-load-results/ootb + # Creates: ./variable-load-results/ootb/ootb.json + + $0 ./variable-load-results/tuned ./results/tuned-consolidated.json + # Creates: ./results/tuned-consolidated.json + +EOF + exit 1 +} + +# Check arguments +if [ $# -lt 1 ]; then + usage +fi + +SESSION_DIR="$1" +OUTPUT_FILE="$2" + +# Validate session directory +if [ ! -d "$SESSION_DIR" ]; then + log_error "Session directory not found: $SESSION_DIR" + exit 1 +fi + +# Extract session name from directory +SESSION_NAME=$(basename "$SESSION_DIR") + +# Set default output file if not provided +if [ -z "$OUTPUT_FILE" ]; then + OUTPUT_FILE="$SESSION_DIR/${SESSION_NAME}.json" +fi + +log_info "Consolidating session: $SESSION_NAME" +log_info "Session directory: $SESSION_DIR" +log_info "Output file: $OUTPUT_FILE" + +# Find all iteration files (excluding phase-specific files) +ITERATION_FILES=($(find "$SESSION_DIR" -maxdepth 1 -name "*-iter[0-9]*.json" ! -name "*_phase*" | sort)) + +if [ ${#ITERATION_FILES[@]} -eq 0 ]; then + log_error "No iteration files found in $SESSION_DIR" + exit 1 +fi + +log_info "Found ${#ITERATION_FILES[@]} iteration files" + +# Consolidate using Python - preserves ALL metrics +python3 - "${ITERATION_FILES[@]}" "$OUTPUT_FILE" "$SESSION_NAME" << 'PYTHON_SCRIPT' +import sys +import json +from pathlib import Path + +def consolidate_iterations(iteration_files, output_file, session_name): + """Consolidate all iteration files into a single JSON preserving ALL metrics.""" + + iterations = [] + metadata = {} + + for iter_file in iteration_files: + try: + with open(iter_file, 'r') as f: + data = json.load(f) + + # Extract metadata from first file + if not metadata: + metadata = { + 'runtime': data.get('runtime', 'unknown'), + 'scenario': data.get('scenario', 'unknown'), + 'session_id': data.get('session_id', session_name) + } + + # Preserve the ENTIRE iteration data structure + # This ensures ALL metrics are kept, not just a subset + iteration_data = dict(data) # Copy all fields + iteration_data['iteration'] = data.get('iteration', len(iterations) + 1) + iterations.append(iteration_data) + + except Exception as e: + print(f"Warning: Failed to process {iter_file}: {e}", file=sys.stderr) + continue + + if not iterations: + print("Error: No valid iteration data found", file=sys.stderr) + sys.exit(1) + + # Create consolidated structure + consolidated = { + 'session_name': session_name, + 'runtime': metadata['runtime'], + 'scenario': metadata['scenario'], + 'session_id': metadata['session_id'], + 'total_iterations': len(iterations), + 'iterations': iterations + } + + # Write output + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, 'w') as f: + json.dump(consolidated, f, indent=2) + + print(f"Consolidated {len(iterations)} iterations into {output_file}") + +if __name__ == '__main__': + if len(sys.argv) < 4: + print("Usage: script ... ", file=sys.stderr) + sys.exit(1) + + iteration_files = sys.argv[1:-2] + output_file = sys.argv[-2] + session_name = sys.argv[-1] + + consolidate_iterations(iteration_files, output_file, session_name) +PYTHON_SCRIPT + +if [ $? -eq 0 ]; then + log_info "✓ Consolidation complete: $OUTPUT_FILE" +else + log_error "✗ Consolidation failed" + exit 1 +fi + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/generate-final-reports.sh b/spring-quarkus-perf-comparison/scripts/results-tools/generate-final-reports.sh new file mode 100755 index 00000000..7b99881a --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/generate-final-reports.sh @@ -0,0 +1,183 @@ +#!/bin/bash + +# ============================================================================ +# Generate Final Reports +# ============================================================================ +# Master script to generate all final output files: +# 1. Individual session JSON files (ootb.json, tuned.json, etc.) +# 2. Combined JSON file (combined.json) +# 3. Comparison text file (comparison.txt) +# 4. Comparison HTML report (comparison-report.html) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +log_step() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] [STEP]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 + +Generates all final output files from session-based test results: + - Individual session JSON files (e.g., ootb.json, tuned.json) + - Combined JSON file (combined.json) + - Comparison text file (comparison.txt) + - Comparison HTML report (comparison-report.html) + +Arguments: + results-directory Directory containing session subdirectories + +Example: + $0 ./variable-load-results + + This will process all sessions in ./variable-load-results/ and generate: + - ./variable-load-results/ootb.json + - ./variable-load-results/tuned.json + - ./variable-load-results/combined.json + - ./variable-load-results/comparison.txt + - ./variable-load-results/comparison-report.html + +EOF + exit 1 +} + +# Check arguments +if [ $# -lt 1 ]; then + usage +fi + +RESULTS_DIR="$1" + +# Validate results directory +if [ ! -d "$RESULTS_DIR" ]; then + log_error "Results directory not found: $RESULTS_DIR" + exit 1 +fi + +log_info "Processing results from: $RESULTS_DIR" +echo "" + +# Find all session directories +SESSION_DIRS=($(find "$RESULTS_DIR" -mindepth 1 -maxdepth 1 -type d | sort)) + +if [ ${#SESSION_DIRS[@]} -eq 0 ]; then + log_error "No session directories found in $RESULTS_DIR" + exit 1 +fi + +log_info "Found ${#SESSION_DIRS[@]} session directories" +for dir in "${SESSION_DIRS[@]}"; do + echo " - $(basename "$dir")" +done +echo "" + +# Step 1: Consolidate each session +log_step "Step 1: Consolidating individual sessions" +SESSION_FILES=() +for session_dir in "${SESSION_DIRS[@]}"; do + session_name=$(basename "$session_dir") + output_file="$RESULTS_DIR/${session_name}.json" + + log_info "Consolidating session: $session_name" + bash "$SCRIPT_DIR/consolidate-session.sh" "$session_dir" "$output_file" + + if [ -f "$output_file" ]; then + SESSION_FILES+=("$output_file") + fi +done +echo "" + +# Step 2: Combine all sessions +log_step "Step 2: Combining all sessions" +COMBINED_FILE="$RESULTS_DIR/combined.json" +bash "$SCRIPT_DIR/combine-sessions.sh" "${SESSION_FILES[@]}" --output "$COMBINED_FILE" +echo "" + +# Step 3: Generate comparison files +log_step "Step 3: Generating comparison reports" + +# Determine session pairs for comparison +if [ ${#SESSION_DIRS[@]} -eq 2 ]; then + SESSION_A="${SESSION_DIRS[0]}" + SESSION_B="${SESSION_DIRS[1]}" + SESSION_A_NAME=$(basename "$SESSION_A") + SESSION_B_NAME=$(basename "$SESSION_B") + + log_info "Comparing: $SESSION_A_NAME vs $SESSION_B_NAME" + + # Generate comparison text file directly + COMPARISON_TXT="$RESULTS_DIR/${SESSION_A_NAME}-vs-${SESSION_B_NAME}-comparison.txt" + + bash "$SCRIPT_DIR/compare-sessions.sh" \ + --session-a "$SESSION_A" \ + --session-b "$SESSION_B" \ + --output "$COMPARISON_TXT" + + # Generate HTML report + bash "$SCRIPT_DIR/generate-session-report.sh" \ + --session-a "$SESSION_A" \ + --session-b "$SESSION_B" \ + --output "$RESULTS_DIR/comparison-report.html" + + # Clean up intermediate JSON comparison file (we only need txt and html) + if [ -f "$COMPARISON_JSON" ]; then + rm "$COMPARISON_JSON" + fi + +else + log_info "Skipping comparison (requires exactly 2 sessions, found ${#SESSION_DIRS[@]})" +fi +echo "" + +# Summary +log_step "Summary of generated files:" +echo "" +echo "Session JSON files:" +for file in "${SESSION_FILES[@]}"; do + if [ -f "$file" ]; then + size=$(du -h "$file" | cut -f1) + echo " ✓ $(basename "$file") ($size)" + fi +done +echo "" + +if [ -f "$COMBINED_FILE" ]; then + size=$(du -h "$COMBINED_FILE" | cut -f1) + echo "Combined JSON:" + echo " ✓ combined.json ($size)" + echo "" +fi + +if [ -f "$COMPARISON_TXT" ]; then + size=$(du -h "$COMPARISON_TXT" | cut -f1) + echo "Comparison files:" + echo " ✓ $(basename "$COMPARISON_TXT") ($size)" +fi + +if [ -f "$RESULTS_DIR/comparison-report.html" ]; then + size=$(du -h "$RESULTS_DIR/comparison-report.html" | cut -f1) + echo " ✓ comparison-report.html ($size)" +fi + +echo "" +log_info "✓ All reports generated successfully!" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/generate-report.sh b/spring-quarkus-perf-comparison/scripts/results-tools/generate-report.sh new file mode 100755 index 00000000..93dfaabd --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/generate-report.sh @@ -0,0 +1,440 @@ +#!/bin/bash + +# ============================================================================ +# Generate HTML Report +# ============================================================================ +# Generates an HTML report from benchmark results + +set -e + +RESULTS_DIR=$1 + +if [ -z "$RESULTS_DIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -d "$RESULTS_DIR" ]; then + echo "ERROR: Results directory not found: $RESULTS_DIR" + exit 1 +fi + +OUTPUT_FILE="${RESULTS_DIR}/report.html" + +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +# Generate HTML report +generate_html() { + log_info "Generating HTML report..." + + # Read metrics.json if it exists + local metrics_data="{}" + if [ -f "${RESULTS_DIR}/metrics.json" ]; then + metrics_data=$(cat "${RESULTS_DIR}/metrics.json" | jq -c .) + fi + + # Read summary.json if it exists + local summary_data="{}" + if [ -f "${RESULTS_DIR}/summary.json" ]; then + summary_data=$(cat "${RESULTS_DIR}/summary.json" | jq -c .) + fi + + cat > "$OUTPUT_FILE" <<'HTMLEOF' + + + + + + Quarkus Performance Benchmark Report + + + +
+

🚀 Quarkus Performance Benchmark Report

+ +
+

Summary

+
+ Timestamp: + +
+
+ Scenarios: + OOTB vs Tuned +
+
+ Iterations: + +
+
+ +
+

📊 Scenario Comparison

+
+
+
+ + + +
+
+

Generated by OpenShift Benchmark Suite

+

For more information, visit the project repository

+
+
+ + + + +HTMLEOF + + # Replace placeholders with actual data + sed -i "s|METRICS_DATA_PLACEHOLDER|${metrics_data}|g" "$OUTPUT_FILE" + sed -i "s|SUMMARY_DATA_PLACEHOLDER|${summary_data}|g" "$OUTPUT_FILE" + + log_info "HTML report generated: $OUTPUT_FILE" +} + +# Main execution +generate_html + +log_info "Report generation completed" +log_info "Open the report: file://${OUTPUT_FILE}" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/generate-session-report.sh b/spring-quarkus-perf-comparison/scripts/results-tools/generate-session-report.sh new file mode 100755 index 00000000..26b67e3b --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/generate-session-report.sh @@ -0,0 +1,523 @@ +#!/bin/bash + +# ============================================================================ +# Generate Session HTML Report +# ============================================================================ +# Generates HTML reports from session-based test results +# Supports both single session reports and cross-session comparisons + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +SESSION_A="" +SESSION_B="" +OUTPUT_FILE="" +REPORT_TYPE="single" # single or comparison + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] [INFO]${NC} $@" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR]${NC} $@" +} + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Generate HTML reports from session-based test results. + +Single Session Report: + --session Session directory to generate report for + --output Output HTML file (default: /report.html) + +Comparison Report: + --session-a First session directory + --session-b Second session directory + --output Output HTML file (default: ./session-comparison.html) + +Options: + --help Show this help message + +Examples: + # Generate report for single session + $0 --session ./variable-load-results/session-baseline + + # Generate comparison report + $0 --session-a ./variable-load-results/session-baseline \\ + --session-b ./variable-load-results/session-optimized \\ + --output comparison-report.html + +Note: Sessions must have aggregated.json files (run aggregate-session.sh first) + +EOF + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --session) + SESSION_A="$2" + REPORT_TYPE="single" + shift 2 + ;; + --session-a) + SESSION_A="$2" + REPORT_TYPE="comparison" + shift 2 + ;; + --session-b) + SESSION_B="$2" + REPORT_TYPE="comparison" + shift 2 + ;; + --output) + OUTPUT_FILE="$2" + shift 2 + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate inputs +if [ "$REPORT_TYPE" = "single" ]; then + if [ -z "$SESSION_A" ]; then + log_error "--session is required for single session report" + usage + fi + if [ ! -d "$SESSION_A" ]; then + log_error "Session directory not found: $SESSION_A" + exit 1 + fi + if [ ! -f "${SESSION_A}/aggregated.json" ]; then + log_error "aggregated.json not found in $SESSION_A" + log_error "Run aggregate-session.sh first" + exit 1 + fi + if [ -z "$OUTPUT_FILE" ]; then + OUTPUT_FILE="${SESSION_A}/report.html" + fi +else + if [ -z "$SESSION_A" ] || [ -z "$SESSION_B" ]; then + log_error "Both --session-a and --session-b are required for comparison report" + usage + fi + if [ ! -d "$SESSION_A" ] || [ ! -d "$SESSION_B" ]; then + log_error "Session directories not found" + exit 1 + fi + if [ ! -f "${SESSION_A}/aggregated.json" ] || [ ! -f "${SESSION_B}/aggregated.json" ]; then + log_error "aggregated.json not found in one or both sessions" + log_error "Run aggregate-session.sh first" + exit 1 + fi + if [ -z "$OUTPUT_FILE" ]; then + OUTPUT_FILE="./session-comparison.html" + fi +fi + +log_info "=========================================" +log_info "Generating Session HTML Report" +log_info "=========================================" +log_info "Report Type: $REPORT_TYPE" +log_info "Output: $OUTPUT_FILE" +log_info "" + +# For now, use the existing comparison tools to generate data, then create HTML +if [ "$REPORT_TYPE" = "single" ]; then + log_info "Generating single session report for: $SESSION_A" + + # Use analyze-session.sh to get variability data + TEMP_ANALYSIS=$(mktemp) + "${SCRIPT_DIR}/analyze-session.sh" --session-dir "$SESSION_A" --format json --output "$TEMP_ANALYSIS" + + # Generate HTML + export SESSION_A OUTPUT_FILE TEMP_ANALYSIS + python3 - "$SESSION_A" "$OUTPUT_FILE" "$TEMP_ANALYSIS" << 'PYTHON_SCRIPT' +import json +import sys +import os + +session_dir = sys.argv[1] +output_file = sys.argv[2] +temp_analysis = sys.argv[3] + +# Load data +with open(os.path.join(session_dir, 'aggregated.json'), 'r') as f: + agg_data = json.load(f) + +with open(temp_analysis, 'r') as f: + analysis_data = json.load(f) + +session_id = agg_data.get('session_id', 'unknown') +runtime = agg_data.get('runtime', 'unknown') +iterations = agg_data.get('total_iterations', 0) + +html = f""" + + + + + Session Report: {session_id} + + + +
+

Session Performance Report

+ +
+

Session ID: {session_id}

+

Runtime: {runtime}

+

Total Iterations: {iterations}

+

Test Type: Multi-Phase Variable Load

+
+ +

Phase Performance Summary

+""" + +for phase in agg_data.get('phases', []): + phase_name = phase['phase_name'] + threads = phase['threads'] + connections = phase['connections'] + + # Find corresponding analysis + phase_analysis = next((p for p in analysis_data['phase_analyses'] if p['phase_name'] == phase_name), None) + + stability_class = "stability-excellent" + if phase_analysis: + stability = phase_analysis.get('overall_stability_score', 0) + if stability < 70: + stability_class = "stability-poor" + elif stability < 80: + stability_class = "stability-moderate" + elif stability < 90: + stability_class = "stability-good" + + html += f""" +
+

Phase: {phase_name}

+

Configuration: {threads} threads, {connections} connections

+ {f'

Stability Score: {phase_analysis["overall_stability_score"]:.1f}/100

' if phase_analysis else ''} + +
+
+

Throughput (req/s)

+
{phase['requests_per_sec']['mean']:.2f}
+
± {phase['requests_per_sec']['stddev']:.2f} (CV: {phase_analysis['requests_per_sec']['cv']:.1f}%)
+
Range: {phase['requests_per_sec']['min']:.2f} - {phase['requests_per_sec']['max']:.2f}
+
+ +
+

Mean Latency (ms)

+
{phase['latency_mean_ms']['mean']:.2f}
+
± {phase['latency_mean_ms']['stddev']:.2f} (CV: {phase_analysis['latency_mean_ms']['cv']:.1f}%)
+
Range: {phase['latency_mean_ms']['min']:.2f} - {phase['latency_mean_ms']['max']:.2f}
+
+ +
+

P99 Latency (ms)

+
{phase['latency_p99_ms']['mean']:.2f}
+
± {phase['latency_p99_ms']['stddev']:.2f} (CV: {phase_analysis['latency_p99_ms']['cv']:.1f}%)
+
Range: {phase['latency_p99_ms']['min']:.2f} - {phase['latency_p99_ms']['max']:.2f}
+
+ +
+

Errors

+
{phase['errors']['mean']:.2f}
+
± {phase['errors']['stddev']:.2f}
+
Range: {phase['errors']['min']:.2f} - {phase['errors']['max']:.2f}
+
+
+
+""" + +html += """ +
+ + +""" + +with open(output_file, 'w') as f: + f.write(html) + +print(f"HTML report generated: {output_file}") + +PYTHON_SCRIPT + + rm -f "$TEMP_ANALYSIS" + +else + log_info "Generating comparison report: $SESSION_A vs $SESSION_B" + + # Use compare-sessions.sh to get comparison data + TEMP_COMPARISON=$(mktemp) + "${SCRIPT_DIR}/compare-sessions.sh" --session-a "$SESSION_A" --session-b "$SESSION_B" --format json --output "$TEMP_COMPARISON" + + # Get variability analysis for both sessions + TEMP_ANALYSIS_A=$(mktemp) + TEMP_ANALYSIS_B=$(mktemp) + "${SCRIPT_DIR}/analyze-session.sh" --session-dir "$SESSION_A" --format json --output "$TEMP_ANALYSIS_A" + "${SCRIPT_DIR}/analyze-session.sh" --session-dir "$SESSION_B" --format json --output "$TEMP_ANALYSIS_B" + + # Generate HTML comparison report with variability data + export SESSION_A SESSION_B OUTPUT_FILE TEMP_COMPARISON TEMP_ANALYSIS_A TEMP_ANALYSIS_B + python3 - "$SESSION_A" "$SESSION_B" "$OUTPUT_FILE" "$TEMP_COMPARISON" "$TEMP_ANALYSIS_A" "$TEMP_ANALYSIS_B" << 'PYTHON_SCRIPT' +import json +import sys + +session_a_dir = sys.argv[1] +session_b_dir = sys.argv[2] +output_file = sys.argv[3] +temp_comparison = sys.argv[4] +temp_analysis_a = sys.argv[5] +temp_analysis_b = sys.argv[6] + +# Load comparison data +with open(temp_comparison, 'r') as f: + comp_data = json.load(f) + +# Load variability analysis data +with open(temp_analysis_a, 'r') as f: + analysis_a = json.load(f) +with open(temp_analysis_b, 'r') as f: + analysis_b = json.load(f) + +# Load aggregated data for stddev and ranges +import os +with open(os.path.join(session_a_dir, 'aggregated.json'), 'r') as f: + agg_a = json.load(f) +with open(os.path.join(session_b_dir, 'aggregated.json'), 'r') as f: + agg_b = json.load(f) + +# Extract session IDs from the comparison data keys +session_ids = [k for k in comp_data.keys() if k not in ['runtime', 'phase_comparisons']] +session_a_id = session_ids[0] if len(session_ids) > 0 else 'Session A' +session_b_id = session_ids[1] if len(session_ids) > 1 else 'Session B' + +runtime = comp_data.get('runtime', 'unknown') +session_a_info = comp_data.get(session_a_id, {}) +session_b_info = comp_data.get(session_b_id, {}) +phase_comparisons = comp_data.get('phase_comparisons', []) + +def format_improvement(value): + if value is None: + return 'N/A' + if value > 0: + return f'+{value:.1f}%' + elif value < 0: + return f'{value:.1f}%' + else: + return '0.0%' + +html = f""" + + + + + Performance Comparison: {session_a_id} vs {session_b_id} + + + +
+

Performance Comparison Report

+

{session_a_id} vs {session_b_id}

+ +
+
+

{session_a_id}

+

Runtime: {runtime}

+

Iterations: {session_a_info.get('iterations', 'N/A')}

+
+
+

{session_b_id}

+

Runtime: {runtime}

+

Iterations: {session_b_info.get('iterations', 'N/A')}

+
+
+ +

Phase-by-Phase Comparison

+""" + +for phase in phase_comparisons: + phase_name = phase['phase_name'] + threads = phase['threads'] + connections = phase['connections'] + + session_a_data = phase.get(session_a_id, {}) + session_b_data = phase.get(session_b_id, {}) + improvements = phase.get('improvement', {}) + + # Find variability data and aggregated data for this phase + phase_analysis_a = next((p for p in analysis_a.get('phase_analyses', []) if p['phase_name'] == phase_name), {}) + phase_analysis_b = next((p for p in analysis_b.get('phase_analyses', []) if p['phase_name'] == phase_name), {}) + phase_agg_a = next((p for p in agg_a.get('phases', []) if p['phase_name'] == phase_name), {}) + phase_agg_b = next((p for p in agg_b.get('phases', []) if p['phase_name'] == phase_name), {}) + + html += f""" +
+
+

Phase: {phase_name}

+ {threads} threads, {connections} connections +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Metric{session_a_id}{session_b_id}Improvement
Throughput (req/s) + {session_a_data.get('requests_per_sec', 0):.2f}
+ ± {phase_agg_a.get('requests_per_sec', {}).get('stddev', 0):.2f} (CV: {phase_analysis_a.get('requests_per_sec', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_a.get('requests_per_sec', {}).get('min', 0):.2f} - {phase_agg_a.get('requests_per_sec', {}).get('max', 0):.2f} + 		+ {session_b_data.get('requests_per_sec', 0):.2f}
+ ± {phase_agg_b.get('requests_per_sec', {}).get('stddev', 0):.2f} (CV: {phase_analysis_b.get('requests_per_sec', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_b.get('requests_per_sec', {}).get('min', 0):.2f} - {phase_agg_b.get('requests_per_sec', {}).get('max', 0):.2f} + 		{format_improvement(improvements.get('requests_per_sec'))}
Mean Latency (ms) + {session_a_data.get('latency_mean_ms', 0):.2f}
+ ± {phase_agg_a.get('latency_mean_ms', {}).get('stddev', 0):.2f} (CV: {phase_analysis_a.get('latency_mean_ms', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_a.get('latency_mean_ms', {}).get('min', 0):.2f} - {phase_agg_a.get('latency_mean_ms', {}).get('max', 0):.2f} + 		+ {session_b_data.get('latency_mean_ms', 0):.2f}
+ ± {phase_agg_b.get('latency_mean_ms', {}).get('stddev', 0):.2f} (CV: {phase_analysis_b.get('latency_mean_ms', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_b.get('latency_mean_ms', {}).get('min', 0):.2f} - {phase_agg_b.get('latency_mean_ms', {}).get('max', 0):.2f} + 		{format_improvement(improvements.get('latency_mean_ms'))}
P99 Latency (ms) + {session_a_data.get('latency_p99_ms', 0):.2f}
+ ± {phase_agg_a.get('latency_p99_ms', {}).get('stddev', 0):.2f} (CV: {phase_analysis_a.get('latency_p99_ms', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_a.get('latency_p99_ms', {}).get('min', 0):.2f} - {phase_agg_a.get('latency_p99_ms', {}).get('max', 0):.2f} + 		+ {session_b_data.get('latency_p99_ms', 0):.2f}
+ ± {phase_agg_b.get('latency_p99_ms', {}).get('stddev', 0):.2f} (CV: {phase_analysis_b.get('latency_p99_ms', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_b.get('latency_p99_ms', {}).get('min', 0):.2f} - {phase_agg_b.get('latency_p99_ms', {}).get('max', 0):.2f} + 		{format_improvement(improvements.get('latency_p99_ms'))}
Errors + {session_a_data.get('errors', 0):.2f}
+ ± {phase_agg_a.get('errors', {}).get('stddev', 0):.2f} (CV: {phase_analysis_a.get('errors', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_a.get('errors', {}).get('min', 0):.2f} - {phase_agg_a.get('errors', {}).get('max', 0):.2f} + 		+ {session_b_data.get('errors', 0):.2f}
+ ± {phase_agg_b.get('errors', {}).get('stddev', 0):.2f} (CV: {phase_analysis_b.get('errors', {}).get('cv', 0):.1f}%)
+ Range: {phase_agg_b.get('errors', {}).get('min', 0):.2f} - {phase_agg_b.get('errors', {}).get('max', 0):.2f} + 		{format_improvement(improvements.get('errors'))}
+
+""" + +html += """ +
+

Note

+

Positive percentages indicate improvement. For throughput, higher is better. For latency and errors, lower is better.

+
+
+ + +""" + +with open(output_file, 'w') as f: + f.write(html) + +print(f"HTML comparison report generated: {output_file}") + +PYTHON_SCRIPT + + rm -f "$TEMP_COMPARISON" "$TEMP_ANALYSIS_A" "$TEMP_ANALYSIS_B" +fi + +log_info "" +log_info "Report generated successfully: $OUTPUT_FILE" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/results-tools/regenerate-metrics.sh b/spring-quarkus-perf-comparison/scripts/results-tools/regenerate-metrics.sh new file mode 100755 index 00000000..fea658e6 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/results-tools/regenerate-metrics.sh @@ -0,0 +1,333 @@ +#!/bin/bash + +# ============================================================================ +# Regenerate Metrics JSON from Individual Results +# ============================================================================ +# This script regenerates the comprehensive metrics.json file from individual +# result JSON files in a results directory. + +set -e + +RESULTS_DIR=$1 + +if [ -z "$RESULTS_DIR" ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 ./results" + echo "" + echo "This will scan the results directory for individual result files" + echo "(e.g., quarkus3-jvm-ootb-1.json) and generate a comprehensive" + echo "metrics.json file in the same directory." + exit 1 +fi + +if [ ! -d "$RESULTS_DIR" ]; then + echo "ERROR: Results directory not found: $RESULTS_DIR" + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +log_info() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [INFO] $@" +} + +log_error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [ERROR] $@" +} + +# Detect scenario from filenames +detect_scenario() { + local first_file=$(ls "$RESULTS_DIR"/*.json 2>/dev/null | head -1) + if [ -z "$first_file" ]; then + echo "ootb" + return + fi + + local basename=$(basename "$first_file") + # Extract scenario from filename: runtime-scenario-iteration.json + local scenario=$(echo "$basename" | sed 's/^[^-]*-//' | sed 's/-[^-]*\.json$//') + echo "$scenario" +} + +# Detect runtimes from filenames +detect_runtimes() { + local runtimes=() + + for file in "$RESULTS_DIR"/*.json; do + if [ -f "$file" ]; then + local basename=$(basename "$file") + # Extract runtime from filename: runtime-scenario-iteration.json + local runtime=$(echo "$basename" | sed 's/-[^-]*-[^-]*\.json$//') + + if [[ ! " ${runtimes[@]} " =~ " ${runtime} " ]]; then + runtimes+=("$runtime") + fi + fi + done + + echo "${runtimes[@]}" +} + +# Count iterations for a runtime +count_iterations() { + local runtime=$1 + local scenario=$2 + local count=0 + + for file in "$RESULTS_DIR"/${runtime}-${scenario}-*.json; do + if [ -f "$file" ]; then + ((count++)) + fi + done + + echo $count +} + +# Generate comprehensive metrics JSON +generate_metrics_json() { + log_info "Generating comprehensive metrics JSON..." + + local metrics_file="${RESULTS_DIR}/metrics.json" + local start_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local stop_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + # Detect scenario and runtimes + SCENARIO=$(detect_scenario) + RUNTIMES=$(detect_runtimes) + + if [ -z "$RUNTIMES" ]; then + log_error "No result files found in $RESULTS_DIR" + exit 1 + fi + + log_info "Detected scenario: $SCENARIO" + log_info "Detected runtimes: $RUNTIMES" + + # Start building JSON structure + cat > "$metrics_file" << 'EOF_START' +{ + "timing": { + "start": "START_TIME_PLACEHOLDER", + "stop": "STOP_TIME_PLACEHOLDER" + }, + "results": { +EOF_START + + # Replace placeholders + sed -i "s/START_TIME_PLACEHOLDER/${start_time}/" "$metrics_file" + sed -i "s/STOP_TIME_PLACEHOLDER/${stop_time}/" "$metrics_file" + + # Parse runtimes + local runtime_array=($RUNTIMES) + local first_runtime=true + local total_iterations=0 + + # Add results for each runtime + for runtime in "${runtime_array[@]}"; do + if [ "$first_runtime" = false ]; then + echo "," >> "$metrics_file" + fi + first_runtime=false + + # Count iterations for this runtime + local iterations=$(count_iterations "$runtime" "$SCENARIO") + if [ $iterations -gt $total_iterations ]; then + total_iterations=$iterations + fi + + log_info "Processing runtime: $runtime (${iterations} iterations)" + + # Aggregate results across iterations + local total_throughput=0 + local total_memory=0 + local total_startup=0 + local total_errors=0 + local count=0 + + # Parse all iteration results for this runtime + for file in "$RESULTS_DIR"/${runtime}-${SCENARIO}-*.json; do + if [ -f "$file" ]; then + # Extract metrics using jq + local throughput=$(jq -r '.tests.load_test.requests_per_sec // .tests."run-load-test".throughput // 0' "$file" 2>/dev/null || echo "0") + local memory=$(jq -r '.tests.memory.rss_mb // .tests."measure-memory".rss_mib // 0' "$file" 2>/dev/null || echo "0") + local startup=$(jq -r '.tests.startup.startup_time_ms // .tests."measure-startup".startup_ms // 0' "$file" 2>/dev/null || echo "0") + local errors=$(jq -r '.tests.load_test.errors // .tests."run-load-test".errors // 0' "$file" 2>/dev/null || echo "0") + + total_throughput=$(awk "BEGIN {print $total_throughput + $throughput}") + total_memory=$(awk "BEGIN {print $total_memory + $memory}") + total_startup=$(awk "BEGIN {print $total_startup + $startup}") + total_errors=$(awk "BEGIN {print $total_errors + $errors}") + ((count++)) + fi + done + + # Calculate averages + local av_throughput=0 + local av_memory=0 + local av_startup=0 + local av_errors=0 + local throughput_density=0 + + if [ $count -gt 0 ]; then + av_throughput=$(awk "BEGIN {printf \"%.2f\", $total_throughput / $count}") + av_memory=$(awk "BEGIN {printf \"%.2f\", $total_memory / $count}") + av_startup=$(awk "BEGIN {printf \"%.2f\", $total_startup / $count}") + av_errors=$(awk "BEGIN {printf \"%.0f\", $total_errors / $count}") + + # Calculate throughput density (throughput per MiB) + if [ "$(awk "BEGIN {print ($av_memory > 0)}")" -eq 1 ]; then + throughput_density=$(awk "BEGIN {printf \"%.6f\", $av_throughput / $av_memory}") + fi + fi + + log_info " Throughput: ${av_throughput} req/s" + log_info " Memory: ${av_memory} MB" + log_info " Startup: ${av_startup} ms" + + # Write runtime results + cat >> "$metrics_file" << EOF + "${runtime}": { + "load": { + "throughput": [${av_throughput}], + "connectionErrors": [${av_errors}], + "requestTimeouts": [0], + "appErrors": [0], + "app4xxErrors": [0], + "app5xxErrors": [0], + "rss": [${av_memory}], + "throughputDensity": [${throughput_density}], + "avThroughput": ${av_throughput}, + "avMaxRss": ${av_memory}, + "maxThroughputDensity": ${throughput_density}, + "avConnectionErrors": ${av_errors}, + "avRequestTimeouts": 0, + "avAppErrors": 0, + "avApp4xxErrors": 0, + "avApp5xxErrors": 0 + }, + "startup": { + "timings": [${av_startup}], + "avStartupTime": ${av_startup} + } + } +EOF + done + + # Add configuration section + cat >> "$metrics_file" << EOF + }, + "config": { + "units": { + "timings": { + "startup": "ms" + }, + "rss": { + "startup": "MiB", + "firstRequest": "MiB", + "load": "MiB" + }, + "load": { + "throughput": "req/s", + "throughputDensity": "req/s per MiB", + "errors": { + "connectionErrors": "absolute number", + "requestTimeouts": "absolute number" + } + } + }, + "jvm": { + "home": "", + "version": "unknown", + "graalvm": { + "home": "", + "version": "unknown" + }, + "memory": "-Xmx512m -Xms512m", + "args": "-XX:+UseParallelGC -XX:+UseNUMA" + }, + "quarkus": { + "version": "unknown", + "build_config_args": "", + "native_build_options": "" + }, + "springboot3": { + "version": "unknown", + "native_build_options": "" + }, + "springboot4": { + "version": "unknown", + "native_build_options": "" + }, + "run": { + "dropOsFilesystemCaches": "false", + "useContainerHostNetwork": "false", + "description": "Spring and Quarkus Performance Comparison - Regenerated", + "identifier": "regenerated-$(date +%Y%m%d-%H%M%S)" + }, + "resources": { + "cpu": { + "app": "1", + "db": "1" + } + }, + "profiler": { + "name": "none", + "events": "cpu" + }, + "repo": { + "branch": "main", + "url": "https://github.com/quarkusio/spring-quarkus-perf-comparison.git", + "scenario": "${SCENARIO}", + "scenarioName": "$(echo ${SCENARIO} | sed 's/.*/\u&/')" + }, + "num_iterations": ${total_iterations} + }, + "env": { + "host": { + "os": "$(cat /etc/os-release 2>/dev/null | grep PRETTY_NAME | cut -d'"' -f2 || echo 'Unknown')", + "type": "$(dmidecode -s system-product-name 2>/dev/null || echo 'Unknown')", + "kernel": "$(uname -r)", + "memory": "$(free -h 2>/dev/null | grep Mem | awk '{print \$2}' || echo 'Unknown')" + }, + "run": { + "host": { + "user": "$(whoami)", + "name": "$(hostname)", + "target": "$(whoami)@$(hostname)" + } + } + } +} +EOF + + log_info "Comprehensive metrics JSON generated: ${metrics_file}" + + # Validate JSON + if command -v jq &> /dev/null; then + if jq empty "$metrics_file" 2>/dev/null; then + log_info "✓ Generated JSON is valid" + else + log_error "✗ Generated JSON is invalid" + return 1 + fi + fi + + echo "" + log_info "Summary:" + log_info " Scenario: ${SCENARIO}" + log_info " Runtimes: ${RUNTIMES}" + log_info " Iterations: ${total_iterations}" + log_info " Output: ${metrics_file}" +} + +# Main execution +log_info "Regenerating metrics.json from: $RESULTS_DIR" +generate_metrics_json + +echo "" +log_info "Done! You can now use the generated metrics.json file." +log_info "To view a comparison, run:" +echo " ${SCRIPT_DIR}/compare-all.sh $(dirname $RESULTS_DIR)" + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/run-load-test.sh b/spring-quarkus-perf-comparison/scripts/run-load-test.sh new file mode 100755 index 00000000..0229b2b1 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/run-load-test.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# ============================================================================ +# Run Load Test - Using JBang + Hyperfoil (Same as Original Repository) +# ============================================================================ +# This script runs load tests using JBang + Hyperfoil CLI from the client +# machine, exactly matching the original repository's approach. +# The only difference: targets OpenShift route URL instead of localhost + +set -e + +# Capture parameters BEFORE sourcing config.env to avoid overwriting +RUNTIME_PARAM=$1 +SCENARIO_PARAM=$2 +RESULT_FILE=$3 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common utilities +source "${SCRIPT_DIR}/common-utils.sh" + +source "${SCRIPT_DIR}/../config.env" + +# Use the parameters passed to the script, not the defaults from config.env +RUNTIME="$RUNTIME_PARAM" +SCENARIO="$SCENARIO_PARAM" + +run_load_test() { + local app_name="${RUNTIME}-${SCENARIO}" + + log_info "Running load test for ${app_name} using JBang + Hyperfoil..." + + # Check JBang installation + if ! check_jbang; then + log_error "Cannot run load test without JBang" + return 1 + fi + + # Get route URL (external access to OpenShift app) + local route_host=$(oc get route ${app_name} -o jsonpath='{.spec.host}') + if [ -z "$route_host" ]; then + log_error "Route not found for ${app_name}" + return 1 + fi + + local url="http://${route_host}" + log_info "Target URL: ${url}" + + # Create temporary directory for Hyperfoil results + local tempdir=$(mktemp -d) + log_info "Hyperfoil results directory: ${tempdir}" + + # Run Hyperfoil load test using JBang + # This is EXACTLY the same command as in the original stress.sh + # Only difference: -PHOST points to OpenShift route instead of localhost + log_info "Starting Hyperfoil load test..." + log_info " Warmup: ${LOAD_TEST_WARMUP}" + log_info " Cooldown: 30s" + log_info " Load test duration: ${LOAD_TEST_DURATION}" + log_info " Connections: ${LOAD_TEST_CONNECTIONS}" + log_info " Threads: 2" + + # EXACT command from original repository's stress.sh + # The benchmark file is passed as the last argument to 'run@hyperfoil' + jbang \ + -Dio.hyperfoil.rootdir=${tempdir} \ + -Dio.hyperfoil.cpu.watchdog.idle.threshold=0.0 \ + run@hyperfoil \ + -PPROTOCOL=http \ + -PHOST=${route_host} \ + -PPORT=80 \ + -PLOAD_DURATION=${LOAD_TEST_DURATION} \ + -PWARMUP_DURATION=${LOAD_TEST_WARMUP} \ + -PWARMUP_PAUSE_DURATION=30s \ + -PCONNECTIONS=${LOAD_TEST_CONNECTIONS} \ + -PTHREADS=2 \ + ${SCRIPT_DIR}/../manifests/fixed-load-hf.yml \ + &> ${tempdir}/hf.log + + log_info "Load test completed" + + # Display results + echo "-------------------------------------------------" + cat ${tempdir}/hf.log + echo "-------------------------------------------------" + + # Parse and save results + parse_and_update_results "${tempdir}/hf.log" + + log_info "Hyperfoil output saved in: ${tempdir}" + log_info "You can review detailed results in ${tempdir}/hf.log" + + return 0 +} + +# Parse results and update JSON file +# Note: parse_hyperfoil_results() is now in common-utils.sh +parse_and_update_results() { + local log_file=$1 + + # Parse using common function + local parsed_json=$(parse_hyperfoil_results "$log_file" "loadTest" 2 ${LOAD_TEST_CONNECTIONS} "${LOAD_TEST_DURATION}") + + # Extract values from parsed JSON + local throughput=$(echo "$parsed_json" | jq -r '.results.requests_per_sec') + local total_requests=$(echo "$parsed_json" | jq -r '.results.requests_total') + local latency_mean=$(echo "$parsed_json" | jq -r '.results.latency_mean_ms') + local latency_p50=$(echo "$parsed_json" | jq -r '.results.latency_p50_ms') + local latency_p90=$(echo "$parsed_json" | jq -r '.results.latency_p90_ms') + local latency_p99=$(echo "$parsed_json" | jq -r '.results.latency_p99_ms') + local errors=$(echo "$parsed_json" | jq -r '.results.errors') + local success_2xx=$(echo "$parsed_json" | jq -r '.results.success_2xx') + local duration_sec=$(echo "$parsed_json" | jq -r '.configuration.duration_seconds') + + log_info "Parsed Results:" + log_info " Total Requests: ${total_requests}" + log_info " Throughput: ${throughput} req/s" + log_info " Latency (mean): ${latency_mean}ms" + log_info " Latency (p50): ${latency_p50}ms" + log_info " Latency (p90): ${latency_p90}ms" + log_info " Latency (p99): ${latency_p99}ms" + log_info " Errors: ${errors}" + log_info " Success (2xx): ${success_2xx}" + + # Update result file with load test results + local temp_file=$(mktemp) + jq ".tests.load_test = { + \"tool\": \"hyperfoil-jbang\", + \"approach\": \"Same as original repository - JBang + Hyperfoil CLI\", + \"duration_s\": ${duration_sec}, + \"warmup_duration\": \"${LOAD_TEST_WARMUP}\", + \"cooldown_duration\": \"30s\", + \"connections\": ${LOAD_TEST_CONNECTIONS}, + \"threads\": 2, + \"requests_total\": ${total_requests}, + \"requests_per_sec\": ${throughput}, + \"latency_mean_ms\": ${latency_mean}, + \"latency_p50_ms\": ${latency_p50}, + \"latency_p90_ms\": ${latency_p90}, + \"latency_p99_ms\": ${latency_p99}, + \"errors\": ${errors}, + \"success_2xx\": ${success_2xx} + }" "$RESULT_FILE" > "$temp_file" && mv "$temp_file" "$RESULT_FILE" +} + +# Main execution +run_load_test + +# Made with Bob diff --git a/spring-quarkus-perf-comparison/scripts/run-variable-load-multi-phase.sh b/spring-quarkus-perf-comparison/scripts/run-variable-load-multi-phase.sh new file mode 100755 index 00000000..8072aae9 --- /dev/null +++ b/spring-quarkus-perf-comparison/scripts/run-variable-load-multi-phase.sh @@ -0,0 +1,733 @@ +#!/bin/bash + +# ============================================================================ +# Run Multi-Phase Variable Load Test with Varying Threads +# ============================================================================ +# This script simulates real-world load by running multiple sequential +# Hyperfoil tests with different thread and connection configurations. +# +# Unlike a single Hyperfoil run (where threads are global), this approach +# runs separate tests for each phase, allowing threads to scale with load. +# +# Real-world simulation: +# - Low traffic periods: Fewer threads (2-4) +# - Moderate traffic: Medium threads (4-8) +# - Peak traffic: High threads (8-16) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source common utilities +source "${SCRIPT_DIR}/common-utils.sh" + +# Default values +RUNTIME="" +SCENARIO="" +NAMESPACE="benchmark" +URL="" +OUTPUT_DIR="./variable-load-results" +SESSION_ID="" # Session identifier for grouping related test runs +ITERATION="" # Iteration number (auto-detected if not specified) +DURATION_MODE="4h" # 1h, 4h, 24h, or custom +PHASE_DURATION="" # Custom duration for each phase (e.g., "3m" for testing) +MAX_THREADS=6 # Maximum threads for peak load (used if specific values not set) + +# Phase-specific defaults (will be calculated from MAX_THREADS if not set) +LOW_THREADS="" +LOW_CONNECTIONS="" +MED_THREADS="" +MED_CONNECTIONS="" +PEAK_THREADS="" +PEAK_CONNECTIONS="" + +# Connection scaling factor (connections per thread) +CONNECTIONS_PER_THREAD=25 # Default: 25 connections per thread + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Usage function +usage() { + cat << EOF +Usage: $0 --runtime [options] + +This script runs multiple sequential Hyperfoil tests with varying thread counts +to simulate real-world load patterns where thread pools scale with traffic. + +Required: + --runtime Runtime name (e.g., quarkus3-jvm, spring3-virtual) + +Optional: + --scenario Scenario name (ootb or tuned) - auto-discovers URL + --url Application URL (alternative to --scenario) + --namespace OpenShift namespace (default: benchmark) + --session-id Session identifier for grouping related test runs + Multiple runs with same session-id are treated as iterations + Output: session-{id}/{runtime}-{scenario}-iter{N}.json + --iteration Explicit iteration number (auto-detected if not specified) + --duration Duration mode: 1h, 4h, 24h, or custom (default: 4h) + --phase-duration