test: performance workflow improvements DHIS2-20175

.github/workflows/run-performance-tests.yml

@@ -1,11 +1,24 @@
 # Performance test workflow to compare a baseline against a candidate DHIS2 version
-# You can run the workflow using the GitHub CLI like so
+# You can run the workflow using the GitHub CLI like so:
+#
+# Basic example (required args only):
 # gh workflow run run-performance-tests.yml \
+# 	--field perf_tests_git_ref="master" \
+# 	--field simulation_class="org.hisp.dhis.test.tracker.TrackerTest" \
+# 	--field dhis2_image_candidate="dhis2/core-dev:latest"
+#
+# With custom baseline:
+# gh workflow run run-performance-tests.yml \
+# 	--field perf_tests_git_ref="master" \
 # 	--field simulation_class="org.hisp.dhis.test.tracker.TrackerTest" \
 # 	--field dhis2_image_baseline="dhis2/core:2.42.1" \
 # 	--field dhis2_image_candidate="dhis2/core-dev:latest"
 #
-# To use a specific database version, provide both dump URL and matching image suffix:
+# With specific database version:
+# gh workflow run run-performance-tests.yml \
+# 	--field perf_tests_git_ref="2.42.2" \
+# 	--field simulation_class="org.hisp.dhis.test.tracker.TrackerTest" \
+# 	--field dhis2_image_candidate="dhis2/core:2.42.2" \
 # 	--field dhis2_db_dump_url="https://databases.dhis2.org/sierra-leone/2.42.2/dhis2-db-sierra-leone.sql.gz" \
 # 	--field dhis2_db_image_suffix="sierra-leone-2.42.2"
 name: Performance tests
@@ -15,6 +28,10 @@ run-name: Performance test comparing ${{ inputs.dhis2_image_baseline }} to ${{ i
 on:
   workflow_dispatch:
     inputs:
+      perf_tests_git_ref:
+        description: 'Git ref (tag/branch/commit) to checkout dhis2-core performance tests'
+        required: true
+        type: string
       simulation_class:
         description: 'Fully qualified Gatling simulation class to run (e.g., org.hisp.dhis.test.TrackerTest)'
         required: true
@@ -24,6 +41,11 @@ on:
         required: false
         default: ''
         type: string
+      prof_args:
+        description: 'Profiler arguments (enables profiling). Options: https://github.com/async-profiler/async-profiler/blob/master/docs/ProfilerOptions.md'
+        required: false
+        default: ''
+        type: string
       # Read https://github.com/dhis2/dhis2-core/blob/master/docker/DOCKERHUB.md on how we publish
       # DHIS2 Docker images
       dhis2_image_baseline:
@@ -57,6 +79,7 @@ jobs:
     - name: Checkout performance tests
       uses: actions/checkout@v5
       with:
+        ref: ${{ inputs.perf_tests_git_ref }}
         sparse-checkout: |
           dhis-2/dhis-test-performance
 
@@ -78,6 +101,7 @@ jobs:
         DHIS2_IMAGE="${{ inputs.dhis2_image_baseline }}" \
         SIMULATION_CLASS="${{ inputs.simulation_class }}" \
         MVN_ARGS="${{ inputs.mvn_args }} -Dgatling.failOnError=false" \
+        PROF_ARGS="${{ inputs.prof_args }}" \
         DHIS2_DB_DUMP_URL="${{ inputs.dhis2_db_dump_url }}" \
         DHIS2_DB_IMAGE_SUFFIX="${{ inputs.dhis2_db_image_suffix }}" \
         ./run-simulation.sh
@@ -87,6 +111,7 @@ jobs:
         DHIS2_IMAGE="${{ inputs.dhis2_image_candidate }}" \
         SIMULATION_CLASS="${{ inputs.simulation_class }}" \
         MVN_ARGS="${{ inputs.mvn_args }}" \
+        PROF_ARGS="${{ inputs.prof_args }}" \
         DHIS2_DB_DUMP_URL="${{ inputs.dhis2_db_dump_url }}" \
         DHIS2_DB_IMAGE_SUFFIX="${{ inputs.dhis2_db_image_suffix }}" \
         ./run-simulation.sh
@@ -115,10 +140,70 @@ jobs:
       run: |
         echo "## 🚀 Performance Test Results" >> $GITHUB_STEP_SUMMARY
         echo "" >> $GITHUB_STEP_SUMMARY
+
+        # Build the command to reproduce this run
+        echo "### 🔄 Reproduce this run" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '```sh' >> $GITHUB_STEP_SUMMARY
+        echo -n 'gh workflow run run-performance-tests.yml' >> $GITHUB_STEP_SUMMARY
+        echo ' \' >> $GITHUB_STEP_SUMMARY
+        echo "  --field perf_tests_git_ref=\"${{ inputs.perf_tests_git_ref }}\" \\" >> $GITHUB_STEP_SUMMARY
+        echo "  --field simulation_class=\"${{ inputs.simulation_class }}\" \\" >> $GITHUB_STEP_SUMMARY
+
+        # Add optional fields only if they're not empty or not default
+        if [ -n "${{ inputs.mvn_args }}" ]; then
+          echo "  --field mvn_args=\"${{ inputs.mvn_args }}\" \\" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        if [ -n "${{ inputs.prof_args }}" ]; then
+          echo "  --field prof_args=\"${{ inputs.prof_args }}\" \\" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        if [ "${{ inputs.dhis2_image_baseline }}" != "dhis2/core-dev:latest" ]; then
+          echo "  --field dhis2_image_baseline=\"${{ inputs.dhis2_image_baseline }}\" \\" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        echo "  --field dhis2_image_candidate=\"${{ inputs.dhis2_image_candidate }}\" \\" >> $GITHUB_STEP_SUMMARY
+
+        if [ -n "${{ inputs.dhis2_db_dump_url }}" ]; then
+          echo "  --field dhis2_db_dump_url=\"${{ inputs.dhis2_db_dump_url }}\" \\" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        if [ "${{ inputs.dhis2_db_image_suffix }}" != "sierra-leone-dev" ]; then
+          echo "  --field dhis2_db_image_suffix=\"${{ inputs.dhis2_db_image_suffix }}\" \\" >> $GITHUB_STEP_SUMMARY
+        fi
+
+        echo "  --ref ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+        echo "### 📊 Test Configuration" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
         echo "Performance tests completed comparing:" >> $GITHUB_STEP_SUMMARY
+        echo "* **Simulation**" >> $GITHUB_STEP_SUMMARY
+        echo "  * **Class**: \`${{ inputs.simulation_class }}\`" >> $GITHUB_STEP_SUMMARY
+        echo "  * **Git ref**: \`${{ inputs.perf_tests_git_ref }}\`" >> $GITHUB_STEP_SUMMARY
         echo "* **Baseline**: \`${{ inputs.dhis2_image_baseline }}\`" >> $GITHUB_STEP_SUMMARY
+
+        # Extract and display DHIS2 labels for baseline
+        OUTPUT=$(docker inspect -f '{{json .Config.Labels}}' "${{ inputs.dhis2_image_baseline }}" 2>/dev/null | \
+          jq -r 'to_entries | map(select(.key | startswith("DHIS2_"))) | sort_by(.key) | .[] | "  * **\(.key)**: `\(.value)`"')
+        if [ -z "$OUTPUT" ]; then
+          echo "  * Failed to find DHIS2 related metadata in Docker image" >> $GITHUB_STEP_SUMMARY
+        else
+          echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY
+        fi
+
         echo "* **Candidate**: \`${{ inputs.dhis2_image_candidate }}\`" >> $GITHUB_STEP_SUMMARY
-        echo "* **Simulation Class**: \`${{ inputs.simulation_class }}\`" >> $GITHUB_STEP_SUMMARY
+
+        # Extract and display DHIS2 labels for candidate
+        OUTPUT=$(docker inspect -f '{{json .Config.Labels}}' "${{ inputs.dhis2_image_candidate }}" 2>/dev/null | \
+          jq -r 'to_entries | map(select(.key | startswith("DHIS2_"))) | sort_by(.key) | .[] | "  * **\(.key)**: `\(.value)`"')
+        if [ -z "$OUTPUT" ]; then
+          echo "  * Failed to find DHIS2 related metadata in Docker image" >> $GITHUB_STEP_SUMMARY
+        else
+          echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY
+        fi
         echo "" >> $GITHUB_STEP_SUMMARY
         echo "### 🖥️ Environment" >> $GITHUB_STEP_SUMMARY
         echo "" >> $GITHUB_STEP_SUMMARY
@@ -148,4 +233,7 @@ jobs:
         echo "  * \`simulation-run.txt\` - Test run metadata (indicates baseline/candidate)" >> $GITHUB_STEP_SUMMARY
         echo "  * \`simulation.log\` - Binary test data (response times, etc.)" >> $GITHUB_STEP_SUMMARY
         echo "  * \`simulation.csv\` - Parsed binary data in CSV format" >> $GITHUB_STEP_SUMMARY
+        echo "  * \`profile.html\` - Flamegraph visualization (if profiling enabled with prof_args)" >> $GITHUB_STEP_SUMMARY
+        echo "  * \`profile.jfr\` - JFR profiler data (if profiling enabled with prof_args)" >> $GITHUB_STEP_SUMMARY
+        echo "  * \`profile.collapsed\` - Collapsed stack traces (if profiling enabled with prof_args)" >> $GITHUB_STEP_SUMMARY
 

dhis-2/dhis-test-performance/README.md

@@ -20,8 +20,9 @@ Test results are saved to `target/gatling/<simulation-class>-<timestamp>/`:
 * `simulation.log` - Binary response times
 * `simulation.csv` - Response times (automated in CI only, [see below](#simulationcsv))
 * `simulation-run.txt` - Run metadata
-* `profile.html` - Flamegraph (when profiling enabled)
-* `profile.jfr` - JFR (Java flight recorder) profiling data (when profiling enabled)
+* `profile.html` - Flamegraph visualization (if profiling enabled with PROF_ARGS)
+* `profile.jfr` - JFR profiler data (if profiling enabled with PROF_ARGS)
+* `profile.collapsed` - Collapsed stack traces (if profiling enabled with PROF_ARGS)
 
 ### simulation.csv
 

dhis-2/dhis-test-performance/docker-compose.yml

@@ -2,6 +2,7 @@ services:
   # https://docs.dhis2.org/en/manage/reference/dhisconf.html
   web:
     image: "${DHIS2_IMAGE:-dhis2/core-dev:latest}"
+    cpus: 4
     mem_limit: 16gb
     environment:
       JAVA_OPTS:
@@ -42,13 +43,15 @@ services:
       context: ./docker
       dockerfile: Dockerfile.postgres
       args:
-        POSTGRES_BASE_IMAGE: postgis/postgis:13-3.5
+        POSTGRES_BASE_IMAGE: ghcr.io/baosystems/postgis:14-3.5
         DHIS2_DB_DUMP_URL: "${DHIS2_DB_DUMP_URL:-https://databases.dhis2.org/sierra-leone/dev/dhis2-db-sierra-leone.sql.gz}"
-    image: dhis2-postgres:13-3.5-${DHIS2_DB_IMAGE_SUFFIX:-sierra-leone-dev}
+    image: localhost/dhis2-postgres:14-3.5-${DHIS2_DB_IMAGE_SUFFIX:-sierra-leone-dev}
+    cpus: 4
     mem_limit: 16gb
     shm_size: 256mb
     volumes:
-      - ./docker/postgresql.conf:/etc/postgresql.conf
+      - ./docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
     environment:
       POSTGRES_USER: &postgres_user dhis
       POSTGRES_DB: &postgres_db dhis
@@ -64,7 +67,7 @@ services:
           'psql --quiet --host=127.0.0.1 --port=5432 --set=application_name=docker --command "/** docker healthcheck **/ SELECT ''ok''" > /dev/null',
         ]
       start_period: 120s
-      interval: 3s
+      interval: 5s
       timeout: 3s
       retries: 5
     ports:

dhis-2/dhis-test-performance/docker/Dockerfile.postgres

@@ -1,4 +1,7 @@
-ARG POSTGRES_BASE_IMAGE=postgis/postgis:13-3.5
+# syntax=docker/dockerfile:1
+# Dockerfile to build the DB dump into the Docker image essentially caching the DB dump using
+# Dockers image/layer caching. The dump will be restored on each container creation.
+ARG POSTGRES_BASE_IMAGE=ghcr.io/baosystems/postgis:14-3.5
 ARG DHIS2_DB_DUMP_URL
 
 FROM busybox AS downloader

dhis-2/dhis-test-performance/docker/dhis.conf

@@ -4,6 +4,6 @@ connection.url = jdbc:postgresql://db/dhis
 connection.username = dhis
 connection.password = dhis
 
-system.update_notifications_enabled = 0ff
+system.update_notifications_enabled = off
 
 tracker.import.preheat.cache.enabled = off

dhis-2/dhis-test-performance/docker/log4j2.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" monitorInterval="10">
+    <!-- Stream logs:
+            docker compose logs -f web
+         Stream logs of a particuar package only:
+            docker compose logs -f web | grep org.hisp.dhis.monitoring.metrics
+         Show timestamps:
+            docker compose logs -t -f web
+    -->
+    <Properties>
+        <Property name="layout">%-5level %c [%t] %msg%n</Property>
+    </Properties>
+
+    <Appenders>
+        <Console name="console" target="SYSTEM_OUT">
+            <PatternLayout pattern="${layout}" />
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <!--
+            This config logs events
+
+            for Loggers declared in packages prefixed with "org.hisp.dhis" i.e. "org.hisp.dhis.security"
+            from level INFO to more severe (WARN, ERROR, ...)
+
+            any Logger declared in a package not prefixed with "org.hisp.dhis" and
+            without a Logger config defined here will log from level WARN to more severe.
+
+            Adapt this config as you see fit.
+
+            Please check https://logging.apache.org/log4j/2.x/manual/configuration.html
+        -->
+        <Logger name="org.hisp.dhis" level="INFO" additivity="true"/>
+
+        <Root level="WARN">
+            <AppenderRef ref="console"/>
+        </Root>
+    </Loggers>
+</Configuration>

dhis-2/dhis-test-performance/run-simulation.sh

@@ -22,7 +22,6 @@ show_usage() {
   echo "                        Options: https://github.com/async-profiler/async-profiler/blob/master/docs/ProfilerOptions.md"
   echo "  MVN_ARGS              Additional Maven arguments passed to mvn gatling:test"
   echo "  HEALTHCHECK_TIMEOUT   Max wait time for DHIS2 startup in seconds (default: 300)"
-  echo "  HEALTHCHECK_INTERVAL  Check interval for DHIS2 startup in seconds (default: 10)"
   echo ""
   echo "EXAMPLES:"
   echo "  # Basic test run"
@@ -52,7 +51,6 @@ MVN_ARGS=${MVN_ARGS:-""}
 DHIS2_DB_DUMP_URL=${DHIS2_DB_DUMP_URL:-"https://databases.dhis2.org/sierra-leone/dev/dhis2-db-sierra-leone.sql.gz"}
 DHIS2_DB_IMAGE_SUFFIX=${DHIS2_DB_IMAGE_SUFFIX:-"sierra-leone-dev"}
 HEALTHCHECK_TIMEOUT=${HEALTHCHECK_TIMEOUT:-300} # default of 5min
-HEALTHCHECK_INTERVAL=${HEALTHCHECK_INTERVAL:-10} # default of 10s
 PROF_ARGS=${PROF_ARGS:=""}
 
 parse_prof_args() {
@@ -86,32 +84,37 @@ cleanup() {
 
 trap cleanup EXIT INT
 
+pull_mutable_image() {
+  # Pull images with mutable tags to ensure we get the latest version. See
+  # https://github.com/dhis2/dhis2-core/blob/master/docker/DOCKERHUB.md for tag types. Mutable tags
+  # (dhis2/core-dev:*, dhis2/core-pr:*) are overwritten multiple times a day. Immutable tags
+  # (dhis2/core:2.42.1) are never rebuilt once published. Docker caches images locally, so without
+  # an explicit pull, we may run an outdated version even when using tags like 'latest' or 'master'.
+  # This is especially important on our self-hosted runner as devs will expect their latest change
+  # to be tested.
+
+  if [[ "$DHIS2_IMAGE" =~ ^dhis2/core-(dev|pr): ]]; then
+    echo "Pulling mutable image tag: $DHIS2_IMAGE"
+    docker pull "$DHIS2_IMAGE"
+  fi
+}
+
 start_containers() {
   echo "Testing with image: $DHIS2_IMAGE"
+  echo "Waiting for containers to be ready..."
+
+  local start_time
+  start_time=$(date +%s)
 
   if [ -n "$PROF_ARGS" ]; then
     docker compose -f docker-compose.yml -f docker-compose.profile.yml down --volumes
-    docker compose -f docker-compose.yml -f docker-compose.profile.yml up --detach
+    docker compose -f docker-compose.yml -f docker-compose.profile.yml up --detach --wait --wait-timeout "$HEALTHCHECK_TIMEOUT"
   else
     docker compose down --volumes
-    docker compose up --detach
+    docker compose up --detach --wait --wait-timeout "$HEALTHCHECK_TIMEOUT"
   fi
-}
-
-wait_for_health() {
-  echo "Waiting for DHIS2 to start..."
-  local start_time
-  start_time=$(date +%s)
 
-  while ! docker compose ps web-healthcheck | grep -q "healthy"; do
-    sleep "$HEALTHCHECK_INTERVAL"
-    echo "Still waiting..."
-    if [ $(($(date +%s) - start_time)) -gt "$HEALTHCHECK_TIMEOUT" ]; then
-      echo "Timeout waiting for DHIS2 to start"
-      exit 1
-    fi
-  done
-  echo "DHIS2 is ready! (took $(($(date +%s) - start_time))s)"
+  echo "All containers ready! (took $(($(date +%s) - start_time))s)"
 }
 
 save_profiler_data() {
@@ -147,10 +150,12 @@ post_process_profiler_data() {
 
   local title="$SIMULATION_CLASS on $DHIS2_IMAGE (async-profiler $PROF_ARGS)"
   # generate flamegraph and collapsed stack traces using jfrconv from async-profiler
+  # shellcheck disable=SC2086
   docker compose exec --workdir /profiler-output web \
-    jfrconv "$jfrconv_flags" --dot --title "$title" profile.jfr profile.html
+    jfrconv $jfrconv_flags --dot --title "$title" profile.jfr profile.html
+  # shellcheck disable=SC2086
   docker compose exec --workdir /profiler-output web \
-    jfrconv "$jfrconv_flags" --dot profile.jfr profile.collapsed
+    jfrconv $jfrconv_flags --dot profile.jfr profile.collapsed
 
   docker compose cp web:/profiler-output/. "$gatling_dir/"
 
@@ -164,12 +169,14 @@ prepare_database() {
 
 start_profiler() {
   if [ -n "$PROF_ARGS" ]; then
+    # shellcheck disable=SC2086
     docker compose exec --workdir /profiler-output web asprof start $PROF_ARGS -f profile.jfr 1 > /dev/null
   fi
 }
 
 run_simulation() {
   echo "Running $SIMULATION_CLASS..."
+  # shellcheck disable=SC2086
   mvn gatling:test \
     -Dgatling.simulationClass="$SIMULATION_CLASS" \
     $MVN_ARGS
@@ -189,7 +196,7 @@ generate_metadata() {
   echo "Generating run metadata..."
   {
     echo "RUN_DIR=$gatling_run_dir"
-    echo "COMMAND=DHIS2_IMAGE=$DHIS2_IMAGE DHIS2_DB_DUMP_URL=$DHIS2_DB_DUMP_URL SIMULATION_CLASS=$SIMULATION_CLASS${MVN_ARGS:+ MVN_ARGS=$MVN_ARGS}${HEALTHCHECK_TIMEOUT:+ HEALTHCHECK_TIMEOUT=$HEALTHCHECK_TIMEOUT}${HEALTHCHECK_INTERVAL:+ HEALTHCHECK_INTERVAL=$HEALTHCHECK_INTERVAL} $0"
+    echo "COMMAND=DHIS2_IMAGE=$DHIS2_IMAGE DHIS2_DB_DUMP_URL=$DHIS2_DB_DUMP_URL SIMULATION_CLASS=$SIMULATION_CLASS${MVN_ARGS:+ MVN_ARGS=$MVN_ARGS}${HEALTHCHECK_TIMEOUT:+ HEALTHCHECK_TIMEOUT=$HEALTHCHECK_TIMEOUT} $0"
     echo "SCRIPT_NAME=$0"
     echo "SCRIPT_ARGS=$*"
     echo "DHIS2_IMAGE=$DHIS2_IMAGE"
@@ -198,15 +205,14 @@ generate_metadata() {
     echo "SIMULATION_CLASS=$SIMULATION_CLASS"
     echo "MVN_ARGS=$MVN_ARGS"
     echo "HEALTHCHECK_TIMEOUT=$HEALTHCHECK_TIMEOUT"
-    echo "HEALTHCHECK_INTERVAL=$HEALTHCHECK_INTERVAL"
     echo "GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo 'unknown')"
     echo "GIT_COMMIT=$(git rev-parse HEAD 2>/dev/null || echo 'unknown')"
     echo "GIT_DIRTY=\$([ -n \"\$(git status --porcelain 2>/dev/null)\" ] && echo 'true' || echo 'false')"
   } > "$simulation_run_file"
 }
 
+pull_mutable_image
 start_containers
-wait_for_health
 prepare_database
 start_profiler
 run_simulation