From 69851e41a9907a36344b8b3b9fe2def8e8b8e935 Mon Sep 17 00:00:00 2001 From: Gerard Date: Wed, 1 Jul 2026 08:14:05 -0400 Subject: [PATCH] fix(perf): TRex ASTF needs --lro-disable on AWS ENA (no hardware TCP_LRO) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 28514788453 (post-#103) proved the harness fix works — TRex now starts in ASTF mode, all DUTs start, tcp-echo binds on the real NIC — but exposed a deeper blocker: TRex ASTF requests hardware TCP_LRO, which AWS ENA does not support, so t-rex-64 dies at startup ('Requested RX offload TCP_LRO is not supported') and the ASTF RPC (:4501) never comes up -> clients get 'Failed to get server response'. UDP/STL never hit this (no LRO requested). - Extract trex_launch_cmd(mode) and pass --lro-disable (harmless for STL). - Readiness check: drop LOG_GROWING as a pass signal — a growing log does not mean TRex is alive; it masked the dead server for the whole run. Require PROCESS_FOUND or API_PORT (:4501). - Regression test asserts ASTF launch includes --astf + --lro-disable. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/perf-tests/test/test_run_perf_tests.sh | 7 +++++++ scripts/run-perf-tests.sh | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/perf-tests/test/test_run_perf_tests.sh b/scripts/perf-tests/test/test_run_perf_tests.sh index 4caf32b..6531563 100755 --- a/scripts/perf-tests/test/test_run_perf_tests.sh +++ b/scripts/perf-tests/test/test_run_perf_tests.sh @@ -74,6 +74,13 @@ SNIP="$(dut_kill_snippet)" if [[ "$SNIP" == *"for i in 1 2 3 4 5 6 7 8 9 10"* ]]; then ok "grace list expands to DUT_KILL_GRACE_SECS ($DUT_KILL_GRACE_SECS)"; else bad "grace list expands to $DUT_KILL_GRACE_SECS"; fi if [[ "$SNIP" == *"rm -rf /var/run/dpdk/"* ]]; then ok "snippet clears the DPDK lock dir"; else bad "snippet clears the DPDK lock dir"; fi +echo "== trex_launch_cmd: ASTF gets --lro-disable (ENA has no hardware TCP_LRO) ==" +ACMD="$(trex_launch_cmd astf)"; SCMD="$(trex_launch_cmd stl)" +if [[ "$ACMD" == *"--astf"* ]]; then ok "astf mode includes --astf"; else bad "astf mode includes --astf"; fi +if [[ "$ACMD" == *"--lro-disable"* ]]; then ok "astf launch includes --lro-disable (else TRex dies on ENA)"; else bad "astf launch includes --lro-disable"; fi +if [[ "$SCMD" == *"--lro-disable"* ]]; then ok "stl launch includes --lro-disable"; else bad "stl launch includes --lro-disable"; fi +if [[ "$SCMD" != *"--astf"* ]]; then ok "stl mode omits --astf"; else bad "stl mode omits --astf"; fi + # ── 3. TRex mode selection: stl for UDP, astf for any *-tcp ─────────────────── echo "== TRex STL/ASTF mode selection ==" for c in "rust-dpdk-tcp,tokio-dpdk-tcp,plain-rust-tcp" "rust-dpdk-tcp"; do diff --git a/scripts/run-perf-tests.sh b/scripts/run-perf-tests.sh index 7e52739..e0ac87c 100755 --- a/scripts/run-perf-tests.sh +++ b/scripts/run-perf-tests.sh @@ -686,10 +686,18 @@ YAMLEOF fi } +# Build the TRex server launch command for a mode (stl|astf). AWS ENA has no +# hardware TCP_LRO; ASTF requests it and TRex dies at startup ("Requested RX +# offload TCP_LRO is not supported"), so pass --lro-disable. STL/UDP does not +# request LRO, so --lro-disable is a harmless no-op there. Unit-tested. +trex_launch_cmd() { + local mode="${1:-stl}" astf="" + [[ "$mode" == "astf" ]] && astf="--astf " + echo "cd /opt/trex && nohup /opt/trex/t-rex-64 -i ${astf}--lro-disable --cfg /etc/trex_cfg.yaml -c 2 /var/log/trex-server.log 2>&1 & disown" +} + start_trex_server() { local trex_mode="${1:-stl}" - local astf_flag="" - [[ "$trex_mode" == "astf" ]] && astf_flag="--astf" CURRENT_TREX_MODE="$trex_mode" log_info "Starting TRex server (mode: $trex_mode)..." local TX_PCI="0000:00:06.0" @@ -726,7 +734,7 @@ start_trex_server() { log_info "Starting TRex server..." local start_cmd_id start_cmd_id=$(ssm_run_command_fire_and_forget "$TREX_INSTANCE_ID" 120 \ - "pkill -f t-rex-64 2>/dev/null || true; sleep 2; rm -rf /var/run/dpdk/ 2>/dev/null || true; cd /opt/trex && nohup /opt/trex/t-rex-64 -i $astf_flag --cfg /etc/trex_cfg.yaml -c 2 /var/log/trex-server.log 2>&1 & disown") + "pkill -f t-rex-64 2>/dev/null || true; sleep 2; rm -rf /var/run/dpdk/ 2>/dev/null || true; $(trex_launch_cmd "$trex_mode")") log_info "TRex start command sent (cmd_id: ${start_cmd_id:-none})" # Wait for TRex to initialize DPDK and start its API server. @@ -740,7 +748,7 @@ start_trex_server() { "LOG_SIZE=\$(wc -c < /var/log/trex-server.log 2>/dev/null || echo 0); echo LOG_SIZE:\$LOG_SIZE; pgrep -f t-rex >/dev/null 2>&1 && echo PROCESS_FOUND; ss -tlnp 2>/dev/null | grep 4501 && echo API_PORT; if [ \$LOG_SIZE -gt 100 ]; then echo LOG_GROWING; fi; tail -10 /var/log/trex-server.log 2>/dev/null" || echo "SSM_CHECK_FAILED") log_info "TRex check: $(echo "$check" | tr '\n' ' ' | head -c 500)" - if [[ "$check" == *"PROCESS_FOUND"* || "$check" == *"API_PORT"* || "$check" == *"LOG_GROWING"* ]]; then + if [[ "$check" == *"PROCESS_FOUND"* || "$check" == *"API_PORT"* ]]; then log_info "TRex server is running" return 0 fi @@ -752,7 +760,7 @@ start_trex_server() { "LOG_SIZE=\$(wc -c < /var/log/trex-server.log 2>/dev/null || echo 0); echo LOG_SIZE:\$LOG_SIZE; pgrep -f t-rex >/dev/null 2>&1 && echo PROCESS_FOUND; ss -tlnp 2>/dev/null | grep 4501 && echo API_PORT; if [ \$LOG_SIZE -gt 100 ]; then echo LOG_GROWING; fi; tail -10 /var/log/trex-server.log 2>/dev/null" || echo "SSM_CHECK_FAILED") log_info "TRex retry check: $(echo "$check" | tr '\n' ' ' | head -c 500)" - if [[ "$check" == *"PROCESS_FOUND"* || "$check" == *"API_PORT"* || "$check" == *"LOG_GROWING"* ]]; then + if [[ "$check" == *"PROCESS_FOUND"* || "$check" == *"API_PORT"* ]]; then log_info "TRex server is running (after retry)" return 0 fi