bench.sh

#!/bin/zsh
# Content Directories vs AssetBundles benchmark orchestrator (URP 3D Sample).
#
# Usage:
#   ./bench.sh setup            # one-time: addressables groups (by type) + scenes + builder config
#   ./bench.sh ab | cd          # switch every Bench-* group to AssetBundles / Content Directories
#   ./bench.sh content <label>  # timed Addressables content build (label e.g. AB-cold1)
#   ./bench.sh player <AB|CD>   # player build into Builds/<variant>/
#   ./bench.sh smoke  <AB|CD>   # one scene-tour run against the built player, verified
#   ./bench.sh builds <AB|CD>   # full build-time campaign (cold x3 / nochange / incr x3)
#   ./bench.sh tours  <AB|CD> N # N scene-tour runs (memory + per-scene load times)
#
# Every Unity/player invocation runs under a hard timeout, and each measurement is
# verified to have produced a CSV row. Failures abort loudly — nothing waits forever.
set -e

ROOT="${0:A:h}"
UNITY="${UNITY:-/Applications/Unity/Hub/Editor/6000.6.0b3/Unity.app/Contents/MacOS/Unity}"
RESULTS="$ROOT/bench-results"
LOGS="$ROOT/bench-logs"
BUILDS="$ROOT/Builds"
MAT_REL="Assets/Scenes/Garden/Art/Terrain/Materials/WaterSurface_Mat.mat"  # incremental-change target
SETUP_TIMEOUT=1800
CONTENT_TIMEOUT=5400   # first build compiles the full URP shader variant space (~50 min on M3 Pro)
PLAYER_TIMEOUT=5400
RUN_TIMEOUT=360        # runner's own watchdog fires at 300s
mkdir -p "$RESULTS" "$LOGS"

fail() { echo "[bench][FAIL] $1" | tee -a "$LOGS/bench-errors.log"; exit 1; }

run_with_timeout() { # <seconds> <label> <cmd...>
  local secs=$1 label=$2; shift 2
  "$@" &
  local pid=$! waited=0
  while kill -0 $pid 2>/dev/null; do
    sleep 5; waited=$((waited + 5))
    if [ $waited -ge $secs ]; then
      echo "[bench][TIMEOUT] $label exceeded ${secs}s — killing pid $pid" | tee -a "$LOGS/bench-errors.log"
      pkill -9 -P $pid 2>/dev/null || true
      kill -9 $pid 2>/dev/null || true
      return 124
    fi
  done
  wait $pid
}

csv_rows() { [ -f "$1" ] && wc -l < "$1" || echo 0; }
verify_row() { # <csvfile> <rows_before> <label>
  [ "$(csv_rows "$1")" -gt "$2" ] || fail "$3: no CSV row appended (step did not complete)"
}

exec_method() { # <timeout> <label> <method> [extra unity args...]
  local secs=$1 label=$2 method=$3; shift 3
  run_with_timeout $secs "$label" \
    "$UNITY" -batchmode -projectPath "$ROOT" -executeMethod "$method" "$@" \
    -logFile "$LOGS/$label-$(date +%H%M%S).log" \
    || fail "$label (see bench-logs/$label-*.log)"
}

content_build() { # <variant> <label>
  local before=$(csv_rows "$RESULTS/build-times.csv")
  echo "[bench] content build $1/$2 ..."
  exec_method $CONTENT_TIMEOUT "content-$1-$2" CDBench.Editor.CDBenchBuild.BuildContent \
    -cdbenchVariant:$1 -cdbenchLabel:$2 -cdbenchOut:"$RESULTS/build-times.csv"
  verify_row "$RESULTS/build-times.csv" $before "content build $1/$2"
}

player_build() { # <variant>
  echo "[bench] player build $1 ..."
  rm -rf "$BUILDS/$1"
  exec_method $PLAYER_TIMEOUT "player-$1" CDBench.Editor.CDBenchBuild.BuildPlayer \
    -cdbenchVariant:$1 -cdbenchOut:"$RESULTS/build-times.csv" \
    -cdbenchPlayerOut:"$BUILDS/$1/CDBenchSample.app"
  [ -d "$BUILDS/$1/CDBenchSample.app" ] || fail "player build $1: app bundle missing"
}

run_tour() { # <variant> <runIndex> [retain]
  local BIN=$(ls "$BUILDS/$1/CDBenchSample.app/Contents/MacOS/"* | head -1)
  local before=$(csv_rows "$RESULTS/tour.csv")
  echo "[bench] tour run $1 #$2 (retain=${3:-1})..."
  run_with_timeout $RUN_TIMEOUT "tour-$1-$2" \
    "$BIN" -cdbench -cdbenchVariant:$1 -cdbenchRun:$2 -cdbenchRetain:${3:-1} \
    -cdbenchOut:"$RESULTS/tour.csv" -logFile "$LOGS/tour-$1-$2.log" \
    || echo "[bench][WARN] tour $1#$2 exited nonzero (row may still be recorded)"
  verify_row "$RESULTS/tour.csv" $before "tour run $1#$2"
  sleep 3
}

snapshot_library() {
  [ -d "$ROOT/Library-pristine" ] && return
  echo "[bench] snapshotting pristine Library..."
  cp -Rc "$ROOT/Library" "$ROOT/Library-pristine"
}

restore_library() {
  echo "[bench] restoring pristine Library..."
  rm -rf "$ROOT/Library"
  cp -Rc "$ROOT/Library-pristine" "$ROOT/Library"
}

touch_asset() { # <value digits> : set _Smoothness to 0.<digits> (use a never-built value; 749 = canonical)
  sed -i '' "s/    - _Smoothness: 0\..*$/    - _Smoothness: 0.$1/" "$ROOT/$MAT_REL"
  echo "[bench] touched $(basename $MAT_REL) -> 0.$1"
}

bench_builds() { # <variant> : cold x3 / nochange / incr x3
  local V=$1
  snapshot_library
  for i in 1 2 3; do restore_library; content_build $V cold$i; done
  content_build $V nochange
  for i in 1 2 3; do touch_asset 5${i}17; content_build $V incr$i; done
  touch_asset 749   # revert to canonical value
  content_build $V incr-revert
}

bench_tours() { # <variant> <N>
  local V=$1 N=${2:-5}
  for i in $(seq 1 $N); do run_tour $V $i 1; done
}

case "$1" in
  setup)   exec_method $SETUP_TIMEOUT setup CDBench.Editor.CDBenchSetup.SetupGroups ;;
  ab)      exec_method $SETUP_TIMEOUT switch-ab CDBench.Editor.CDBenchSetup.UseAssetBundles ;;
  cd)      exec_method $SETUP_TIMEOUT switch-cd CDBench.Editor.CDBenchSetup.UseContentDirectories ;;
  content) content_build ${3:-NA} ${2:?label} ;;
  player)  player_build ${2:?variant} ;;
  smoke)   run_tour ${2:?variant} smoke 1 ;;
  builds)  bench_builds ${2:?variant} ;;
  tours)   bench_tours ${2:?variant} ${3:-5} ;;
  *) echo "usage: $0 {setup|ab|cd|content|player|smoke|builds|tours} ..."; exit 1 ;;
esac
echo "[bench] done: $*"