diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 10bef6e..14b65b2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -380,6 +380,18 @@ jobs: # changes with it and the rule above selects the right members. # So a tools/ edit alone selects nothing rather than forcing a # full workspace rebuild. + # The timing table decides how work is DISTRIBUTED across + # shards, never what gets built — no member's result can change + # because a measured number moved. It used to fall through to + # the `*)` catch-all below and force a full run, which is the + # most expensive way in this workflow to test nothing: the next + # full run reads the new numbers anyway. + # + # Note this makes the file invisible to CI. A malformed row is + # then silent, and `plan_shards` prices anything it cannot parse + # at the median — the exact failure mode that overflowed a shard + # here. Guard it in `lint` if that ever bites. + tests/member-timings.tsv) : ;; *.md|docs/*|.agents/*|.github/*|tools/*) : ;; *) full "unclassified change: $f" ;; esac @@ -441,38 +453,69 @@ jobs: # platform and take one shard per ~70 minutes, which leaves ~20 # minutes of headroom under the 90-minute cap for a cold cache. # - # The caps are where runner concurrency comes back in: linux 3 (the - # measured concurrency — a 4th shard would queue), macOS 2, windows - # 2. Past those, more shards buy fit that is already there and pay - # another checkout + mcpp download + cache restore. + # The caps are where runner concurrency comes back in: linux 4, + # macOS 2, windows 2. Past those, more shards buy fit that is + # already there and pay another checkout + mcpp download + cache + # restore. # - # Full run, from the table: linux 13570s over 3 -> ~75min/shard - # (observed slowest 77), macOS 6922s over 2 -> ~57, windows 8043s - # over 2 -> ~67. linux is the tight one — its cap binds before the - # ~70-minute target does, so it is the first place to look if a - # cold full run starts brushing 90 again. The levers, in order: - # raise the linux cap to 4 (costs a queued runner), then the job + # linux was 3 — "the measured concurrency, a 4th shard would queue". + # That cap outlived its premise twice over: the workspace grew, and + # the second toolchain leg means linux emits 2 x N jobs, so it + # queues at any N. It is now 4, which is what the formula asks for. + # + # Full run, from the table: linux 15891s over 4 -> ~74min slowest + # shard, macOS 6922s over 2 -> ~57, windows 8043s over 2 -> ~67. + # + # linux is still the tight one, and the number to watch is the + # SLOWEST shard against the 90-minute job cap, not the average. + # Simulated on the current table: 3-way 98min, 4-way 74, 5-way 60. + # The levers, in order: raise the linux cap to 5, then the job # timeout. + # + # Keep tests/member-timings.tsv fresh — that is not housekeeping. An + # untimed member is priced at the MEDIAN, so a heavy newcomer packs + # like a trivial one. `mysql-connector-cpp` (881s) and + # `libmysqlclient` (329s) landed on shard 0 next to grpc-codegen + # exactly that way and pushed it past the cap, and the stale table + # then hid the overflow by under-counting the total. + # `plan_shards.lua 0 0` reports the platform's total cost. + # + # This used to sum the timing table's rows inline, which was the same + # answer only while the table listed every member. It now lists just + # the heavy ones, so summing rows measures a fraction of the work: + # 10965s instead of 16095s on linux, i.e. 3 shards where 4 are + # needed, i.e. a slowest shard of 107 minutes against a 90-minute + # cap. Asking plan_shards keeps ONE definition of both "which members + # are there" and "what does an untimed one cost". + # + # The estimate is tuned on linux (16095s vs 15891s measured, +1.3%). + # macOS and windows members are cheaper on average than the default, + # so their totals come out high — harmless here because both are + # already at their cap, and erring toward more shards is the safe + # direction. Revisit if either cap moves. shards_for() { # platform cap -> shard count - local secs - secs=$(XPLAT="$1" lua5.4 -e ' - local plat = os.getenv("XPLAT") - local sel, all = {}, (os.getenv("MEMBERS") == "__ALL__") - if not all then - for m in (os.getenv("MEMBERS") or ""):gmatch("%S+") do sel[m] = true end - end - local total = 0 - for line in io.lines("tests/member-timings.tsv") do - local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)$") - if p == plat and m and (all or sel[m]) then total = total + tonumber(s) end - end - print(total)') + local secs sel="" + [ "$MEMBERS" = "__ALL__" ] || sel="$MEMBERS" + secs=$(lua5.4 tests/plan_shards.lua "$1" 0 0 $sel) local n=$(( secs / 4200 + 1 )) [ "$n" -gt "$2" ] && n="$2" echo "$1 work: ${secs}s (measured) -> $n shard(s)" >&2 echo "$n" } - ln=$(shards_for linux 3) + # linux 3 -> 4. The cap was the binding constraint, not the ~70min + # target: `secs / 4200 + 1` asks for 4 on the refreshed table + # (15891s) and has been getting clamped back to 3. Simulated on that + # table, slowest shard: 3-way 98min (OVER the 90min job cap — and + # `linux default 0/3` was in fact cancelled at 90min on run + # 31266814148), 4-way 74min, 5-way 60min. 4 is the formula's own + # answer and leaves 16 minutes for a cold cache; 5 is there if that + # stops being enough. + # + # This does cross the concurrency line the old comment drew: with two + # toolchain legs linux now emits 8 jobs against a measured runner + # concurrency of 3, so shards queue. Queuing is the right trade — + # back-to-back shards still finish, a shard over the cap does not. + ln=$(shards_for linux 4) mn=$(shards_for macos 2) wn=$(shards_for windows 2) { @@ -945,25 +988,37 @@ jobs: # silently absorb a one-off slow runner. Refresh it deliberately — # download this artifact and replace tests/member-timings.tsv when # the numbers have actually moved. + # Top 10 per platform, not every member. + # + # plan_shards prices anything absent at a fixed default, and the + # packing decisions are made by the heavy members anyway — on linux + # the top 10 of 67 are 69% of the total, and dropping the other 57 + # moves the slowest shard by 0 minutes. A table that lists everyone + # goes stale the moment someone adds a test; one that lists ten only + # goes stale when the heavy set actually changes. That is the whole + # point: adding a member must not require touching this file. { - echo "# \t\t — from run ${{ github.run_id }}" - echo "# refresh: download the member-timings artifact and replace this file" + echo "# \t\t — the HEAVY members only, top 10" + echo "# per platform, from run ${{ github.run_id }}. Everything else is" + echo "# priced at plan_shards.lua's fixed default; adding a test does NOT" + echo "# require touching this file. Refresh when a member becomes heavy" + echo "# enough to enter the top 10, or one of these numbers moves." + # ONLY the default-toolchain leg feeds this table. The llvm leg + # runs the SAME members again, so globbing every leg would put two + # rows per (platform, member) into the file — `sort -u` keeps both, + # since the seconds differ. Both legs are planned from these + # numbers, so the default leg is the right single baseline. + # + # Then: heaviest first, keep 10. `sort -u` before that so a member + # reported by two shards cannot occupy two of the ten slots. for plat in linux macos windows; do - # ONLY the default-toolchain leg feeds this table. The llvm leg - # runs the SAME members again, so globbing every leg would put - # two rows per (platform, member) into the file — `sort -u` - # keeps both, since the seconds differ — and shards_for sums - # every matching row. Linux work would read as roughly double - # and its shard count would be permanently pinned at the cap. - # Both legs are planned from these numbers, so the default leg - # is the right single baseline. for f in timings/timings-$plat-default-*/timings.tsv; do [ -f "$f" ] || continue awk -F'\t' -v p="$plat" '{ printf "%s\t%s\t%s\n", p, $2, $1 }' "$f" - done + done | sort -u | sort -t"$(printf '\t')" -k3,3nr | head -10 done - } | sort -u > member-timings.tsv - echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows)" + } > member-timings.tsv + echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows, top 10/platform)" - name: Upload the timing table for the next run's sharding if: always() && hashFiles('member-timings.tsv') != '' diff --git a/tests/member-timings.tsv b/tests/member-timings.tsv index 8c844db..b193700 100644 --- a/tests/member-timings.tsv +++ b/tests/member-timings.tsv @@ -1,188 +1,43 @@ -linux abseil 111 -linux archive 49 -linux asio-module 37 -linux asio-ssl 49 -linux boost-ext.ut 30 -linux build-mcpp 28 -linux c-ares 28 -linux catch2 81 -linux catch2-main 53 -linux catch2-v2 13 -linux catch2-v2-main 15 -linux cjson 2 -linux core 67 -linux curl 26 -linux eigen 25 -linux eui-neo 151 -linux eui-neo-app-main 140 -linux eui-neo-markdown 143 -linux eui-neo-sdl2 119 -linux eui-neo-vulkan 154 -linux eui-neo-window 128 -linux ffmpeg 288 -linux ffmpeg-module 343 -linux fmtlib.fmt 5 -linux freetype 10 -linux glad 2 -linux godot-cpp 573 -linux godot-cpp-module 549 -linux godot-cpp-module-v10 392 -linux godot-cpp-v10 504 -linux grpc-codegen 3563 -linux grpc-module 1701 -linux gui-stack 81 -linux imgui 6 -linux imgui-module 91 -linux imgui-window 91 -linux libpng 6 -linux llamacpp 101 -linux llamacpp-metal 0 -linux magic_enum 4 -linux marzer.tomlplusplus 7 -linux md4c 3 -linux nlohmann.json 10 -linux openblas 0 -linux opencv-module 602 -linux opencv-module-dnn 741 -linux opencv-module-unifont 671 -linux openssl 0 -linux protobuf 259 -linux protobuf-gzip 158 -linux protobuf-protoc 923 -linux protobuf-upb 263 -linux re2 15 -linux sdl2 63 -linux spdlog 9 -linux spdlog-compiled 13 -linux tinyhttps 13 -linux tray 3 -linux vulkan 13 -linux websocket 19 -linux websocket-features 23 -linux yyjson 3 -macos abseil 75 -macos archive 49 -macos asio-module 30 -macos asio-ssl 78 -macos boost-ext.ut 27 -macos build-mcpp 33 -macos c-ares 27 -macos catch2 58 -macos catch2-main 28 -macos catch2-v2 10 -macos catch2-v2-main 9 -macos cjson 3 -macos core 58 -macos curl 103 -macos eigen 7 -macos eui-neo 52 -macos eui-neo-app-main 57 -macos eui-neo-markdown 76 -macos eui-neo-sdl2 142 -macos eui-neo-vulkan 59 -macos eui-neo-window 53 -macos ffmpeg 148 -macos ffmpeg-module 130 -macos fmtlib.fmt 4 -macos freetype 12 -macos glad 3 -macos godot-cpp 333 -macos godot-cpp-module 256 -macos godot-cpp-module-v10 215 -macos godot-cpp-v10 230 -macos grpc-codegen 1715 -macos grpc-module 880 -macos gui-stack 1 -macos imgui 4 -macos imgui-module 1 -macos imgui-window 1 -macos libpng 7 -macos llamacpp 66 -macos llamacpp-metal 75 -macos magic_enum 3 -macos marzer.tomlplusplus 5 -macos md4c 4 -macos nlohmann.json 9 -macos openblas 1 -macos opencv-module 198 -macos opencv-module-dnn 399 -macos opencv-module-unifont 304 -macos openssl 1 -macos protobuf 100 -macos protobuf-gzip 87 -macos protobuf-protoc 458 -macos protobuf-upb 143 -macos re2 8 -macos sdl2 22 -macos spdlog 5 -macos spdlog-compiled 6 -macos tinyhttps 13 -macos tray 4 -macos vulkan 7 -macos websocket 12 -macos websocket-features 12 -macos yyjson 6 -# — measured, run 31034885938 -# refresh: download the member-timings artifact from a full run and replace this file -windows abseil 133 -windows archive 76 -windows asio-module 37 -windows asio-ssl 24 -windows boost-ext.ut 40 -windows build-mcpp 27 -windows c-ares 33 -windows catch2 83 -windows catch2-main 62 -windows catch2-v2 12 -windows catch2-v2-main 12 -windows cjson 3 -windows core 98 -windows curl 31 -windows eigen 10 -windows eui-neo 93 -windows eui-neo-app-main 119 -windows eui-neo-markdown 84 -windows eui-neo-sdl2 150 -windows eui-neo-vulkan 84 -windows eui-neo-window 74 -windows ffmpeg 454 -windows ffmpeg-module 532 -windows fmtlib.fmt 5 -windows freetype 15 -windows glad 2 -windows godot-cpp 742 -windows godot-cpp-module 636 -windows godot-cpp-module-v10 684 -windows godot-cpp-v10 566 -windows grpc-codegen 31 -windows grpc-module 1 -windows gui-stack 1 -windows imgui 6 -windows imgui-module 0 -windows imgui-window 0 -windows libpng 10 -windows llamacpp 125 -windows llamacpp-metal 1 -windows magic_enum 4 -windows marzer.tomlplusplus 6 -windows md4c 3 -windows nlohmann.json 10 -windows openblas 11 -windows opencv-module 747 -windows opencv-module-dnn 1174 -windows opencv-module-unifont 1 -windows openssl 1 -windows protobuf 215 -windows protobuf-gzip 141 -windows protobuf-protoc 227 -windows protobuf-upb 251 -windows re2 15 -windows sdl2 46 -windows spdlog 6 -windows spdlog-compiled 14 -windows tinyhttps 14 -windows tray 2 -windows vulkan 7 -windows websocket 24 -windows websocket-features 26 -windows yyjson 2 +# \t\t — the HEAVY members only, top 10 per +# platform. Everything else is priced at plan_shards.lua's fixed default; +# on linux the 57 members not listed here average 86s and are 31% of the +# total, and listing them changes the slowest shard by 0 minutes. +# +# So: adding a test does NOT require touching this file. Refresh it when +# a member becomes heavy enough to enter the top 10, or when one of these +# numbers moves materially — download the member-timings artifact from a +# full run and keep the top 10 per platform. +# +# Sources: run 31260545520; libmysqlclient / mysql-connector-cpp measured +# from run 31266814148's linux default 0/3 step durations, whose shard was +# cancelled at the 90m cap before its timings artifact could upload. +linux grpc-codegen 3463 +linux grpc-module 1771 +linux protobuf-protoc 885 +linux mysql-connector-cpp 881 +linux opencv-module-dnn 875 +linux opencv-module-unifont 655 +linux opencv-module 649 +linux godot-cpp-module 627 +linux eui-neo-sdl2 591 +linux godot-cpp-module-v10 568 +macos grpc-codegen 1156 +macos grpc-module 719 +macos opencv-module-dnn 325 +macos protobuf-protoc 314 +macos godot-cpp-module 276 +macos opencv-module 223 +macos godot-cpp-v10 206 +macos godot-cpp-module-v10 195 +macos opencv-module-unifont 164 +macos eui-neo-sdl2 138 +windows opencv-module 868 +windows opencv-module-dnn 828 +windows godot-cpp-module 707 +windows ffmpeg 693 +windows godot-cpp-module-v10 657 +windows godot-cpp-v10 655 +windows ffmpeg-module 526 +windows protobuf-gzip 238 +windows protobuf-protoc 152 +windows protobuf-upb 151 diff --git a/tests/plan_shards.lua b/tests/plan_shards.lua index 59be162..6766313 100644 --- a/tests/plan_shards.lua +++ b/tests/plan_shards.lua @@ -26,9 +26,16 @@ -- among shards whose load is close, prefer the one already holding -- members with overlapping dependencies. -- --- Missing timing → the median, so a newly added member is neither assumed --- free nor assumed huge. No table at all → falls back to round-robin, which --- is worse but never wrong. +-- Missing timing → a FIXED default (see DEFAULT_SECONDS), so a newly added +-- member is neither assumed free nor assumed huge. No table at all → every +-- member prices the same, which degenerates to round-robin: worse, never +-- wrong. +-- +-- The table deliberately holds only the HEAVY members. They are where the +-- packing decisions actually get made — on linux the top 10 of 67 are 69% of +-- the total — and listing the other 57 buys a table that goes stale every +-- time someone adds a test. Verified equivalent: full table and top-10 table +-- both land the slowest linux shard at 74 minutes. -- -- Measured on the real workspace (linux, 3 shards), LPT against round-robin: -- @@ -71,24 +78,58 @@ end -- ── measured times ──────────────────────────────────────────────────────── -- Format: \t\t -local times, samples = {}, {} +local times = {} local tsv = read_file("tests/member-timings.tsv") if tsv then for line in tsv:gmatch("[^\n]+") do if not line:match("^#") then local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)") - if p == platform and m then - times[m] = tonumber(s) - samples[#samples + 1] = tonumber(s) - end + if p == platform and m then times[m] = tonumber(s) end end end end -local median = 60 -if #samples > 0 then - table.sort(samples) - median = samples[math.ceil(#samples / 2)] +-- Price for a member with no row. A FIXED constant, deliberately, and not the +-- median of whatever the table happens to hold. +-- +-- The table only carries the HEAVY members now (see its header). Deriving the +-- default from those samples would take the median OF THE HEAVYWEIGHTS — +-- measured at 875s on the linux set — and charge every small member that, +-- which packs far worse than having no table at all. Simulated on the full +-- 67-member measurement, 4 linux shards, slowest shard: +-- +-- full 67-row table 74 min +-- top-10 table, median-derived default 90 min <- at the job cap +-- top-10 table, fixed default 74 min +-- +-- 86s is the mean of the 57 untimed linux members (median 24s); 90 rounds it. +-- The choice is not delicate: across defaults from 30s to 300s the slowest +-- shard stays between 72 and 84 minutes, all under the 90-minute cap. That +-- insensitivity is the point — it is what lets the table sit untouched while +-- small members come and go. +-- +-- What it does NOT absorb is a new member that belongs in the heavy set. +-- `mysql-connector-cpp` (881s) priced at the default is 13 minutes of work +-- that the packer cannot see. Adding a heavy newcomer stays a real edit. +local DEFAULT_SECONDS = 90 +local median = DEFAULT_SECONDS -- name kept: read as "the price of unknown" + +-- ` ` of `0 0` prints this platform's TOTAL cost instead of a +-- shard's members. +-- +-- The workflow's shards_for needs exactly that number to pick a shard count, +-- and it needs it computed the same way: same member enumeration, same price +-- for an untimed member. It used to sum the table's rows directly, which was +-- equivalent only while the table listed every member. It no longer does — +-- summing 10 heavy rows and calling it the workload asks for 3 linux shards +-- where the real work needs 4, and 3 lands the slowest shard at 107 minutes +-- against a 90-minute cap. Two copies of "how much work is there" drift; this +-- is the one copy. +if shardCount == 0 then + local total = 0 + for _, m in ipairs(members) do total = total + (times[m] or DEFAULT_SECONDS) end + print(total) + return end -- ── run order: cheapest first ─────────────────────────────────────────────