From 4d6f0c80ec3c6472e7ece43f00618852ec7e3dba Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 20 Mar 2026 10:44:19 +1300 Subject: [PATCH] [ML] Fix analytics steps failing when not all platforms are built The check_build_regression, ingest_build_timings, and analyze_build_timings pipeline steps had hardcoded depends_on entries for all four platform build step keys. When a PR build only builds a subset of platforms (controlled by CI labels or comments), the missing step keys cause Buildkite to reject the pipeline upload with "Step dependencies not found". Fix by having the pipeline generators (pipeline.json.py and branch.json.py) compute which step keys will actually exist and pass them as ML_BUILD_STEP_KEYS / ML_TEST_STEP_KEYS environment variables. The shell scripts then emit depends_on entries dynamically from these env vars. Made-with: Cursor --- .buildkite/branch.json.py | 22 +++++++++++++++++ .buildkite/pipeline.json.py | 24 +++++++++++++++---- .../pipelines/analyze_build_timings.yml.sh | 18 +++++++++----- .../pipelines/check_build_regression.yml.sh | 22 ++++++++++++----- .../pipelines/ingest_build_timings.yml.sh | 20 ++++++++++------ 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py index 4916a89cc..4f5a8ac95 100755 --- a/.buildkite/branch.json.py +++ b/.buildkite/branch.json.py @@ -30,6 +30,27 @@ def main(): ".buildkite/pipelines/format_and_validation.yml.sh")) config = buildConfig.Config() config.parse() + + build_step_keys = [] + test_step_keys = [] + if config.build_linux and config.build_aarch64: + build_step_keys.append("build_test_linux-aarch64-RelWithDebInfo") + test_step_keys.append("test_linux-aarch64-RelWithDebInfo") + if config.build_linux and config.build_x86_64: + build_step_keys.append("build_test_linux-x86_64-RelWithDebInfo") + test_step_keys.append("test_linux-x86_64-RelWithDebInfo") + if config.build_macos and config.build_aarch64: + build_step_keys.append("build_test_macos-aarch64-RelWithDebInfo") + test_step_keys.append("test_macos-aarch64-RelWithDebInfo") + if config.build_windows and config.build_x86_64: + build_step_keys.append("build_test_Windows-x86_64-RelWithDebInfo") + test_step_keys.append("test_Windows-x86_64-RelWithDebInfo") + + env = { + "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), + "ML_TEST_STEP_KEYS": ",".join(test_step_keys), + } + if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64) pipeline_steps.append(build_windows) @@ -55,6 +76,7 @@ def main(): pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS", ".buildkite/pipelines/upload_dra_to_gcs.yml.sh")) + pipeline["env"] = env pipeline["steps"] = pipeline_steps print(json.dumps(pipeline, indent=2)) diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 1796a665b..73e54b3ea 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -23,11 +23,6 @@ config as buildConfig, ) -# Ensure VERSION_QUALIFIER is always empty for PR builds -env = { - "VERSION_QUALIFIER": "" -} - def main(): pipeline = {} pipeline_steps = step.PipelineStep([]) @@ -39,6 +34,25 @@ def main(): ".buildkite/pipelines/format_and_validation.yml.sh")) config = buildConfig.Config() config.parse() + + # Compute which build step keys will exist so that analytics steps + # can emit a correct depends_on list (not all platforms are built + # for every PR, depending on labels/comments). + build_step_keys = [] + if config.build_linux and config.build_aarch64: + build_step_keys.append("build_test_linux-aarch64-RelWithDebInfo") + if config.build_linux and config.build_x86_64: + build_step_keys.append("build_test_linux-x86_64-RelWithDebInfo") + if config.build_macos and config.build_aarch64: + build_step_keys.append("build_test_macos-aarch64-RelWithDebInfo") + if config.build_windows and config.build_x86_64: + build_step_keys.append("build_test_Windows-x86_64-RelWithDebInfo") + + env = { + "VERSION_QUALIFIER": "", + "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), + } + if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64) pipeline_steps.append(build_windows) diff --git a/.buildkite/pipelines/analyze_build_timings.yml.sh b/.buildkite/pipelines/analyze_build_timings.yml.sh index 79bf79e2f..dac129751 100755 --- a/.buildkite/pipelines/analyze_build_timings.yml.sh +++ b/.buildkite/pipelines/analyze_build_timings.yml.sh @@ -8,17 +8,23 @@ # compliance with the Elastic License 2.0 and the foregoing additional # limitation. -cat </dev/null || true" - - "python3 dev-tools/ingest_build_timings.py --pipeline \$BUILDKITE_PIPELINE_SLUG --build \$BUILDKITE_BUILD_NUMBER" - depends_on: - - "build_test_linux-aarch64-RelWithDebInfo" - - "build_test_linux-x86_64-RelWithDebInfo" - - "build_test_macos-aarch64-RelWithDebInfo" - - "build_test_Windows-x86_64-RelWithDebInfo" + - "python3 dev-tools/ingest_build_timings.py --pipeline $BUILDKITE_PIPELINE_SLUG --build $BUILDKITE_BUILD_NUMBER" +EOL + +if [ -n "${ML_BUILD_STEP_KEYS:-}" ]; then + echo ' depends_on:' + IFS=',' read -ra STEP_KEYS <<< "$ML_BUILD_STEP_KEYS" + for key in "${STEP_KEYS[@]}"; do + echo " - \"${key}\"" + done +fi + +cat <<'EOL' allow_dependency_failure: true soft_fail: true agents: