diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py index 4916a89cc..4f5a8ac95 100755 --- a/.buildkite/branch.json.py +++ b/.buildkite/branch.json.py @@ -30,6 +30,27 @@ def main(): ".buildkite/pipelines/format_and_validation.yml.sh")) config = buildConfig.Config() config.parse() + + build_step_keys = [] + test_step_keys = [] + if config.build_linux and config.build_aarch64: + build_step_keys.append("build_test_linux-aarch64-RelWithDebInfo") + test_step_keys.append("test_linux-aarch64-RelWithDebInfo") + if config.build_linux and config.build_x86_64: + build_step_keys.append("build_test_linux-x86_64-RelWithDebInfo") + test_step_keys.append("test_linux-x86_64-RelWithDebInfo") + if config.build_macos and config.build_aarch64: + build_step_keys.append("build_test_macos-aarch64-RelWithDebInfo") + test_step_keys.append("test_macos-aarch64-RelWithDebInfo") + if config.build_windows and config.build_x86_64: + build_step_keys.append("build_test_Windows-x86_64-RelWithDebInfo") + test_step_keys.append("test_Windows-x86_64-RelWithDebInfo") + + env = { + "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), + "ML_TEST_STEP_KEYS": ",".join(test_step_keys), + } + if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64) pipeline_steps.append(build_windows) @@ -55,6 +76,7 @@ def main(): pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS", ".buildkite/pipelines/upload_dra_to_gcs.yml.sh")) + pipeline["env"] = env pipeline["steps"] = pipeline_steps print(json.dumps(pipeline, indent=2)) diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 1796a665b..73e54b3ea 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -23,11 +23,6 @@ config as buildConfig, ) -# Ensure VERSION_QUALIFIER is always empty for PR builds -env = { - "VERSION_QUALIFIER": "" -} - def main(): pipeline = {} pipeline_steps = step.PipelineStep([]) @@ -39,6 +34,25 @@ def main(): ".buildkite/pipelines/format_and_validation.yml.sh")) config = buildConfig.Config() config.parse() + + # Compute which build step keys will exist so that analytics steps + # can emit a correct depends_on list (not all platforms are built + # for every PR, depending on labels/comments). + build_step_keys = [] + if config.build_linux and config.build_aarch64: + build_step_keys.append("build_test_linux-aarch64-RelWithDebInfo") + if config.build_linux and config.build_x86_64: + build_step_keys.append("build_test_linux-x86_64-RelWithDebInfo") + if config.build_macos and config.build_aarch64: + build_step_keys.append("build_test_macos-aarch64-RelWithDebInfo") + if config.build_windows and config.build_x86_64: + build_step_keys.append("build_test_Windows-x86_64-RelWithDebInfo") + + env = { + "VERSION_QUALIFIER": "", + "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), + } + if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64) pipeline_steps.append(build_windows) diff --git a/.buildkite/pipelines/analyze_build_timings.yml.sh b/.buildkite/pipelines/analyze_build_timings.yml.sh index 79bf79e2f..dac129751 100755 --- a/.buildkite/pipelines/analyze_build_timings.yml.sh +++ b/.buildkite/pipelines/analyze_build_timings.yml.sh @@ -8,17 +8,23 @@ # compliance with the Elastic License 2.0 and the foregoing additional # limitation. -cat </dev/null || true" - - "python3 dev-tools/ingest_build_timings.py --pipeline \$BUILDKITE_PIPELINE_SLUG --build \$BUILDKITE_BUILD_NUMBER" - depends_on: - - "build_test_linux-aarch64-RelWithDebInfo" - - "build_test_linux-x86_64-RelWithDebInfo" - - "build_test_macos-aarch64-RelWithDebInfo" - - "build_test_Windows-x86_64-RelWithDebInfo" + - "python3 dev-tools/ingest_build_timings.py --pipeline $BUILDKITE_PIPELINE_SLUG --build $BUILDKITE_BUILD_NUMBER" +EOL + +if [ -n "${ML_BUILD_STEP_KEYS:-}" ]; then + echo ' depends_on:' + IFS=',' read -ra STEP_KEYS <<< "$ML_BUILD_STEP_KEYS" + for key in "${STEP_KEYS[@]}"; do + echo " - \"${key}\"" + done +fi + +cat <<'EOL' allow_dependency_failure: true soft_fail: true agents: