Skip to content

wip: new 1es pools #3412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tjones60 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

wip: new 1es pools #3412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e1a9746
use gen2 images and new skus for runners
tjones60 May 1, 2026
dd713fa
work folder
tjones60 May 1, 2026
8506ced
dont always use x2apic
tjones60 May 2, 2026
b5fd4e6
specify temp folder
tjones60 May 2, 2026
5df8fe5
fix temp
tjones60 May 2, 2026
6e99976
filter and shorter
tjones60 May 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,110 changes: 414 additions & 696 deletions .github/workflows/openvmm-ci.yaml
View file
Open in desktop

Large diffs are not rendered by default.

1,043 changes: 381 additions & 662 deletions .github/workflows/openvmm-pr-release.yaml
View file
Open in desktop

Large diffs are not rendered by default.

1,054 changes: 386 additions & 668 deletions .github/workflows/openvmm-pr.yaml
View file
Open in desktop

Large diffs are not rendered by default.

36 changes: 18 additions & 18 deletions ci-flowey/openvmm-pr.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 32 additions & 16 deletions flowey/flowey_hvlite/src/pipelines/checkin_gates.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1183,7 +1183,7 @@ impl IntoPipeline for CheckinGatesCli {
.map_err(|missing| {
anyhow::anyhow!("missing required windows-amd-snp vmm_tests artifact: {missing}")
})?;
let vmm_tests_artifacts_linux_mshv_x86 = vmm_tests_artifacts_linux_musl_x86
let _vmm_tests_artifacts_linux_mshv_x86 = vmm_tests_artifacts_linux_musl_x86
.finish()
.map_err(|missing| {
anyhow::anyhow!("missing required linux-mshv (musl) vmm_tests artifact: {missing}")
Expand All @@ -1204,6 +1204,7 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch,
gh_pool: GhRunner,
ado_pool: Option<AdoPool>,
temp_dir: Option<PathBuf>,
label: &'a str,
target: CommonTriple,
resolve_vmm_tests_artifacts: vmm_tests_artifact_builders::ResolveVmmTestsDepArtifacts,
Expand Down Expand Up @@ -1297,11 +1298,14 @@ impl IntoPipeline for CheckinGatesCli {
KnownTestArtifacts::VmgsWith16kTpm,
];

let temp_dir_from = |work_folder: &str| Some(PathBuf::from(work_folder).join("temp"));

for VmmTestJobParams {
platform,
arch,
gh_pool,
ado_pool,
temp_dir,
label,
target,
resolve_vmm_tests_artifacts,
Expand All @@ -1315,6 +1319,7 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch::X86_64,
gh_pool: gh_pools::windows_intel_1es(),
ado_pool: Some(ado_pools::windows_intel_1es()),
temp_dir: temp_dir_from(gh_pools::WINDOWS_WORK_FOLDER),
label: "x64-windows-intel",
target: CommonTriple::X86_64_WINDOWS_MSVC,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_intel_x86,
Expand All @@ -1328,6 +1333,7 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch::X86_64,
gh_pool: gh_pools::windows_tdx_self_hosted_baremetal(),
ado_pool: None,
temp_dir: None,
label: "x64-windows-intel-tdx",
target: CommonTriple::X86_64_WINDOWS_MSVC,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_intel_tdx_x86,
Expand All @@ -1341,10 +1347,14 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch::X86_64,
gh_pool: gh_pools::windows_amd_1es(),
ado_pool: Some(ado_pools::windows_amd_1es()),
temp_dir: temp_dir_from(gh_pools::WINDOWS_WORK_FOLDER),
label: "x64-windows-amd",
target: CommonTriple::X86_64_WINDOWS_MSVC,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_amd_x86,
nextest_filter_expr: standard_filter.clone(),
// tmk and sidecar requires x2apic, which causes our amd nested runners to crash
nextest_filter_expr: format!(
"{standard_filter} & !(test(hyperv_openhcl) & (test(sidecar) + test(tmk)))"
),
test_artifacts: standard_x64_test_artifacts.clone(),
needs_prep_run: false,
hugetlb_2mb_overcommit_pages: None,
Expand All @@ -1354,6 +1364,7 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch::X86_64,
gh_pool: gh_pools::windows_snp_self_hosted_baremetal(),
ado_pool: None,
temp_dir: None,
label: "x64-windows-amd-snp",
target: CommonTriple::X86_64_WINDOWS_MSVC,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_amd_snp_x86,
Expand All @@ -1367,6 +1378,7 @@ impl IntoPipeline for CheckinGatesCli {
arch: FlowArch::X86_64,
gh_pool: gh_pools::linux_amd_1es(),
ado_pool: Some(ado_pools::linux_amd_1es()),
temp_dir: temp_dir_from(gh_pools::LINUX_WORK_FOLDER),
label: "x64-linux-amd-kvm",
target: CommonTriple::X86_64_LINUX_GNU,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_linux_x86,
Expand All @@ -1376,25 +1388,27 @@ impl IntoPipeline for CheckinGatesCli {
needs_prep_run: false,
hugetlb_2mb_overcommit_pages: Some(HUGETLB_2MB_OVERCOMMIT_PAGES),
},
VmmTestJobParams {
platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::AzureLinux),
arch: FlowArch::X86_64,
gh_pool: gh_pools::linux_mshv_1es(),
ado_pool: None,
label: "x64-linux-intel-mshv",
target: CommonTriple::X86_64_LINUX_MUSL,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_linux_mshv_x86,
// - No legal way to obtain gen1 pcat blobs on non-msft linux machines
nextest_filter_expr: format!("{standard_filter} & !test(pcat_x64)"),
test_artifacts: standard_x64_test_artifacts.clone(),
needs_prep_run: false,
hugetlb_2mb_overcommit_pages: None,
},
// VmmTestJobParams {
// platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::AzureLinux),
// arch: FlowArch::X86_64,
// gh_pool: gh_pools::linux_mshv_1es(),
// ado_pool: None,
// temp_dir: temp_dir_from(gh_pools::WINDOWS_WORK_FOLDER),
// label: "x64-linux-intel-mshv",
// target: CommonTriple::X86_64_LINUX_MUSL,
// resolve_vmm_tests_artifacts: vmm_tests_artifacts_linux_mshv_x86,
// // - No legal way to obtain gen1 pcat blobs on non-msft linux machines
// nextest_filter_expr: format!("{standard_filter} & !test(pcat_x64)"),
// test_artifacts: standard_x64_test_artifacts.clone(),
// needs_prep_run: false,
// hugetlb_2mb_overcommit_pages: None,
// },
Comment thread
tjones60 marked this conversation as resolved.
VmmTestJobParams {
Comment on lines +1391 to 1406
platform: FlowPlatform::Windows,
arch: FlowArch::Aarch64,
gh_pool: gh_pools::windows_arm_self_hosted_baremetal(),
ado_pool: None,
temp_dir: None,
label: "aarch64-windows",
target: CommonTriple::AARCH64_WINDOWS_MSVC,
resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_aarch64,
Expand Down Expand Up @@ -1452,6 +1466,7 @@ impl IntoPipeline for CheckinGatesCli {
artifact_dir: pub_vmm_tests_results.map(|x| ctx.publish_artifact(x)),
needs_prep_run,
hugetlb_2mb_overcommit_pages,
temp_dir,
done: ctx.new_done_handle(),
}
});
Expand Down Expand Up @@ -1594,6 +1609,7 @@ impl IntoPipeline for CheckinGatesCli {
artifact_dir: pub_mi_secure_test_results.map(|x| ctx.publish_artifact(x)),
needs_prep_run: false,
hugetlb_2mb_overcommit_pages: None,
temp_dir: temp_dir_from(gh_pools::WINDOWS_WORK_FOLDER),
done: ctx.new_done_handle(),
}
});
Expand Down
34 changes: 19 additions & 15 deletions flowey/flowey_hvlite/src/pipelines_shared/gh_pools.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,46 +5,50 @@

use flowey::pipeline::prelude::*;

pub const AMD_POOL_1ES: &str = "openvmm-gh-amd-westus3";
pub const INTEL_POOL_1ES: &str = "openvmm-gh-intel-westus3";
pub const ARM_POOL_1ES: &str = "openvmm-gh-arm-westus2";
pub const AMD_POOL_1ES: &str = "openvmm-gh-amd";
pub const INTEL_POOL_1ES: &str = "openvmm-gh-intel";
pub const ARM_POOL_1ES: &str = "openvmm-gh-arm";

pub const WINDOWS_IMAGE_AMD64: &str = "win-amd64";
pub const WINDOWS_IMAGE_ARM64: &str = "win-arm64";
pub const LINUX_IMAGE_AMD64: &str = "ubuntu2404-amd64";
pub const LINUX_IMAGE_ARM64: &str = "ubuntu2404-arm64";
pub const MSHV_IMAGE_AMD64: &str = "azurelinux3-amd64-dom0";
pub const WINDOWS_IMAGE_AMD64: &str = "win-amd64-v2";
pub const WINDOWS_IMAGE_ARM64: &str = "win-arm64-v2";
pub const LINUX_IMAGE_AMD64: &str = "ubuntu2404-amd64-v2";
pub const LINUX_IMAGE_ARM64: &str = "ubuntu2404-arm64-v2";
pub const MSHV_IMAGE_AMD64: &str = "azurelinux3-amd64-dom0-v2";

fn gh_pool_with_image_1es(pool: &str, image: &str) -> GhRunner {
pub const WINDOWS_WORK_FOLDER: &str = "E:\\_work";
pub const LINUX_WORK_FOLDER: &str = "/mnt/azure_nvme_temp/_work";

fn gh_pool_with_image_1es(pool: &str, image: &str, work_folder: &str) -> GhRunner {
GhRunner::SelfHosted(vec![
"self-hosted".to_string(),
format!("1ES.Pool={pool}"),
format!("1ES.ImageOverride={image}"),
format!("1ES.WorkFolder={work_folder}"),
])
}

pub fn windows_amd_1es() -> GhRunner {
gh_pool_with_image_1es(AMD_POOL_1ES, WINDOWS_IMAGE_AMD64)
gh_pool_with_image_1es(AMD_POOL_1ES, WINDOWS_IMAGE_AMD64, WINDOWS_WORK_FOLDER)
}

pub fn windows_intel_1es() -> GhRunner {
gh_pool_with_image_1es(INTEL_POOL_1ES, WINDOWS_IMAGE_AMD64)
gh_pool_with_image_1es(INTEL_POOL_1ES, WINDOWS_IMAGE_AMD64, WINDOWS_WORK_FOLDER)
}

pub fn windows_arm_1es() -> GhRunner {
gh_pool_with_image_1es(ARM_POOL_1ES, WINDOWS_IMAGE_ARM64)
gh_pool_with_image_1es(ARM_POOL_1ES, WINDOWS_IMAGE_ARM64, WINDOWS_WORK_FOLDER)
}

pub fn linux_arm_1es() -> GhRunner {
gh_pool_with_image_1es(ARM_POOL_1ES, LINUX_IMAGE_ARM64)
gh_pool_with_image_1es(ARM_POOL_1ES, LINUX_IMAGE_ARM64, LINUX_WORK_FOLDER)
}

pub fn linux_amd_1es() -> GhRunner {
gh_pool_with_image_1es(AMD_POOL_1ES, LINUX_IMAGE_AMD64)
gh_pool_with_image_1es(AMD_POOL_1ES, LINUX_IMAGE_AMD64, LINUX_WORK_FOLDER)
}

pub fn linux_mshv_1es() -> GhRunner {
gh_pool_with_image_1es(INTEL_POOL_1ES, MSHV_IMAGE_AMD64)
gh_pool_with_image_1es(INTEL_POOL_1ES, MSHV_IMAGE_AMD64, LINUX_WORK_FOLDER)
}

pub fn windows_x64_gh() -> GhRunner {
Expand Down
6 changes: 5 additions & 1 deletion flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ flowey_request! {
pub fail_job_on_test_fail: bool,
/// If provided, also publish junit.xml test results as an artifact.
pub artifact_dir: Option<ReadVar<PathBuf>>,
/// Specify where to store temporary files
pub temp_dir: Option<PathBuf>,
pub done: WriteVar<SideEffect>,
}
}
Expand Down Expand Up @@ -100,6 +102,7 @@ impl SimpleFlowNode for Node {
needs_prep_run,
hugetlb_2mb_overcommit_pages,
artifact_dir,
temp_dir,
done,
} = request;

Expand Down Expand Up @@ -172,6 +175,8 @@ impl SimpleFlowNode for Node {

let extra_env = ctx.reqv(|v| crate::init_vmm_tests_env::Request {
test_content_dir,
disk_images_dir: Some(disk_images_dir),
temp_dir,
vmm_tests_target: target.clone(),
register_openvmm,
register_openvmm_vhost,
Expand All @@ -185,7 +190,6 @@ impl SimpleFlowNode for Node {
register_tpm_guest_tests_windows,
register_tpm_guest_tests_linux,
register_test_igvm_agent_rpc_server,
disk_images_dir: Some(disk_images_dir),
register_openhcl_igvm_files,
get_test_log_path: Some(get_test_log_path),
get_env: v,
Expand Down
Loading
Loading