From 4c5fe0a0e39c4fc09e203dac67463e6c913eead3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 6 Feb 2026 13:50:06 +0100 Subject: [PATCH 1/9] ci: use custom VM images --- .buildkite/pipeline.package.mbp.yml | 12 ++++++------ .buildkite/pipeline.yml | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline.package.mbp.yml b/.buildkite/pipeline.package.mbp.yml index 21636efaff..130e524c97 100644 --- a/.buildkite/pipeline.package.mbp.yml +++ b/.buildkite/pipeline.package.mbp.yml @@ -1,8 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "fleet server package mbp" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" + IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins @@ -91,7 +91,7 @@ steps: command: ".buildkite/scripts/package.sh snapshot" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + image: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.2xlarge" artifact_paths: - build/distributions/** @@ -106,7 +106,7 @@ steps: .buildkite/scripts/package.sh staging agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + image: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.2xlarge" artifact_paths: - build/distributions/** @@ -122,7 +122,7 @@ steps: FIPS: "true" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + image: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.2xlarge" artifact_paths: - build/distributions/** @@ -139,7 +139,7 @@ steps: FIPS: "true" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + image: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.2xlarge" artifact_paths: - build/distributions/** diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d9e5d8a3ce..c8e8bf67b5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,9 +3,9 @@ env: DOCKER_COMPOSE_VERSION: "1.25.5" TERRAFORM_VERSION: "1.6.4" - IMAGE_UBUNTU_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1767879728" - IMAGE_UBUNTU_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1767879728" - IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" + IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" + IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1770381074" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins @@ -173,7 +173,7 @@ steps: GO_DISTRO: "microsoft" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_X86_64_FIPS}" + image: "${IMAGE_UBUNTU_X86_64_FIPS}" instanceType: "m5.xlarge" artifact_paths: - build/*.xml @@ -187,7 +187,7 @@ steps: GO_DISTRO: "stdlib" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_X86_64_FIPS}" + image: "${IMAGE_UBUNTU_X86_64_FIPS}" instanceType: "m5.xlarge" artifact_paths: - build/*.xml From 50dc9a30222e0a155d121261d442d179c9b1b1c2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 6 Feb 2026 14:48:03 +0100 Subject: [PATCH 2/9] test --- magefile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/magefile.go b/magefile.go index 06aecf9878..d9fe0f637d 100644 --- a/magefile.go +++ b/magefile.go @@ -706,7 +706,7 @@ func (Build) Local() error { outFile = filepath.Join("bin", binaryExe) } - return sh.RunWithV(env, "go", "build", "-tags="+getTagsString(), "-gcflags="+getGCFlags(), "-ldflags="+getLDFlags(), "-o", outFile, ".") + return sh.RunWithV(env, "go", "build", "-buildvcs=false", "-tags="+getTagsString(), "-gcflags="+getGCFlags(), "-ldflags="+getLDFlags(), "-o", outFile, ".") } // Binary builds release binaries for the specified platforms. @@ -753,6 +753,7 @@ func goBuild(osArg, archArg string, cover bool) error { args := []string{ "build", + "-buildvcs=false", "-tags=" + getTagsString(), "-gcflags=" + getGCFlags(), "-ldflags=" + getLDFlags(), From a5f2192753bf7c67e268c30e30a3c9167d8855be Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Fri, 20 Feb 2026 13:42:57 +0100 Subject: [PATCH 3/9] Add method to mount the git alternates paths in the fleet container in the mage file --- magefile.go | 61 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/magefile.go b/magefile.go index d9fe0f637d..21f3833cff 100644 --- a/magefile.go +++ b/magefile.go @@ -974,6 +974,8 @@ func (Docker) Release() error { } // dockerRun runs the target on a container produced by docker:builder. +// When .git/objects/info/alternates exists and points to paths on the host (e.g. CI git mirrors), +// those paths are mounted into the container at the same location so Go's VCS stamping can read them. func dockerRun(target string) error { userInfo, err := user.Current() if err != nil { @@ -983,17 +985,56 @@ func dockerRun(target string) error { if err != nil { return fmt.Errorf("unable to get wd: %w", err) } - return sh.RunV("docker", "run", "--rm", - "-u", userInfo.Uid+":"+userInfo.Gid, + args := []string{"run", "--rm", + "-u", userInfo.Uid + ":" + userInfo.Gid, "--env=GOCACHE=/go/cache", - "--volume", pwd+":/fleet-server/", - "-e", envPlatforms+"="+strings.Join(getPlatforms(), ","), - "-e", envDev+"="+strconv.FormatBool(isDEV()), - "-e", envFIPS+"="+strconv.FormatBool(isFIPS()), - "-e", envSnapshot+"="+strconv.FormatBool(isSnapshot()), - "-e", envVersionQualifier+"="+os.Getenv(envVersionQualifier), - dockerBuilderName+":"+getGoVersion(), target, - ) + "--volume", pwd + ":/fleet-server/", + "-e", envPlatforms + "=" + strings.Join(getPlatforms(), ","), + "-e", envDev + "=" + strconv.FormatBool(isDEV()), + "-e", envFIPS + "=" + strconv.FormatBool(isFIPS()), + "-e", envSnapshot + "=" + strconv.FormatBool(isSnapshot()), + "-e", envVersionQualifier + "=" + os.Getenv(envVersionQualifier), + } + for _, vol := range gitAlternatesVolumeMounts() { + args = append(args, "--volume", vol) + } + args = append(args, dockerBuilderName+":"+getGoVersion(), target) + return sh.RunV("docker", args...) +} + +// gitAlternatesVolumeMounts returns docker --volume args for paths listed in +// .git/objects/info/alternates that exist on the host, so the container can +// access them at the same path (required for Go VCS stamping when using git alternates). +func gitAlternatesVolumeMounts() []string { + alternatesPath := filepath.Join(".git", "objects", "info", "alternates") + data, err := os.ReadFile(alternatesPath) + if err != nil { + return nil + } + infoDir := filepath.Dir(alternatesPath) + var mounts []string + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + path := strings.TrimSpace(scanner.Text()) + if path == "" || strings.HasPrefix(path, "#") { + continue + } + // Path in alternates is relative to .git/objects/info or absolute. + if !filepath.IsAbs(path) { + path = filepath.Join(infoDir, path) + } + path, err = filepath.Abs(path) + if err != nil { + continue + } + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + continue + } + // Mount host path to same path in container so Go's VCS resolution works. + mounts = append(mounts, path+":"+path) + } + return mounts } // Binary builds binaries within a docker image produced by docker:builder. From 2b728e5d54d56d51dceab9c11f03dd9fd7a2f1ae Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 14:32:12 +0100 Subject: [PATCH 4/9] Set right fleet server ubuntu image --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c8e8bf67b5..640e5a821f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ env: DOCKER_COMPOSE_VERSION: "1.25.5" TERRAFORM_VERSION: "1.6.4" - IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1770381074" IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1770381074" From 781e91f4fced915c66efda10e7471b7a65e1735d Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 14:52:40 +0100 Subject: [PATCH 5/9] Revert "Set right fleet server ubuntu image" This reverts commit 2b728e5d54d56d51dceab9c11f03dd9fd7a2f1ae. --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 640e5a821f..c8e8bf67b5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ env: DOCKER_COMPOSE_VERSION: "1.25.5" TERRAFORM_VERSION: "1.6.4" - IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1770381074" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1770381074" From 25476f2cfa9f529ddd79e1cd2092e2b68b851f74 Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 19:20:43 +0100 Subject: [PATCH 6/9] Setting last qa generated images --- .buildkite/pipeline.package.mbp.yml | 4 ++-- .buildkite/pipeline.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.package.mbp.yml b/.buildkite/pipeline.package.mbp.yml index 130e524c97..e84f631788 100644 --- a/.buildkite/pipeline.package.mbp.yml +++ b/.buildkite/pipeline.package.mbp.yml @@ -1,8 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "fleet server package mbp" env: - IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" - IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1772547557" + IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c8e8bf67b5..ef7b2b1394 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,9 +3,9 @@ env: DOCKER_COMPOSE_VERSION: "1.25.5" TERRAFORM_VERSION: "1.6.4" - IMAGE_UBUNTU_X86_64: "platform-ingest-fleets-ubuntu-2204-1770381074" - IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1770381074" - IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1770381074" + IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1772547557" + IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" + IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1772547557" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins From f695f258dbd82a2eeb07e2ae5a42ae1b7f8ca267 Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 19:38:40 +0100 Subject: [PATCH 7/9] Try to use qa images --- .buildkite/pipeline.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ef7b2b1394..7ad679a665 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -7,6 +7,11 @@ env: IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1772547557" +providers: + gcp: + defaults: + imageProject: "elastic-images-qa" + # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins common: From 880f732d9a56513583874f5a2f765b66cdaede8e Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 19:52:03 +0100 Subject: [PATCH 8/9] Set qa images project in each agemt specs --- .buildkite/pipeline.package.mbp.yml | 6 ++++++ .buildkite/pipeline.yml | 9 ++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.package.mbp.yml b/.buildkite/pipeline.package.mbp.yml index e84f631788..4c12b91cbe 100644 --- a/.buildkite/pipeline.package.mbp.yml +++ b/.buildkite/pipeline.package.mbp.yml @@ -32,6 +32,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/distributions/** plugins: @@ -46,6 +47,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/distributions/** plugins: @@ -62,6 +64,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/distributions/** plugins: @@ -79,6 +82,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/distributions/** plugins: @@ -155,6 +159,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" plugins: - *oidc_plugin - *docker_elastic_login_plugin @@ -180,6 +185,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" plugins: - *oidc_plugin - *docker_elastic_login_plugin diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7ad679a665..fad548a5df 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -7,11 +7,6 @@ env: IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1772547557" -providers: - gcp: - defaults: - imageProject: "elastic-images-qa" - # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins common: @@ -64,6 +59,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" plugins: - *oidc_plugin - *dockerhub_login_plugin @@ -80,6 +76,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" plugins: - *oidc_plugin - *dockerhub_login_plugin @@ -215,6 +212,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/*.xml plugins: @@ -233,6 +231,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" + imageProject: "elastic-images-qa" artifact_paths: - build/*.xml - build/e2e-coverage.out From e19fa01db199e7879e7a3d9896f9651a78bebe42 Mon Sep 17 00:00:00 2001 From: "fr4nc1sc0.r4m0n" Date: Tue, 3 Mar 2026 20:01:15 +0100 Subject: [PATCH 9/9] Set the test-prefix for arm aws images --- .buildkite/pipeline.package.mbp.yml | 2 +- .buildkite/pipeline.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.package.mbp.yml b/.buildkite/pipeline.package.mbp.yml index 4c12b91cbe..818e2cabda 100644 --- a/.buildkite/pipeline.package.mbp.yml +++ b/.buildkite/pipeline.package.mbp.yml @@ -2,7 +2,7 @@ name: "fleet server package mbp" env: IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1772547557" - IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" + IMAGE_UBUNTU_ARM_64: "test-platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fad548a5df..c87fdcf9ae 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,8 +4,8 @@ env: DOCKER_COMPOSE_VERSION: "1.25.5" TERRAFORM_VERSION: "1.6.4" IMAGE_UBUNTU_X86_64: "platform-ingest-fleet-server-ubuntu-2204-1772547557" - IMAGE_UBUNTU_ARM_64: "platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" - IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1772547557" + IMAGE_UBUNTU_ARM_64: "test-platform-ingest-fleet-server-ubuntu-2204-aarch64-1772547557" + IMAGE_UBUNTU_X86_64_FIPS: "test-platform-ingest-fleet-server-ubuntu-2204-fips-1772547557" # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins