From 386737b2c6242e91cd78ea74523dfba6843f72a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Tue, 28 Jul 2026 16:13:55 +0200 Subject: [PATCH 1/9] fix: Improved publish flow --- .../tests/test_build_images_functionality.py | 13 +++- .../workflows/build_images/build_images.py | 68 ++++--------------- 2 files changed, 24 insertions(+), 57 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 6fd1a6cc..473294eb 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -333,9 +333,6 @@ async def call_and_test_compile_image_and_publish( platforms = ["linux/amd64"] platforms_to_build = ["linux/amd64"] - mocker.patch.object(ciap_builder, "_get_existing_platforms") - ciap_builder._get_existing_platforms.return_value = [] - mocker.patch.object(ciap_builder, "test_image") ciap_builder_test_image_mock = ciap_builder.test_image ciap_builder_test_image_mock.return_value = "Mock test image result" @@ -344,14 +341,24 @@ async def call_and_test_compile_image_and_publish( mocker.patch.object(ctx_mock, "publish") ctx_mock_publish_mock = ctx_mock.publish + subprocess_run_mock = mocker.patch("subprocess.run") + subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0) + await ciap_builder.compile_image_and_publish( ctx_mock, build_args, secrets, dockerfile, image, platforms_to_build, platforms ) if publish: ctx_mock_publish_mock.assert_called_with(image, platform_variants=ANY) + subprocess_run_mock.assert_called_once() + args, _ = subprocess_run_mock.call_args + assert args[0] == [ + "docker", "buildx", "imagetools", "create", + "--append", f"{image}@Mock publish result", image + ] else: ctx_mock_publish_mock.assert_not_called() + subprocess_run_mock.assert_not_called() if container_structure_filename is not None: ciap_builder_test_image_mock.assert_called_with(ctx_mock) diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index e4334b0b..08b17687 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -1,5 +1,4 @@ import datetime -import json import re import os import sys @@ -330,16 +329,6 @@ async def compile_image_and_publish( variants = [] other_platforms = [p for p in platforms if p not in platforms_to_build] - # Preserve any platforms that already exist in the registry manifest - existing_platforms = [] - if self.publish: - existing_platforms = await anyio.to_thread.run_sync( - self._get_existing_platforms, image - ) - for p in existing_platforms: - if p not in platforms_to_build and p not in other_platforms: - other_platforms.append(p) - if len(other_platforms) > 0: logger.info( f"Not building for these platforms as they are not in the filtered list: {other_platforms}, but including them as variants in the published multi-platform manifest list." @@ -381,7 +370,20 @@ async def compile_image_and_publish( await self.test_image(variant) if self.publish: - await ctx.container().publish(image, platform_variants=variants) + published_ref = await ctx.container().publish(image, platform_variants=variants) + digest = published_ref.split("@")[-1] + new_ref = f"{image}@{digest}" + try: + subprocess.run( + ["docker", "buildx", "imagetools", "create", "--append", new_ref, image], + capture_output=True, text=True, check=True, timeout=60 + ) + except subprocess.CalledProcessError as e: + logger.warning( + f"Failed to append {new_ref} to existing manifest for {image}: " + f"{e.stderr}. The image has been published but may only contain " + "the platforms from this build." + ) # Define a coroutine function to execute the compilation process # for all flavors @@ -640,48 +642,6 @@ def get_extra_tags_for_registry(self, registry_address, extra_tags): return extra_full_registry_addresses - def _get_existing_platforms(self, image): - try: - proc = subprocess.run( - ['docker', 'manifest', 'inspect', image], - capture_output=True, text=True, - timeout=30 - ) - except (FileNotFoundError, OSError): - logger.info(f"Docker CLI not available, skipping registry manifest inspection for {image}") - return [] - except subprocess.TimeoutExpired: - logger.info(f"Timeout inspecting manifest for {image}, skipping") - return [] - if proc.returncode != 0: - return [] - try: - manifest = json.loads(proc.stdout) - except json.JSONDecodeError: - logger.info(f"Failed to parse manifest for {image}: non-JSON output") - return [] - platforms = [] - for m in manifest.get('manifests', []): - p = m.get('platform', {}) - os_val = p.get('os', 'linux') - arch = p.get('architecture') - variant = p.get('variant') - if arch: - platform_str = f"{os_val}/{arch}" - if variant: - platform_str = f"{platform_str}/{variant}" - platforms.append(platform_str) - - if not platforms: - # Single-arch manifest (schema2) — architecture is in config - config = manifest.get('config', {}) - arch = config.get('architecture') - os_val = config.get('os', 'linux') - if arch: - platforms.append(f"{os_val}/{arch}") - - return platforms - def is_auto_build(self): return self.flavors is None or len(self.flavors) == 0 From 7cc86906e4741b784486446227471b5a72a3311e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Tue, 28 Jul 2026 16:27:41 +0200 Subject: [PATCH 2/9] fix: Tests --- firestarter/tests/test_build_images_functionality.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 473294eb..7d3c48a9 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -338,8 +338,7 @@ async def call_and_test_compile_image_and_publish( ciap_builder_test_image_mock.return_value = "Mock test image result" ctx_mock = DaggerContextMock() - mocker.patch.object(ctx_mock, "publish") - ctx_mock_publish_mock = ctx_mock.publish + ctx_mock_publish_mock = mocker.spy(ctx_mock, "publish") subprocess_run_mock = mocker.patch("subprocess.run") subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0) From c36ddebe0b9f0e6cc28cef076c015637735c82dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Tue, 28 Jul 2026 16:32:48 +0200 Subject: [PATCH 3/9] fix: Tests --- firestarter/tests/test_build_images_functionality.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 7d3c48a9..1359fc91 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -338,7 +338,9 @@ async def call_and_test_compile_image_and_publish( ciap_builder_test_image_mock.return_value = "Mock test image result" ctx_mock = DaggerContextMock() - ctx_mock_publish_mock = mocker.spy(ctx_mock, "publish") + async def _publish(*args, **kwargs): + return "Mock publish result" + ctx_mock_publish_mock = mocker.patch.object(ctx_mock, "publish", side_effect=_publish) subprocess_run_mock = mocker.patch("subprocess.run") subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0) From f369963e2d4fad889bda0b8a88c04ff9ba9da520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 09:10:05 +0200 Subject: [PATCH 4/9] fix: Error when uploading already existing platforms --- firestarter/workflows/build_images/build_images.py | 2 +- firestarter/workflows/build_images/resources/schema.json | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 08b17687..5f130f6c 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -591,7 +591,7 @@ def get_flavor_data(self, flavor): dockerfile = flavor_data.dockerfile or "" extra_registries = flavor_data.extra_registries or [] extra_tags = flavor_data.extra_tags or [] - platforms = flavor_data.platforms or ["linux/amd64"] + platforms = list(dict.fromkeys(flavor_data.platforms or ["linux/amd64"])) return ( flavor_registry_data["name"], diff --git a/firestarter/workflows/build_images/resources/schema.json b/firestarter/workflows/build_images/resources/schema.json index 8c583c2e..12bf8469 100644 --- a/firestarter/workflows/build_images/resources/schema.json +++ b/firestarter/workflows/build_images/resources/schema.json @@ -82,7 +82,8 @@ "type": "string", "pattern": "^(linux/)?(amd64|arm64)$" }, - "minItems": 1 + "minItems": 1, + "uniqueItems": true } }, "required": ["dockerfile"], @@ -174,7 +175,8 @@ "type": "string", "pattern": "^(linux/)?(amd64|arm64)$" }, - "minItems": 1 + "minItems": 1, + "uniqueItems": true } }, "required": ["dockerfile"], From ce16061f38a51f19ccf87e7161d744bdaf97a756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 09:20:53 +0200 Subject: [PATCH 5/9] fix: Error when uploading already existing platforms --- firestarter/workflows/build_images/build_images.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 5f130f6c..283c4a52 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -338,6 +338,14 @@ async def compile_image_and_publish( v = ctx.container(platform=dagger.Platform(p)).from_(image) try: await v.sync() + pulled_platform = await v.platform() + if pulled_platform != dagger.Platform(p): + logger.info( + f"Platform mismatch: requested {p} but registry image " + f"resolved to {pulled_platform}. This variant will " + f"not be included." + ) + continue variants.append(v) except Exception as e: logger.info( From 2fc427cd926a8eb6289a4a3a92b915096fd0e121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 09:40:40 +0200 Subject: [PATCH 6/9] fix: Removing non built platforms --- .../tests/test_build_images_functionality.py | 14 ++- .../workflows/build_images/build_images.py | 98 +++++++++++-------- 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 1359fc91..d705576e 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -330,7 +330,6 @@ async def call_and_test_compile_image_and_publish( secrets = { "test_secret": "b" } dockerfile = "/path/to/dockerfile" image = "image_tag" - platforms = ["linux/amd64"] platforms_to_build = ["linux/amd64"] mocker.patch.object(ciap_builder, "test_image") @@ -345,17 +344,26 @@ async def _publish(*args, **kwargs): subprocess_run_mock = mocker.patch("subprocess.run") subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0) + if publish: + mock_existing = mocker.patch.object( + ciap_builder, "get_existing_platform_digests", + return_value={"linux/arm64": "sha256:oldarm64digest"} + ) + await ciap_builder.compile_image_and_publish( - ctx_mock, build_args, secrets, dockerfile, image, platforms_to_build, platforms + ctx_mock, build_args, secrets, dockerfile, image, platforms_to_build ) if publish: ctx_mock_publish_mock.assert_called_with(image, platform_variants=ANY) + mock_existing.assert_called_once_with(image) subprocess_run_mock.assert_called_once() args, _ = subprocess_run_mock.call_args assert args[0] == [ "docker", "buildx", "imagetools", "create", - "--append", f"{image}@Mock publish result", image + "--tag", image, + f"{image}@Mock publish result", + f"{image}@sha256:oldarm64digest" ] else: ctx_mock_publish_mock.assert_not_called() diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 283c4a52..7df2a1a8 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -319,38 +319,44 @@ async def test_image(self, ctx): os.remove(file_name) + def get_existing_platform_digests(self, image): + """Get a mapping of platform -> digest from the existing manifest list in the registry. + + Returns an empty dict if the image doesn't exist or is a single-platform manifest. + """ + try: + result = subprocess.run( + ["docker", "buildx", "imagetools", "inspect", image], + capture_output=True, text=True, check=True, timeout=30 + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + return {} + + output = result.stdout + + if "Manifests:" not in output: + return {} + + existing = {} + lines = output.split('\n') + current_digest = None + + for line in lines: + line = line.strip() + name_match = re.match(r'Name:\s+\S+@(sha256:[a-f0-9]+)', line) + if name_match: + current_digest = name_match.group(1) + platform_match = re.match(r'Platform:\s+(\S+)', line) + if platform_match and current_digest: + existing[platform_match.group(1)] = current_digest + + return existing + # Define a coroutine function to compile an image using Docker async def compile_image_and_publish( - self, ctx, build_args, secrets, dockerfile, image, platforms_to_build, platforms + self, ctx, build_args, secrets, dockerfile, image, platforms_to_build ): - # If there are platforms that are not being built for this flavor, log them - # and create container variants for them without building, - # so that they can be included in the published multi-platform manifest list variants = [] - other_platforms = [p for p in platforms if p not in platforms_to_build] - - if len(other_platforms) > 0: - logger.info( - f"Not building for these platforms as they are not in the filtered list: {other_platforms}, but including them as variants in the published multi-platform manifest list." - ) - for p in other_platforms: - logger.info(f"Creating container variant for platform {p} without building...") - v = ctx.container(platform=dagger.Platform(p)).from_(image) - try: - await v.sync() - pulled_platform = await v.platform() - if pulled_platform != dagger.Platform(p): - logger.info( - f"Platform mismatch: requested {p} but registry image " - f"resolved to {pulled_platform}. This variant will " - f"not be included." - ) - continue - variants.append(v) - except Exception as e: - logger.info( - f"Failed to create container variant for platform {p} using image {image}. Error: {e}. This variant will not be included in the published multi-platform manifest list." - ) # Set a current working directory src = ctx.host().directory(".") @@ -378,20 +384,31 @@ async def compile_image_and_publish( await self.test_image(variant) if self.publish: + existing_platforms = self.get_existing_platform_digests(image) + platforms_built = set(platforms_to_build) + old_refs = [ + f"{image}@{d}" + for p, d in existing_platforms.items() + if p not in platforms_built + ] + published_ref = await ctx.container().publish(image, platform_variants=variants) digest = published_ref.split("@")[-1] new_ref = f"{image}@{digest}" - try: - subprocess.run( - ["docker", "buildx", "imagetools", "create", "--append", new_ref, image], - capture_output=True, text=True, check=True, timeout=60 - ) - except subprocess.CalledProcessError as e: - logger.warning( - f"Failed to append {new_ref} to existing manifest for {image}: " - f"{e.stderr}. The image has been published but may only contain " - "the platforms from this build." - ) + + if old_refs: + all_refs = [new_ref] + old_refs + try: + subprocess.run( + ["docker", "buildx", "imagetools", "create", "--tag", image] + all_refs, + capture_output=True, text=True, check=True, timeout=60 + ) + except subprocess.CalledProcessError as e: + logger.warning( + f"Failed to merge existing platforms into manifest for {image}: " + f"{e.stderr}. The image has been published but may only contain " + "the platforms from this build." + ) # Define a coroutine function to execute the compilation process # for all flavors @@ -531,8 +548,7 @@ async def compile_images_for_all_flavors(self): secrets, dockerfile, image, - platforms_to_build, - platforms + platforms_to_build ) image_tag = image.split(":")[1] From 2882597ef4660a48589f3bc7f0b2f034f5b17948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 10:56:29 +0200 Subject: [PATCH 7/9] fix: Platforms not being correctly published --- firestarter/tests/test_build_images_functionality.py | 4 ++-- firestarter/workflows/build_images/build_images.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index d705576e..a6f25a79 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -347,7 +347,7 @@ async def _publish(*args, **kwargs): if publish: mock_existing = mocker.patch.object( ciap_builder, "get_existing_platform_digests", - return_value={"linux/arm64": "sha256:oldarm64digest"} + return_value={"__unknown__": "sha256:oldsingledigest"} ) await ciap_builder.compile_image_and_publish( @@ -363,7 +363,7 @@ async def _publish(*args, **kwargs): "docker", "buildx", "imagetools", "create", "--tag", image, f"{image}@Mock publish result", - f"{image}@sha256:oldarm64digest" + f"{image}@sha256:oldsingledigest" ] else: ctx_mock_publish_mock.assert_not_called() diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 7df2a1a8..2d24b15b 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -320,9 +320,10 @@ async def test_image(self, ctx): def get_existing_platform_digests(self, image): - """Get a mapping of platform -> digest from the existing manifest list in the registry. + """Get a mapping of platform -> digest from the existing manifest in the registry. - Returns an empty dict if the image doesn't exist or is a single-platform manifest. + Returns {"__unknown__": digest} for single-platform manifests, or an empty dict + if the image doesn't exist. """ try: result = subprocess.run( @@ -335,6 +336,9 @@ def get_existing_platform_digests(self, image): output = result.stdout if "Manifests:" not in output: + digest_match = re.search(r'Digest:\s+(sha256:[a-f0-9]+)', output) + if digest_match: + return {"__unknown__": digest_match.group(1)} return {} existing = {} @@ -389,7 +393,7 @@ async def compile_image_and_publish( old_refs = [ f"{image}@{d}" for p, d in existing_platforms.items() - if p not in platforms_built + if p not in platforms_built or p == "__unknown__" ] published_ref = await ctx.container().publish(image, platform_variants=variants) From 536d9fe9d7922b156425d50845804d0721fecb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 12:30:46 +0200 Subject: [PATCH 8/9] fix: Copilot reviews --- .../tests/test_build_images_functionality.py | 13 +++++-------- firestarter/workflows/build_images/build_images.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index a6f25a79..531848d2 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -357,14 +357,11 @@ async def _publish(*args, **kwargs): if publish: ctx_mock_publish_mock.assert_called_with(image, platform_variants=ANY) mock_existing.assert_called_once_with(image) - subprocess_run_mock.assert_called_once() - args, _ = subprocess_run_mock.call_args - assert args[0] == [ - "docker", "buildx", "imagetools", "create", - "--tag", image, - f"{image}@Mock publish result", - f"{image}@sha256:oldsingledigest" - ] + subprocess_run_mock.assert_called_once_with( + ["docker", "buildx", "imagetools", "create", "--tag", image, + f"{image}@Mock publish result", f"{image}@sha256:oldsingledigest"], + capture_output=True, text=True, check=True, timeout=60 + ) else: ctx_mock_publish_mock.assert_not_called() subprocess_run_mock.assert_not_called() diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 2d24b15b..9416d815 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -397,11 +397,15 @@ async def compile_image_and_publish( ] published_ref = await ctx.container().publish(image, platform_variants=variants) - digest = published_ref.split("@")[-1] - new_ref = f"{image}@{digest}" - if old_refs: - all_refs = [new_ref] + old_refs + if "@" not in published_ref: + logger.warning( + f"Publish result {published_ref} did not return a digest reference; " + "the image has been published but the manifest merge was skipped." + ) + elif old_refs: + digest = published_ref.split("@")[-1] + all_refs = [f"{image}@{digest}"] + old_refs try: subprocess.run( ["docker", "buildx", "imagetools", "create", "--tag", image] + all_refs, From 5432bd0562d861782c744abc7f17cedefd04ba9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20V=C3=A1zquez=20Gil?= Date: Wed, 29 Jul 2026 12:44:08 +0200 Subject: [PATCH 9/9] fix: Tests --- firestarter/tests/test_build_images_functionality.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 531848d2..11ed7966 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -337,8 +337,9 @@ async def call_and_test_compile_image_and_publish( ciap_builder_test_image_mock.return_value = "Mock test image result" ctx_mock = DaggerContextMock() + publish_digest = "sha256:mockedpublishdigest" async def _publish(*args, **kwargs): - return "Mock publish result" + return f"{image}@{publish_digest}" ctx_mock_publish_mock = mocker.patch.object(ctx_mock, "publish", side_effect=_publish) subprocess_run_mock = mocker.patch("subprocess.run") @@ -359,7 +360,7 @@ async def _publish(*args, **kwargs): mock_existing.assert_called_once_with(image) subprocess_run_mock.assert_called_once_with( ["docker", "buildx", "imagetools", "create", "--tag", image, - f"{image}@Mock publish result", f"{image}@sha256:oldsingledigest"], + f"{image}@{publish_digest}", f"{image}@sha256:oldsingledigest"], capture_output=True, text=True, check=True, timeout=60 ) else: