diff --git a/firestarter/tests/test_build_images_functionality.py b/firestarter/tests/test_build_images_functionality.py index 45c9a281..11ed7966 100644 --- a/firestarter/tests/test_build_images_functionality.py +++ b/firestarter/tests/test_build_images_functionality.py @@ -128,58 +128,41 @@ def test_dereference_from_input(mocker) -> None: SHORT_SHA_INPUT = "6a32377" BRANCH_INPUT = "test_branch" - completed_process_mock = subprocess.CompletedProcess - completed_process_mock.check_return_code = mocker.MagicMock( - name="completed_process.check_return_code.mock", - return_value=True - ) - - subprocess_mock_tag_return_value = completed_process_mock( - args=None, returncode=0 - ) - subprocess_mock_tag_return_value.stdout = TAG_INPUT.encode("windows-1252") + mock_tag = mocker.MagicMock(returncode=0) + mock_tag.stdout = TAG_INPUT.encode() - subprocess_mock_empty_return_value = completed_process_mock( - args=None, returncode=0 - ) - subprocess_mock_empty_return_value.stdout = "".encode("windows-1252") + mock_empty = mocker.MagicMock(returncode=0) + mock_empty.stdout = b"" - subprocess_mock_sha_return_value = completed_process_mock( - args=None, returncode=0 - ) - subprocess_mock_sha_return_value.stdout = LONG_SHA_INPUT.encode( - "windows-1252" - ) + mock_sha = mocker.MagicMock(returncode=0) + mock_sha.stdout = LONG_SHA_INPUT.encode() - # Test tag input - subprocess_mock = subprocess - subprocess_mock.run = mocker.MagicMock( - name="subprocess.run.mock", - side_effect=[ - subprocess_mock_tag_return_value, - subprocess_mock_empty_return_value, - subprocess_mock_sha_return_value, - ] - ) + # Test tag input with snapshots type (builder defaults to snapshots) + # Flow: git tag -l → mock_tag, git rev-parse tag^{commit} → mock_sha + mocker.patch('subprocess.run', side_effect=[mock_tag, mock_sha]) + result = builder.dereference_from_input(TAG_INPUT) + assert result == SHORT_SHA_INPUT - tag_input_dereference = builder.dereference_from_input(TAG_INPUT) + # Test long sha input — no subprocess calls + result = builder.dereference_from_input(LONG_SHA_INPUT) + assert result == SHORT_SHA_INPUT - assert tag_input_dereference == TAG_INPUT - - # Test long sha input - long_sha_input_dereference = builder.dereference_from_input(LONG_SHA_INPUT) - - assert long_sha_input_dereference == SHORT_SHA_INPUT - - # Test short sha input - short_sha_input_dereference = builder.dereference_from_input(SHORT_SHA_INPUT) - - assert short_sha_input_dereference == SHORT_SHA_INPUT + # Test short sha input — no subprocess calls + result = builder.dereference_from_input(SHORT_SHA_INPUT) + assert result == SHORT_SHA_INPUT # Test branch input - branch_input_dereference = builder.dereference_from_input(BRANCH_INPUT) + # Flow: git tag -l → mock_empty, git rev-parse origin/branch → mock_sha + mocker.patch('subprocess.run', side_effect=[mock_empty, mock_sha]) + result = builder.dereference_from_input(BRANCH_INPUT) + assert result == SHORT_SHA_INPUT - assert branch_input_dereference == SHORT_SHA_INPUT + # Test tag input with releases type — tag returned as-is, no rev-parse + # Flow: git tag -l → mock_tag only (no additional dereference) + builder._type = "releases" + mocker.patch('subprocess.run', side_effect=[mock_tag]) + result = builder.dereference_from_input(TAG_INPUT) + assert result == TAG_INPUT # Secrets are correctly solved, using the corresponding SecretResolver @@ -347,7 +330,6 @@ async def call_and_test_compile_image_and_publish( secrets = { "test_secret": "b" } dockerfile = "/path/to/dockerfile" image = "image_tag" - platforms = ["linux/amd64"] platforms_to_build = ["linux/amd64"] mocker.patch.object(ciap_builder, "test_image") @@ -355,17 +337,35 @@ async def call_and_test_compile_image_and_publish( ciap_builder_test_image_mock.return_value = "Mock test image result" ctx_mock = DaggerContextMock() - mocker.patch.object(ctx_mock, "publish") - ctx_mock_publish_mock = ctx_mock.publish + publish_digest = "sha256:mockedpublishdigest" + async def _publish(*args, **kwargs): + return f"{image}@{publish_digest}" + ctx_mock_publish_mock = mocker.patch.object(ctx_mock, "publish", side_effect=_publish) + + subprocess_run_mock = mocker.patch("subprocess.run") + subprocess_run_mock.return_value = subprocess.CompletedProcess(args=[], returncode=0) + + if publish: + mock_existing = mocker.patch.object( + ciap_builder, "get_existing_platform_digests", + return_value={"__unknown__": "sha256:oldsingledigest"} + ) await ciap_builder.compile_image_and_publish( - ctx_mock, build_args, secrets, dockerfile, image, platforms_to_build, platforms + ctx_mock, build_args, secrets, dockerfile, image, platforms_to_build ) if publish: ctx_mock_publish_mock.assert_called_with(image, platform_variants=ANY) + mock_existing.assert_called_once_with(image) + subprocess_run_mock.assert_called_once_with( + ["docker", "buildx", "imagetools", "create", "--tag", image, + f"{image}@{publish_digest}", f"{image}@sha256:oldsingledigest"], + capture_output=True, text=True, check=True, timeout=60 + ) else: ctx_mock_publish_mock.assert_not_called() + subprocess_run_mock.assert_not_called() if container_structure_filename is not None: ciap_builder_test_image_mock.assert_called_with(ctx_mock) diff --git a/firestarter/workflows/build_images/build_images.py b/firestarter/workflows/build_images/build_images.py index 554e42bd..9416d815 100644 --- a/firestarter/workflows/build_images/build_images.py +++ b/firestarter/workflows/build_images/build_images.py @@ -1,5 +1,4 @@ import datetime -import json import re import os import sys @@ -37,6 +36,7 @@ def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self._secrets = self.resolve_secrets(self.secrets) + self._type = self.vars.get('type', 'snapshots') # We checkout the correct sha/tag self._from = self.dereference_from_input(self.vars.get('from')) @@ -47,7 +47,6 @@ def __init__(self, **kwargs) -> None: self._releases_registry_creds = self.vars.get('releases_registry_creds', None) self._auth_strategy = self.vars.get('auth_strategy', None) self._output_results = self.vars.get('output_results', 'results.yaml') - self._type = self.vars.get('type', 'snapshots') self._workflow_run_id = self.vars.get('workflow_run_id', None) self._workflow_run_url = self.vars.get('workflow_run_url', None) self._service_path = self.vars.get('service_path', '') @@ -225,11 +224,23 @@ def dereference_from_input(self, input_value): git_output = proc.stdout.decode('utf-8').strip() if git_output: + # git tag -l uses glob matching; filter to exact match + git_output = input_value if input_value in git_output.split('\n') else None + + if git_output: + if self.type == 'snapshots': + proc = subprocess.run( + ['git', 'rev-parse', f"{git_output}^{{commit}}"], + stdout=subprocess.PIPE + ) + proc.check_returncode() + return proc.stdout.decode('utf-8')[:7] return git_output # if the input value is a branch, we need to get the sha of the branch proc = subprocess.run( - ['git', 'rev-parse', f"origin/{input_value}"], stdout=subprocess.PIPE + ['git', 'rev-parse', f"origin/{input_value}"], + stdout=subprocess.PIPE ) proc.check_returncode() return proc.stdout.decode('utf-8')[:7] @@ -308,29 +319,48 @@ async def test_image(self, ctx): os.remove(file_name) + def get_existing_platform_digests(self, image): + """Get a mapping of platform -> digest from the existing manifest in the registry. + + Returns {"__unknown__": digest} for single-platform manifests, or an empty dict + if the image doesn't exist. + """ + try: + result = subprocess.run( + ["docker", "buildx", "imagetools", "inspect", image], + capture_output=True, text=True, check=True, timeout=30 + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + return {} + + output = result.stdout + + if "Manifests:" not in output: + digest_match = re.search(r'Digest:\s+(sha256:[a-f0-9]+)', output) + if digest_match: + return {"__unknown__": digest_match.group(1)} + return {} + + existing = {} + lines = output.split('\n') + current_digest = None + + for line in lines: + line = line.strip() + name_match = re.match(r'Name:\s+\S+@(sha256:[a-f0-9]+)', line) + if name_match: + current_digest = name_match.group(1) + platform_match = re.match(r'Platform:\s+(\S+)', line) + if platform_match and current_digest: + existing[platform_match.group(1)] = current_digest + + return existing + # Define a coroutine function to compile an image using Docker async def compile_image_and_publish( - self, ctx, build_args, secrets, dockerfile, image, platforms_to_build, platforms + self, ctx, build_args, secrets, dockerfile, image, platforms_to_build ): - # If there are platforms that are not being built for this flavor, log them - # and create container variants for them without building, - # so that they can be included in the published multi-platform manifest list variants = [] - other_platforms = [p for p in platforms if p not in platforms_to_build] - if len(other_platforms) > 0: - logger.info( - f"Not building for these platforms as they are not in the filtered list: {other_platforms}, but including them as variants in the published multi-platform manifest list." - ) - for p in other_platforms: - logger.info(f"Creating container variant for platform {p} without building...") - v = ctx.container(platform=dagger.Platform(p)).from_(image) - try: - await v.sync() - variants.append(v) - except Exception as e: - logger.info( - f"Failed to create container variant for platform {p} using image {image}. Error: {e}. This variant will not be included in the published multi-platform manifest list." - ) # Set a current working directory src = ctx.host().directory(".") @@ -358,7 +388,35 @@ async def compile_image_and_publish( await self.test_image(variant) if self.publish: - await ctx.container().publish(image, platform_variants=variants) + existing_platforms = self.get_existing_platform_digests(image) + platforms_built = set(platforms_to_build) + old_refs = [ + f"{image}@{d}" + for p, d in existing_platforms.items() + if p not in platforms_built or p == "__unknown__" + ] + + published_ref = await ctx.container().publish(image, platform_variants=variants) + + if "@" not in published_ref: + logger.warning( + f"Publish result {published_ref} did not return a digest reference; " + "the image has been published but the manifest merge was skipped." + ) + elif old_refs: + digest = published_ref.split("@")[-1] + all_refs = [f"{image}@{digest}"] + old_refs + try: + subprocess.run( + ["docker", "buildx", "imagetools", "create", "--tag", image] + all_refs, + capture_output=True, text=True, check=True, timeout=60 + ) + except subprocess.CalledProcessError as e: + logger.warning( + f"Failed to merge existing platforms into manifest for {image}: " + f"{e.stderr}. The image has been published but may only contain " + "the platforms from this build." + ) # Define a coroutine function to execute the compilation process # for all flavors @@ -498,8 +556,7 @@ async def compile_images_for_all_flavors(self): secrets, dockerfile, image, - platforms_to_build, - platforms + platforms_to_build ) image_tag = image.split(":")[1] @@ -566,7 +623,7 @@ def get_flavor_data(self, flavor): dockerfile = flavor_data.dockerfile or "" extra_registries = flavor_data.extra_registries or [] extra_tags = flavor_data.extra_tags or [] - platforms = flavor_data.platforms or ["linux/amd64"] + platforms = list(dict.fromkeys(flavor_data.platforms or ["linux/amd64"])) return ( flavor_registry_data["name"], @@ -617,7 +674,6 @@ def get_extra_tags_for_registry(self, registry_address, extra_tags): return extra_full_registry_addresses - def is_auto_build(self): return self.flavors is None or len(self.flavors) == 0 diff --git a/firestarter/workflows/build_images/resources/schema.json b/firestarter/workflows/build_images/resources/schema.json index 8c583c2e..12bf8469 100644 --- a/firestarter/workflows/build_images/resources/schema.json +++ b/firestarter/workflows/build_images/resources/schema.json @@ -82,7 +82,8 @@ "type": "string", "pattern": "^(linux/)?(amd64|arm64)$" }, - "minItems": 1 + "minItems": 1, + "uniqueItems": true } }, "required": ["dockerfile"], @@ -174,7 +175,8 @@ "type": "string", "pattern": "^(linux/)?(amd64|arm64)$" }, - "minItems": 1 + "minItems": 1, + "uniqueItems": true } }, "required": ["dockerfile"],