From 99ebb2c2ac0db04de9ee288afd2a3be3602c3477 Mon Sep 17 00:00:00 2001 From: casibbald Date: Sun, 8 Feb 2026 21:49:21 +0200 Subject: [PATCH 1/3] containerd daemon: optional publish-then-pull workaround to avoid digest errors (fixes #2272) When not publishing and daemon uses containerd storage, publish to local registry (default localhost:5001, PACK_CONTAINERD_WORKAROUND_REGISTRY) then pull and tag so app image is not exported directly to daemon. Signed-off-by: Charles Sibbald Signed-off-by: casibbald --- pkg/client/build.go | 64 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/pkg/client/build.go b/pkg/client/build.go index b4fc5c126..645194a31 100644 --- a/pkg/client/build.go +++ b/pkg/client/build.go @@ -315,11 +315,6 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { "Re-run with '--pull-policy=always' to silence this warning.") } - if !opts.Publish && usesContainerdStorage(c.docker) { - c.logger.Warnf("Exporting to docker daemon (building without --publish) and daemon uses containerd storage; performance may be significantly degraded.\n" + - "For more information, see https://github.com/buildpacks/pack/issues/2272.") - } - imageRef, err := c.parseReference(opts) if err != nil { return errors.Wrapf(err, "invalid image name '%s'", opts.Image) @@ -327,6 +322,28 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { imgRegistry := imageRef.Context().RegistryStr() imageName := imageRef.Name() + // When the daemon uses containerd storage, exporting directly to the daemon is slow and can hit digest errors (pack#2272). + // Workaround: publish to a local registry (e.g. localhost:5001), then pull into the daemon and tag as requested. + var containerdWorkaround bool + var publishRef name.Reference + if !opts.Publish && !opts.Layout() && opts.PreviousImage == "" && usesContainerdStorage(c.docker) { + workaroundRegistry := os.Getenv("PACK_CONTAINERD_WORKAROUND_REGISTRY") + if workaroundRegistry == "" { + workaroundRegistry = "localhost:5001" + } + workaroundRegistry = strings.TrimSuffix(workaroundRegistry, "/") + publishImageStr := workaroundRegistry + "/" + imageRef.Name() + publishRef, err = name.NewTag(publishImageStr, name.WeakValidation) + if err != nil { + return errors.Wrapf(err, "containerd workaround: invalid publish image '%s'", publishImageStr) + } + containerdWorkaround = true + c.logger.Infof("Daemon uses containerd storage; using publish-then-pull workaround (registry: %s). See https://github.com/buildpacks/pack/issues/2272.", workaroundRegistry) + } else if !opts.Publish && usesContainerdStorage(c.docker) { + c.logger.Warnf("Exporting to docker daemon (building without --publish) and daemon uses containerd storage; performance may be significantly degraded.\n" + + "For more information, see https://github.com/buildpacks/pack/issues/2272.") + } + if opts.Layout() { pathsConfig, err = c.processLayoutPath(opts.LayoutConfig.InputImage, opts.LayoutConfig.PreviousInputImage) if err != nil { @@ -634,16 +651,29 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { } } + effectiveImageRef := imageRef + effectivePublish := opts.Publish + effectiveInsecureRegistries := opts.InsecureRegistries + if containerdWorkaround { + effectiveImageRef = publishRef + effectivePublish = true + workaroundRegistry := publishRef.Context().RegistryStr() + regSet := stringset.FromSlice(effectiveInsecureRegistries) + if _, ok := regSet[workaroundRegistry]; !ok { + effectiveInsecureRegistries = append([]string{workaroundRegistry}, effectiveInsecureRegistries...) + } + } + lifecycleOpts := build.LifecycleOptions{ AppPath: appPath, - Image: imageRef, + Image: effectiveImageRef, Builder: ephemeralBuilder, BuilderImage: builderRef.Name(), LifecycleImage: ephemeralBuilder.Name(), RunImage: runImageName, ProjectMetadata: projectMetadata, ClearCache: opts.ClearCache, - Publish: opts.Publish, + Publish: effectivePublish, TrustBuilder: opts.TrustBuilder(opts.Builder), UseCreator: useCreator, UseCreatorWithExtensions: supportsCreatorWithExtensions(lifecycleVersion), @@ -671,7 +701,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { Keychain: c.keychain, EnableUsernsHost: opts.EnableUsernsHost, ExecutionEnvironment: opts.CNBExecutionEnv, - InsecureRegistries: opts.InsecureRegistries, + InsecureRegistries: effectiveInsecureRegistries, } switch { @@ -834,6 +864,24 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { if err = c.lifecycleExecutor.Execute(ctx, lifecycleOpts); err != nil { return fmt.Errorf("executing lifecycle: %w", err) } + + if containerdWorkaround { + // Pull the image from the registry into the daemon, then tag as the user-requested name. + rdr, pullErr := c.docker.ImagePull(ctx, publishRef.String(), client.ImagePullOptions{}) + if pullErr != nil { + return fmt.Errorf("containerd workaround: pulling image into daemon: %w", pullErr) + } + _, _ = io.Copy(io.Discard, rdr) + rdr.Close() + + if publishRef.String() != imageRef.String() { + _, tagErr := c.docker.ImageTag(ctx, client.ImageTagOptions{Source: publishRef.String(), Target: imageRef.String()}) + if tagErr != nil { + return fmt.Errorf("containerd workaround: tagging image: %w", tagErr) + } + } + } + return c.logImageNameAndSha(ctx, opts.Publish, imageRef, opts.InsecureRegistries) } From be07f29a9a334242e95354d2d56879a05db8a32b Mon Sep 17 00:00:00 2001 From: casibbald Date: Tue, 17 Feb 2026 13:30:02 +0200 Subject: [PATCH 2/3] docs: add PR description --- PR_DESCRIPTION.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 PR_DESCRIPTION.md diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 000000000..98a09d4a7 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,13 @@ +## PR Description: Fix Digest Errors on Publish-Pull + +### Problem +When using `pack` as a library within `octopilot-pipeline-tools` (`op`), we encountered digest mismatch errors during the `publish` phase, specifically in environments using `containerd` or when attempting to immediately pull a just-published image. + +This issue (referenced as #2272 in upstream discussions) prevents reliable multi-arch builds and promotions. + +### Changes +- **Publish-Then-Pull Workaround**: Implemented an optional logic to handle the publish-then-pull sequence more robustly. +- **Library Exposure**: Exposed internal `BuildOptions` and registry handling logic to allow `op` to configure authentication and lifecycle behavior programmatically. + +### Verification +Verified integration within `op`. The tool can now successfully build images using buildpacks and push them to a registry without encountering digest errors, even in diverse container runtime environments. From 59c81a3c9764f095ab78c90aa72dbd925e45cb04 Mon Sep 17 00:00:00 2001 From: casibbald Date: Sun, 1 Mar 2026 19:15:35 +0200 Subject: [PATCH 3/3] fix: containerd workaround use repository path to avoid double-prefix image ref --- pkg/client/build.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/client/build.go b/pkg/client/build.go index 645194a31..cacc66b6b 100644 --- a/pkg/client/build.go +++ b/pkg/client/build.go @@ -324,6 +324,8 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { // When the daemon uses containerd storage, exporting directly to the daemon is slow and can hit digest errors (pack#2272). // Workaround: publish to a local registry (e.g. localhost:5001), then pull into the daemon and tag as requested. + // Use only the repository path + tag (Context().RepositoryStr() + Identifier()), not the full image name, so we + // don't double-prefix when the requested image is already e.g. localhost:5001/ghcr.io/org/app:latest. var containerdWorkaround bool var publishRef name.Reference if !opts.Publish && !opts.Layout() && opts.PreviousImage == "" && usesContainerdStorage(c.docker) { @@ -332,7 +334,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { workaroundRegistry = "localhost:5001" } workaroundRegistry = strings.TrimSuffix(workaroundRegistry, "/") - publishImageStr := workaroundRegistry + "/" + imageRef.Name() + publishImageStr := workaroundRegistry + "/" + imageRef.Context().RepositoryStr() + ":" + imageRef.Identifier() publishRef, err = name.NewTag(publishImageStr, name.WeakValidation) if err != nil { return errors.Wrapf(err, "containerd workaround: invalid publish image '%s'", publishImageStr)