From 589086a579d44454ce50c89ff86e22a698bcccc0 Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Thu, 5 Sep 2024 23:26:46 +0800 Subject: [PATCH 1/2] fix: try fetch lost layer when unpack image If someone removes the content or snapshot by-pass the cri plugin, cir in-memory metadata think the image exists. When we create sandbox or container, we can't find snapshot and unpack from the content layer with the `discard_unpacked_layers` cri config. In that case, we can try fetch the lost image layer to complete create container. Signed-off-by: Yadong Ding --- image.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/image.go b/image.go index 40dc3ff6cb73c..75f22b8b24e2f 100644 --- a/image.go +++ b/image.go @@ -384,7 +384,19 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...Unpa for _, layer := range layers { unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, config.SnapshotOpts, config.ApplyOpts) if err != nil { - return err + // layer not found in content, try to fetch layer and retry unpack + if errdefs.IsNotFound(err) { + if _, err2 := i.client.Pull(ctx, i.Name()+"@"+layer.Blob.Digest.String()); err2 != nil { + return fmt.Errorf("fetch lost layer failed: %w : %w", err2, err) + } + // retry + unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, config.SnapshotOpts, config.ApplyOpts) + if err != nil { + return err + } + } else { + return err + } } if unpacked { From 84f3a9498064b5bdbb7b367f7ef0c3517cc6909d Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Thu, 12 Sep 2024 04:23:36 +0800 Subject: [PATCH 2/2] feat: allows clean layers after fetching in unpack When CRI has config `discardUnpackedLayers`, we should allows GC to clean layers up from the content store after unpacking. Use ctx to notificate unpack image to gc layer content when retry fetch lost layers. Signed-off-by: Yadong Ding --- image.go | 9 ++++++++- pkg/cri/server/container_create.go | 3 +++ pkg/cri/server/sandbox_run.go | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/image.go b/image.go index 75f22b8b24e2f..76b77ee4cb6e5 100644 --- a/image.go +++ b/image.go @@ -336,6 +336,8 @@ func WithUnpackDuplicationSuppressor(suppressor kmutex.KeyedLocker) UnpackOpt { } } +type DiscardUnpackedLayersKey struct{} + func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...UnpackOpt) error { ctx, done, err := i.client.WithLease(ctx) if err != nil { @@ -386,7 +388,12 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...Unpa if err != nil { // layer not found in content, try to fetch layer and retry unpack if errdefs.IsNotFound(err) { - if _, err2 := i.client.Pull(ctx, i.Name()+"@"+layer.Blob.Digest.String()); err2 != nil { + pullOpts := []RemoteOpt{} + if discardUnpackedLayers, ok := ctx.Value(DiscardUnpackedLayersKey{}).(bool); ok && discardUnpackedLayers { + pullOpts = append(pullOpts, WithChildLabelMap(images.ChildGCLabelsFilterLayers)) + } + + if _, err2 := i.client.Pull(ctx, i.Name(), pullOpts...); err2 != nil { return fmt.Errorf("fetch lost layer failed: %w : %w", err2, err) } // retry diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index 9483778d19e00..48929bf14a8a7 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -259,6 +259,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta }() var cntr containerd.Container + if c.config.ContainerdConfig.DiscardUnpackedLayers { + ctx = context.WithValue(ctx, containerd.DiscardUnpackedLayersKey{}, true) + } if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil { return nil, fmt.Errorf("failed to create containerd container: %w", err) } diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index ebe9facda760d..461a9d99f389f 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -173,6 +173,9 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata), containerd.WithRuntime(ociRuntime.Type, runtimeOpts)} + if c.config.ContainerdConfig.DiscardUnpackedLayers { + ctx = context.WithValue(ctx, containerd.DiscardUnpackedLayersKey{}, true) + } container, err := c.client.NewContainer(ctx, id, opts...) if err != nil { return nil, fmt.Errorf("failed to create containerd container: %w", err)