From ddf782719eba23ad181cd29947476572c9365f77 Mon Sep 17 00:00:00 2001 From: Valentin Delaye Date: Wed, 4 Mar 2026 06:48:21 +0100 Subject: [PATCH] Add CopyOption and add option for deep copy (index of index) Signed-off-by: Valentin Delaye --- src/main/java/land/oras/ContainerRef.java | 9 ++ src/main/java/land/oras/CopyUtils.java | 122 +++++++++++++----- src/main/java/land/oras/OCI.java | 17 +++ src/test/java/land/oras/DockerIoITCase.java | 12 +- .../oras/GitHubContainerRegistryITCase.java | 2 +- src/test/java/land/oras/OCILayoutTest.java | 18 +-- src/test/java/land/oras/RegistryTest.java | 58 ++++++++- 7 files changed, 187 insertions(+), 51 deletions(-) diff --git a/src/main/java/land/oras/ContainerRef.java b/src/main/java/land/oras/ContainerRef.java index 7b886e1d..9085ffe5 100644 --- a/src/main/java/land/oras/ContainerRef.java +++ b/src/main/java/land/oras/ContainerRef.java @@ -213,6 +213,15 @@ public ContainerRef withDigest(String digest) { return new ContainerRef(registry, unqualified, namespace, repository, tag, digest); } + /** + * Return a copy of reference with the given tag + * @param tag The tag + * @return The container reference with the given tag + */ + public ContainerRef withTag(String tag) { + return new ContainerRef(registry, unqualified, namespace, repository, tag, digest); + } + @Override public SupportedAlgorithm getAlgorithm() { // Default if not set diff --git a/src/main/java/land/oras/CopyUtils.java b/src/main/java/land/oras/CopyUtils.java index 302a6330..f9da4976 100644 --- a/src/main/java/land/oras/CopyUtils.java +++ b/src/main/java/land/oras/CopyUtils.java @@ -43,16 +43,41 @@ private CopyUtils() { // Utils class } + /** + * Options for copy. + * @param includeReferrers Whether to include referrers in the copy + */ + public record CopyOptions(boolean includeReferrers) { + + /** + * The default copy options with includeReferrers to false + * @return The default copy options + */ + public static CopyOptions shallow() { + return new CopyOptions(false); + } + + /** + * The copy options with includeReferrers and recursive set to true. + * @return The copy options with includeReferrers and recursive set to true + */ + public static CopyOptions deep() { + return new CopyOptions(true); + } + } + /** * Copy a container from source to target. + * @deprecated Use {@link #copy(OCI, Ref, OCI, Ref, CopyOptions)} instead. This method will be removed in a future release. * @param source The source OCI * @param sourceRef The source reference * @param target The target OCI * @param targetRef The target reference - * @param recursive Whether to copy referrers recursively + * @param recursive Copy refferers * @param The source reference type * @param The target reference type */ + @Deprecated(forRemoval = true) public static , TargetRefType extends Ref<@NonNull TargetRefType>> void copy( OCI source, @@ -60,6 +85,28 @@ void copy( OCI target, TargetRefType targetRef, boolean recursive) { + copy(source, sourceRef, target, targetRef, recursive ? CopyOptions.deep() : CopyOptions.shallow()); + } + + /** + * Copy a container from source to target. + * @param source The source OCI + * @param sourceRef The source reference + * @param target The target OCI + * @param targetRef The target reference + * @param options The copy option + * @param The source reference type + * @param The target reference type + */ + public static , TargetRefType extends Ref<@NonNull TargetRefType>> + void copy( + OCI source, + SourceRefType sourceRef, + OCI target, + TargetRefType targetRef, + CopyOptions options) { + + boolean includeReferrers = options.includeReferrers(); Descriptor descriptor = source.probeDescriptor(sourceRef); @@ -79,22 +126,22 @@ void copy( SourceRefType effectiveSourceRef = sourceRef.forTarget(source).forTarget(resolveSourceRegistry); TargetRefType effectiveTargetRef = targetRef.forTarget(target).forTarget(effectiveTargetRegistry); - // Write all layer - for (Layer layer : source.collectLayers(effectiveSourceRef, contentType, true)) { - Objects.requireNonNull(layer.getDigest(), "Layer digest is required for streaming copy"); - Objects.requireNonNull(layer.getSize(), "Layer size is required for streaming copy"); - LOG.debug("Copying layer {}", layer.getDigest()); - target.pushBlob( - effectiveTargetRef.withDigest(layer.getDigest()), - layer.getSize(), - () -> source.fetchBlob(effectiveSourceRef.withDigest(layer.getDigest())), - layer.getAnnotations()); - LOG.debug("Copied layer {}", layer.getDigest()); - } - // Single manifest if (source.isManifestMediaType(contentType)) { + // Write all layer + for (Layer layer : source.collectLayers(effectiveSourceRef, contentType, true)) { + Objects.requireNonNull(layer.getDigest(), "Layer digest is required for streaming copy"); + Objects.requireNonNull(layer.getSize(), "Layer size is required for streaming copy"); + LOG.debug("Copying layer {}", layer.getDigest()); + target.pushBlob( + effectiveTargetRef.withDigest(layer.getDigest()), + layer.getSize(), + () -> source.fetchBlob(effectiveSourceRef.withDigest(layer.getDigest())), + layer.getAnnotations()); + LOG.debug("Copied layer {}", layer.getDigest()); + } + // Write manifest as any blob Manifest manifest = source.getManifest(effectiveSourceRef); String tag = effectiveSourceRef.getTag(); @@ -109,18 +156,20 @@ void copy( target.pushManifest(effectiveTargetRef.withDigest(tag), manifest); LOG.debug("Copied manifest {}", manifestDigest); - if (recursive) { - LOG.debug("Recursively copy referrers"); + if (includeReferrers) { + LOG.debug("Including referrers on copy of manifest {}", manifestDigest); Referrers referrers = source.getReferrers(effectiveSourceRef.withDigest(manifestDigest), null); for (ManifestDescriptor referer : referrers.getManifests()) { - LOG.info("Copy reference {}", referer.getDigest()); + LOG.debug("Copy reference from referrers {}", referer.getDigest()); copy( source, effectiveSourceRef.withDigest(referer.getDigest()), target, effectiveTargetRef, - recursive); + options); } + } else { + LOG.debug("Not including referrers on copy of manifest {}", manifestDigest); } } @@ -132,21 +181,36 @@ else if (source.isIndexMediaType(contentType)) { // Write all manifests and their config for (ManifestDescriptor manifestDescriptor : index.getManifests()) { - Manifest manifest = source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest())); - - // Push config - copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef); - // Push the manifest - LOG.debug("Copying manifest {}", manifestDigest); - target.pushManifest( - effectiveTargetRef.withDigest(manifest.getDigest()), - manifest.withDescriptor(manifestDescriptor)); - LOG.debug("Copied manifest {}", manifestDigest); + // Copy manifest + if (source.isManifestMediaType(manifestDescriptor.getMediaType())) { + Manifest manifest = + source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest())); + + // Push config + copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef); + + // Push the manifest + LOG.debug("Copying nested manifest {}", manifestDescriptor.getDigest()); + target.pushManifest( + effectiveTargetRef.withDigest(manifest.getDigest()), + manifest.withDescriptor(manifestDescriptor)); + LOG.debug("Copied nested manifest {}", manifestDescriptor.getDigest()); + } else if (source.isIndexMediaType(manifestDescriptor.getMediaType())) { + // Copy index of index + LOG.debug("Copying nested index {}", manifestDescriptor.getDigest()); + copy( + source, + effectiveSourceRef.withDigest(manifestDescriptor.getDigest()), + target, + effectiveTargetRef.withDigest(manifestDescriptor.getDigest()), + options); + LOG.debug("Copied nested index {}", manifestDescriptor.getDigest()); + } } LOG.debug("Copying index {}", manifestDigest); - target.pushIndex(effectiveTargetRef.withDigest(tag), index); + index = target.pushIndex(effectiveTargetRef.withDigest(tag), index); LOG.debug("Copied index {}", manifestDigest); } else { diff --git a/src/main/java/land/oras/OCI.java b/src/main/java/land/oras/OCI.java index f8be7dd2..1b8fa43a 100644 --- a/src/main/java/land/oras/OCI.java +++ b/src/main/java/land/oras/OCI.java @@ -133,6 +133,23 @@ protected List collectLayers(T ref, String contentType, boolean includeAl } Index index = getIndex(ref); for (ManifestDescriptor manifestDescriptor : index.getManifests()) { + String manifestContentType = manifestDescriptor.getMediaType(); + // We just skip unknown media type descriptor + if (!isManifestMediaType(manifestContentType) && !isIndexMediaType(manifestContentType)) { + LOG.info( + "Unrecognized content type {}, skipping descriptor {}", + manifestContentType, + manifestDescriptor.getDigest()); + continue; + } + // Shallow + if (isIndexMediaType(manifestContentType)) { + LOG.debug( + "Not collecting layers for index media type manifest descriptor: {}", + manifestDescriptor.getDigest()); + continue; + } + // Collect layer for each manifest List manifestLayers = getManifest(ref.withDigest(manifestDescriptor.getDigest())).getLayers(); for (Layer manifestLayer : manifestLayers) { diff --git a/src/test/java/land/oras/DockerIoITCase.java b/src/test/java/land/oras/DockerIoITCase.java index 4d374a9f..d013829e 100644 --- a/src/test/java/land/oras/DockerIoITCase.java +++ b/src/test/java/land/oras/DockerIoITCase.java @@ -99,13 +99,8 @@ void shouldCopyTagToInternalRegistry() { ContainerRef containerTarget = ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry())); - // CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true); - // assertTrue(targetRegistry.exists(containerTarget)); - - Index index = targetRegistry.getIndex(containerSource); - - // Ensure standard platform matching - assertTrue(index.getManifests().stream().anyMatch(m -> m.getPlatform().equals(Platform.linux386()))); + CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep()); + assertTrue(targetRegistry.exists(containerTarget)); } @Test @@ -156,7 +151,8 @@ void shouldCopyTagToInternalRegistryViaAlias(@TempDir Path homeDir) throws Excep ContainerRef containerTarget = ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry())); - CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true); + CopyUtils.copy( + sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep()); assertTrue(targetRegistry.exists(containerTarget)); }); } diff --git a/src/test/java/land/oras/GitHubContainerRegistryITCase.java b/src/test/java/land/oras/GitHubContainerRegistryITCase.java index 6740d08a..16f588fc 100644 --- a/src/test/java/land/oras/GitHubContainerRegistryITCase.java +++ b/src/test/java/land/oras/GitHubContainerRegistryITCase.java @@ -120,7 +120,7 @@ void shouldCopyTagToInternalRegistry() { ContainerRef containerTarget = ContainerRef.parse("%s/docker/library/oras:main".formatted(unsecureRegistry.getRegistry())); - CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true); + CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep()); assertTrue(targetRegistry.exists(containerTarget)); } } diff --git a/src/test/java/land/oras/OCILayoutTest.java b/src/test/java/land/oras/OCILayoutTest.java index 0767c71d..ebd6d0cf 100644 --- a/src/test/java/land/oras/OCILayoutTest.java +++ b/src/test/java/land/oras/OCILayoutTest.java @@ -705,7 +705,7 @@ void testShouldCopyArtifactFromRegistryIntoOciLayout() throws IOException { registry.attachArtifact(containerRef, ArtifactType.from("application/foo"), LocalPath.of(file2)); // Copy to oci layout - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, false); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.shallow()); assertOciLayout(layoutPath); @@ -736,7 +736,7 @@ void testShouldCopyFromOciLayoutIntoOciLayoutRecursive() throws IOException { OCILayout target = OCILayout.builder().defaults(targetRef.getFolder()).build(); // Copy to oci layout - CopyUtils.copy(source, sourceRef, target, targetRef, true); + CopyUtils.copy(source, sourceRef, target, targetRef, CopyUtils.CopyOptions.deep()); // Assertion assertOciLayout(ociLayoutPath); @@ -766,7 +766,7 @@ void testShouldCopyFromOciLayoutIntoOciLayoutNonRecursive() throws IOException { OCILayout target = OCILayout.builder().defaults(targetRef.getFolder()).build(); // Copy to oci layout - CopyUtils.copy(source, sourceRef, target, targetRef, false); + CopyUtils.copy(source, sourceRef, target, targetRef, CopyUtils.CopyOptions.shallow()); // Assertion assertOciLayout(ociLayoutPath); @@ -814,7 +814,7 @@ void testShouldCopyRecursivelyArtifactFromRegistryIntoOciLayout() throws IOExcep LocalPath.of(file3)); // Copy to oci layout - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); assertOciLayout(layoutPath); @@ -865,7 +865,7 @@ void testShouldCopyImageIntoOciLayoutWithoutIndexAndTag() { Manifest pushedManifest = registry.pushManifest(containerRef, emptyManifest); // Copy to oci layout - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); assertOciLayout(layoutPath); @@ -895,7 +895,7 @@ void testShouldCopyImageIntoOciLayoutWithoutIndexAndTag() { assertLayerExists(layoutPath, layer2); // Copy to oci layout again - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); // Check manifest exists assertBlobExists(layoutPath, pushedManifest.getDescriptor().getDigest()); @@ -941,7 +941,7 @@ void testShouldCopyImageIntoOciLayoutWithIndex() { Index index = registry.pushIndex(containerRef, Index.fromManifests(List.of(pushedManifest.getDescriptor()))); // Copy to oci layout - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); assertOciLayout(layoutPathIndex); @@ -971,7 +971,7 @@ void testShouldCopyImageIntoOciLayoutWithIndex() { assertBlobExists(layoutPathIndex, pushedManifest.getDescriptor().getDigest()); // Copy to oci layout again - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); // Check manifest exists assertLayerExists(layoutPathIndex, layer1); @@ -1007,7 +1007,7 @@ void testShouldCopyIntoOciLayoutWithBlobConfig() throws IOException { containerRef, ArtifactType.from("my/artifact"), Annotations.empty(), config, LocalPath.of(file1)); // Copy to oci layout - CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, true); + CopyUtils.copy(registry, containerRef, ociLayout, layoutRef, CopyUtils.CopyOptions.deep()); assertOciLayout(layoutPath); diff --git a/src/test/java/land/oras/RegistryTest.java b/src/test/java/land/oras/RegistryTest.java index 3aea7370..33fa80a1 100644 --- a/src/test/java/land/oras/RegistryTest.java +++ b/src/test/java/land/oras/RegistryTest.java @@ -936,6 +936,55 @@ void testShouldFailReferrerWithoutDigest() { "Digest is required to get referrers"); } + @Test + void testShouldCopyIndexOfIndexArtifactFromRegistryIntoRegistryWithDeepCopy() throws IOException { + // Copy to same registry + Registry registry = Registry.Builder.builder() + .defaults("myuser", "mypass") + .withInsecure(true) + .build(); + + ContainerRef containerSource = + ContainerRef.parse("%s/library/artifact-source".formatted(this.registry.getRegistry())); + + // Create files + Path file1 = blobDir.resolve("source1.txt"); + Files.writeString(file1, "foobar"); + Path file2 = blobDir.resolve("source2.txt"); + Files.writeString(file2, "barfoo"); + + // Push individual manifests + Manifest manifest1 = registry.pushArtifact(containerSource.withTag("manifest1"), LocalPath.of(file1)); + Manifest manifest2 = registry.pushArtifact(containerSource.withTag("manifest2"), LocalPath.of(file2)); + + assertNotNull(manifest1.getDescriptor(), "Manifest 1 descriptor should not be null"); + assertNotNull(manifest2.getDescriptor(), "Manifest 2 descriptor should not be null"); + + Index index1 = Index.fromManifests(List.of(manifest1.getDescriptor())); + + // Push first index + index1 = registry.pushIndex(containerSource.withTag("index1"), index1); + assertNotNull(index1.getDescriptor(), "Index 1 descriptor should not be null"); + + Index index2 = Index.fromManifests(List.of(manifest2.getDescriptor(), index1.getDescriptor())); + index2 = registry.pushIndex(containerSource.withTag("index2"), index2); + + // Copy to other registry + try (RegistryContainer otherRegistryContainer = new RegistryContainer()) { + otherRegistryContainer.start(); + ContainerRef containerTarget = + ContainerRef.parse("%s/library/artifact-target".formatted(otherRegistryContainer.getRegistry())); + CopyUtils.copy( + registry, + containerSource.withTag("index2"), + registry, + containerTarget.withTag("index2"), + CopyUtils.CopyOptions.deep()); + Index index = registry.getIndex(containerTarget.withTag("index2")); + assertEquals(2, index.getManifests().size(), "Index should have 1 manifests due to shallow copy"); + } + } + @Test void testShouldCopySingleArtifactFromRegistryIntoRegistry() throws IOException { // Copy to same registry @@ -958,7 +1007,7 @@ void testShouldCopySingleArtifactFromRegistryIntoRegistry() throws IOException { otherRegistryContainer.start(); ContainerRef containerTarget = ContainerRef.parse("%s/library/artifact-target".formatted(otherRegistryContainer.getRegistry())); - CopyUtils.copy(registry, containerSource, registry, containerTarget, false); + CopyUtils.copy(registry, containerSource, registry, containerTarget, CopyUtils.CopyOptions.shallow()); registry.pullArtifact(containerTarget, artifactDir, true); assertEquals("foobar", Files.readString(artifactDir.resolve("source.txt"))); } @@ -1051,7 +1100,8 @@ void testShouldCopyFromAliasToAlias(@TempDir Path homeDir) throws Exception { // Copy to other registry ContainerRef containerTarget = ContainerRef.parse("the-target"); - CopyUtils.copy(registry, containerSource, registry, containerTarget, false); + CopyUtils.copy( + registry, containerSource, registry, containerTarget, CopyUtils.CopyOptions.shallow()); registry.pullArtifact(containerTarget, artifactDir, true); assertEquals("foobar", Files.readString(artifactDir.resolve("source.txt"))); } catch (Exception e) { @@ -1076,7 +1126,7 @@ void testShouldCopyFromOciLayoutToRegistryNonRecursive() throws IOException { OCILayout ociLayout = OCILayout.Builder.builder().defaults(layoutRef.getFolder()).build(); - CopyUtils.copy(ociLayout, layoutRef, registry, targetRef, false); + CopyUtils.copy(ociLayout, layoutRef, registry, targetRef, CopyUtils.CopyOptions.shallow()); // Pull Path extractPath = artifactDir.resolve("testShouldCopyFromOciLayoutToRegistryNonRecursive"); @@ -1113,7 +1163,7 @@ void testShouldCopyFromOciLayoutToRegistryRecursive() throws IOException { OCILayout ociLayout = OCILayout.Builder.builder().defaults(layoutRef.getFolder()).build(); - CopyUtils.copy(ociLayout, layoutRef, registry, targetRef, true); + CopyUtils.copy(ociLayout, layoutRef, registry, targetRef, CopyUtils.CopyOptions.deep()); // Pull Path extractPath = artifactDir.resolve("testShouldCopyFromOciLayoutToRegistryRecursive");