Skip to content

Add CopyOption and add option for deep copy (index of index) #610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jonesbusy wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add CopyOption and add option for deep copy (index of index) #610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/main/java/land/oras/ContainerRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,15 @@ public ContainerRef withDigest(String digest) {
return new ContainerRef(registry, unqualified, namespace, repository, tag, digest);
}

/**
* Return a copy of reference with the given tag
* @param tag The tag
* @return The container reference with the given tag
*/
public ContainerRef withTag(String tag) {
return new ContainerRef(registry, unqualified, namespace, repository, tag, digest);
}

@Override
public SupportedAlgorithm getAlgorithm() {
// Default if not set
Expand Down
126 changes: 110 additions & 16 deletions src/main/java/land/oras/CopyUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,90 @@ private CopyUtils() {
// Utils class
}

/**
* Options for copy.
* @param includeReferrers Whether to include referrers in the copy
* @param recursive Recursive copy that include index of index. If false will copy manifests media type when encountered inside index
*/
public record CopyOptions(boolean includeReferrers, boolean recursive) {

/**
* The default copy options with includeReferrers and recursive set to false.
* @return The default copy options
*/
public static CopyOptions shallow() {
return new CopyOptions(false, false);
}

/**
* The copy options with includeReferrers and recursive set to true.
* @return The copy options with includeReferrers and recursive set to true
*/
public static CopyOptions deep() {
return new CopyOptions(true, true);
}

/**
* Create a new copy options with default values.
* @param includeReferrers Whether to include referrers in the copy
* @return The copy options
*/
public CopyOptions withIncludeReferrers(boolean includeReferrers) {
return new CopyOptions(includeReferrers, this.recursive);
}

/**
* Create a new copy options with default values.
* @param recursive Recursive copy that include index of index. If false will copy manifests media type when encountered inside index
* @return The copy options
*/
public CopyOptions withRecursive(boolean recursive) {
return new CopyOptions(this.includeReferrers, recursive);
}
}

/**
* Copy a container from source to target.
* @deprecated Use {@link #copy(OCI, Ref, OCI, Ref, CopyOptions)} instead. This method will be removed in a future release.
* @param source The source OCI
* @param sourceRef The source reference
* @param target The target OCI
* @param targetRef The target reference
* @param recursive Whether to copy referrers recursively
* @param recursive Copy refferers
* @param <SourceRefType> The source reference type
* @param <TargetRefType> The target reference type
*/
@Deprecated(forRemoval = true)
public static <SourceRefType extends Ref<@NonNull SourceRefType>, TargetRefType extends Ref<@NonNull TargetRefType>>
void copy(
OCI<SourceRefType> source,
SourceRefType sourceRef,
OCI<TargetRefType> target,
TargetRefType targetRef,
boolean recursive) {
copy(source, sourceRef, target, targetRef, CopyOptions.shallow().withRecursive(recursive));
}

/**
* Copy a container from source to target.
* @param source The source OCI
* @param sourceRef The source reference
* @param target The target OCI
* @param targetRef The target reference
* @param options The copy option
* @param <SourceRefType> The source reference type
* @param <TargetRefType> The target reference type
*/
public static <SourceRefType extends Ref<@NonNull SourceRefType>, TargetRefType extends Ref<@NonNull TargetRefType>>
void copy(
OCI<SourceRefType> source,
SourceRefType sourceRef,
OCI<TargetRefType> target,
TargetRefType targetRef,
CopyOptions options) {

boolean recursive = options.recursive();
boolean includeReferrers = options.includeReferrers();

Descriptor descriptor = source.probeDescriptor(sourceRef);

Expand All @@ -80,7 +147,7 @@ void copy(
TargetRefType effectiveTargetRef = targetRef.forTarget(target).forTarget(effectiveTargetRegistry);

// Write all layer
for (Layer layer : source.collectLayers(effectiveSourceRef, contentType, true)) {
for (Layer layer : source.collectLayers(effectiveSourceRef, contentType, true, false)) {
Objects.requireNonNull(layer.getDigest(), "Layer digest is required for streaming copy");
Objects.requireNonNull(layer.getSize(), "Layer size is required for streaming copy");
LOG.debug("Copying layer {}", layer.getDigest());
Expand Down Expand Up @@ -109,18 +176,20 @@ void copy(
target.pushManifest(effectiveTargetRef.withDigest(tag), manifest);
LOG.debug("Copied manifest {}", manifestDigest);

if (recursive) {
LOG.debug("Recursively copy referrers");
if (includeReferrers) {
LOG.debug("Including referrers on copy of manifest {}", manifestDigest);
Referrers referrers = source.getReferrers(effectiveSourceRef.withDigest(manifestDigest), null);
for (ManifestDescriptor referer : referrers.getManifests()) {
LOG.info("Copy reference {}", referer.getDigest());
LOG.debug("Copy reference from referrers {}", referer.getDigest());
copy(
source,
effectiveSourceRef.withDigest(referer.getDigest()),
target,
effectiveTargetRef,
recursive);
options);
}
} else {
LOG.debug("Not including referrers on copy of manifest {}", manifestDigest);
}

}
Expand All @@ -132,21 +201,46 @@ else if (source.isIndexMediaType(contentType)) {

// Write all manifests and their config
for (ManifestDescriptor manifestDescriptor : index.getManifests()) {
Manifest manifest = source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest()));

// Push config
copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef);
if (!recursive && source.isIndexMediaType(manifestDescriptor.getMediaType())) {
LOG.debug(
"Encountered index media type {} in index {}, skipping manifest {} due to non-recursive copy",
manifestDescriptor.getMediaType(),
manifestDigest,
manifestDescriptor.getDigest());
index = index.removeDescriptor(manifestDescriptor);
continue;
}

// Copy manifest
if (source.isManifestMediaType(manifestDescriptor.getMediaType())) {
Manifest manifest =
source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest()));

// Push config
copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef);

// Push the manifest
LOG.debug("Copying manifest {}", manifestDigest);
target.pushManifest(
effectiveTargetRef.withDigest(manifest.getDigest()),
manifest.withDescriptor(manifestDescriptor));
LOG.debug("Copied manifest {}", manifestDigest);
// Push the manifest
LOG.debug("Copying nested manifest {}", manifestDigest);
target.pushManifest(
effectiveTargetRef.withDigest(manifest.getDigest()),
manifest.withDescriptor(manifestDescriptor));
LOG.debug("Copied nested manifest {}", manifestDigest);
} else if (source.isIndexMediaType(manifestDescriptor.getMediaType())) {
// Copy index of index
LOG.debug("Copying nested index {}", manifestDigest);
copy(
source,
effectiveSourceRef.withDigest(manifestDescriptor.getDigest()),
target,
effectiveTargetRef.withDigest(manifestDescriptor.getDigest()),
options);
LOG.debug("Copied nested index {}", manifestDigest);
}
}

LOG.debug("Copying index {}", manifestDigest);
target.pushIndex(effectiveTargetRef.withDigest(tag), index);
index = target.pushIndex(effectiveTargetRef.withDigest(tag), index);
LOG.debug("Copied index {}", manifestDigest);

} else {
Expand Down
24 changes: 24 additions & 0 deletions src/main/java/land/oras/Index.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,30 @@ public String getArtifactTypeAsString() {
return artifactType;
}

/**
* Return a new index with the given manifest descriptor removed from the index
* @param descriptor The manifest descriptor to remove
* @return The index with the manifest descriptor removed
*/
public Index removeDescriptor(ManifestDescriptor descriptor) {
List<ManifestDescriptor> newManifests = new LinkedList<>();
for (ManifestDescriptor d : manifests) {
if (!d.getDigest().equals(descriptor.getDigest())) {
newManifests.add(ManifestDescriptor.fromJson(d.toJson()));
}
}
return new Index(
schemaVersion,
mediaType,
ArtifactType.from(artifactType),
newManifests,
annotations,
subject,
this.descriptor,
registry,
json);
}

/**
* Return a new index with new manifest added to index
* @param manifest The manifest
Expand Down
50 changes: 38 additions & 12 deletions src/main/java/land/oras/OCI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,28 +124,54 @@ public Layer pushBlob(T ref, InputStream input) {
* @param ref The ref
* @param contentType The content type
* @param includeAll Include all layers or only the ones with title annotation
* @param deep Whether to collect layers of index of index
* @return The layers
*/
protected List<Layer> collectLayers(T ref, String contentType, boolean includeAll) {
protected List<Layer> collectLayers(T ref, String contentType, boolean includeAll, boolean deep) {
List<Layer> layers = new LinkedList<>();
if (isManifestMediaType(contentType)) {
return getManifest(ref).getLayers();
}
Index index = getIndex(ref);
for (ManifestDescriptor manifestDescriptor : index.getManifests()) {
List<Layer> manifestLayers =
getManifest(ref.withDigest(manifestDescriptor.getDigest())).getLayers();
for (Layer manifestLayer : manifestLayers) {
if (manifestLayer.getAnnotations().isEmpty()
|| !manifestLayer.getAnnotations().containsKey(Const.ANNOTATION_TITLE)) {
if (includeAll) {
LOG.debug("Including layer without title annotation: {}", manifestLayer.getDigest());
layers.add(manifestLayer);
String manifestContentType = manifestDescriptor.getMediaType();
// We just skip unknown media type descriptor
if (!isManifestMediaType(manifestContentType) && !isIndexMediaType(manifestContentType)) {
LOG.info(
"Unrecognized content type {}, skipping descriptor {}",
manifestContentType,
manifestDescriptor.getDigest());
continue;
}
// Shallow
if (!deep && isIndexMediaType(manifestContentType)) {
LOG.debug(
"Skipping index of index due to shallow copy of manifest with digest {}",
manifestDescriptor.getDigest());
continue;
}
// Deep copy
if (isIndexMediaType(manifestContentType)) {
LOG.debug("Collecting layers from index of index with digest {}", manifestDescriptor.getDigest());
layers.addAll(collectLayers(
ref.withDigest(manifestDescriptor.getDigest()), manifestContentType, includeAll, deep));
}
// Collect layer for each manifest
else {
List<Layer> manifestLayers = getManifest(ref.withDigest(manifestDescriptor.getDigest()))
.getLayers();
for (Layer manifestLayer : manifestLayers) {
if (manifestLayer.getAnnotations().isEmpty()
|| !manifestLayer.getAnnotations().containsKey(Const.ANNOTATION_TITLE)) {
if (includeAll) {
LOG.debug("Including layer without title annotation: {}", manifestLayer.getDigest());
layers.add(manifestLayer);
}
LOG.debug("Skipping layer without title annotation: {}", manifestLayer.getDigest());
continue;
}
LOG.debug("Skipping layer without title annotation: {}", manifestLayer.getDigest());
continue;
layers.add(manifestLayer);
}
layers.add(manifestLayer);
}
}
return layers;
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/land/oras/Registry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ public void pullArtifact(ContainerRef containerRef, Path path, boolean overwrite
}
// Only collect layer that are files
String contentType = getContentType(ref);
List<Layer> layers = collectLayers(ref, contentType, false);
List<Layer> layers = collectLayers(ref, contentType, false, false);
if (layers.isEmpty()
|| layers.stream().noneMatch(layer -> layer.getAnnotations().containsKey(Const.ANNOTATION_TITLE))) {
LOG.info("Skipped pulling layers without file name in '{}'", Const.ANNOTATION_TITLE);
Expand Down
12 changes: 4 additions & 8 deletions src/test/java/land/oras/DockerIoITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,8 @@ void shouldCopyTagToInternalRegistry() {
ContainerRef containerTarget =
ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry()));

// CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
// assertTrue(targetRegistry.exists(containerTarget));

Index index = targetRegistry.getIndex(containerSource);

// Ensure standard platform matching
assertTrue(index.getManifests().stream().anyMatch(m -> m.getPlatform().equals(Platform.linux386())));
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
assertTrue(targetRegistry.exists(containerTarget));
}

@Test
Expand Down Expand Up @@ -156,7 +151,8 @@ void shouldCopyTagToInternalRegistryViaAlias(@TempDir Path homeDir) throws Excep
ContainerRef containerTarget =
ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry()));

CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
CopyUtils.copy(
sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
assertTrue(targetRegistry.exists(containerTarget));
});
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/land/oras/GitHubContainerRegistryITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ void shouldCopyTagToInternalRegistry() {
ContainerRef containerTarget =
ContainerRef.parse("%s/docker/library/oras:main".formatted(unsecureRegistry.getRegistry()));

CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
assertTrue(targetRegistry.exists(containerTarget));
}
}
Loading
Loading