Skip to content

Commit

Permalink
copy: implement instanceCopyClone for zstd compression
Browse files Browse the repository at this point in the history
* copy.Options now contains a new field `EnsureCompressionVariantExists
  map[string]int` which allows users to specify if they want to clone
images with specified `compression`.

* copyMultipleImages now implements `instanceCopyClone` and extends
  function specifically for `zstd` compression.

Signed-off-by: Aditya R <[email protected]>
  • Loading branch information
flouthoc committed Jun 30, 2023
1 parent c7f692d commit 97d07ec
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 14 deletions.
10 changes: 10 additions & 0 deletions copy/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache"
compression "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
"github.com/containers/image/v5/transports"
Expand All @@ -38,6 +39,11 @@ var (
maxParallelDownloads = uint(6)
)

type CopyOptionCompressionVariant struct {
Algorithm *compression.Algorithm
Level int
}

const (
// CopySystemImage is the default value which, when set in
// Options.ImageListSelection, indicates that the caller expects only one
Expand Down Expand Up @@ -126,6 +132,10 @@ type Options struct {
// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
// to not indicate "nondistributable".
DownloadForeignLayers bool

// Contains a map of compression algorithm and compression level, where c/image will ensure that selected
// images in the manifest list will be replicated with provided compression algorithms.
EnsureCompressionVariantExists []CopyOptionCompressionVariant
}

// copier allows us to keep track of diffID values for blobs, and other
Expand Down
61 changes: 52 additions & 9 deletions copy/multiple.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/containers/image/v5/internal/image"
internalManifest "github.com/containers/image/v5/internal/manifest"
"github.com/containers/image/v5/manifest"
compressionTypes "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/signature"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
Expand All @@ -26,25 +27,42 @@ const (
)

type instanceCopy struct {
op instanceCopyKind
sourceDigest digest.Digest
op instanceCopyKind
sourceDigest digest.Digest
compressionVariant CopyOptionCompressionVariant
}

// prepareInstanceCopies prepares a list of instances which needs to copied to the manifest list.
func prepareInstanceCopies(instanceDigests []digest.Digest, options *Options) []instanceCopy {
func prepareInstanceCopies(instanceDigests []digest.Digest, options *Options) ([]instanceCopy, error) {
res := []instanceCopy{}
for i, instanceDigest := range instanceDigests {
if options.ImageListSelection == CopySpecificImages &&
!slices.Contains(options.Instances, instanceDigest) {
logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests))
continue
}
for _, compressionVariant := range options.EnsureCompressionVariantExists {
switch compressionVariant.Algorithm.Name() {
case compressionTypes.ZstdAlgorithmName:
res = append(res, instanceCopy{
op: instanceCopyClone,
sourceDigest: instanceDigest,
compressionVariant: compressionVariant,
})
case compressionTypes.GzipAlgorithmName:
continue
default:
return res, fmt.Errorf("unsupported compression algorithm for instance %s: %s", instanceDigest, compressionVariant.Algorithm.Name())
}
}
noCompressionVariant := CopyOptionCompressionVariant{Algorithm: nil, Level: -1}
res = append(res, instanceCopy{
op: instanceCopyCopy,
sourceDigest: instanceDigest,
op: instanceCopyCopy,
sourceDigest: instanceDigest,
compressionVariant: noCompressionVariant,
})
}
return res
return res, nil
}

// copyMultipleImages copies some or all of an image list's instances, using
Expand Down Expand Up @@ -119,7 +137,10 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
// Copy each image, or just the ones we want to copy, in turn.
instanceDigests := updatedList.Instances()
instanceEdits := []internalManifest.ListEdit{}
instanceCopyList := prepareInstanceCopies(instanceDigests, options)
instanceCopyList, err := prepareInstanceCopies(instanceDigests, options)
if err != nil {
return nil, fmt.Errorf("while preparing instances for copy: %w", err)
}
c.Printf("Copying %d of %d images in list\n", len(instanceCopyList), len(instanceDigests))
for i, instance := range instanceCopyList {
// Update instances to be edited by their `ListOperation` and
Expand All @@ -140,8 +161,30 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
UpdateDigest: updatedManifestDigest,
UpdateSize: int64(len(updatedManifest)),
UpdateMediaType: updatedManifestType})
default:
return nil, fmt.Errorf("copying image: invalid copy operation %d", instance.op)
case instanceCopyClone:
c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest)
addCompressionAlgorithms := []compressionTypes.Algorithm{}
if instance.compressionVariant.Algorithm != nil {
// copy options with new compression format
optionsClone := options
optionsClone.DestinationCtx.CompressionFormat = instance.compressionVariant.Algorithm
options.DestinationCtx.CompressionLevel = &instance.compressionVariant.Level
addCompressionAlgorithms = append(addCompressionAlgorithms, *instance.compressionVariant.Algorithm)
}
addManifest, addManifestType, addManifestDigest, err := c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, &instanceCopyList[i].sourceDigest)
if err != nil {
return nil, fmt.Errorf("Replicating image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err)
}
// Record the result of a possible conversion here.
instanceEdits = append(instanceEdits, internalManifest.ListEdit{
ListOperation: internalManifest.ListOpAdd,
AddDigest: addManifestDigest,
AddSize: int64(len(addManifest)),
AddMediaType: addManifestType,
AddCompressionAlgorithms: addCompressionAlgorithms,
})
}
}

Expand Down
12 changes: 7 additions & 5 deletions copy/multiple_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,24 @@ func TestPrepareCopyInstances(t *testing.T) {
digest.Digest("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"),
}

instancesToCopy := prepareInstanceCopies(sourceInstances, &Options{})
instancesToCopy, _ := prepareInstanceCopies(sourceInstances, &Options{})
compare := []instanceCopy{}
noCompressionVariant := CopyOptionCompressionVariant{Algorithm: nil, Level: -1}
for _, instance := range sourceInstances {
compare = append(compare, instanceCopy{op: instanceCopyCopy,
sourceDigest: instance})
sourceDigest: instance, compressionVariant: noCompressionVariant})
}
assert.Equal(t, instancesToCopy, compare)

// Test with CopySpecific Images
copyOnly := []digest.Digest{
digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
}
instancesToCopy = prepareInstanceCopies(sourceInstances, &Options{
instancesToCopy, _ = prepareInstanceCopies(sourceInstances, &Options{
Instances: copyOnly,
ImageListSelection: CopySpecificImages})
assert.Equal(t, instancesToCopy, []instanceCopy{{
op: instanceCopyCopy,
sourceDigest: copyOnly[0]}})
op: instanceCopyCopy,
sourceDigest: copyOnly[0],
compressionVariant: noCompressionVariant}})
}

0 comments on commit 97d07ec

Please sign in to comment.