Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

copy: implement instanceCopyClone for zstd compression #1987

Merged
merged 11 commits into from
Jul 26, 2023
Merged
22 changes: 22 additions & 0 deletions copy/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache"
compression "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
"github.com/containers/image/v5/transports"
Expand Down Expand Up @@ -126,6 +127,21 @@ type Options struct {
// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
// to not indicate "nondistributable".
DownloadForeignLayers bool

// Contains slice of OptionCompressionVariant, where copy will ensure that for each platform
// in the manifest list, a variant with the requested compression will exist.
// Invalid when copying a non-multi-architecture image. That will probably
// change in the future.
EnsureCompressionVariantsExist []OptionCompressionVariant
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
}

// OptionCompressionVariant allows to supply information about
// selected compression algorithm and compression level by the
// end-user. Refer to EnsureCompressionVariantsExist to know
// more about its usage.
type OptionCompressionVariant struct {
flouthoc marked this conversation as resolved.
Show resolved Hide resolved
Algorithm compression.Algorithm
Level *int // Only used when we are creating a new image instance using the specified algorithm, not when the image already contains such an instance
}

// copier allows us to keep track of diffID values for blobs, and other
Expand Down Expand Up @@ -250,13 +266,19 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
}

if !multiImage {
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
// The simple case: just copy a single image.
single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
if err != nil {
return nil, err
}
copiedManifest = single.manifest
} else if c.options.ImageListSelection == CopySystemImage {
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
// This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that
// matches the current system to copy, and copy it.
mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx)
Expand Down
131 changes: 127 additions & 4 deletions copy/multiple.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"

"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
internalManifest "github.com/containers/image/v5/internal/manifest"
"github.com/containers/image/v5/internal/set"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/compression"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)

Expand All @@ -27,23 +31,118 @@ const (
type instanceCopy struct {
op instanceCopyKind
sourceDigest digest.Digest

// Fields which can be used by callers when operation
// is `instanceCopyClone`
cloneCompressionVariant OptionCompressionVariant
clonePlatform *imgspecv1.Platform
cloneAnnotations map[string]string
}

// internal type only to make imgspecv1.Platform comparable
type platformComparable struct {
architecture string
os string
osVersion string
osFeatures string
variant string
}

// Converts imgspecv1.Platform to a comparable format.
func platformV1ToPlatformComparable(platform *imgspecv1.Platform) platformComparable {
if platform == nil {
return platformComparable{}
}
osFeatures := slices.Clone(platform.OSFeatures)
sort.Strings(osFeatures)
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
return platformComparable{architecture: platform.Architecture,
os: platform.OS,
// This is strictly speaking ambiguous, fields of OSFeatures can contain a ','. Probably good enough for now.
osFeatures: strings.Join(osFeatures, ","),
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
osVersion: platform.OSVersion,
variant: platform.Variant,
}
}

// platformCompressionMap prepares a mapping of platformComparable -> CompressionAlgorithmNames for given digests
func platformCompressionMap(list internalManifest.List, instanceDigests []digest.Digest) (map[platformComparable]*set.Set[string], error) {
res := make(map[platformComparable]*set.Set[string])
for _, instanceDigest := range instanceDigests {
instanceDetails, err := list.Instance(instanceDigest)
if err != nil {
return nil, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
}
platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
platformSet, ok := res[platform]
if !ok {
platformSet = set.New[string]()
res[platform] = platformSet
}
platformSet.AddSlice(instanceDetails.ReadOnly.CompressionAlgorithmNames)
}
return res, nil
}

func validateCompressionVariantExists(input []OptionCompressionVariant) error {
for _, option := range input {
_, err := compression.AlgorithmByName(option.Algorithm.Name())
if err != nil {
return fmt.Errorf("invalid algorithm %q in option.EnsureCompressionVariantsExist: %w", option.Algorithm.Name(), err)
}
}
return nil
}

// prepareInstanceCopies prepares a list of instances which needs to copied to the manifest list.
func prepareInstanceCopies(instanceDigests []digest.Digest, options *Options) []instanceCopy {
func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.Digest, options *Options) ([]instanceCopy, error) {
res := []instanceCopy{}
if options.ImageListSelection == CopySpecificImages && len(options.EnsureCompressionVariantsExist) > 0 {
// List can already contain compressed instance for a compression selected in `EnsureCompressionVariantsExist`
// It’s unclear what it means when `CopySpecificImages` includes an instance in options.Instances,
// EnsureCompressionVariantsExist asks for an instance with some compression,
// an instance with that compression already exists, but is not included in options.Instances.
// We might define the semantics and implement this in the future.
return res, fmt.Errorf("EnsureCompressionVariantsExist is not implemented for CopySpecificImages")
}
err := validateCompressionVariantExists(options.EnsureCompressionVariantsExist)
if err != nil {
return res, err
}
compressionsByPlatform, err := platformCompressionMap(list, instanceDigests)
if err != nil {
return nil, err
}
for i, instanceDigest := range instanceDigests {
if options.ImageListSelection == CopySpecificImages &&
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
!slices.Contains(options.Instances, instanceDigest) {
logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests))
continue
}
instanceDetails, err := list.Instance(instanceDigest)
if err != nil {
return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
}
platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
compressionList := compressionsByPlatform[platform]
for _, compressionVariant := range options.EnsureCompressionVariantsExist {
if !compressionList.Contains(compressionVariant.Algorithm.Name()) {
res = append(res, instanceCopy{
op: instanceCopyClone,
sourceDigest: instanceDigest,
cloneCompressionVariant: compressionVariant,
clonePlatform: instanceDetails.ReadOnly.Platform,
cloneAnnotations: maps.Clone(instanceDetails.ReadOnly.Annotations),
})
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
// add current compression to the list so that we don’t create duplicate clones
compressionList.Add(compressionVariant.Algorithm.Name())
}
}
res = append(res, instanceCopy{
op: instanceCopyCopy,
sourceDigest: instanceDigest,
})
}
return res
return res, nil
}

// copyMultipleImages copies some or all of an image list's instances, using
Expand Down Expand Up @@ -118,8 +217,11 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
// Copy each image, or just the ones we want to copy, in turn.
instanceDigests := updatedList.Instances()
instanceEdits := []internalManifest.ListEdit{}
instanceCopyList := prepareInstanceCopies(instanceDigests, c.options)
c.Printf("Copying %d of %d images in list\n", len(instanceCopyList), len(instanceDigests))
instanceCopyList, err := prepareInstanceCopies(updatedList, instanceDigests, c.options)
if err != nil {
return nil, fmt.Errorf("preparing instances for copy: %w", err)
}
c.Printf("Copying %d images generated from %d images in list\n", len(instanceCopyList), len(instanceDigests))
for i, instance := range instanceCopyList {
// Update instances to be edited by their `ListOperation` and
// populate necessary fields.
Expand All @@ -140,6 +242,27 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
UpdateSize: int64(len(updated.manifest)),
UpdateCompressionAlgorithms: updated.compressionAlgorithms,
UpdateMediaType: updated.manifestMIMEType})
case instanceCopyClone:
logrus.Debugf("Replicating instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList))
c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest)
updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{
requireCompressionFormatMatch: true,
compressionFormat: &instance.cloneCompressionVariant.Algorithm,
compressionLevel: instance.cloneCompressionVariant.Level})
if err != nil {
return nil, fmt.Errorf("replicating image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err)
}
// Record the result of a possible conversion here.
instanceEdits = append(instanceEdits, internalManifest.ListEdit{
ListOperation: internalManifest.ListOpAdd,
AddDigest: updated.manifestDigest,
AddSize: int64(len(updated.manifest)),
AddMediaType: updated.manifestMIMEType,
AddPlatform: instance.clonePlatform,
AddAnnotations: instance.cloneAnnotations,
AddCompressionAlgorithms: updated.compressionAlgorithms,
})
default:
return nil, fmt.Errorf("copying image: invalid copy operation %d", instance.op)
}
Expand Down
Loading