Merge pull request #1987 from flouthoc/copy-add-zstd-annotation

copy: implement `instanceCopyClone` for `zstd` compression

copy/copy.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/image/v5/internal/private"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/blobinfocache"
+	compression "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/signature"
 	"github.com/containers/image/v5/signature/signer"
 	"github.com/containers/image/v5/transports"
@@ -126,6 +127,21 @@ type Options struct {
 	// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
 	// to not indicate "nondistributable".
 	DownloadForeignLayers bool
+
+	// Contains slice of OptionCompressionVariant, where copy will ensure that for each platform
+	// in the manifest list, a variant with the requested compression will exist.
+	// Invalid when copying a non-multi-architecture image. That will probably
+	// change in the future.
+	EnsureCompressionVariantsExist []OptionCompressionVariant
+}
+
+// OptionCompressionVariant allows to supply information about
+// selected compression algorithm and compression level by the
+// end-user. Refer to EnsureCompressionVariantsExist to know
+// more about its usage.
+type OptionCompressionVariant struct {
+	Algorithm compression.Algorithm
+	Level     *int // Only used when we are creating a new image instance using the specified algorithm, not when the image already contains such an instance
 }
 
 // copier allows us to keep track of diffID values for blobs, and other
@@ -250,13 +266,19 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 	}
 
 	if !multiImage {
+		if len(options.EnsureCompressionVariantsExist) > 0 {
+			return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
+		}
 		// The simple case: just copy a single image.
 		single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
 		if err != nil {
 			return nil, err
 		}
 		copiedManifest = single.manifest
 	} else if c.options.ImageListSelection == CopySystemImage {
+		if len(options.EnsureCompressionVariantsExist) > 0 {
+			return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
+		}
 		// This is a manifest list, and we weren't asked to copy multiple images.  Choose a single image that
 		// matches the current system to copy, and copy it.
 		mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx)

copy/multiple.go

@@ -5,15 +5,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/internal/image"
 	internalManifest "github.com/containers/image/v5/internal/manifest"
+	"github.com/containers/image/v5/internal/set"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 )
 
@@ -27,23 +31,118 @@ const (
 type instanceCopy struct {
 	op           instanceCopyKind
 	sourceDigest digest.Digest
+
+	// Fields which can be used by callers when operation
+	// is `instanceCopyClone`
+	cloneCompressionVariant OptionCompressionVariant
+	clonePlatform           *imgspecv1.Platform
+	cloneAnnotations        map[string]string
+}
+
+// internal type only to make imgspecv1.Platform comparable
+type platformComparable struct {
+	architecture string
+	os           string
+	osVersion    string
+	osFeatures   string
+	variant      string
+}
+
+// Converts imgspecv1.Platform to a comparable format.
+func platformV1ToPlatformComparable(platform *imgspecv1.Platform) platformComparable {
+	if platform == nil {
+		return platformComparable{}
+	}
+	osFeatures := slices.Clone(platform.OSFeatures)
+	sort.Strings(osFeatures)
+	return platformComparable{architecture: platform.Architecture,
+		os: platform.OS,
+		// This is strictly speaking ambiguous, fields of OSFeatures can contain a ','. Probably good enough for now.
+		osFeatures: strings.Join(osFeatures, ","),
+		osVersion:  platform.OSVersion,
+		variant:    platform.Variant,
+	}
+}
+
+// platformCompressionMap prepares a mapping of platformComparable -> CompressionAlgorithmNames for given digests
+func platformCompressionMap(list internalManifest.List, instanceDigests []digest.Digest) (map[platformComparable]*set.Set[string], error) {
+	res := make(map[platformComparable]*set.Set[string])
+	for _, instanceDigest := range instanceDigests {
+		instanceDetails, err := list.Instance(instanceDigest)
+		if err != nil {
+			return nil, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
+		}
+		platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
+		platformSet, ok := res[platform]
+		if !ok {
+			platformSet = set.New[string]()
+			res[platform] = platformSet
+		}
+		platformSet.AddSlice(instanceDetails.ReadOnly.CompressionAlgorithmNames)
+	}
+	return res, nil
+}
+
+func validateCompressionVariantExists(input []OptionCompressionVariant) error {
+	for _, option := range input {
+		_, err := compression.AlgorithmByName(option.Algorithm.Name())
+		if err != nil {
+			return fmt.Errorf("invalid algorithm %q in option.EnsureCompressionVariantsExist: %w", option.Algorithm.Name(), err)
+		}
+	}
+	return nil
 }
 
 // prepareInstanceCopies prepares a list of instances which needs to copied to the manifest list.
-func prepareInstanceCopies(instanceDigests []digest.Digest, options *Options) []instanceCopy {
+func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.Digest, options *Options) ([]instanceCopy, error) {
 	res := []instanceCopy{}
+	if options.ImageListSelection == CopySpecificImages && len(options.EnsureCompressionVariantsExist) > 0 {
+		// List can already contain compressed instance for a compression selected in `EnsureCompressionVariantsExist`
+		// It’s unclear what it means when `CopySpecificImages` includes an instance in options.Instances,
+		// EnsureCompressionVariantsExist asks for an instance with some compression,
+		// an instance with that compression already exists, but is not included in options.Instances.
+		// We might define the semantics and implement this in the future.
+		return res, fmt.Errorf("EnsureCompressionVariantsExist is not implemented for CopySpecificImages")
+	}
+	err := validateCompressionVariantExists(options.EnsureCompressionVariantsExist)
+	if err != nil {
+		return res, err
+	}
+	compressionsByPlatform, err := platformCompressionMap(list, instanceDigests)
+	if err != nil {
+		return nil, err
+	}
 	for i, instanceDigest := range instanceDigests {
 		if options.ImageListSelection == CopySpecificImages &&
 			!slices.Contains(options.Instances, instanceDigest) {
 			logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests))
 			continue
 		}
+		instanceDetails, err := list.Instance(instanceDigest)
+		if err != nil {
+			return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
+		}
+		platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
+		compressionList := compressionsByPlatform[platform]
+		for _, compressionVariant := range options.EnsureCompressionVariantsExist {
+			if !compressionList.Contains(compressionVariant.Algorithm.Name()) {
+				res = append(res, instanceCopy{
+					op:                      instanceCopyClone,
+					sourceDigest:            instanceDigest,
+					cloneCompressionVariant: compressionVariant,
+					clonePlatform:           instanceDetails.ReadOnly.Platform,
+					cloneAnnotations:        maps.Clone(instanceDetails.ReadOnly.Annotations),
+				})
+				// add current compression to the list so that we don’t create duplicate clones
+				compressionList.Add(compressionVariant.Algorithm.Name())
+			}
+		}
 		res = append(res, instanceCopy{
 			op:           instanceCopyCopy,
 			sourceDigest: instanceDigest,
 		})
 	}
-	return res
+	return res, nil
 }
 
 // copyMultipleImages copies some or all of an image list's instances, using
@@ -118,8 +217,11 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
 	// Copy each image, or just the ones we want to copy, in turn.
 	instanceDigests := updatedList.Instances()
 	instanceEdits := []internalManifest.ListEdit{}
-	instanceCopyList := prepareInstanceCopies(instanceDigests, c.options)
-	c.Printf("Copying %d of %d images in list\n", len(instanceCopyList), len(instanceDigests))
+	instanceCopyList, err := prepareInstanceCopies(updatedList, instanceDigests, c.options)
+	if err != nil {
+		return nil, fmt.Errorf("preparing instances for copy: %w", err)
+	}
+	c.Printf("Copying %d images generated from %d images in list\n", len(instanceCopyList), len(instanceDigests))
 	for i, instance := range instanceCopyList {
 		// Update instances to be edited by their `ListOperation` and
 		// populate necessary fields.
@@ -140,6 +242,27 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
 				UpdateSize:                  int64(len(updated.manifest)),
 				UpdateCompressionAlgorithms: updated.compressionAlgorithms,
 				UpdateMediaType:             updated.manifestMIMEType})
+		case instanceCopyClone:
+			logrus.Debugf("Replicating instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList))
+			c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
+			unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest)
+			updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{
+				requireCompressionFormatMatch: true,
+				compressionFormat:             &instance.cloneCompressionVariant.Algorithm,
+				compressionLevel:              instance.cloneCompressionVariant.Level})
+			if err != nil {
+				return nil, fmt.Errorf("replicating image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err)
+			}
+			// Record the result of a possible conversion here.
+			instanceEdits = append(instanceEdits, internalManifest.ListEdit{
+				ListOperation:            internalManifest.ListOpAdd,
+				AddDigest:                updated.manifestDigest,
+				AddSize:                  int64(len(updated.manifest)),
+				AddMediaType:             updated.manifestMIMEType,
+				AddPlatform:              instance.clonePlatform,
+				AddAnnotations:           instance.cloneAnnotations,
+				AddCompressionAlgorithms: updated.compressionAlgorithms,
+			})
 		default:
 			return nil, fmt.Errorf("copying image: invalid copy operation %d", instance.op)
 		}