Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

manifest, push: implement --add-compression to push with compressed variants. #4912

Merged
merged 4 commits into from
Jul 31, 2023
Merged

manifest, push: implement --add-compression to push with compressed variants. #4912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5122735
vendor: bump c/image to v5.26.1-0.20230726142307-8c387a14f4ac
flouthoc Jul 27, 2023
071f851
vendor: bump c/common to v0.55.1-0.20230727095721-647ed1d4d79a
flouthoc Jul 27, 2023
d27a823
manifest/inspect: add support for tls-verify and authfile
flouthoc Jul 27, 2023
b9ac8d5
manifest/push: add support for --add-compression
flouthoc Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions cmd/buildah/manifest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ type manifestAnnotateOpts = struct {
os, arch, variant, osVersion string
features, osFeatures, annotations []string
}
type manifestInspectOpts = struct{}
type manifestInspectOpts = struct {
authfile string
tlsVerify bool
}

func init() {
var (
Expand Down Expand Up @@ -199,6 +202,9 @@ func init() {
Example: `buildah manifest inspect mylist:v1.11`,
Args: cobra.MinimumNArgs(1),
}
flags = manifestInspectCommand.Flags()
flags.StringVar(&manifestInspectOpts.authfile, "authfile", auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
flags.BoolVar(&manifestInspectOpts.tlsVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry. TLS verification cannot be used when talking to an insecure registry.")
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
manifestInspectCommand.SetUsageTemplate(UsageTemplate())
manifestCommand.AddCommand(manifestInspectCommand)

Expand All @@ -221,6 +227,7 @@ func init() {
flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
flags.StringVar(&manifestPushOpts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting digest to the file")
flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing")
flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images")
flags.StringVar(&manifestPushOpts.signBy, "sign-by", "", "sign the image using a GPG key with the specified `FINGERPRINT`")
flags.StringVar(&manifestPushOpts.signaturePolicy, "signature-policy", "", "`pathname` of signature policy file (not usually used)")
Expand Down Expand Up @@ -696,6 +703,11 @@ func manifestAnnotateCmd(c *cobra.Command, args []string, opts manifestAnnotateO
}

func manifestInspectCmd(c *cobra.Command, args []string, opts manifestInspectOpts) error {
if c.Flag("authfile").Changed {
if err := auth.CheckAuthFile(opts.authfile); err != nil {
return err
}
}
imageSpec := ""
switch len(args) {
case 0:
Expand Down Expand Up @@ -831,7 +843,7 @@ func manifestPushCmd(c *cobra.Command, args []string, opts pushOptions) error {
return errors.New("At least a source list ID must be specified")
case 1:
listImageSpec = args[0]
destSpec = "docker://"+listImageSpec
destSpec = "docker://" + listImageSpec
case 2:
listImageSpec = args[0]
destSpec = args[1]
Expand Down Expand Up @@ -897,6 +909,7 @@ func manifestPush(systemContext *types.SystemContext, store storage.Store, listI
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signBy,
ManifestType: manifestType,
AddCompression: opts.addCompression,
}
if opts.all {
options.ImageListSelection = cp.CopyAllImages
Expand Down
1 change: 1 addition & 0 deletions cmd/buildah/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ type pushOptions struct {
encryptionKeys []string
encryptLayers []int
insecure bool
addCompression []string
}

func init() {
Expand Down
11 changes: 11 additions & 0 deletions docs/buildah-manifest-inspect.1.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,17 @@ Displays the manifest list or image index stored using the specified image name.

A formatted JSON representation of the manifest list or image index.

## OPTIONS

**--authfile** *path*
flouthoc marked this conversation as resolved.
Show resolved Hide resolved

Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `buildah login`.
If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
vrothberg marked this conversation as resolved.
Show resolved Hide resolved

**--tls-verify** *bool-value*

Require HTTPS and verification of certificates when talking to container registries (defaults to true). TLS verification cannot be used when talking to an insecure registry.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: the new flags are not tested in the commit adding them but the one after.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was not planning to add these flags but they were needed for the --add-compression test, I can add independent tests for these but they will be just clone of --add-compression test without the --add-compression flag, I decided not to add because of duplication since these two flags are already getting tested in the test added for --add-compression. But I could be wrong 😅


## EXAMPLE

```
Expand Down
9 changes: 9 additions & 0 deletions docs/buildah-manifest-push.1.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ The list image's ID and the digest of the image's manifest.

## OPTIONS

**--add-compression** *compression*

Makes sure that requested compression variant for each platform is added to the manifest list keeping original instance
intact in the same manifest list. Supported values are (`gzip`, `zstd` and `zstd:chunked`)

Note: This is different than `--compression` which replaces the instance with requested with specified compression
while `--add-compression` makes sure than each instance has it variant added to manifest list without modifying the
original instance.

**--all**

Push the images mentioned in the manifest list or image index, in addition to
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ require (
github.com/containerd/containerd v1.7.3
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.3.0
github.com/containers/common v0.55.1-0.20230721175448-664d013a6ae2
github.com/containers/image/v5 v5.26.1
github.com/containers/common v0.55.1-0.20230727095721-647ed1d4d79a
github.com/containers/image/v5 v5.26.1-0.20230727122416-da7899237198
github.com/containers/ocicrypt v1.1.7
github.com/containers/storage v1.48.1-0.20230721123825-4a3a3019d765
github.com/cyphar/filepath-securejoin v0.2.3
Expand Down Expand Up @@ -115,7 +115,7 @@ require (
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/vbatts/tar-split v0.11.5 // indirect
github.com/vbauerster/mpb/v8 v8.5.2 // indirect
github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
github.com/vishvananda/netns v0.0.4 // indirect
Expand Down
16 changes: 6 additions & 10 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
Expand Down Expand Up @@ -51,10 +50,10 @@ github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl3
github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM=
github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0=
github.com/containers/common v0.55.1-0.20230721175448-664d013a6ae2 h1:4B42HUIAghFGSqej5RADTNf0WlOBFiGGzmGjNa3Do78=
github.com/containers/common v0.55.1-0.20230721175448-664d013a6ae2/go.mod h1:O/JSRY1dLfwgBxVvn3yJfKvF63KEjbNJcJAtjpNvO90=
github.com/containers/image/v5 v5.26.1 h1:8y3xq8GO/6y8FR+nAedHPsAFiAtOrab9qHTBpbqaX8g=
github.com/containers/image/v5 v5.26.1/go.mod h1:IwlOGzTkGnmfirXxt0hZeJlzv1zVukE03WZQ203Z9GA=
github.com/containers/common v0.55.1-0.20230727095721-647ed1d4d79a h1:w314+pqd43cO35iRzJW8dB3tX6JOQH8aSP8a2wMoT2g=
github.com/containers/common v0.55.1-0.20230727095721-647ed1d4d79a/go.mod h1:kZWakZy8Ep9Zhdg+qrPRyWIZj4AyborpXqjuxt5ADfk=
github.com/containers/image/v5 v5.26.1-0.20230727122416-da7899237198 h1:H1YlMbFrkoBOWzgChilbuSZkPj149U9v/zkBAnyC1XU=
github.com/containers/image/v5 v5.26.1-0.20230727122416-da7899237198/go.mod h1:Zg7m6YHPZRl/wbUDZ6vt+yAyXAjAvALVUelmsIPpMcE=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U=
Expand Down Expand Up @@ -359,7 +358,6 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
Expand Down Expand Up @@ -398,9 +396,8 @@ github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
github.com/vbauerster/mpb/v8 v8.5.2 h1:zanzt1cZpSEG5uGNYKcv43+97f0IgEnXpuBFaMxKbM0=
github.com/vbauerster/mpb/v8 v8.5.2/go.mod h1:YqKyR4ZR6Gd34yD3cDHPMmQxc+uUQMwjgO/LkxiJQ6I=
github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
Expand Down Expand Up @@ -510,7 +507,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
61 changes: 61 additions & 0 deletions tests/bud.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,34 @@ load helpers
run_buildah build $BUDFILES/stdio
}

@test "bud: build manifest list and --add-compression zstd" {
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
mkdir -p $contextdir

cat > $contextdir/Dockerfile1 << _EOF
FROM alpine
_EOF

start_registry
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
run_buildah build $WITH_POLICY_JSON -t image2 --platform linux/arm64 -f $contextdir/Dockerfile1

run_buildah manifest create foo
run_buildah manifest add foo image1
run_buildah manifest add foo image2

run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list

run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
list="$output"

validate_instance_compression "0" "$list" "amd64" "gzip"
validate_instance_compression "1" "$list" "arm64" "gzip"
validate_instance_compression "2" "$list" "amd64" "zstd"
validate_instance_compression "3" "$list" "arm64" "zstd"
}

@test "bud with --dns* flags" {
_prefetch alpine

Expand Down Expand Up @@ -2103,6 +2131,39 @@ function _test_http() {
run_buildah from ${target}
}

# Helper function for several of the tests which verifies compression.
#
# Usage: validate_instance_compression INDEX MANIFEST ARCH COMPRESSION
#
# INDEX instance which needs to be verified in
# provided manifest list.
#
# MANIFEST OCI manifest specification in json format
#
# ARCH instance architecture
#
# COMPRESSION compression algorithm name; e.g "zstd".
#
function validate_instance_compression {
case $4 in

gzip)
run jq -r '.manifests['$1'].annotations' <<< $2
# annotation is `null` for gzip compression
assert "$output" = "null" ".manifests[$1].annotations (null means gzip)"
;;

zstd)
# annotation `'"io.github.containers.compression.zstd": "true"'` must be there for zstd compression
run jq -r '.manifests['$1'].annotations."io.github.containers.compression.zstd"' <<< $2
assert "$output" = "true" ".manifests[$1].annotations.'io.github.containers.compression.zstd' (io.github.containers.compression.zstd must be set)"
;;
esac

run jq -r '.manifests['$1'].platform.architecture' <<< $2
assert "$output" = $3 ".manifests[$1].platform.architecture"
}

@test "bud-http-Dockerfile" {
_test_http from-scratch Containerfile
}
Expand Down
82 changes: 0 additions & 82 deletions ...ntainer-orchestrated-devices/container-device-interface/internal/multierror/multierror.go
View file
Open in desktop

This file was deleted.

57 changes: 0 additions & 57 deletions ...ner-orchestrated-devices/container-device-interface/internal/validation/k8s/objectmeta.go
View file
Open in desktop

This file was deleted.

Loading
Loading