diff --git a/cmd/podman/artifact/add.go b/cmd/podman/artifact/add.go index a92018fb3f..fad131d80f 100644 --- a/cmd/podman/artifact/add.go +++ b/cmd/podman/artifact/add.go @@ -27,6 +27,7 @@ var ( type artifactAddOptions struct { ArtifactType string Annotations []string + Append bool } var ( @@ -47,6 +48,10 @@ func init() { addTypeFlagName := "type" flags.StringVar(&addOpts.ArtifactType, addTypeFlagName, "", "Use type to describe an artifact") _ = addCmd.RegisterFlagCompletionFunc(addTypeFlagName, completion.AutocompleteNone) + + appendFlagName := "append" + flags.BoolVarP(&addOpts.Append, appendFlagName, "a", false, "Append files to an existing artifact") + _ = addCmd.RegisterFlagCompletionFunc(appendFlagName, completion.AutocompleteNone) } func add(cmd *cobra.Command, args []string) error { @@ -58,6 +63,8 @@ func add(cmd *cobra.Command, args []string) error { } opts.Annotations = annots opts.ArtifactType = addOpts.ArtifactType + opts.Append = addOpts.Append + report, err := registry.ImageEngine().ArtifactAdd(registry.Context(), args[0], args[1:], opts) if err != nil { return err diff --git a/docs/source/markdown/podman-artifact-add.1.md.in b/docs/source/markdown/podman-artifact-add.1.md.in index 35a99f9958..c2a728427f 100644 --- a/docs/source/markdown/podman-artifact-add.1.md.in +++ b/docs/source/markdown/podman-artifact-add.1.md.in @@ -21,6 +21,10 @@ added. @@option annotation.manifest +#### **--append**, **-a** + +Append files to an existing artifact + #### **--help** Print usage statement. diff --git a/pkg/domain/entities/artifact.go b/pkg/domain/entities/artifact.go index 5103401e8e..41185bd528 100644 --- a/pkg/domain/entities/artifact.go +++ b/pkg/domain/entities/artifact.go @@ -12,6 +12,7 @@ import ( type ArtifactAddOptions struct { Annotations map[string]string ArtifactType string + Append bool } type ArtifactInspectOptions struct { diff --git a/pkg/domain/infra/abi/artifact.go b/pkg/domain/infra/abi/artifact.go index fe1c3c12c8..0da9ecad50 100644 --- a/pkg/domain/infra/abi/artifact.go +++ b/pkg/domain/infra/abi/artifact.go @@ -162,6 +162,7 @@ func (ir *ImageEngine) ArtifactAdd(ctx context.Context, name string, paths []str addOptions := types.AddOptions{ Annotations: opts.Annotations, ArtifactType: opts.ArtifactType, + Append: opts.Append, } artifactDigest, err := artStore.Add(ctx, name, paths, &addOptions) diff --git a/pkg/libartifact/store/store.go b/pkg/libartifact/store/store.go index 951bb8a496..1ffe488e1e 100644 --- a/pkg/libartifact/store/store.go +++ b/pkg/libartifact/store/store.go @@ -29,6 +29,7 @@ import ( var ( ErrEmptyArtifactName = errors.New("artifact name cannot be empty") + SchemaVersion = 2 ) type ArtifactStore struct { @@ -162,13 +163,10 @@ func (as ArtifactStore) Push(ctx context.Context, src, dest string, opts libimag // Add takes one or more local files and adds them to the local artifact store. The empty // string input is for possible custom artifact types. func (as ArtifactStore) Add(ctx context.Context, dest string, paths []string, options *libartTypes.AddOptions) (*digest.Digest, error) { - annots := maps.Clone(options.Annotations) if len(dest) == 0 { return nil, ErrEmptyArtifactName } - artifactManifestLayers := make([]specV1.Descriptor, 0) - // Check if artifact already exists artifacts, err := as.getArtifacts(ctx, nil) if err != nil { @@ -177,10 +175,21 @@ func (as ArtifactStore) Add(ctx context.Context, dest string, paths []string, op // Check if artifact exists; in GetByName not getting an // error means it exists - if _, _, err := artifacts.GetByNameOrDigest(dest); err == nil { + artifact, _, err := artifacts.GetByNameOrDigest(dest) + if err == nil && !options.Append { return nil, fmt.Errorf("artifact %s already exists", dest) } + var instanceDigest *digest.Digest + currentArtifactLayers := make([]specV1.Descriptor, 0) + if artifact != nil && options.Append { + currentArtifactLayers = artifact.Manifest.Layers + instanceDigest, err = artifact.GetDigest() + if err != nil { + return nil, err + } + } + ir, err := layout.NewReference(as.storePath, dest) if err != nil { return nil, err @@ -192,66 +201,32 @@ func (as ArtifactStore) Add(ctx context.Context, dest string, paths []string, op } defer imageDest.Close() - for _, path := range paths { - // currently we don't allow override of the filename ; if a user requirement emerges, - // we could seemingly accommodate but broadens possibilities of something bad happening - // for things like `artifact extract` - if _, hasTitle := options.Annotations[specV1.AnnotationTitle]; hasTitle { - return nil, fmt.Errorf("cannot override filename with %s annotation", specV1.AnnotationTitle) - } - - // get the new artifact into the local store - newBlobDigest, newBlobSize, err := layout.PutBlobFromLocalFile(ctx, imageDest, path) - if err != nil { - return nil, err - } - detectedType, err := determineManifestType(path) - if err != nil { - return nil, err - } - - annots[specV1.AnnotationTitle] = filepath.Base(path) - - newLayer := specV1.Descriptor{ - MediaType: detectedType, - Digest: newBlobDigest, - Size: newBlobSize, - Annotations: annots, - } - - artifactManifestLayers = append(artifactManifestLayers, newLayer) - } - - artifactManifest := specV1.Manifest{ - Versioned: specs.Versioned{SchemaVersion: 2}, - MediaType: specV1.MediaTypeImageManifest, - // TODO This should probably be configurable once the CLI is capable - Config: specV1.DescriptorEmptyJSON, - Layers: artifactManifestLayers, + newArtifactLayers, err := appendNewLocalFiles(ctx, currentArtifactLayers, imageDest, paths, options.Annotations) + if err != nil { + return nil, err } - artifactManifest.ArtifactType = options.ArtifactType - - rawData, err := json.Marshal(artifactManifest) + artifactManifest := getNewArtifactManifest(newArtifactLayers, options.ArtifactType) + rawData, artifactManifestDigest, err := marshalArtifactManifest(artifactManifest) if err != nil { return nil, err } - if err := imageDest.PutManifest(ctx, rawData, nil); err != nil { + + if err := imageDest.PutManifest(ctx, rawData, instanceDigest); err != nil { return nil, err } + unparsed := newUnparsedArtifactImage(ir, artifactManifest) if err := imageDest.Commit(ctx, unparsed); err != nil { return nil, err } - artifactManifestDigest := digest.FromBytes(rawData) - // the config is an empty JSON stanza i.e. '{}'; if it does not yet exist, it needs // to be created if err := createEmptyStanza(filepath.Join(as.storePath, specV1.ImageBlobsDir, artifactManifestDigest.Algorithm().String(), artifactManifest.Config.Digest.Encoded())); err != nil { logrus.Errorf("failed to check or write empty stanza file: %v", err) } - return &artifactManifestDigest, nil + return artifactManifestDigest, nil } // readIndex is currently unused but I want to keep this around until @@ -269,7 +244,7 @@ func (as ArtifactStore) readIndex() (*specV1.Index, error) { //nolint:unused func (as ArtifactStore) createEmptyManifest() error { index := specV1.Index{ MediaType: specV1.MediaTypeImageIndex, - Versioned: specs.Versioned{SchemaVersion: 2}, + Versioned: specs.Versioned{SchemaVersion: SchemaVersion}, } rawData, err := json.Marshal(&index) if err != nil { @@ -360,3 +335,66 @@ func determineManifestType(path string) (string, error) { } return http.DetectContentType(b[:n]), nil } + +func appendNewLocalFiles(ctx context.Context, currentArtifactLayers []specV1.Descriptor, imageDest types.ImageDestination, paths []string, optionsAnnotations map[string]string) ([]specV1.Descriptor, error) { + // currently we don't allow override of the filename ; if a user requirement emerges, + // we could seemingly accommodate but broadens possibilities of something bad happening + // for things like `artifact extract` + if _, hasTitle := optionsAnnotations[specV1.AnnotationTitle]; hasTitle { + return nil, fmt.Errorf("cannot override filename with %s annotation", specV1.AnnotationTitle) + } + + annotations := maps.Clone(optionsAnnotations) + + fileNames := map[string]struct{}{} + for _, layer := range currentArtifactLayers { + fileNames[layer.Annotations[specV1.AnnotationTitle]] = struct{}{} + } + + for _, path := range paths { + fileName := filepath.Base(path) + if _, ok := fileNames[fileName]; ok { + return nil, fmt.Errorf("file: %s already exist in artifact", fileName) + } + // get the new artifact into the local store + newBlobDigest, newBlobSize, err := layout.PutBlobFromLocalFile(ctx, imageDest, path) + if err != nil { + return nil, err + } + detectedType, err := determineManifestType(path) + if err != nil { + return nil, err + } + annotations[specV1.AnnotationTitle] = fileName + newLayer := specV1.Descriptor{ + MediaType: detectedType, + Digest: newBlobDigest, + Size: newBlobSize, + Annotations: annotations, + } + + fileNames[fileName] = struct{}{} + currentArtifactLayers = append(currentArtifactLayers, newLayer) + } + return currentArtifactLayers, nil +} + +func getNewArtifactManifest(newArtifactLayers []specV1.Descriptor, artifactType string) specV1.Manifest { + return specV1.Manifest{ + Versioned: specs.Versioned{SchemaVersion: SchemaVersion}, + MediaType: specV1.MediaTypeImageManifest, + ArtifactType: artifactType, + // TODO This should probably be configurable once the CLI is capable + Config: specV1.DescriptorEmptyJSON, + Layers: newArtifactLayers, + } +} + +func marshalArtifactManifest(artifactManifest specV1.Manifest) ([]byte, *digest.Digest, error) { + rawData, err := json.Marshal(artifactManifest) + if err != nil { + return nil, nil, err + } + artifactManifestDigest := digest.FromBytes(rawData) + return rawData, &artifactManifestDigest, nil +} diff --git a/pkg/libartifact/types/config.go b/pkg/libartifact/types/config.go index c458e646a6..e86e592164 100644 --- a/pkg/libartifact/types/config.go +++ b/pkg/libartifact/types/config.go @@ -8,4 +8,5 @@ type GetArtifactOptions struct{} type AddOptions struct { Annotations map[string]string `json:"annotations,omitempty"` ArtifactType string `json:",omitempty"` + Append bool `json:",omitempty"` } diff --git a/test/e2e/artifact_test.go b/test/e2e/artifact_test.go index c1ad2721e3..6da58ee009 100644 --- a/test/e2e/artifact_test.go +++ b/test/e2e/artifact_test.go @@ -3,10 +3,9 @@ package integration import ( - "encoding/json" "fmt" + "path/filepath" - "github.com/containers/podman/v5/pkg/libartifact" . "github.com/containers/podman/v5/test/utils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -64,12 +63,8 @@ var _ = Describe("Podman artifact", func() { artifact1Name := "localhost/test/artifact1" podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File}...) - inspectSingleSession := podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifact1Name}...) + a := podmanTest.InspectArtifact(artifact1Name) - a := libartifact.Artifact{} - inspectOut := inspectSingleSession.OutputToString() - err = json.Unmarshal([]byte(inspectOut), &a) - Expect(err).ToNot(HaveOccurred()) Expect(a.Name).To(Equal(artifact1Name)) // Adding an artifact with an existing name should fail @@ -89,10 +84,8 @@ var _ = Describe("Podman artifact", func() { annotation2 := "flavor=lemon" podmanTest.PodmanExitCleanly([]string{"artifact", "add", "--type", artifactType, "--annotation", annotation1, "--annotation", annotation2, artifact1Name, artifact1File}...) - inspectSingleSession := podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifact1Name}...) - a := libartifact.Artifact{} - err = json.Unmarshal([]byte(inspectSingleSession.OutputToString()), &a) - Expect(err).ToNot(HaveOccurred()) + + a := podmanTest.InspectArtifact(artifact1Name) Expect(a.Name).To(Equal(artifact1Name)) Expect(a.Manifest.ArtifactType).To(Equal(artifactType)) Expect(a.Manifest.Layers[0].Annotations["color"]).To(Equal("blue")) @@ -114,13 +107,7 @@ var _ = Describe("Podman artifact", func() { podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File1, artifact1File2}...) - inspectSingleSession := podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifact1Name}...) - - a := libartifact.Artifact{} - inspectOut := inspectSingleSession.OutputToString() - err = json.Unmarshal([]byte(inspectOut), &a) - Expect(err).ToNot(HaveOccurred()) - Expect(a.Name).To(Equal(artifact1Name)) + a := podmanTest.InspectArtifact(artifact1Name) Expect(a.Manifest.Layers).To(HaveLen(2)) }) @@ -144,12 +131,8 @@ var _ = Describe("Podman artifact", func() { podmanTest.PodmanExitCleanly([]string{"artifact", "pull", "--tls-verify=false", artifact1Name}...) - inspectSingleSession := podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifact1Name}...) + a := podmanTest.InspectArtifact(artifact1Name) - a := libartifact.Artifact{} - inspectOut := inspectSingleSession.OutputToString() - err = json.Unmarshal([]byte(inspectOut), &a) - Expect(err).ToNot(HaveOccurred()) Expect(a.Name).To(Equal(artifact1Name)) }) @@ -190,4 +173,102 @@ var _ = Describe("Podman artifact", func() { podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifactDigest[:12]}...) }) + + It("podman artifact simple add --append", func() { + artifact1File, err := createArtifactFile(1024) + Expect(err).ToNot(HaveOccurred()) + + artifact2File, err := createArtifactFile(2048) + Expect(err).ToNot(HaveOccurred()) + + artifact1Name := "localhost/test/artifact1" + podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File}...) + + _ = podmanTest.InspectArtifact(artifact1Name) + + podmanTest.PodmanExitCleanly([]string{"artifact", "add", "--append", artifact1Name, artifact2File}...) + + a := podmanTest.InspectArtifact(artifact1Name) + + Expect(a.Manifest.Layers).To(HaveLen(2)) + + listSession := podmanTest.PodmanExitCleanly([]string{"artifact", "ls"}...) + Expect(listSession.OutputToStringArray()).To(HaveLen(2)) + }) + + It("podman artifact add --append multiple", func() { + artifact1File, err := createArtifactFile(1024) + Expect(err).ToNot(HaveOccurred()) + + artifact2File, err := createArtifactFile(2048) + Expect(err).ToNot(HaveOccurred()) + + artifact3File, err := createArtifactFile(4192) + Expect(err).ToNot(HaveOccurred()) + + artifact1Name := "localhost/test/artifact1" + podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File}...) + + podmanTest.PodmanExitCleanly([]string{"artifact", "add", "--append", artifact1Name, artifact2File, artifact3File}...) + + a := podmanTest.InspectArtifact(artifact1Name) + + Expect(a.Manifest.Layers).To(HaveLen(3)) + }) + + It("podman artifact add --append modified file", func() { + artifact1File, err := createArtifactFile(1024) + Expect(err).ToNot(HaveOccurred()) + + artifact1Name := "localhost/test/artifact1" + podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File}...) + + err = modifyArtifactFile(artifact1File, 4192) + Expect(err).ToNot(HaveOccurred()) + + appendFail := podmanTest.Podman([]string{"artifact", "add", "--append", artifact1Name, artifact1File}) + appendFail.WaitWithDefaultTimeout() + Expect(appendFail).Should(Exit(125)) + Expect(appendFail.ErrorToString()).Should(Equal(fmt.Sprintf("Error: file: %s already exist in artifact", filepath.Base(artifact1File)))) + + a := podmanTest.InspectArtifact(artifact1Name) + + Expect(a.Manifest.Layers).To(HaveLen(1)) + Expect(a.TotalSizeBytes()).To(Equal(int64(524288))) + }) + + It("podman artifact add file already exists in artifact", func() { + artifact1File, err := createArtifactFile(2048) + Expect(err).ToNot(HaveOccurred()) + + artifact1Name := "localhost/test/artifact1" + + addFail := podmanTest.Podman([]string{"artifact", "add", artifact1Name, artifact1File, artifact1File}) + addFail.WaitWithDefaultTimeout() + Expect(addFail).Should(Exit(125)) + Expect(addFail.ErrorToString()).Should(Equal(fmt.Sprintf("Error: file: %s already exist in artifact", filepath.Base(artifact1File)))) + + inspectFail := podmanTest.Podman([]string{"artifact", "inspect", artifact1Name}) + inspectFail.WaitWithDefaultTimeout() + Expect(inspectFail).Should(Exit(125)) + Expect(inspectFail.ErrorToString()).Should(Equal(fmt.Sprintf("Error: no artifact found with name or digest of %s", artifact1Name))) + }) + + It("podman artifact add --append file already exists in artifact", func() { + artifact1File, err := createArtifactFile(2048) + Expect(err).ToNot(HaveOccurred()) + + artifact1Name := "localhost/test/artifact1" + podmanTest.PodmanExitCleanly([]string{"artifact", "add", artifact1Name, artifact1File}...) + + appendFail := podmanTest.Podman([]string{"artifact", "add", "--append", artifact1Name, artifact1File}) + appendFail.WaitWithDefaultTimeout() + Expect(appendFail).Should(Exit(125)) + Expect(appendFail.ErrorToString()).Should(Equal(fmt.Sprintf("Error: file: %s already exist in artifact", filepath.Base(artifact1File)))) + + a := podmanTest.InspectArtifact(artifact1Name) + + Expect(a.Manifest.Layers).To(HaveLen(1)) + Expect(a.TotalSizeBytes()).To(Equal(int64(1048576))) + }) }) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 898efbc889..87a6361031 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -5,6 +5,7 @@ package integration import ( "bufio" "bytes" + "encoding/json" "errors" "fmt" "io" @@ -26,6 +27,7 @@ import ( "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/pkg/inspect" + "github.com/containers/podman/v5/pkg/libartifact" . "github.com/containers/podman/v5/test/utils" "github.com/containers/podman/v5/utils" "github.com/containers/storage/pkg/lockfile" @@ -488,6 +490,15 @@ func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { return i } +// InspectArtifactToJSON takes the session output of an artifact inspect and returns json +func (s *PodmanSessionIntegration) InspectArtifactToJSON() libartifact.Artifact { + a := libartifact.Artifact{} + inspectOut := s.OutputToString() + err := json.Unmarshal([]byte(inspectOut), &a) + Expect(err).ToNot(HaveOccurred()) + return a +} + // PodmanExitCleanly runs a podman command with args, and expects it to ExitCleanly within the default timeout. // It returns the session (to allow consuming output if desired). func (p *PodmanTestIntegration) PodmanExitCleanly(args ...string) *PodmanSessionIntegration { @@ -514,6 +525,15 @@ func (p *PodmanTestIntegration) InspectContainer(name string) []define.InspectCo return session.InspectContainerToJSON() } +// InspectArtifact returns an artifact's inspect data in JSON format +func (p *PodmanTestIntegration) InspectArtifact(name string) libartifact.Artifact { + cmd := []string{"artifact", "inspect", name} + session := p.Podman(cmd) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + return session.InspectArtifactToJSON() +} + // Pull a single field from a container using `podman inspect --format {{ field }}`, // and verify it against the given expected value. func (p *PodmanTestIntegration) CheckContainerSingleField(name, field, expected string) { @@ -1604,3 +1624,13 @@ func createArtifactFile(numBytes int64) (string, error) { } return outFile, nil } + +func modifyArtifactFile(path string, newNumBytes int64) error { + dir, last := filepath.Split(path) + session := podmanTest.Podman([]string{"run", "-v", fmt.Sprintf("%s:/artifacts:z", dir), ALPINE, "dd", "if=/dev/urandom", fmt.Sprintf("of=%s", filepath.Join("/artifacts", last)), "bs=1b", fmt.Sprintf("count=%d", newNumBytes)}) + session.WaitWithDefaultTimeout() + if session.ExitCode() != 0 { + return errors.New("unable to generate artifact file") + } + return nil +}