diff --git a/.gitignore b/.gitignore index a0b951aa0..29f8fa84c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ secrets.env # Dotenv environment file .env +.env_* .env.test # Files to be excluded. diff --git a/DOCS.md b/DOCS.md index 940ee02ba..95ba914a5 100644 --- a/DOCS.md +++ b/DOCS.md @@ -56,6 +56,12 @@ echo "VELA_SCM_CLIENT=" >> .env echo "VELA_SCM_SECRET=" >> .env ``` +* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela: + +```bash + sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts' +```` + ## Start **NOTE: Please review the [setup section](#setup) before moving forward.** diff --git a/api/admin/storage.go b/api/admin/storage.go new file mode 100644 index 000000000..588f6ed80 --- /dev/null +++ b/api/admin/storage.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/storage" + "github.com/go-vela/server/util" +) + +// swagger:operation POST /api/v1/admin/storage/bucket admin CreateBucket +// +// Create a new bucket +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: The bucket name to be created +// required: true +// schema: +// type: object +// properties: +// bucketName: +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the bucket +// '400': +// description: Invalid request payload +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// CreateBucket represents the API handler to create a new bucket. +func CreateBucket(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + + l.Debug("platform admin: creating bucket") + + // capture body from API request + input := new(types.Bucket) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for bucket %s: %w", input.BucketName, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + l.Debugf("bucket name: %s", input.BucketName) + + err = storage.FromGinContext(c).CreateBucket(ctx, input) + if err != nil { + retErr := fmt.Errorf("unable to create bucket: %w", err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.Status(http.StatusCreated) +} + +// swagger:operation GET /api/v1/admin/storage/presign admin GetPresignedURL +// +// # Generate a presigned URL for an object +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: bucketName +// description: The name of the bucket +// required: true +// type: string +// - in: query +// name: objectName +// description: The name of the object +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully generated the presigned URL +// '400': +// description: Invalid request payload +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// GetPresignedURL represents the API handler to generate a presigned URL for an object. +func GetPresignedURL(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + + l.Debug("platform admin: generating presigned URL") + + // capture query parameters from API request + bucketName := c.Query("bucketName") + objectName := c.Query("objectName") + + if bucketName == "" || objectName == "" { + retErr := fmt.Errorf("bucketName and objectName are required") + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + input := &types.Object{ + Bucket: types.Bucket{BucketName: bucketName}, + ObjectName: objectName, + } + + url, err := storage.FromGinContext(c).PresignedGetObject(ctx, input) + if err != nil || url == "" { + retErr := fmt.Errorf("unable to generate presigned URL: %w", err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.JSON(http.StatusOK, url) +} diff --git a/api/artifact/create.go b/api/artifact/create.go new file mode 100644 index 000000000..f213feeac --- /dev/null +++ b/api/artifact/create.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database" + "github.com/go-vela/server/util" +) + +func CreateArtifact(c *gin.Context) { + // capture middleware values + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + + // capture the artifact from the request body + input := new(types.Artifact) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new Artifact: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure build_id is defined + if input.GetBuildID() <= 0 { + util.HandleError(c, http.StatusBadRequest, fmt.Errorf("build_id must set and greater than 0")) + return + } + + input.SetCreatedAt(time.Now().UTC().Unix()) + + l.Debugf("creating new artifact") + // create the artifact in the database using the input from request + a, err := database.FromContext(c).CreateArtifact(ctx, input) + if err != nil { + retErr := fmt.Errorf("unable to create new artifact: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusCreated, a) +} diff --git a/api/artifact/get.go b/api/artifact/get.go new file mode 100644 index 000000000..bdb657ae3 --- /dev/null +++ b/api/artifact/get.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + artifact "github.com/go-vela/server/router/middleware/artifact" + "github.com/go-vela/server/router/middleware/build" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/artifact/{artifact} artifacts GetArtifact +// +// Get an artifact +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the organization +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repository +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: artifact +// description: artifact ID +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the artifact +// type: json +// schema: +// "$ref": "#/definitions/Artifact" +// '400': +// description: Invalid request payload or path +// schema: +// "$ref": "#/definitions/Error" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Not found +// schema: +// "$ref": "#/definitions/Error" + +// GetArtifact represents the API handler to get +// an artifact for a build. +func GetArtifact(c *gin.Context) { + // capture middleware values + l := c.MustGet("logger").(*logrus.Entry) + b := build.Retrieve(c) + a := artifact.Retrieve(c) + + l.Debugf("getting artifact %d for build %d", a.GetID(), b.GetNumber()) + + // return the artifact with presigned URL + response := gin.H{ + "id": a.GetID(), + "build_id": a.GetBuildID(), + "file_name": a.GetFileName(), + "file_type": a.GetFileType(), + "file_size": a.GetFileSize(), + "object_path": a.GetObjectPath(), + "presigned_url": a.GetPresignedURL(), + "created_at": a.GetCreatedAt(), + } + + c.JSON(http.StatusOK, response) +} diff --git a/api/artifact/list.go b/api/artifact/list.go new file mode 100644 index 000000000..55b75966f --- /dev/null +++ b/api/artifact/list.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/util" +) + +func ListArtifactsForBuild(c *gin.Context) { + // capture middleware values + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + b := build.Retrieve(c) + + l.Debugf("listing artifacts for build %d", b.GetNumber()) + + // retrieve artifacts from the database + artifacts, err := database.FromContext(c).ListArtifactsByBuildID(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to list artifacts for build %d: %w", b.GetNumber(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, artifacts) +} diff --git a/api/storage/doc.go b/api/storage/doc.go new file mode 100644 index 000000000..aae3e99dd --- /dev/null +++ b/api/storage/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package storage provides the storage handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/storage" +package storage diff --git a/api/storage/storage.go b/api/storage/storage.go new file mode 100644 index 000000000..311cb23fe --- /dev/null +++ b/api/storage/storage.go @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/storage" + "github.com/go-vela/server/util" +) + +// swagger:operation GET /api/v1/storage/info storage StorageInfo +// +// Get storage credentials +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved storage credentials +// schema: +// "$ref": "#/definitions/StorageInfo" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" + +// Info represents the API handler to +// retrieve storage credentials as part of worker onboarding. +func Info(c *gin.Context) { + l := c.MustGet("logger").(*logrus.Entry) + + enable := c.MustGet("storage-enable").(bool) + if !enable { + l.Info("storage is not enabled, sending storage disabled response") + + e := c.MustGet("storage-enable").(bool) + wr := types.StorageInfo{ + StorageEnabled: &e, + } + + c.JSON(http.StatusOK, wr) + + return + } + + l.Info("requesting storage credentials with registration token") + + // extract the storage-enable that was packed into gin context + e := c.MustGet("storage-enable").(bool) + + // extract the public key that was packed into gin context + k := c.MustGet("storage-access-key").(string) + + // extract the storage-address that was packed into gin context + a := c.MustGet("storage-address").(string) + + // extract the secret key that was packed into gin context + s := c.MustGet("storage-secret-key").(string) + + // extract bucket name that was packed into gin context + b := c.MustGet("storage-bucket").(string) + + wr := types.StorageInfo{ + StorageEnabled: &e, + StorageAccessKey: &k, + StorageAddress: &a, + StorageSecretKey: &s, + StorageBucket: &b, + } + + c.JSON(http.StatusOK, wr) +} + +// swagger:operation GET /api/v1/storage/{bucket}/objects storage ListObjects +// +// List objects in a bucket +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: bucket +// description: Name of the bucket +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully listed objects in the bucket +// schema: +// type: array +// items: +// type: string +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// ListObjects represents the API handler to list objects in a bucket. +func ListObjects(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + + l.Debug("listing objects in bucket") + + // extract the bucket name from the request + bucketName := util.PathParameter(c, "bucket") + + // create a new bucket object + b := &types.Bucket{ + BucketName: bucketName, + Recursive: true, + } + + // list objects in the bucket + objects, err := storage.FromGinContext(c).ListObjects(c.Request.Context(), b) + if err != nil { + l.Errorf("unable to list objects in bucket %s: %v", bucketName, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + c.JSON(http.StatusOK, gin.H{"objects": objects}) +} + +// ListObjectNames represents the API handler to list only the names of objects in a bucket. +func ListObjectNames(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + l.Debug("listing object names in bucket") + + // extract the bucket name from the request + bucketName := util.PathParameter(c, "bucket") + + // create a new bucket object + b := &types.Bucket{ + BucketName: bucketName, + Recursive: true, + } + + // list objects in the bucket + objects, err := storage.FromGinContext(c).ListObjects(c.Request.Context(), b) + if err != nil { + l.Errorf("unable to list objects in bucket %s: %v", bucketName, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + // Extract just the names from the objects + names := make([]string, 0, len(objects)) + for _, obj := range objects { + names = append(names, obj.Key) + } + + c.JSON(http.StatusOK, gin.H{"names": names}) +} + +// ListBuildObjectNames represents the API handler to list object names for a specific build. +func ListBuildObjectNames(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + + // Extract path parameters + bucketName := util.PathParameter(c, "bucket") + org := util.PathParameter(c, "org") + repo := util.PathParameter(c, "repo") + buildNum := util.PathParameter(c, "build") + + // Validate parameters + if org == "" || repo == "" || buildNum == "" { + l.Error("missing required parameters (org, repo, or build)") + c.JSON(http.StatusBadRequest, gin.H{"error": "missing required parameters"}) + + return + } + + l.Debugf("listing object names in bucket %s for %s/%s build #%s", bucketName, org, repo, buildNum) + + // Create a new bucket object + b := &types.Bucket{ + BucketName: bucketName, + Recursive: true, + } + + // Call the ListBuildObjectNames method that handles prefix filtering + objectNames, err := storage.FromGinContext(c).ListBuildObjectNames( + c.Request.Context(), + b, + org, + repo, + buildNum, + ) + if err != nil { + l.Errorf("unable to list objects for %s/%s build #%s: %v", org, repo, buildNum, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + c.JSON(http.StatusOK, gin.H{"names": objectNames}) +} diff --git a/api/types/artifact.go b/api/types/artifact.go new file mode 100644 index 000000000..ef9220b9d --- /dev/null +++ b/api/types/artifact.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import "fmt" + +// Artifact is the API representation of an artifact. +// +// swagger:model Artifact +type Artifact struct { + ID *int64 `json:"id,omitempty"` + BuildID *int64 `json:"build_id,omitempty"` + FileName *string `json:"file_name,omitempty"` + ObjectPath *string `json:"object_path,omitempty"` + FileSize *int64 `json:"file_size,omitempty"` + FileType *string `json:"file_type,omitempty"` + PresignedURL *string `json:"presigned_url,omitempty"` + CreatedAt *int64 `json:"created_at,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetID() int64 { + // return zero value if Artifact type or ID field is nil + if a == nil || a.ID == nil { + return 0 + } + + return *a.ID +} + +// GetBuildID returns the BuildID field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetBuildID() int64 { + // return zero value if Artifact type or BuildID field is nil + if a == nil || a.BuildID == nil { + return 0 + } + + return *a.BuildID +} + +// GetFileName returns the FileName field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetFileName() string { + // return zero value if Artifact type or FileName field is nil + if a == nil || a.FileName == nil { + return "" + } + + return *a.FileName +} + +// GetObjectPath returns the FilePath field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetObjectPath() string { + // return zero value if Artifact type or FilePath field is nil + if a == nil || a.ObjectPath == nil { + return "" + } + + return *a.ObjectPath +} + +// GetFileSize returns the FileSize field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetFileSize() int64 { + // return zero value if Artifact type or FileSize field is nil + if a == nil || a.FileSize == nil { + return 0 + } + + return *a.FileSize +} + +// GetFileType returns the FileType field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetFileType() string { + // return zero value if Artifact type or FileType field is nil + if a == nil || a.FileType == nil { + return "" + } + + return *a.FileType +} + +// GetPresignedURL returns the PresignedUrl field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetPresignedURL() string { + // return zero value if Artifact type or PresignedUrl field is nil + if a == nil || a.PresignedURL == nil { + return "" + } + + return *a.PresignedURL +} + +// GetCreatedAt returns the CreatedAt field. +// +// When the provided Artifact type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Artifact) GetCreatedAt() int64 { + // return zero value if Artifact type or CreatedAt field is nil + if a == nil || a.CreatedAt == nil { + return 0 + } + + return *a.CreatedAt +} + +// SetID sets the ID field. +func (a *Artifact) SetID(v int64) { + // return if Artifact type is nil + if a == nil { + return + } + // set the ID field + a.ID = &v +} + +// SetBuildID sets the BuildID field. +func (a *Artifact) SetBuildID(v int64) { + // return if Artifact type is nil + if a == nil { + return + } + // set the BuildID field + a.BuildID = &v +} + +// SetFileName sets the FileName field. +func (a *Artifact) SetFileName(v string) { + // return if Artifact type is nil + if a == nil { + return + } + // set the FileName field + a.FileName = &v +} + +// SetObjectPath sets the ObjectPath field. +func (a *Artifact) SetObjectPath(v string) { + // return if Artifact type is nil + if a == nil { + return + } + // set the ObjectPath field + a.ObjectPath = &v +} + +// SetFileSize sets the FileSize field. +func (a *Artifact) SetFileSize(v int64) { + // return if Artifact type is nil + if a == nil { + return + } + // set the FileSize field + a.FileSize = &v +} + +// SetFileType sets the FileType field. +func (a *Artifact) SetFileType(v string) { + // return if Artifact type is nil + if a == nil { + return + } + // set the FileType field + a.FileType = &v +} + +// SetPresignedURL sets the PresignedUrl field. +func (a *Artifact) SetPresignedURL(v string) { + // return if Artifact type is nil + if a == nil { + return + } + // set the PresignedUrl field + a.PresignedURL = &v +} + +// SetCreatedAt sets the CreatedAt field. +func (a *Artifact) SetCreatedAt(v int64) { + // return if Artifact type is nil + if a == nil { + return + } + // set the CreatedAt field + a.CreatedAt = &v +} + +// String implements the Stringer interface for the Artifact type. +func (a *Artifact) String() string { + return fmt.Sprintf("FileName: %s, ObjectPath: %s, FileType: %s", a.GetFileName(), a.GetObjectPath(), a.GetFileType()) +} diff --git a/api/types/pipeline.go b/api/types/pipeline.go index dd0ff617d..0a1464847 100644 --- a/api/types/pipeline.go +++ b/api/types/pipeline.go @@ -24,6 +24,7 @@ type Pipeline struct { Stages *bool `json:"stages,omitempty"` Steps *bool `json:"steps,omitempty"` Templates *bool `json:"templates,omitempty"` + Artifact *bool `json:"artifact,omitempty"` Warnings *[]string `json:"warnings,omitempty"` // swagger:strfmt base64 Data *[]byte `json:"data,omitempty"` @@ -237,6 +238,19 @@ func (p *Pipeline) GetData() []byte { return *p.Data } +// GetArtifact returns the Artifact flag. +// +// When the provided Pipeline type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (p *Pipeline) GetArtifact() bool { + // return zero value if Pipeline type or Artifact field is nil + if p == nil || p.Artifact == nil { + return false + } + + return *p.Artifact +} + // SetID sets the ID field. // // When the provided Pipeline type is nil, it @@ -419,6 +433,19 @@ func (p *Pipeline) SetTemplates(v bool) { p.Templates = &v } +// SetArtifact sets the Artifact field. +// +// When the provided Pipeline type is nil, it +// will set nothing and immediately return. +func (p *Pipeline) SetArtifact(v bool) { + // return if Pipeline type is nil + if p == nil { + return + } + + p.Artifact = &v +} + // SetWarnings sets the Warnings field. // // When the provided Pipeline type is nil, it @@ -461,6 +488,7 @@ func (p *Pipeline) String() string { Stages: %t, Steps: %t, Templates: %t, + Artifacts: %t, Type: %s, Version: %s, Warnings: %v, @@ -478,6 +506,7 @@ func (p *Pipeline) String() string { p.GetStages(), p.GetSteps(), p.GetTemplates(), + p.GetArtifact(), p.GetType(), p.GetVersion(), p.GetWarnings(), diff --git a/api/types/pipeline_test.go b/api/types/pipeline_test.go index 8c14c5af2..1c59019d0 100644 --- a/api/types/pipeline_test.go +++ b/api/types/pipeline_test.go @@ -214,6 +214,7 @@ func TestAPI_Pipeline_String(t *testing.T) { Stages: %t, Steps: %t, Templates: %t, + Artifacts: %t, Type: %s, Version: %s, Warnings: %v, @@ -231,6 +232,7 @@ func TestAPI_Pipeline_String(t *testing.T) { p.GetStages(), p.GetSteps(), p.GetTemplates(), + p.GetArtifact(), p.GetType(), p.GetVersion(), p.GetWarnings(), @@ -263,6 +265,7 @@ func testPipeline() *Pipeline { p.SetStages(false) p.SetSteps(true) p.SetTemplates(false) + p.SetArtifact(true) p.SetData(testPipelineData()) p.SetWarnings([]string{"42:this is a warning"}) diff --git a/api/types/storage.go b/api/types/storage.go new file mode 100644 index 000000000..0c6913282 --- /dev/null +++ b/api/types/storage.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "github.com/minio/minio-go/v7" +) + +// Bucket is the API types representation of an object storage. +// +// swagger:model CreateBucket +type Bucket struct { + BucketName string `json:"bucket_name,omitempty"` + MakeBucketOptions minio.MakeBucketOptions `json:"make_bucket_options,omitempty"` + ListObjectsOptions minio.ListObjectsOptions `json:"list_objects_options,omitempty"` + Recursive bool `json:"recursive"` +} + +type Object struct { + ObjectName string `json:"object_name,omitempty"` + Bucket Bucket `json:"bucket,omitempty"` + FilePath string `json:"file_path,omitempty"` +} diff --git a/api/types/storage_info.go b/api/types/storage_info.go new file mode 100644 index 000000000..6e426b98a --- /dev/null +++ b/api/types/storage_info.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +// StorageInfo is the API representation of a StorageInfo. +// +// swagger:model StorageInfo +type StorageInfo struct { + StorageEnabled *bool `json:"storage_enabled,omitempty"` + StorageAccessKey *string `json:"storage_access_key,omitempty"` + StorageSecretKey *string `json:"storage_secret_key,omitempty"` + StorageAddress *string `json:"storage_address,omitempty"` + StorageBucket *string `json:"storage_bucket,omitempty"` +} + +// GetEnabled returns the StorageEnabled field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns false for the field. +func (w *StorageInfo) GetEnabled() bool { + // return zero value if StorageInfo type or StorageEnabled field is nil + if w == nil || w.StorageEnabled == nil { + return false + } + + return *w.StorageEnabled +} + +// GetAccessKey returns the StorageAccessKey field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetAccessKey() string { + // return zero value if StorageInfo type or StorageAccessKey field is nil + if w == nil || w.StorageAccessKey == nil { + return "" + } + + return *w.StorageAccessKey +} + +// GetSecretKey returns the StorageSecretKey field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetSecretKey() string { + // return zero value if StorageInfo type or StorageSecretKey field is nil + if w == nil || w.StorageSecretKey == nil { + return "" + } + + return *w.StorageSecretKey +} + +// GetStorageAddress returns the StorageAddress field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetStorageAddress() string { + // return zero value if StorageInfo type or StorageAddress field is nil + if w == nil || w.StorageAddress == nil { + return "" + } + + return *w.StorageAddress +} + +// GetStorageBucket returns the StorageBucket field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetStorageBucket() string { + // return zero value if StorageInfo type or StorageBucket field is nil + if w == nil || w.StorageBucket == nil { + return "" + } + + return *w.StorageBucket +} + +// SetEnabled sets the StorageEnabled field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetEnabled(v bool) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageEnabled = &v +} + +// SetAccessKey sets the StorageAccessKey field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetAccessKey(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageAccessKey = &v +} + +// SetSecretKey sets the StorageSecretKey field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetSecretKey(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageSecretKey = &v +} + +// SetStorageAddress sets the StorageAddress field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetStorageAddress(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageAddress = &v +} + +// SetStorageBucket sets the StorageBucket field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetStorageBucket(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageBucket = &v +} diff --git a/api/types/storage_info_test.go b/api/types/storage_info_test.go new file mode 100644 index 000000000..56f53f775 --- /dev/null +++ b/api/types/storage_info_test.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "testing" +) + +func TestTypes_StorageInfo_Getters(t *testing.T) { + // setup tests + tests := []struct { + sI *StorageInfo + want *StorageInfo + }{ + { + sI: testStorageInfo(), + want: testStorageInfo(), + }, + { + sI: new(StorageInfo), + want: new(StorageInfo), + }, + } + + // run tests + for _, test := range tests { + if test.sI.GetAccessKey() != test.want.GetAccessKey() { + t.Errorf("GetAccessKey is %v, want %v", test.sI.GetAccessKey(), test.want.GetAccessKey()) + } + + if test.sI.GetSecretKey() != test.want.GetSecretKey() { + t.Errorf("GetSecretKey is %v, want %v", test.sI.GetSecretKey(), test.want.GetSecretKey()) + } + + if test.sI.GetStorageAddress() != test.want.GetStorageAddress() { + t.Errorf("GetStorageAddress is %v, want %v", test.sI.GetStorageAddress(), test.want.GetStorageAddress()) + } + + if test.sI.GetStorageBucket() != test.want.GetStorageBucket() { + t.Errorf("GetStorageBucket is %v, want %v", test.sI.GetStorageBucket(), test.want.GetStorageBucket()) + } + } +} + +func TestTypes_StorageInfo_Setters(t *testing.T) { + // setup types + var sI *StorageInfo + + // setup tests + tests := []struct { + sI *StorageInfo + want *StorageInfo + }{ + { + sI: testStorageInfo(), + want: testStorageInfo(), + }, + { + sI: sI, + want: new(StorageInfo), + }, + } + + // run tests + for _, test := range tests { + test.sI.SetAccessKey(test.want.GetAccessKey()) + test.sI.SetSecretKey(test.want.GetSecretKey()) + test.sI.SetStorageAddress(test.want.GetStorageAddress()) + test.sI.SetStorageBucket(test.want.GetStorageBucket()) + + if test.sI.GetAccessKey() != test.want.GetAccessKey() { + t.Errorf("GetAccessKey is %v, want %v", test.sI.GetAccessKey(), test.want.GetAccessKey()) + } + + if test.sI.GetSecretKey() != test.want.GetSecretKey() { + t.Errorf("GetSecretKey is %v, want %v", test.sI.GetSecretKey(), test.want.GetSecretKey()) + } + + if test.sI.GetStorageAddress() != test.want.GetStorageAddress() { + t.Errorf("GetStorageAddress is %v, want %v", test.sI.GetStorageAddress(), test.want.GetStorageAddress()) + } + } +} + +// testStorageInfo is a test helper function to register a StorageInfo +// type with all fields set to a fake value. +func testStorageInfo() *StorageInfo { + sI := new(StorageInfo) + sI.SetAccessKey("fakeAccessKey") + sI.SetSecretKey("fakeSecretKey") + sI.SetStorageAddress("http://localhost:8080") + + return sI +} diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 1fdae07e7..b06974327 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -18,6 +18,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/scm" "github.com/go-vela/server/secret" + "github.com/go-vela/server/storage" "github.com/go-vela/server/tracing" "github.com/go-vela/server/version" ) @@ -62,6 +63,9 @@ func main() { // Add Tracing Flags cmd.Flags = append(cmd.Flags, tracing.Flags...) + // Add S3 Flags + cmd.Flags = append(cmd.Flags, storage.Flags...) + if err = cmd.Run(context.Background(), os.Args); err != nil { logrus.Fatal(err) } diff --git a/cmd/vela-server/metadata.go b/cmd/vela-server/metadata.go index f23e309c1..347c972b4 100644 --- a/cmd/vela-server/metadata.go +++ b/cmd/vela-server/metadata.go @@ -45,6 +45,13 @@ func setupMetadata(c *cli.Command) (*internal.Metadata, error) { m.Vela = vela + storage, err := metadataStorage(c) + if err != nil { + return nil, err + } + + m.Storage = storage + return m, nil } @@ -93,6 +100,21 @@ func metadataSource(c *cli.Command) (*internal.Source, error) { }, nil } +// helper function to capture the queue metadata from the CLI arguments. +func metadataStorage(c *cli.Command) (*internal.Storage, error) { + logrus.Trace("creating storage metadata from CLI configuration") + + u, err := url.Parse(c.String("storage.addr")) + if err != nil { + return nil, err + } + + return &internal.Storage{ + Driver: c.String("storage.driver"), + Host: u.Host, + }, nil +} + // helper function to capture the Vela metadata from the CLI arguments. // //nolint:unparam // ignore unparam for now diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 8c3870088..34175a7a5 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -27,6 +27,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" + "github.com/go-vela/server/storage" "github.com/go-vela/server/tracing" ) @@ -113,6 +114,11 @@ func server(ctx context.Context, cmd *cli.Command) error { return err } + st, err := storage.FromCLICommand(ctx, cmd) + if err != nil { + return err + } + metadata, err := setupMetadata(cmd) if err != nil { return err @@ -197,6 +203,7 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.Secret(cmd.String("vela-secret")), middleware.Secrets(secrets), middleware.Scm(scm), + middleware.Storage(st), middleware.QueueSigningPrivateKey(cmd.String("queue.private-key")), middleware.QueueSigningPublicKey(cmd.String("queue.public-key")), middleware.QueueAddress(cmd.String("queue.addr")), @@ -213,6 +220,11 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.ScheduleFrequency(cmd.Duration("schedule-minimum-frequency")), middleware.TracingClient(tc), middleware.TracingInstrumentation(tc), + middleware.StorageAddress(cmd.String("storage.addr")), + middleware.StorageAccessKey(cmd.String("storage.access.key")), + middleware.StorageSecretKey(cmd.String("storage.secret.key")), + middleware.StorageBucket(cmd.String("storage.bucket.name")), + middleware.StorageEnable(cmd.Bool("storage.enable")), ) addr, err := url.Parse(cmd.String("server-addr")) diff --git a/compiler/native/validate.go b/compiler/native/validate.go index ff416cf24..34feadcd5 100644 --- a/compiler/native/validate.go +++ b/compiler/native/validate.go @@ -108,7 +108,7 @@ func validateYAMLSteps(s yaml.StepSlice) error { if len(step.Commands) == 0 && len(step.Environment) == 0 && len(step.Parameters) == 0 && len(step.Secrets) == 0 && - len(step.Template.Name) == 0 && !step.Detach { + len(step.Template.Name) == 0 && len(step.Artifacts.Paths) == 0 && !step.Detach { return fmt.Errorf("no commands, environment, parameters, secrets or template provided for step %s", step.Name) } } diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 8c9a2bf67..dbea19de7 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -743,3 +743,33 @@ func TestNative_Validate_Steps_StepNameConflict(t *testing.T) { t.Errorf("Validate should have returned err") } } + +func TestNative_Validate_Artifact(t *testing.T) { + // setup types + str := "foo" + p := &yaml.Build{ + Version: "v1", + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: raw.StringSlice{"echo hello"}, + Image: "alpine", + Name: str, + Pull: "always", + Artifacts: yaml.Artifacts{ + Paths: []string{"results.xml", "artifacts.png"}, + }, + }, + }, + } + + // run test + compiler, err := FromCLICommand(context.Background(), testCommand(t, "http://foo.example.com")) + if err != nil { + t.Errorf("Unable to create new compiler: %v", err) + } + + err = compiler.ValidateYAML(p) + if err != nil { + t.Errorf("Validate returned err: %v", err) + } +} diff --git a/compiler/types/pipeline/artifact.go b/compiler/types/pipeline/artifact.go new file mode 100644 index 000000000..0f9f11eb9 --- /dev/null +++ b/compiler/types/pipeline/artifact.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +// ArtifactSlice is the pipeline representation +// of a slice of artifacts. +// +// swagger:model PipelineArtifactSlice +type ArtifactSlice []*Artifacts + +// Artifact is the pipeline representation +// of artifacts for a pipeline. +// +// swagger:model PipelineArtifact +type Artifacts struct { + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` +} + +// Empty returns true if the provided Artifact is empty. +func (a *Artifacts) Empty() bool { + // return true if paths field is empty + if len(a.Paths) == 0 { + return true + } + + // return false if Paths are provided + return false +} diff --git a/compiler/types/pipeline/artifact_test.go b/compiler/types/pipeline/artifact_test.go new file mode 100644 index 000000000..bb0ae397f --- /dev/null +++ b/compiler/types/pipeline/artifact_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +import "testing" + +func TestPipeline_Artifacts_Empty(t *testing.T) { + // setup tests + tests := []struct { + artifacts *Artifacts + want bool + }{ + { + artifacts: &Artifacts{Paths: []string{"foo"}}, + want: false, + }, + { + artifacts: new(Artifacts), + want: true, + }, + } + + // run tests + for _, test := range tests { + got := test.artifacts.Empty() + + if got != test.want { + t.Errorf("Empty is %v, want %t", got, test.want) + } + } +} diff --git a/compiler/types/pipeline/container.go b/compiler/types/pipeline/container.go index 34a10dad1..e83b17c95 100644 --- a/compiler/types/pipeline/container.go +++ b/compiler/types/pipeline/container.go @@ -48,6 +48,7 @@ type ( Pull string `json:"pull,omitempty" yaml:"pull,omitempty"` Ruleset Ruleset `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` Secrets StepSecretSlice `json:"secrets,omitempty" yaml:"secrets,omitempty"` + Artifacts Artifacts `json:"artifacts,omitempty" yaml:"artifacts,omitempty"` Ulimits UlimitSlice `json:"ulimits,omitempty" yaml:"ulimits,omitempty"` Volumes VolumeSlice `json:"volumes,omitempty" yaml:"volumes,omitempty"` User string `json:"user,omitempty" yaml:"user,omitempty"` @@ -140,7 +141,8 @@ func (c *Container) Empty() bool { len(c.Volumes) == 0 && len(c.User) == 0 && len(c.ReportAs) == 0 && - len(c.IDRequest) == 0 { + len(c.IDRequest) == 0 && + reflect.DeepEqual(c.Artifacts, Artifacts{}) { return true } diff --git a/compiler/types/yaml/artifacts.go b/compiler/types/yaml/artifacts.go new file mode 100644 index 000000000..02a150fb1 --- /dev/null +++ b/compiler/types/yaml/artifacts.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import "github.com/go-vela/server/compiler/types/pipeline" + +// Artifact represents the structure for artifacts configuration. +type Artifacts struct { + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` +} + +// ToPipeline converts the Artifact type +// to a pipeline Artifact type. +func (a *Artifacts) ToPipeline() *pipeline.Artifacts { + return &pipeline.Artifacts{ + Paths: a.Paths, + } +} + +// UnmarshalYAML implements the Unmarshaler interface for the Artifact type. +func (a *Artifacts) UnmarshalYAML(unmarshal func(interface{}) error) error { + // artifacts we try unmarshalling to + artifacts := new(struct { + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` + }) + + // attempt to unmarshal artifacts type + err := unmarshal(artifacts) + if err != nil { + return err + } + + // set the paths field + a.Paths = artifacts.Paths + + return nil +} diff --git a/compiler/types/yaml/secret.go b/compiler/types/yaml/secret.go index bd2b2bb04..37f408fb7 100644 --- a/compiler/types/yaml/secret.go +++ b/compiler/types/yaml/secret.go @@ -158,7 +158,7 @@ func (o *Origin) Empty() bool { // MergeEnv takes a list of environment variables and attempts // to set them in the secret environment. If the environment -// variable already exists in the secret, than this will +// variable already exists in the secret, then this will // overwrite the existing environment variable. func (o *Origin) MergeEnv(environment map[string]string) error { // check if the secret container is empty diff --git a/compiler/types/yaml/step.go b/compiler/types/yaml/step.go index 405e6d598..d588222d7 100644 --- a/compiler/types/yaml/step.go +++ b/compiler/types/yaml/step.go @@ -24,6 +24,7 @@ type ( Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Command to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-entrypoint-key"` Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Sensitive variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` Template StepTemplate `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"oneof_required=template,description=Name of template to expand in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Artifacts Artifacts `yaml:"artifacts,omitempty" json:"artifacts,omitempty" jsonschema:"description=Artifacts configuration for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-artifacts-key"` Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` Volumes VolumeSlice `yaml:"volumes,omitempty" json:"volumes,omitempty" jsonschema:"description=Mount volumes for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"oneof_required=image,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` @@ -59,6 +60,7 @@ func (s *StepSlice) ToPipeline() *pipeline.ContainerSlice { Pull: step.Pull, Ruleset: *step.Ruleset.ToPipeline(), Secrets: *step.Secrets.ToPipeline(), + Artifacts: *step.Artifacts.ToPipeline(), Ulimits: *step.Ulimits.ToPipeline(), Volumes: *step.Volumes.ToPipeline(), User: step.User, diff --git a/compiler/types/yaml/step_test.go b/compiler/types/yaml/step_test.go index b570c0c7d..e4c1901a6 100644 --- a/compiler/types/yaml/step_test.go +++ b/compiler/types/yaml/step_test.go @@ -77,6 +77,9 @@ func TestYaml_StepSlice_ToPipeline(t *testing.T) { AccessMode: "ro", }, }, + Artifacts: Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, }, }, want: &pipeline.ContainerSlice{ @@ -136,6 +139,9 @@ func TestYaml_StepSlice_ToPipeline(t *testing.T) { AccessMode: "ro", }, }, + Artifacts: pipeline.Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, }, }, }, @@ -215,6 +221,14 @@ func TestYaml_StepSlice_UnmarshalYAML(t *testing.T) { }, }, }, + { + Name: "artifact", + Image: "golang:1.20", + Pull: "always", + Artifacts: Artifacts{ + Paths: []string{"test-results/*.xml", "screenshots/**/*.png", " video/*.mp4"}, + }, + }, }, }, { diff --git a/compiler/types/yaml/testdata/step.yml b/compiler/types/yaml/testdata/step.yml index 1d6d9cc93..a115689f1 100644 --- a/compiler/types/yaml/testdata/step.yml +++ b/compiler/types/yaml/testdata/step.yml @@ -43,4 +43,10 @@ vars: registry: index.docker.io repo: github/octocat - tags: [ latest, dev ] + tags: [latest, dev] + +- name: artifact + image: golang:1.20 + pull: true + artifacts: + paths: ['test-results/*.xml', 'screenshots/**/*.png', ' video/*.mp4'] diff --git a/constants/driver.go b/constants/driver.go index e42b924c5..fb716e435 100644 --- a/constants/driver.go +++ b/constants/driver.go @@ -62,3 +62,11 @@ const ( // DriverGitLab defines the driver type when integrating with a Gitlab source code system. DriverGitlab = "gitlab" ) + +// Server storage drivers. +const ( + // DriverMinio defines the driver type when integrating with a local storage system. + DriverMinio = "minio" + // DriverAws defines the driver type when integrating with an AWS S3 storage system. + DriverAws = "aws" +) diff --git a/constants/filetypes.go b/constants/filetypes.go new file mode 100644 index 000000000..69f4cf177 --- /dev/null +++ b/constants/filetypes.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +package constants + +// TextFileExtensions and MediaFileExtensions package-level vars (Go doesn't allow const slices). +var TextFileExtensions = []string{".xml", ".json", ".html", ".txt"} +var MediaFileExtensions = []string{".png", ".jpg", ".jpeg", ".gif", ".mp4", ".mov"} + +// AllAllowedExtensions is the union of test + media. +var AllAllowedExtensions = append(TextFileExtensions, MediaFileExtensions...) diff --git a/constants/table.go b/constants/table.go index cd1000469..fe0b73d88 100644 --- a/constants/table.go +++ b/constants/table.go @@ -31,6 +31,9 @@ const ( // TableRepo defines the table type for the database repos table. TableRepo = "repos" + // TableArtifact defines the table type for the database artifacts table. + TableArtifact = "artifacts" + // TableSchedule defines the table type for the database schedules table. TableSchedule = "schedules" diff --git a/database/artifact/artifact.go b/database/artifact/artifact.go new file mode 100644 index 000000000..a9759a957 --- /dev/null +++ b/database/artifact/artifact.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" + + "github.com/go-vela/server/constants" +) + +type ( + // config represents the settings required to create the engine that implements the ArtifactInterface interface. + config struct { + // specifies the encryption key to use for the Artifact engine + EncryptionKey string + // specifies to skip creating tables and indexes for the Artifact engine + SkipCreation bool + } + + // engine represents the artifacts functionality that implements the ArtifactsInterface interface. + Engine struct { + // engine configuration settings used in artifacts functions + config *config + + ctx context.Context + + // gorm.io/gorm database client used in artifacts functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in artifacts functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with artifacts in the database. +// + +func New(opts ...EngineOpt) (*Engine, error) { + // create new Artifact engine + e := new(Engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating artifacts database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of artifacts table and indexes") + + return e, nil + } + + // create the artifacts table + err := e.CreateArtifactTable(e.ctx, e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableArtifact, err) + } + + // create the indexes for the artifacts table + err = e.CreateArtifactIndexes(e.ctx) + if err != nil { + return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableArtifact, err) + } + + return e, nil +} diff --git a/database/artifact/artifact_test.go b/database/artifact/artifact_test.go new file mode 100644 index 000000000..bc2db6a40 --- /dev/null +++ b/database/artifact/artifact_test.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/go-vela/server/database/testutils" +) + +func TestArtifact_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := testutils.TestPostgresGormInit(_sql) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *Engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + logger: logger, + skipCreation: false, + want: &Engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + logger: logger, + skipCreation: false, + want: &Engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*Engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres dashboard engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *Engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite dashboard engine: %v", err) + } + + return _engine +} diff --git a/database/artifact/count.go b/database/artifact/count.go new file mode 100644 index 000000000..6762caff9 --- /dev/null +++ b/database/artifact/count.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/go-vela/server/constants" +) + +// CountArtifacts gets the count of all artifacts from the database. +func (e *Engine) CountArtifacts(ctx context.Context) (int64, error) { + e.logger.Tracef("getting count of all artifacts") + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Count(&s). + Error + + return s, err +} diff --git a/database/artifact/count_test.go b/database/artifact/count_test.go new file mode 100644 index 000000000..c2a66ca7e --- /dev/null +++ b/database/artifact/count_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/database/testutils" +) + +func TestArtifact_Engine_Count(t *testing.T) { + // setup types + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + ctx := context.TODO() + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query for the artifacts table + _mock.ExpectQuery(`SELECT count(*) FROM "artifacts"`). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateArtifact(ctx, _artifact) + if err != nil { + t.Errorf("unable to create artifact for sqlite: %v", err) + } + // setup tests + tests := []struct { + failure bool + name string + database *Engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 1, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountArtifacts(ctx) + if test.failure { + if err == nil { + t.Errorf("Count for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Count for %s returned err: %v", test.name, err) + } + + if got != test.want { + t.Errorf("Count for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/artifact/create.go b/database/artifact/create.go new file mode 100644 index 000000000..a8788aa88 --- /dev/null +++ b/database/artifact/create.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// CreateArtifact creates a new artifact in the database. +func (e *Engine) CreateArtifact(ctx context.Context, r *api.Artifact) (*api.Artifact, error) { + e.logger.WithFields(logrus.Fields{ + "build": r.GetID(), + }).Tracef("creating artifact %d", r.GetID()) + + artifact := types.ArtifactFromAPI(r) + + err := artifact.Validate() + if err != nil { + return nil, err + } + + // send query to the database + err = e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Create(artifact).Error + if err != nil { + return nil, err + } + + result := artifact.ToAPI() + result.SetBuildID(r.GetBuildID()) + + return result, nil +} diff --git a/database/artifact/create_test.go b/database/artifact/create_test.go new file mode 100644 index 000000000..06ec4fed1 --- /dev/null +++ b/database/artifact/create_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/database/testutils" +) + +func TestEngine_CreateArtifact(t *testing.T) { + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "artifacts" ("build_id","file_name","object_path","file_size","file_type","presigned_url","created_at","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). + WithArgs(1, "foo", "foo/bar", 1, "xml", "foobar", 1, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CreateArtifact(context.TODO(), _artifact) + + if test.failure { + if err == nil { + t.Errorf("Create for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Create for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, _artifact) { + t.Errorf("Create for %s returned %v, want %v", test.name, got, _artifact) + } + }) + } +} diff --git a/database/artifact/delete.go b/database/artifact/delete.go new file mode 100644 index 000000000..c2ff1e978 --- /dev/null +++ b/database/artifact/delete.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// DeleteArtifact deletes an existing artifact from the database. +func (e *Engine) DeleteArtifact(ctx context.Context, r *api.Artifact) error { + e.logger.WithFields(logrus.Fields{ + "artifact": r.GetID(), + }).Tracef("deleting artifact %d", r.GetID()) + + // cast the API type to database type + artifact := types.ArtifactFromAPI(r) + + // send query to the database + return e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Delete(artifact). + Error +} diff --git a/database/artifact/delete_test.go b/database/artifact/delete_test.go new file mode 100644 index 000000000..c656baf51 --- /dev/null +++ b/database/artifact/delete_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/database/testutils" +) + +func TestArtifact_Engine_Delete(t *testing.T) { + // setup types + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + ctx := context.TODO() + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(`DELETE FROM "artifacts" WHERE "artifacts"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.DeleteArtifact(ctx, _artifact) + + if test.failure { + if err == nil { + t.Errorf("Delete for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Delete for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/artifact/get.go b/database/artifact/get.go new file mode 100644 index 000000000..934e0ec65 --- /dev/null +++ b/database/artifact/get.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// GetArtifact gets an artifact by ID from the database. +func (e *Engine) GetArtifact(ctx context.Context, id int64) (*api.Artifact, error) { + e.logger.WithFields(logrus.Fields{ + "artifact_id": id, + }).Tracef("getting artifact %d", id) + + // variable to store query results + r := new(types.Artifact) + + // send query to the database + err := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Where("id = ?", id). + Take(r). + Error + if err != nil { + return nil, fmt.Errorf("unable to get artifact: %w", err) + } + + return r.ToAPI(), nil +} diff --git a/database/artifact/get_build.go b/database/artifact/get_build.go new file mode 100644 index 000000000..0803cdd88 --- /dev/null +++ b/database/artifact/get_build.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// GetArtifactForBuild gets an artifact by number and build ID from the database. +func (e *Engine) GetArtifactForBuild(ctx context.Context, b *api.Build) (*api.Artifact, error) { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("getting artifact") + + // variable to store query results + tr := new(types.Artifact) + + // send query to the database and store result in variable + err := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Where("build_id = ?", b.GetID()). + Take(tr). + Error + if err != nil { + return nil, err + } + + return tr.ToAPI(), nil +} diff --git a/database/artifact/get_test.go b/database/artifact/get_test.go new file mode 100644 index 000000000..b8803f021 --- /dev/null +++ b/database/artifact/get_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" +) + +func TestEngine_GetArtifact(t *testing.T) { + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := testutils.CreateMockRows([]any{*types.ArtifactFromAPI(_artifact)}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "artifacts" WHERE id = $1 LIMIT $2`).WithArgs(1, 1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateArtifact(context.TODO(), _artifact) + if err != nil { + t.Errorf("unable to create artifact for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + want *api.Artifact + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _artifact, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _artifact, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetArtifact(context.TODO(), 1) + + if test.failure { + if err == nil { + t.Errorf("GetArtifact for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetArtifact for %s returned err: %v", test.name, err) + } + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("GetArtifact mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/database/artifact/index.go b/database/artifact/index.go new file mode 100644 index 000000000..978b056cb --- /dev/null +++ b/database/artifact/index.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import "context" + +const ( + // CreateBuildIDIndex represents a query to create an + // index on the artifacts table for the build_id column. + CreateBuildIDIndex = ` +CREATE INDEX +IF NOT EXISTS +artifacts_build_id +ON artifacts (build_id); +` +) + +// CreateArtifactIndexes creates the indexes for the artifacts table in the database. +func (e *Engine) CreateArtifactIndexes(ctx context.Context) error { + e.logger.Tracef("creating indexes for artifacts table") + + // create the build_id column index for the artifacts table + return e.client. + WithContext(ctx). + Exec(CreateBuildIDIndex).Error +} diff --git a/database/artifact/index_test.go b/database/artifact/index_test.go new file mode 100644 index 000000000..ebaf202e2 --- /dev/null +++ b/database/artifact/index_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestArtifact_Engine_CreateArtifactIndexes(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateArtifactIndexes(context.TODO()) + + if test.failure { + if err == nil { + t.Errorf("CreateBuildIDIndex for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuildIDIndex for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/artifact/interface.go b/database/artifact/interface.go new file mode 100644 index 000000000..acab781aa --- /dev/null +++ b/database/artifact/interface.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// ArtifactInterface represents the Vela interface for artifacts +// functions with the supported Database backends. +// + +type ArtifactInterface interface { + // Artifact Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateArtifactIndexes defines a function that creates the indexes for the artifacts table. + CreateArtifactIndexes(context.Context) error + // CreateArtifactTable defines a function that creates the artifacts table. + CreateArtifactTable(context.Context, string) error + + // Artifact Management Functions + + // CountArtifacts returns the count of all artifacts. + CountArtifacts(context.Context) (int64, error) + + // CreateArtifact creates a new artifact. + CreateArtifact(context.Context, *api.Artifact) (*api.Artifact, error) + + // DeleteArtifact removes an artifact by ID. + DeleteArtifact(context.Context, *api.Artifact) error + + // GetArtifact returns an artifact by ID. + GetArtifact(context.Context, int64) (*api.Artifact, error) + + // GetArtifactForBuild defines a function that gets an artifact by number and build ID. + GetArtifactForBuild(context.Context, *api.Build) (*api.Artifact, error) + + // ListArtifacts returns a list of all artifacts. + ListArtifacts(context.Context) ([]*api.Artifact, error) + + // ListArtifactsByBuildID returns a list of artifacts by build ID. + ListArtifactsByBuildID(context.Context, int64) ([]*api.Artifact, error) + + // UpdateArtifact updates an artifact by ID. + UpdateArtifact(context.Context, *api.Artifact) (*api.Artifact, error) +} diff --git a/database/artifact/list.go b/database/artifact/list.go new file mode 100644 index 000000000..ed032b133 --- /dev/null +++ b/database/artifact/list.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// ListArtifacts returns a list of artifacts from the database. +func (e *Engine) ListArtifacts(ctx context.Context) ([]*api.Artifact, error) { + e.logger.Trace("listing artifacts from the database") + + // variables to store query results and return value + t := new([]types.Artifact) + + var reports []*api.Artifact + + // send query to the database and store result in variable + err := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Order("created_at DESC"). + Find(&t). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, report := range *t { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := report + + reports = append(reports, tmp.ToAPI()) + } + + return reports, nil +} diff --git a/database/artifact/list_build.go b/database/artifact/list_build.go new file mode 100644 index 000000000..da06b3476 --- /dev/null +++ b/database/artifact/list_build.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// ListArtifactsByBuildID returns a list of artifacts for a specific build ID from the database. +func (e *Engine) ListArtifactsByBuildID(ctx context.Context, buildID int64) ([]*api.Artifact, error) { + e.logger.WithFields(logrus.Fields{ + "build_id": buildID, + }).Trace("listing artifacts for build from the database") + + // variables to store query results and return value + t := new([]types.Artifact) + + var artifacts []*api.Artifact + + // send query to the database and store result in variable + err := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Where("build_id = ?", buildID). + Order("artifacts.created_at DESC"). + Find(&t). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, artifact := range *t { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := artifact + + artifacts = append(artifacts, tmp.ToAPI()) + } + + return artifacts, nil +} diff --git a/database/artifact/list_build_test.go b/database/artifact/list_build_test.go new file mode 100644 index 000000000..253721766 --- /dev/null +++ b/database/artifact/list_build_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" +) + +func TestArtifact_Engine_ListArtifactsByBuildID(t *testing.T) { + // setup types + ctx := context.Background() + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // create SQLite tables for relationship testing with correct table names + err := _sqlite.client.Exec(` + CREATE TABLE IF NOT EXISTS artifacts ( + id INTEGER PRIMARY KEY, + build_id INTEGER, + created_at INTEGER + )`).Error + if err != nil { + t.Errorf("unable to create artifacts table for sqlite: %v", err) + } + + err = _sqlite.client.Exec(` + CREATE TABLE IF NOT EXISTS artifacts ( + id INTEGER PRIMARY KEY, + build_id INTEGER, + file_name TEXT, + object_path TEXT, + file_size INTEGER, + file_type TEXT, + presigned_url TEXT, + created_at INTEGER + )`).Error + if err != nil { + t.Errorf("unable to create artifacts table for sqlite: %v", err) + } + + // create the artifact in sqlite + _, err = _sqlite.CreateArtifact(ctx, _artifact) + if err != nil { + t.Errorf("unable to create artifact for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + buildID int64 + want []*api.Artifact + }{ + { + failure: false, + name: "sqlite3", + database: _sqlite, + buildID: 1, + want: []*api.Artifact{_artifact}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListArtifactsByBuildID(ctx, test.buildID) + + if test.failure { + if err == nil { + t.Errorf("ListArtifactsByBuildID should have returned err") + } + + return + } + + if err != nil { + t.Errorf("ListArtifactsByBuildID returned err: %v", err) + } + + if len(got) != len(test.want) { + t.Errorf("ListArtifactsByBuildID for %s returned %d artifacts, want %d", test.name, len(got), len(test.want)) + return + } + + if len(got) > 0 { + // check artifact fields + if !reflect.DeepEqual(got[0].GetID(), test.want[0].GetID()) || + !reflect.DeepEqual(got[0].GetCreatedAt(), test.want[0].GetCreatedAt()) { + t.Errorf("ListArtifactsByBuildID for %s returned %v, want %v", test.name, got[0], test.want[0]) + } + } + }) + } +} diff --git a/database/artifact/list_test.go b/database/artifact/list_test.go new file mode 100644 index 000000000..997ee209e --- /dev/null +++ b/database/artifact/list_test.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" +) + +func TestArtifact_Engine_ListArtifacts(t *testing.T) { + // setup types + ctx := context.Background() + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := testutils.CreateMockRows([]any{*types.ArtifactFromAPI(_artifact)}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "artifacts" ORDER BY created_at DESC`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // Create the artifact in sqlite + _, err := _sqlite.CreateArtifact(ctx, _artifact) + if err != nil { + t.Errorf("unable to create artifact for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + want []*api.Artifact + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*api.Artifact{_artifact}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*api.Artifact{_artifact}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListArtifacts(ctx) + + if test.failure { + if err == nil { + t.Errorf("ListArtifacts should have returned err") + } + + return + } + + if err != nil { + t.Errorf("ListArtifacts returned err: %v", err) + } + + if len(got) != len(test.want) { + t.Errorf("ListArtifacts for %s returned %d artifacts, want %d", test.name, len(got), len(test.want)) + return + } + + if len(got) > 0 { + // Check report fields + if !reflect.DeepEqual(got[0].GetID(), test.want[0].GetID()) || + !reflect.DeepEqual(got[0].GetCreatedAt(), test.want[0].GetCreatedAt()) { + t.Errorf("ListArtifacts for %s returned unexpected artifacts values: got %v, want %v", + test.name, got[0], test.want[0]) + } + } + }) + } +} diff --git a/database/artifact/opts.go b/database/artifact/opts.go new file mode 100644 index 000000000..68b7ab5e3 --- /dev/null +++ b/database/artifact/opts.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for artifacts. +type EngineOpt func(*Engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for artifacts. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *Engine) error { + // set the gorm.io/gorm client in the artifacts engine + e.client = client + + return nil + } +} + +// WithEncryptionKey sets the encryption key in the database engine for artifacts. +func WithEncryptionKey(key string) EngineOpt { + return func(e *Engine) error { + // set the encryption key in the artifacts engine + e.config.EncryptionKey = key + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for artifacts. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *Engine) error { + // set the github.com/sirupsen/logrus logger in the artifacts engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for artifacts. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *Engine) error { + // set to skip creating tables and indexes in the artifacts engine + e.config.SkipCreation = skipCreation + + return nil + } +} + +// WithContext sets the context in the database engine for artifacts. +func WithContext(ctx context.Context) EngineOpt { + return func(e *Engine) error { + e.ctx = ctx + + return nil + } +} diff --git a/database/artifact/opts_test.go b/database/artifact/opts_test.go new file mode 100644 index 000000000..015eba1f0 --- /dev/null +++ b/database/artifact/opts_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "reflect" + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestHook_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &Engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestHook_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &Engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestHook_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &Engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} + +func TestHook_EngineOpt_WithContext(t *testing.T) { + // setup types + e := &Engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + ctx context.Context + want context.Context + }{ + { + failure: false, + name: "context set to TODO", + ctx: context.TODO(), + want: context.TODO(), + }, + { + failure: false, + name: "context set to nil", + ctx: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithContext(test.ctx)(e) + + if test.failure { + if err == nil { + t.Errorf("WithContext for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithContext returned err: %v", err) + } + + if !reflect.DeepEqual(e.ctx, test.want) { + t.Errorf("WithContext is %v, want %v", e.ctx, test.want) + } + }) + } +} diff --git a/database/artifact/table.go b/database/artifact/table.go new file mode 100644 index 000000000..3b7ce2dd5 --- /dev/null +++ b/database/artifact/table.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/go-vela/server/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres artifacts table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +artifacts ( + id BIGSERIAL PRIMARY KEY, + build_id BIGINT, + created_at BIGINT, + file_name VARCHAR(1000), + object_path VARCHAR(1000), + file_size INTEGER, + file_type TEXT, + presigned_url VARCHAR(2000) +); +` + + // // CreatePostgresTable represents a query to create the Postgres artifacts table. + // CreatePostgresTable = ` + // CREATE TABLE + // IF NOT EXISTS + // artifacts ( + // id BIGSERIAL PRIMARY KEY, + // build_id BIGINT, + // created_at BIGINT, + // file_name VARCHAR(1000), + // object_path VARCHAR(1000), + // file_size INTEGER, + // file_type TEXT, + // presigned_url VARCHAR(2000), + // ); + // `. + + // CreateSqliteTable represents a query to create the Sqlite artifacts table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + build_id BIGINT, + created_at BIGINT, + file_name TEXT, + object_path TEXT, + file_size INTEGER, + file_type TEXT, + presigned_url VARCHAR(2000) +); +` + +// // CreateSqliteTable represents a query to create the Sqlite artifacts table. +// CreateSqliteTable = ` +// +// CREATE TABLE +// IF NOT EXISTS +// artifacts ( +// +// id INTEGER PRIMARY KEY AUTOINCREMENT, +// build_id INTEGER, +// created_at INTEGER, +// file_name TEXT, +// object_path TEXT, +// file_size INTEGER, +// file_type TEXT, +// presigned_url VARCHAR(2000), +// +// ); +// ` +) + +// CreateArtifactTable creates the artifacts table in the database. +func (e *Engine) CreateArtifactTable(ctx context.Context, driver string) error { + e.logger.Tracef("creating artifacts table") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the artifacts table for Postgres + return e.client. + WithContext(ctx). + Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the artifacts table for Sqlite + return e.client. + WithContext(ctx). + Exec(CreateSqliteTable).Error + } +} diff --git a/database/artifact/table_test.go b/database/artifact/table_test.go new file mode 100644 index 000000000..ba8781fd5 --- /dev/null +++ b/database/artifact/table_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestArtifact_Engine_CreateArtifactTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateArtifactTable(context.TODO(), test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateArtifactTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateArtifactTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/artifact/update.go b/database/artifact/update.go new file mode 100644 index 000000000..04bb4a660 --- /dev/null +++ b/database/artifact/update.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database/types" +) + +// UpdateArtifact updates an existing artifact in the database. +func (e *Engine) UpdateArtifact(ctx context.Context, t *api.Artifact) (*api.Artifact, error) { + e.logger.WithFields(logrus.Fields{ + "testattchment": t.GetID(), + }).Tracef("updating artifact %d in the database", t.GetID()) + + artifact := types.ArtifactFromAPI(t) + + err := artifact.Validate() + if err != nil { + return nil, err + } + + // send query to the database + result := e.client. + WithContext(ctx). + Table(constants.TableArtifact). + Save(artifact) + + return artifact.ToAPI(), result.Error +} diff --git a/database/artifact/update_test.go b/database/artifact/update_test.go new file mode 100644 index 000000000..f2b6c114a --- /dev/null +++ b/database/artifact/update_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" +) + +func TestArtifact_Engine_Update(t *testing.T) { + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("foo") + _owner.SetToken("bar") + + _artifact := testutils.APIArtifact() + _artifact.SetID(1) + _artifact.SetBuildID(1) + _artifact.SetFileName("foo") + _artifact.SetObjectPath("foo/bar") + _artifact.SetFileSize(1) + _artifact.SetFileType("xml") + _artifact.SetPresignedURL("foobar") + _artifact.SetCreatedAt(1) + + _postgres, _mock := testPostgres(t) + ctx := context.TODO() + + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query for the artifacts table + _mock.ExpectExec(`UPDATE "artifacts" SET "build_id"=$1,"file_name"=$2,"object_path"=$3,"file_size"=$4,"file_type"=$5,"presigned_url"=$6,"created_at"=$7 WHERE "id" = $8`). + WithArgs(1, "foo", "foo/bar", 1, "xml", "foobar", 1, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateArtifact(ctx, _artifact) + if err != nil { + t.Errorf("unable to update artifact for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *Engine + want *api.Artifact + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _artifact, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _artifact, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.UpdateArtifact(ctx, _artifact) + + if test.failure { + if err == nil { + t.Errorf("Update for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Update for %s returned err: %v", test.name, err) + } + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("GetArtifact mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/database/database.go b/database/database.go index 63c7e9cb5..9d5c64445 100644 --- a/database/database.go +++ b/database/database.go @@ -14,6 +14,7 @@ import ( "gorm.io/gorm" "github.com/go-vela/server/constants" + artifact "github.com/go-vela/server/database/artifact" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -91,6 +92,7 @@ type ( log.LogInterface pipeline.PipelineInterface repo.RepoInterface + artifact.ArtifactInterface schedule.ScheduleInterface secret.SecretInterface service.ServiceInterface diff --git a/database/interface.go b/database/interface.go index e2a6343c9..9079f1542 100644 --- a/database/interface.go +++ b/database/interface.go @@ -3,6 +3,7 @@ package database import ( + artifact "github.com/go-vela/server/database/artifact" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -69,6 +70,9 @@ type Interface interface { // RepoInterface defines the interface for repos stored in the database. repo.RepoInterface + // ArtifactInterface defines the interface for artifacts stored in the database. + artifact.ArtifactInterface + // ScheduleInterface defines the interface for schedules stored in the database. schedule.ScheduleInterface diff --git a/database/resource.go b/database/resource.go index c52444017..40f4f04a1 100644 --- a/database/resource.go +++ b/database/resource.go @@ -5,6 +5,7 @@ package database import ( "context" + artifact "github.com/go-vela/server/database/artifact" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -220,5 +221,17 @@ func (e *engine) NewResources(ctx context.Context) error { return err } + // create the database agnostic engine for artifacts + e.ArtifactInterface, err = artifact.New( + artifact.WithContext(ctx), + artifact.WithClient(e.client), + artifact.WithEncryptionKey(e.config.EncryptionKey), + artifact.WithLogger(e.logger), + artifact.WithSkipCreation(e.config.SkipCreation), + ) + if err != nil { + return err + } + return nil } diff --git a/database/resource_test.go b/database/resource_test.go index c852c8bf0..b1d07ff9d 100644 --- a/database/resource_test.go +++ b/database/resource_test.go @@ -8,6 +8,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" + artifact "github.com/go-vela/server/database/artifact" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -80,9 +81,13 @@ func TestDatabase_Engine_NewResources(t *testing.T) { // ensure the mock expects the worker queries _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // Add these expectations for artifact queries + _mock.ExpectExec(artifact.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(artifact.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // create a test database without mocking the call _unmocked, _ := testPostgres(t) + defer _unmocked.Close() _sqlite := testSqlite(t) defer _sqlite.Close() diff --git a/database/testutils/api_resources.go b/database/testutils/api_resources.go index 649920d5c..e07020e7b 100644 --- a/database/testutils/api_resources.go +++ b/database/testutils/api_resources.go @@ -300,6 +300,19 @@ func APIDashboardRepo() *api.DashboardRepo { } } +func APIArtifact() *api.Artifact { + return &api.Artifact{ + ID: new(int64), + BuildID: new(int64), + FileName: new(string), + ObjectPath: new(string), + FileSize: new(int64), + FileType: new(string), + PresignedURL: new(string), + CreatedAt: new(int64), + } +} + func JWK() jwk.RSAPublicKey { privateRSAKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { diff --git a/database/types/artifact.go b/database/types/artifact.go new file mode 100644 index 000000000..5c0cb2f09 --- /dev/null +++ b/database/types/artifact.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "errors" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/util" +) + +var ( + // ErrEmptyBuildID defines the error type when a + // Artifact type has an empty BuildID field provided. + ErrEmptyBuildID = errors.New("empty build_id provided") + // ErrEmptyFileName defines the error type when a + // Artifact type has an empty FileName field provided. + ErrEmptyFileName = errors.New("empty file_name provided") + // ErrEmptyObjectPath defines the error type when a + // Artifact type has an empty ObjectPath field provided. + ErrEmptyObjectPath = errors.New("empty object_path provided") + // ErrEmptyFileSize defines the error type when a + // Artifact type has an empty FileSize field provided. + ErrEmptyFileSize = errors.New("empty file_size provided") + // ErrEmptyFileType defines the error type when a + // Artifact type has an empty FileType field provided. + ErrEmptyFileType = errors.New("empty file_type provided") + // ErrEmptyPresignedURL defines the error type when a + // Artifact type has an empty PresignedUrl field provided. + ErrEmptyPresignedURL = errors.New("empty presigned_url provided") +) + +type Artifact struct { + ID sql.NullInt64 `sql:"id"` + BuildID sql.NullInt64 `sql:"build_id"` + FileName sql.NullString `sql:"file_name"` + ObjectPath sql.NullString `sql:"object_path"` + FileSize sql.NullInt64 `sql:"file_size"` + FileType sql.NullString `sql:"file_type"` + PresignedURL sql.NullString `sql:"presigned_url"` + CreatedAt sql.NullInt64 `sql:"created_at"` +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// When a field within the Artifact type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (a *Artifact) Nullify() *Artifact { + if a == nil { + return nil + } + + // check if the ID field should be false + if a.ID.Int64 == 0 { + a.ID.Valid = false + } + + // check if the BuildID field should be false + if a.BuildID.Int64 == 0 { + a.BuildID.Valid = false + } + + // check if the Created field should be false + if a.CreatedAt.Int64 == 0 { + a.CreatedAt.Valid = false + } + + return a +} + +// ToAPI converts the Artifact type +// to the API representation of the type. +func (a *Artifact) ToAPI() *api.Artifact { + artifact := new(api.Artifact) + artifact.SetID(a.ID.Int64) + + // var tr *api.Artifacts + // if a.Artifacts.ID.Valid { + // tr = a.Artifacts.ToAPI() + // } else { + // tr = new(api.Artifacts) + // tr.SetID(a.BuildID.Int64) + // } + + artifact.SetBuildID(a.BuildID.Int64) + artifact.SetFileName(a.FileName.String) + artifact.SetObjectPath(a.ObjectPath.String) + artifact.SetFileSize(a.FileSize.Int64) + artifact.SetFileType(a.FileType.String) + artifact.SetPresignedURL(a.PresignedURL.String) + artifact.SetCreatedAt(a.CreatedAt.Int64) + + return artifact +} + +// Validate ensures the Artifact type is valid +// by checking if the required fields are set. +func (a *Artifact) Validate() error { + // verify the BuildID field is populated + if !a.BuildID.Valid || a.BuildID.Int64 <= 0 { + return ErrEmptyBuildID + } + + // verify the FileName field is populated + if !a.FileName.Valid || len(a.FileName.String) == 0 { + return ErrEmptyFileName + } + + // verify the ObjectPath field is populated + if !a.ObjectPath.Valid || len(a.ObjectPath.String) == 0 { + return ErrEmptyObjectPath + } + + // verify the FileType field is populated + if !a.FileType.Valid || len(a.FileType.String) == 0 { + return ErrEmptyFileType + } + + // Note: FileSize and PresignedUrl are optional during creation + // They may be set later in the workflow + + // ensure that all Artifact string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + a.FileName = sql.NullString{String: util.Sanitize(a.FileName.String), Valid: a.FileName.Valid} + a.ObjectPath = sql.NullString{String: util.Sanitize(a.ObjectPath.String), Valid: a.ObjectPath.Valid} + a.FileType = sql.NullString{String: util.Sanitize(a.FileType.String), Valid: a.FileType.Valid} + + // Only sanitize PresignedUrl if it's provided + if a.PresignedURL.Valid { + a.PresignedURL = sql.NullString{String: util.Sanitize(a.PresignedURL.String), Valid: a.PresignedURL.Valid} + } + + return nil +} + +// ArtifactFromAPI converts the API Artifact type +// to a database report artifact type. +func ArtifactFromAPI(a *api.Artifact) *Artifact { + artifact := &Artifact{ + ID: sql.NullInt64{Int64: a.GetID(), Valid: a.GetID() > 0}, + BuildID: sql.NullInt64{Int64: a.GetBuildID(), Valid: a.GetBuildID() > 0}, + FileName: sql.NullString{String: a.GetFileName(), Valid: len(a.GetFileName()) > 0}, + ObjectPath: sql.NullString{String: a.GetObjectPath(), Valid: len(a.GetObjectPath()) > 0}, + FileSize: sql.NullInt64{Int64: a.GetFileSize(), Valid: a.GetFileSize() > 0}, + FileType: sql.NullString{String: a.GetFileType(), Valid: len(a.GetFileType()) > 0}, + PresignedURL: sql.NullString{String: a.GetPresignedURL(), Valid: len(a.GetPresignedURL()) > 0}, + CreatedAt: sql.NullInt64{Int64: a.GetCreatedAt(), Valid: a.GetCreatedAt() > 0}, + } + + return artifact.Nullify() +} diff --git a/docker-compose.yml b/docker-compose.yml index c6ffb64c9..4e88ed627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,13 @@ services: VELA_OTEL_TRACING_ENABLE: true VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: false + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9000' # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -69,12 +76,17 @@ services: # https://go-vela.github.io/docs/administration/worker/ worker: container_name: worker - image: target/vela-worker:latest + image: worker:local + # image: target/vela-worker:latest + build: + context: ../worker + dockerfile: ${VELA_WORKER_DOCKERFILE:-Dockerfile} networks: - vela environment: EXECUTOR_DRIVER: linux QUEUE_DRIVER: redis + QUEUE_ADDR: 'redis://redis:6379' VELA_BUILD_LIMIT: 1 VELA_BUILD_TIMEOUT: 30m VELA_LOG_LEVEL: trace @@ -86,6 +98,11 @@ services: VELA_SERVER_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' WORKER_ADDR: 'http://worker:8080' WORKER_CHECK_IN: 2m + VELA_EXECUTOR_OUTPUTS_IMAGE: 'alpine:latest' + VELA_STORAGE_ENABLE: 'false' + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: 'http://minio:9000' + VELA_STORAGE_BUCKET: vela restart: always ports: - '8081:8080' @@ -156,18 +173,18 @@ services: # # https://www.vaultproject.io/ vault: - image: hashicorp/vault:latest - container_name: vault - command: server -dev - networks: - - vela - environment: - VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - VAULT_DEV_ROOT_TOKEN_ID: vela - ports: - - '8200:8200' - cap_add: - - IPC_LOCK + image: hashicorp/vault:latest + container_name: vault + command: server -dev + networks: + - vela + environment: + VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + VAULT_DEV_ROOT_TOKEN_ID: vela + ports: + - '8200:8200' + cap_add: + - IPC_LOCK jaeger: image: jaegertracing/all-in-one:latest @@ -180,5 +197,35 @@ services: - '16686:16686' - '4318:4318' + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - '9000:9000' + - '9001:9001' + networks: + - vela + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + # - MINIO_SERVER_URL=http://minio + # - MINIO_BROWSER_REDIRECT_URL=http:// + command: minio server --address ":9000" --console-address ":9001" /data + + # The `nginx` compose service hosts the NGINX reverse proxy. + nginx: + image: nginx:latest + container_name: minio_nginx + restart: always + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - '80:80' + depends_on: + - minio + networks: + - vela + networks: vela: diff --git a/go.mod b/go.mod index d19ed6a0e..88009399e 100644 --- a/go.mod +++ b/go.mod @@ -32,10 +32,12 @@ require ( github.com/lestrrat-go/jwx/v3 v3.0.10 github.com/lib/pq v1.10.9 github.com/microcosm-cc/bluemonday v1.0.27 + github.com/minio/minio-go/v7 v7.0.83 github.com/prometheus/client_golang v1.23.0 github.com/redis/go-redis/v9 v9.12.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.14.0 + github.com/stretchr/testify v1.10.0 github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 github.com/urfave/cli/v3 v3.4.1 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.62.0 @@ -82,12 +84,15 @@ require ( github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -114,6 +119,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect @@ -124,6 +130,7 @@ require ( github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.28 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -133,9 +140,11 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/go.sum b/go.sum index 359be795b..bf4a23827 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,9 @@ github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCy github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -88,6 +89,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -106,6 +109,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -200,6 +205,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -238,6 +244,10 @@ github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEu github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= +github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -258,8 +268,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -272,6 +283,8 @@ github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgv github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= diff --git a/internal/metadata.go b/internal/metadata.go index 92616ed83..eb92e1508 100644 --- a/internal/metadata.go +++ b/internal/metadata.go @@ -23,6 +23,12 @@ type ( Host string `json:"host"` } + // Storage is the extra set of Storage data passed to the compiler. + Storage struct { + Driver string `json:"driver"` + Host string `json:"host"` + } + // Vela is the extra set of Vela data passed to the compiler. Vela struct { Address string `json:"address"` @@ -41,5 +47,6 @@ type ( Queue *Queue `json:"queue"` Source *Source `json:"source"` Vela *Vela `json:"vela"` + Storage *Storage `json:"storage"` } ) diff --git a/mock/server/artifact.go b/mock/server/artifact.go new file mode 100644 index 000000000..4a66bb1d5 --- /dev/null +++ b/mock/server/artifact.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +// ArtifactResp is the mock response for creating an artifact. +const ArtifactResp = `{ + "id": 1, + "build_id": 1, + "file_name": "test-results.xml", + "object_path": "builds/1/test-results.xml", + "file_size": 1024, + "file_type": "xml", + "presigned_url": "https://storage.example.com/builds/1/test-results.xml", + "created_at": 1750710551 +}` + +func addArtifact(c *gin.Context) { + data := []byte(ArtifactResp) + + var body api.Artifact + + _ = json.Unmarshal(data, &body) + c.JSON(http.StatusCreated, body) +} diff --git a/mock/server/pipeline.go b/mock/server/pipeline.go index d3723fad2..5ef840605 100644 --- a/mock/server/pipeline.go +++ b/mock/server/pipeline.go @@ -169,6 +169,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifact": false, "warnings": [ "42:this is a warning" ], @@ -244,6 +245,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifact": false, "data": "LS0tCnZlcnNpb246ICIxIgoKc3RlcHM6CiAgLSBuYW1lOiBlY2hvCiAgICBpbWFnZTogYWxwaW5lOmxhdGVzdAogICAgY29tbWFuZHM6IFtlY2hvIGZvb10=" }, { @@ -313,6 +315,7 @@ templates: "stages": false, "steps": true, "templates": false, + "artifact": false, "data": "LS0tCnZlcnNpb246ICIxIgoKc3RlcHM6CiAgLSBuYW1lOiBlY2hvCiAgICBpbWFnZTogYWxwaW5lOmxhdGVzdAogICAgY29tbWFuZHM6IFtlY2hvIGZvb10=" } ]` diff --git a/mock/server/server.go b/mock/server/server.go index 0d17c368f..853b4906a 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -159,5 +159,11 @@ func FakeHandler() http.Handler { // mock endpoint for queue credentials e.GET("/api/v1/queue/info", getQueueCreds) + // mock endpoint for storage credentials + e.GET("/api/v1/storage/info", getStorageCreds) + + // mock endpoint for artifact calls + e.PUT("/api/v1/repos/org/repo/builds/1/artifact", addArtifact) + return e } diff --git a/mock/server/worker.go b/mock/server/worker.go index 5b452fbf7..b79f4b0fb 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -188,6 +188,15 @@ const ( "queue_public_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", "queue_address": "redis://redis:6000" }` + + // StorageInfoResp represents a JSON return for an admin requesting a storage registration info. + // + //not actual credentials. + StorageInfoResp = `{ + "storage_access_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_secret_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_address": "http://storage:9000" + }` ) // getWorkers returns mock JSON for a http GET. @@ -341,3 +350,25 @@ func getQueueCreds(c *gin.Context) { c.JSON(http.StatusCreated, body) } + +// getStorageCreds returns mock JSON for a http GET. +// Pass "" to Authorization header to test receiving a http 401 response. +func getStorageCreds(c *gin.Context) { + token := c.Request.Header.Get("Authorization") + // verify token if empty + if token == "" { + msg := "unable get storage credentials; invalid registration token" + + c.AbortWithStatusJSON(http.StatusUnauthorized, api.Error{Message: &msg}) + + return + } + + data := []byte(StorageInfoResp) + + var body api.StorageInfo + + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000..04eac4ae2 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,41 @@ +events {} + +http{ + upstream minio { + least_conn; + server minio:9000; + } + + server { + + listen 80; + listen [::]:80; + server_name minio; + + + # Allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + proxy_request_buffering off; + + location / { + # Disable buffering + proxy_buffering off; + proxy_pass_request_headers on; + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + # proxy_set_header Connection ""; + chunked_transfer_encoding off; + proxy_set_header Host minio:9000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + proxy_pass "http://minio"; # This uses the upstream directive definition to load balance + } + } +} diff --git a/router/admin.go b/router/admin.go index e127850bc..cd30105ba 100644 --- a/router/admin.go +++ b/router/admin.go @@ -25,7 +25,9 @@ import ( // POST /api/v1/admin/workers/:worker/register // GET /api/v1/admin/settings // PUT /api/v1/admin/settings -// DELETE /api/v1/admin/settings. +// DELETE /api/v1/admin/settings +// PUT /api/v1/admin/storage/bucket +// GET /api/v1/admin/storage/presign. func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) @@ -63,6 +65,12 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin step endpoint _admin.PUT("/step", admin.UpdateStep) + // Admin storage bucket endpoints + _admin.PUT("/storage/bucket", admin.CreateBucket) + + // Admin storage presign endpoints + _admin.GET("/storage/presign", admin.GetPresignedURL) + // Admin user endpoint _admin.PUT("/user", admin.UpdateUser) diff --git a/router/artifact.go b/router/artifact.go new file mode 100644 index 000000000..9b5b98215 --- /dev/null +++ b/router/artifact.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package router + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/artifact" + tamiddleware "github.com/go-vela/server/router/middleware/artifact" +) + +// ArtifactHandlers is a function that extends the provided base router group +// with the API handlers for artifact functionality. +// +// GET /api/v1/repos/:org/:repo/builds/:build/artifact +// GET /api/v1/repos/:org/:repo/builds/:build/artifact/:artifact +// PUT /api/v1/repos/:org/:repo/builds/:build/artifact . +func ArtifactHandlers(base *gin.RouterGroup) { + // artifact endpoints + _artifact := base.Group("/artifact") + { + _artifact.GET("", artifact.ListArtifactsForBuild) + _artifact.PUT("", artifact.CreateArtifact) + + // Individual artifact endpoints + a := _artifact.Group("/:artifact", tamiddleware.Establish()) + { + a.GET("", artifact.GetArtifact) + } // end of individual artifact endpoints + } // end of artifact endpoints +} diff --git a/router/build.go b/router/build.go index aeeefa0b2..6c6396506 100644 --- a/router/build.go +++ b/router/build.go @@ -76,6 +76,9 @@ func BuildHandlers(base *gin.RouterGroup) { // Step endpoints // * Log endpoints StepHandlers(b) + + // Artifact endpoints + ArtifactHandlers(b) } // end of build endpoints } // end of builds endpoints } diff --git a/router/middleware/artifact/artifact.go b/router/middleware/artifact/artifact.go new file mode 100644 index 000000000..d29058f59 --- /dev/null +++ b/router/middleware/artifact/artifact.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" +) + +// Retrieve gets the artifact in the given context. +func Retrieve(c *gin.Context) *api.Artifact { + return FromContext(c) +} + +// Establish sets the artifact in the given context. +func Establish() gin.HandlerFunc { + return func(c *gin.Context) { + // capture middleware values + l := c.MustGet("logger").(*logrus.Entry) + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + ctx := c.Request.Context() + + if r == nil { + retErr := fmt.Errorf("repo %s/%s not found", o, util.PathParameter(c, "repo")) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + if b == nil { + retErr := fmt.Errorf("build %s not found for repo %s", util.PathParameter(c, "build"), r.GetFullName()) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + l.Debugf("reading artifact") + + a, err := database.FromContext(c).GetArtifactForBuild(ctx, b) + if err != nil { + retErr := fmt.Errorf("unable to read artifact %s/%d: %w", r.GetFullName(), b.GetNumber(), err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + l = l.WithFields(logrus.Fields{ + "artifact_id": a.GetID(), + }) + + // update the logger with the new fields + c.Set("logger", l) + + ToContext(c, a) + c.Next() + } +} diff --git a/router/middleware/artifact/context.go b/router/middleware/artifact/context.go new file mode 100644 index 000000000..4829adc57 --- /dev/null +++ b/router/middleware/artifact/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package artifact + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +const key = "artifact" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the Artifact associated with this context. +func FromContext(c context.Context) *api.Artifact { + value := c.Value(key) + if value == nil { + return nil + } + + a, ok := value.(*api.Artifact) + if !ok { + return nil + } + + return a +} + +// ToContext adds the Artifact to this context if it supports +// the Setter interface. +func ToContext(c Setter, a *api.Artifact) { + c.Set(key, a) +} diff --git a/router/middleware/signing.go b/router/middleware/signing.go index 6b9dd4c1f..3946eb502 100644 --- a/router/middleware/signing.go +++ b/router/middleware/signing.go @@ -32,3 +32,48 @@ func QueueAddress(address string) gin.HandlerFunc { c.Next() } } + +// StorageAccessKey is a middleware function that attaches the access key used +// to open the connection to the storage. +func StorageAccessKey(key string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-access-key", key) + c.Next() + } +} + +// StorageSecretKey is a middleware function that attaches the secret key used +// to open the connection to the storage. +func StorageSecretKey(key string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-secret-key", key) + c.Next() + } +} + +// StorageAddress is a middleware function that attaches the storage address used +// to open the connection to the storage. +func StorageAddress(address string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-address", address) + c.Next() + } +} + +// StorageBucket is a middleware function that attaches the bucket name used +// to open the connection to the storage. +func StorageBucket(bucket string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-bucket", bucket) + c.Next() + } +} + +// StorageEnable is a middleware function that sets a flag in the context +// to determined if storage is enabled. +func StorageEnable(enabled bool) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-enable", enabled) + c.Next() + } +} diff --git a/router/middleware/signing_test.go b/router/middleware/signing_test.go index 1e85dd682..eca62722f 100644 --- a/router/middleware/signing_test.go +++ b/router/middleware/signing_test.go @@ -106,3 +106,161 @@ func TestMiddleware_QueueAddress(t *testing.T) { t.Errorf("QueueAddress is %v, want %v", got, want) } } + +func TestMiddleware_StorageAddress(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageAddress(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-address").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageAddress returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageAddress is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageAccessKey(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageAccessKey(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-access-key").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageAccessKey returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageAccessKey is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageSecretKey(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageSecretKey(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-secret-key").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageSecretKey returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageSecretKey is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageBucket(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageBucket(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-bucket").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageBucket returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageBucket is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageEnable(t *testing.T) { + // setup types + got := false + want := true + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + // setup mock server + engine.Use(StorageEnable(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-enable").(bool) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageEnable returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageEnable is %v, want %v", got, want) + } +} diff --git a/router/middleware/storage.go b/router/middleware/storage.go new file mode 100644 index 000000000..c27c1820b --- /dev/null +++ b/router/middleware/storage.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" +) + +// Storage is a middleware function that initializes the object storage and +// attaches to the context of every http.Request. +func Storage(q storage.Storage) gin.HandlerFunc { + return func(c *gin.Context) { + // attach the object storage to the context + storage.ToContext(c, q) + + c.Next() + } +} diff --git a/router/middleware/storage_test.go b/router/middleware/storage_test.go new file mode 100644 index 000000000..3ef2c6996 --- /dev/null +++ b/router/middleware/storage_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" + "github.com/go-vela/server/storage/minio" +) + +func TestMiddleware_Storage(t *testing.T) { + // setup types + var got storage.Storage + + want, _ := minio.NewTest("", "", "", "", false) + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequestWithContext(t.Context(), http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(Storage(want)) + engine.GET("/health", func(c *gin.Context) { + got = storage.FromGinContext(c) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Storage returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Storage is %v, want %v", got, want) + } +} diff --git a/router/router.go b/router/router.go index ca131ae63..1b1fb57d4 100644 --- a/router/router.go +++ b/router/router.go @@ -155,6 +155,9 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { // Queue endpoints QueueHandlers(baseAPI) + + // Storage endpoints + StorageHandlers(baseAPI) } // end of api return r diff --git a/router/storage.go b/router/storage.go new file mode 100644 index 000000000..93aca72c0 --- /dev/null +++ b/router/storage.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +package router + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/storage" + "github.com/go-vela/server/router/middleware/perm" +) + +// StorageHandlers is a function that extends the provided base router group +// with the API handlers for storage functionality. +// +// GET /api/v1/storage/info . +func StorageHandlers(base *gin.RouterGroup) { + // Storage endpoints + _storage := base.Group("/storage") + { + _storage.GET("/info", perm.MustWorkerRegisterToken(), storage.Info) + _storage.GET("/:bucket/objects", storage.ListObjects) + _storage.GET("/:bucket/names", storage.ListObjectNames) + _storage.GET("/:bucket/:org/:repo/builds/:build/names", storage.ListBuildObjectNames) + } // end of storage endpoints +} diff --git a/storage/context.go b/storage/context.go new file mode 100644 index 000000000..2172ee632 --- /dev/null +++ b/storage/context.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + + "github.com/gin-gonic/gin" +) + +// key is the key used to store minio service in context. +const key = "minio" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext retrieves minio service from the context. +func FromContext(ctx context.Context) Storage { + // get minio value from context.Context + v := ctx.Value(key) + if v == nil { + return nil + } + + // cast minio value to expected Storage type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// FromGinContext retrieves the S3 Service from the gin.Context. +func FromGinContext(c *gin.Context) Storage { + // get minio value from gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Get + v, ok := c.Get(key) + if !ok { + return nil + } + + // cast minio value to expected Service type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// ToContext adds the secret Service to this +// context if it supports the Setter interface. +func ToContext(c Setter, s Storage) { + c.Set(key, s) +} + +// WithContext adds the minio Storage to the context. +func WithContext(ctx context.Context, s Storage) context.Context { + // set the storage Service in the context.Context + // + // https://pkg.go.dev/context?tab=doc#WithValue + // + return context.WithValue(ctx, key, s) +} + +// WithGinContext inserts the minio Storage into the gin.Context. +func WithGinContext(c *gin.Context, s Storage) { + // set the minio Storage in the gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Set + c.Set(key, s) +} diff --git a/storage/context_test.go b/storage/context_test.go new file mode 100644 index 000000000..025f35274 --- /dev/null +++ b/storage/context_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "reflect" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestExecutor_FromContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context context.Context + want Storage + }{ + { + + context: context.WithValue(context.Background(), key, _service), + want: _service, + }, + { + context: context.Background(), + want: nil, + }, + { + + context: context.WithValue(context.Background(), key, "foo"), + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := FromContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_FromGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context *gin.Context + value interface{} + want Storage + }{ + { + context: new(gin.Context), + value: _service, + want: _service, + }, + { + context: new(gin.Context), + value: nil, + want: nil, + }, + { + context: new(gin.Context), + value: "foo", + want: nil, + }, + } + + // run tests + for _, test := range tests { + if test.value != nil { + test.context.Set(key, test.value) + } + + got := FromGinContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromGinContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_WithContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + want := context.WithValue(context.Background(), key, _service) + + // run test + got := WithContext(context.Background(), _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithContext is %v, want %v", got, want) + } +} + +func TestExecutor_WithGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + want := new(gin.Context) + want.Set(key, _service) + + // run test + got := new(gin.Context) + WithGinContext(got, _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithGinContext is %v, want %v", got, want) + } +} diff --git a/storage/flags.go b/storage/flags.go new file mode 100644 index 000000000..a0776fa77 --- /dev/null +++ b/storage/flags.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "fmt" + "strings" + + "github.com/urfave/cli/v3" +) + +var Flags = []cli.Flag{ + // STORAGE Flags + + &cli.BoolFlag{ + Name: "storage.enable", + Usage: "enable object storage", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ENABLE"), + cli.File("vela/storage/enable"), + ), + }, + &cli.StringFlag{ + Name: "storage.driver", + Usage: "object storage driver", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_DRIVER"), + cli.EnvVar("STORAGE_DRIVER"), + cli.File("vela/storage/driver"), + ), + }, + &cli.StringFlag{ + Name: "storage.addr", + Usage: "set the storage endpoint (ex. scheme://host:port)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ADDRESS"), + cli.EnvVar("STORAGE_ADDRESS"), + cli.File("vela/storage/addr"), + ), + Action: func(_ context.Context, _ *cli.Command, v string) error { + // check if the storage address has a scheme + if !strings.Contains(v, "://") { + return fmt.Errorf("storage address must be fully qualified (://)") + } + + // check if the queue address has a trailing slash + if strings.HasSuffix(v, "/") { + return fmt.Errorf("storage address must not have trailing slash") + } + + return nil + }, + }, + + &cli.StringFlag{ + Name: "storage.access.key", + Usage: "set storage access key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ACCESS_KEY"), + cli.EnvVar("STORAGE_ACCESS_KEY"), + cli.File("vela/storage/access_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.secret.key", + Usage: "set storage secret key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_SECRET_KEY"), + cli.EnvVar("STORAGE_SECRET_KEY"), + cli.File("vela/storage/secret_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.bucket.name", + Usage: "set storage bucket name", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_BUCKET"), + cli.File("vela/storage/bucket"), + ), + }, + &cli.BoolFlag{ + Name: "storage.use.ssl", + Usage: "enable storage to use SSL", + Value: false, + Sources: cli.EnvVars("VELA_STORAGE_USE_SSL"), + }, +} diff --git a/storage/flags_test.go b/storage/flags_test.go new file mode 100644 index 000000000..09bef4e6c --- /dev/null +++ b/storage/flags_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "maps" + "testing" + + "github.com/urfave/cli/v3" +) + +func TestStorage_Flags(t *testing.T) { + // deep copy flags since they are global variables and will hold onto modifications during testing + deepCopyFlags := func(flags []cli.Flag) []cli.Flag { + copiedFlags := make([]cli.Flag, len(flags)) + for i, flag := range flags { + switch f := flag.(type) { + case *cli.StringFlag: + copyFlag := *f + copiedFlags[i] = ©Flag + case *cli.BoolFlag: + copyFlag := *f + copiedFlags[i] = ©Flag + default: + t.Fatalf("unsupported flag type: %T", f) + } + } + + return copiedFlags + } + + validFlags := map[string]string{ + "storage.enable": "true", + "storage.driver": "s3", + "storage.addr": "https://s3.amazonaws.com", + "storage.access.key": "test-access-key", + "storage.secret.key": "test-secret-key", + "storage.bucket.name": "test-bucket", + "storage.use.ssl": "true", + } + + // Define test cases + tests := []struct { + name string + override map[string]string + wantErr bool + }{ + { + name: "happy path", + wantErr: false, + }, + { + name: "invalid storage addr - no scheme", + override: map[string]string{ + "storage.addr": "s3.amazonaws.com", + }, + wantErr: true, + }, + { + name: "invalid storage addr - trailing slash", + override: map[string]string{ + "storage.addr": "https://s3.amazonaws.com/", + }, + wantErr: true, + }, + { + name: "valid storage addr with port", + override: map[string]string{ + "storage.addr": "https://localhost:9000", + }, + wantErr: false, + }, + { + name: "valid storage addr with http scheme", + override: map[string]string{ + "storage.addr": "http://minio.local:9000", + }, + wantErr: false, + }, + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Create a new command with a deep copy of the Flags slice + cmd := cli.Command{ + Name: "test", + Action: func(_ context.Context, _ *cli.Command) error { + return nil + }, + Flags: deepCopyFlags(Flags), + } + + copyMap := maps.Clone(validFlags) + + maps.Copy(copyMap, test.override) + + args := []string{"test"} + // Set command line arguments + for key, value := range copyMap { + if len(value) == 0 { + continue + } + + args = append(args, `--`+key+"="+value) + } + + // Run command + err := cmd.Run(context.Background(), args) + + // Check the result + if (err != nil) != test.wantErr { + t.Errorf("error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/storage/minio/bucket_exists.go b/storage/minio/bucket_exists.go new file mode 100644 index 000000000..ce1f5fa26 --- /dev/null +++ b/storage/minio/bucket_exists.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// BucketExists checks if a bucket exists in MinIO. +func (c *Client) BucketExists(ctx context.Context, bucket *api.Bucket) (bool, error) { + c.Logger.Tracef("checking if bucket %s exists", bucket.BucketName) + + exists, err := c.client.BucketExists(ctx, bucket.BucketName) + if err != nil { + return false, err + } + + return exists, nil +} diff --git a/storage/minio/bucket_exists_test.go b/storage/minio/bucket_exists_test.go new file mode 100644 index 000000000..1351da44a --- /dev/null +++ b/storage/minio/bucket_exists_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_BucketExists(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock bucket exists call + engine.HEAD("/foo/", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + exists, err := client.BucketExists(ctx, &api.Bucket{BucketName: "foo"}) + + if resp.Code != http.StatusOK { + t.Errorf("BucketExists returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("BucketExists returned err: %v", err) + } + + if !exists { + t.Errorf("BucketExists returned %v, want %v", exists, true) + } +} + +func TestMinioClient_BucketExists_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.HEAD("/foo/", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + exists, err := client.BucketExists(ctx, &api.Bucket{BucketName: "bar"}) + + if resp.Code != http.StatusOK { + t.Errorf("BucketExists returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("BucketExists returned err: %v", err) + } + + if exists { + t.Errorf("BucketExists returned %v, want %v", exists, false) + } +} diff --git a/storage/minio/create_bucket.go b/storage/minio/create_bucket.go new file mode 100644 index 000000000..d41784498 --- /dev/null +++ b/storage/minio/create_bucket.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + api "github.com/go-vela/server/api/types" +) + +// CreateBucket creates a new bucket in MinIO. +func (c *Client) CreateBucket(ctx context.Context, bucket *api.Bucket) error { + c.Logger.Tracef("create new bucket: %s", bucket.BucketName) + + exists, errBucketExists := c.BucketExists(ctx, bucket) + if errBucketExists != nil && exists { + c.Logger.Tracef("Bucket %s already exists", bucket.BucketName) + + return fmt.Errorf("bucket %s already exists", bucket.BucketName) + } + + err := c.client.MakeBucket(ctx, bucket.BucketName, bucket.MakeBucketOptions) + if err != nil { + c.Logger.Errorf("unable to create bucket %s: %v", bucket.BucketName, err) + return err + } + + return nil +} diff --git a/storage/minio/create_bucket_test.go b/storage/minio/create_bucket_test.go new file mode 100644 index 000000000..a33b08136 --- /dev/null +++ b/storage/minio/create_bucket_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_CreateBucket_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + err := client.CreateBucket(ctx, b) + + if resp.Code != http.StatusOK { + t.Errorf("CreateBucket returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } +} diff --git a/storage/minio/doc.go b/storage/minio/doc.go new file mode 100644 index 000000000..e1b1afd52 --- /dev/null +++ b/storage/minio/doc.go @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio diff --git a/storage/minio/get_bucket.go b/storage/minio/get_bucket.go new file mode 100644 index 000000000..9a09e055d --- /dev/null +++ b/storage/minio/get_bucket.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" +) + +func (c *Client) GetBucket(context.Context) string { + // GetBucket returns the bucket name for the MinIO client. + return c.config.Bucket +} diff --git a/storage/minio/get_bucket_test.go b/storage/minio/get_bucket_test.go new file mode 100644 index 000000000..abcbf4305 --- /dev/null +++ b/storage/minio/get_bucket_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestMinioClient_GetBucket_ReturnsConfiguredBucket(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + got := client.GetBucket(context.TODO()) + want := "foo" + + if got != want { + t.Fatalf("GetBucket() = %q, want %q", got, want) + } +} + +func TestMinioClient_GetBucket_EmptyWhenUnset(t *testing.T) { + gin.SetMode(gin.TestMode) + + _, engine := gin.CreateTestContext(httptest.NewRecorder()) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("failed to create minio test client: %v", err) + } + + client.config.Bucket = "" + + got := client.GetBucket(context.TODO()) + if got != "" { + t.Fatalf("GetBucket() = %q, want empty string", got) + } +} diff --git a/storage/minio/list_bucket.go b/storage/minio/list_bucket.go new file mode 100644 index 000000000..8547a7af0 --- /dev/null +++ b/storage/minio/list_bucket.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" +) + +// ListBuckets lists all buckets in MinIO. +func (c *Client) ListBuckets(ctx context.Context) ([]string, error) { + c.Logger.Trace("listing all buckets") + + buckets, err := c.client.ListBuckets(ctx) + if err != nil { + return nil, err + } + + bucketNames := make([]string, len(buckets)) + for i, bucket := range buckets { + bucketNames[i] = bucket.Name + } + + return bucketNames, nil +} diff --git a/storage/minio/list_bucket_test.go b/storage/minio/list_bucket_test.go new file mode 100644 index 000000000..9b7e64e5b --- /dev/null +++ b/storage/minio/list_bucket_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_ListBuckets_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock list buckets call + // Match root with or without query params + engine.GET("/", func(c *gin.Context) { + // Raw S3-compatible XML response for ListBuckets + xmlResponse := ` + + + minio + minio + + + + foo + 2025-03-20T19:01:40.968Z + + +` + c.Data(http.StatusOK, "application/xml", []byte(xmlResponse)) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + b := new(api.Bucket) + b.BucketName = "foo" + + buckets, err := client.ListBuckets(ctx) + if err != nil { + t.Errorf("ListBuckets returned err: %v", err) + } + + // Ignore for now as xmlDecoder from minio-go is does not parse correctly with sample data + // check if buckets are correct + expectedBuckets := []string{"foo"} + if len(buckets) != len(expectedBuckets) { + t.Errorf("Expected %d buckets, got %d", len(expectedBuckets), len(buckets)) + } + + for i, bucket := range buckets { + if bucket != expectedBuckets[i] { + t.Errorf("Expected bucket %v, got %v", expectedBuckets[i], bucket) + } + } +} + +func TestMinioClient_ListBuckets_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // mock list buckets call + engine.GET("/minio/buckets", func(c *gin.Context) { + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + _, err := client.ListBuckets(ctx) + if err == nil { + t.Errorf("Expected error, got nil") + } +} diff --git a/storage/minio/list_objects.go b/storage/minio/list_objects.go new file mode 100644 index 000000000..9de01280d --- /dev/null +++ b/storage/minio/list_objects.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// ListObjects lists the objects in a bucket. +func (c *Client) ListObjects(ctx context.Context, b *api.Bucket) ([]minio.ObjectInfo, error) { + c.Logger.Tracef("listing objects in bucket %s", b.BucketName) + + opts := minio.ListObjectsOptions{ + Recursive: b.Recursive, + } + + objectCh := c.client.ListObjects(ctx, b.BucketName, opts) + + var objects []minio.ObjectInfo + + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + + objects = append(objects, object) + } + + return objects, nil +} + +// ListObjectNames lists only the names of objects in a bucket. +func (c *Client) ListObjectNames(ctx context.Context, b *api.Bucket) ([]string, error) { + c.Logger.Tracef("listing object names in bucket %s", b.BucketName) + + // Set ListObjectsOptions with Recursive flag from the Bucket type + opts := minio.ListObjectsOptions{ + Recursive: b.Recursive, + } + + objectCh := c.client.ListObjects(ctx, b.BucketName, opts) + + var objectNames []string + + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + + objectNames = append(objectNames, object.Key) + } + + return objectNames, nil +} + +// ListBuildObjectNames lists the names of objects in a bucket for a specific build. +func (c *Client) ListBuildObjectNames(ctx context.Context, b *api.Bucket, org, repo, build string) ([]string, error) { + // Construct the prefix path for filtering + prefix := org + "/" + repo + "/" + build + "/" + + c.Logger.Tracef("listing object names in bucket %s with prefix %s", b.BucketName, prefix) + + // Set ListObjectsOptions with Recursive flag and prefix + opts := minio.ListObjectsOptions{ + Recursive: b.Recursive, + Prefix: prefix, + } + + objectCh := c.client.ListObjects(ctx, b.BucketName, opts) + + var objectNames []string + + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + + objectNames = append(objectNames, object.Key) + } + + return objectNames, nil +} diff --git a/storage/minio/list_objects_test.go b/storage/minio/list_objects_test.go new file mode 100644 index 000000000..15e3e25a7 --- /dev/null +++ b/storage/minio/list_objects_test.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_ListObjects_Success(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + engine.GET("/foo/", func(c *gin.Context) { + objects := []gin.H{ + {"etag": "982beba05db8083656a03f544c8c7927", + "name": "test.xml", + "lastModified": "2025-03-20T19:01:40.968Z", + "size": 558677, + "contentType": "", + "expires": "0001-01-01T00:00:00Z", + "metadata": "null", + "UserTagCount": 0, + "Owner": gin.H{ + "owner": gin.H{ + "Space": "http://s3.amazonaws.com/doc/2006-03-01/", + "Local": "Owner", + }, + "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "id": "minio", + }, + "Grant": "null", + "storageClass": "STANDARD", + "IsLatest": false, + "IsDeleteMarker": false, + "VersionID": "", + "ReplicationStatus": "", + "ReplicationReady": false, + "Expiration": "0001-01-01T00:00:00Z", + "ExpirationRuleID": "", + "Restore": "null", + "ChecksumCRC32": "", + "ChecksumCRC32C": "", + "ChecksumSHA1": "", + "ChecksumSHA256": "", + "ChecksumCRC64NVME": "", + "Internal": "null"}, + } + + c.Stream(func(w io.Writer) bool { + _, err := w.Write([]byte(objects[0]["name"].(string))) + if err != nil { + return false + } + + c.XML(http.StatusOK, objects) + c.Status(http.StatusOK) + + return false + }) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // For now, passing if listing objects returns no error + _, err = client.ListObjects(ctx, b) + if err != nil { + t.Errorf("ListObject returned err: %v", err) + } + + // + //expected := "test.xml" + //found := false + //for _, result := range results { + // if result.Key == expected { + // found = true + // } + //} + //if !found { + // t.Errorf("Object %v not found in list %v", expected, results) + //} +} + +func TestMinioClient_ListObjects_Failure(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + engine.GET("/foo/", func(c *gin.Context) { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"}) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // run test + _, err = client.ListObjects(ctx, b) + if err == nil { + t.Errorf("ListObject should have returned an error") + } +} + +func TestMinioClient_ListBuildObjectNames_Success(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock: create bucket + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "octocat/hello-world/1/test.xml", + }) + }) + + // handle GET with any query params (/foo, /foo/, /foo?prefix=..., etc.) + engine.GET("/foo/*any", func(c *gin.Context) { + t.Logf("Incoming URL: %s", c.Request.URL.String()) + + // bucket location probe (SDK does ?location=) + if _, ok := c.GetQuery("location"); ok { + c.Data(http.StatusOK, "application/xml", []byte(`us-east-1`)) + return + } + + // real object listing + prefix := c.Query("prefix") + t.Logf("Received prefix: %s", prefix) + + if prefix != "octocat/hello-world/1/" { + t.Logf("Invalid prefix received: %s", prefix) + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid prefix"}) + + return + } + + // return raw XML (valid S3 ListObjectsV2 response) + xmlResponse := ` + + foo + ` + prefix + ` + 2 + 1000 + false + + octocat/hello-world/1/test.xml + "etag-test" + 558677 + 2025-03-20T19:01:40.968Z + + + octocat/hello-world/1/coverage.xml + "etag-coverage" + 123456 + 2025-03-20T19:02:40.968Z + +` + + c.Data(http.StatusOK, "application/xml", []byte(xmlResponse)) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + b.Recursive = true + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Fatalf("Failed to create MinIO client: %v", err) + } + + // Run the test + t.Logf("Running ListBuildObjectNames with org=octocat, repo=hello-world, build=1") + + results, err := client.ListBuildObjectNames(ctx, b, "octocat", "hello-world", "1") + if err != nil { + t.Fatalf("ListBuildObjectNames returned err: %v", err) + } + + // Check results + if len(results) != 2 { + t.Fatalf("Expected 2 results, got %d", len(results)) + } + + expectedNames := []string{ + "octocat/hello-world/1/test.xml", + "octocat/hello-world/1/coverage.xml", + } + + for _, expected := range expectedNames { + found := false + + for _, name := range results { + if name == expected { + found = true + break + } + } + + if !found { + t.Errorf("Expected object name %q not found in results", expected) + } + } +} + +func TestMinioClient_ListBuildObjectNames_Failure(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock bucket endpoint + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + // Return error for GET request + engine.GET("/foo/", func(c *gin.Context) { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"}) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + b.Recursive = true + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // Run test + _, err = client.ListBuildObjectNames(ctx, b, "octocat", "hello-world", "1") + if err == nil { + t.Errorf("ListBuildObjectNames should have returned an error") + } +} diff --git a/storage/minio/minio.go b/storage/minio/minio.go new file mode 100644 index 000000000..dc2fbb650 --- /dev/null +++ b/storage/minio/minio.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +// config holds the configuration for the MinIO client. +type config struct { + Enable bool + Endpoint string + AccessKey string + SecretKey string + Bucket string + Secure bool +} + +// Client implements the Storage interface using MinIO. +type Client struct { + config *config + client *minio.Client + Options *minio.Options + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + Logger *logrus.Entry +} + +// New creates a new MinIO client. +func New(endpoint string, opts ...ClientOpt) (*Client, error) { + // create new Minio client + c := new(Client) + + // default to secure connection + var urlEndpoint string + + // create new fields + c.config = new(config) + c.Options = new(minio.Options) + + // create new logger for the client + logger := logrus.StandardLogger() + c.Logger = logrus.NewEntry(logger).WithField("minio", "minio") + + // apply all provided configuration options + for _, opt := range opts { + err := opt(c) + if err != nil { + return nil, err + } + } + + c.Options.Creds = credentials.NewStaticV4(c.config.AccessKey, c.config.SecretKey, "") + c.Options.Secure = c.config.Secure + logrus.Debugf("secure: %v", c.config.Secure) + + if len(endpoint) > 0 { + useSSL := strings.HasPrefix(endpoint, "https://") + + if !useSSL { + if !strings.HasPrefix(endpoint, "http://") { + return nil, fmt.Errorf("invalid server %s: must to be a HTTP URI", endpoint) + } + + urlEndpoint = endpoint[7:] + } else { + urlEndpoint = endpoint[8:] + } + } + + // create the Minio client from the provided endpoint and options + minioClient, err := minio.New(urlEndpoint, c.Options) + if err != nil { + return nil, err + } + + c.client = minioClient + + return c, nil +} + +// NewTest returns a Storage implementation that +// integrates with a local MinIO instance. +// +// This function is intended for running tests only. +// + +func NewTest(endpoint, accessKey, secretKey, bucket string, secure bool) (*Client, error) { + return New(endpoint, WithAccessKey(accessKey), WithSecretKey(secretKey), WithSecure(secure), WithBucket(bucket)) +} diff --git a/storage/minio/minio_test.go b/storage/minio/minio_test.go new file mode 100644 index 000000000..bc4064796 --- /dev/null +++ b/storage/minio/minio_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "testing" +) + +var ( + endpoint = "http://localhost:9000" + _accessKey = "minio_access_user" + _secretKey = "minio_secret_key" + _bucket = "minio_bucket" + _useSSL = false +) + +func TestMinio_New(t *testing.T) { + // setup types + // create a local fake MinIO instance + // + // https://pkg.go.dev/github.com/minio/minio-go/v7#New + // setup tests + tests := []struct { + failure bool + endpoint string + }{ + { + failure: false, + endpoint: endpoint, + }, + { + failure: true, + endpoint: "", + }, + } + + // run tests + for _, test := range tests { + _, err := New( + test.endpoint, + WithAccessKey(_accessKey), + WithSecretKey(_secretKey), + WithSecure(_useSSL), + WithBucket(_bucket), + ) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +} diff --git a/storage/minio/opts.go b/storage/minio/opts.go new file mode 100644 index 000000000..b719487ed --- /dev/null +++ b/storage/minio/opts.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "fmt" +) + +// ClientOpt represents a configuration option to initialize the MinIO client. +type ClientOpt func(client *Client) error + +// WithEnable sets the enable flag in the MinIO client. +func WithEnable(enable bool) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring enable flag in minio client") + + // set the enable flag in the minio client + c.config.Enable = enable + + return nil + } +} + +// WithAccessKey sets the access key in the MinIO client. +func WithAccessKey(accessKey string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring access key in minio client") + + // check if the access key provided is empty + if len(accessKey) == 0 { + return fmt.Errorf("no MinIO access key provided") + } + + // set the access key in the minio client + c.config.AccessKey = accessKey + + return nil + } +} + +// WithSecretKey sets the secret key in the MinIO client. +func WithSecretKey(secretKey string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring secret key in minio client") + + // check if the secret key provided is empty + if len(secretKey) == 0 { + return fmt.Errorf("no MinIO secret key provided") + } + + // set the secret key in the minio client + c.config.SecretKey = secretKey + + return nil + } +} + +// WithSecure sets the secure connection mode in the MinIO client. +func WithSecure(secure bool) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring secure connection mode in minio client") + + // set the secure connection mode in the minio client + c.config.Secure = secure + + return nil + } +} + +// WithBucket sets the bucket name in the MinIO client. +func WithBucket(bucket string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring bucket name in minio client") + + // check if the bucket name provided is empty + if len(bucket) == 0 { + return fmt.Errorf("no MinIO bucket name provided") + } + + // set the bucket name in the minio client + c.config.Bucket = bucket + + return nil + } +} diff --git a/storage/minio/opts_test.go b/storage/minio/opts_test.go new file mode 100644 index 000000000..fa4d10691 --- /dev/null +++ b/storage/minio/opts_test.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "reflect" + "testing" +) + +func TestWithAccessKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + accessKey string + want string + }{ + { + failure: false, + accessKey: "validAccessKey", + want: "validAccessKey", + }, + { + failure: true, + accessKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + test.accessKey, + "miniosecret", + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithAddress should have returned err") + } + + continue + } + + if err != nil && test.accessKey != "" { + t.Errorf("WithAccessKey returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.AccessKey, test.want) { + t.Errorf("WithAccessKey is %v, want %v", client.config.AccessKey, test.want) + } + } +} + +func TestWithSecretKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secretKey string + want string + }{ + { + failure: false, + secretKey: "validSecretKey", + want: "validSecretKey", + }, + { + failure: true, + secretKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + test.secretKey, + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithSecretKey should have returned err") + } + + continue + } + + if err != nil && test.secretKey != "" { + t.Errorf("WithSecretKey returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.SecretKey, test.want) { + t.Errorf("WithSecretKey is %v, want %v", client.config.SecretKey, test.want) + } + } +} + +func TestWithSecure(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secure bool + want bool + }{ + { + failure: false, + secure: true, + want: true, + }, + { + failure: false, + secure: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + "foo", + test.secure) + + if test.failure { + if err == nil { + t.Errorf("WithSecure should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithSecure returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.Secure, test.want) { + t.Errorf("WithSecure is %v, want %v", client.config.Secure, test.want) + } + } +} + +func TestWithBucket(t *testing.T) { + // setup tests + tests := []struct { + failure bool + bucket string + want string + }{ + { + failure: false, + bucket: "validBucket", + want: "validBucket", + }, + { + failure: true, + bucket: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + test.bucket, + false) + + if test.failure { + if err == nil { + t.Errorf("WithBucket should have returned err") + } + + continue + } + + if err != nil && test.bucket != "" { + t.Errorf("WithBucket returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.Bucket, test.want) { + t.Errorf("WithBucket is %v, want %v", client.config.Bucket, test.want) + } + } +} diff --git a/storage/minio/presigned_get_object.go b/storage/minio/presigned_get_object.go new file mode 100644 index 000000000..58879658c --- /dev/null +++ b/storage/minio/presigned_get_object.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" +) + +// TODO hide URL behind a different name +// PresignedGetObject generates a presigned URL for downloading an object. +func (c *Client) PresignedGetObject(ctx context.Context, object *api.Object) (string, error) { + c.Logger.Tracef("generating presigned URL for object %s in bucket %s", object.ObjectName, object.Bucket.BucketName) + + var url string + // collect metadata on the object + // make sure the object exists before generating the presigned URL + objInfo, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if objInfo.Key == "" { + logrus.Errorf("unable to get object info %s from bucket %s: %v", object.ObjectName, object.Bucket.BucketName, err) + return "", err + } + + // Generate presigned URL for downloading the object. + // The URL is valid for 7 days. + presignedURL, err := c.client.PresignedGetObject(ctx, object.Bucket.BucketName, object.ObjectName, 7*24*time.Hour, nil) + if err != nil { + return "", err + } + + url = presignedURL.String() + + // replace minio:9000 with minio + // for local development + if strings.Contains(url, "minio:9000") { + // replace with minio:9002 + url = strings.Replace(url, "minio:9000", "minio", 1) + } + + return url, nil +} diff --git a/storage/minio/presigned_get_object_test.go b/storage/minio/presigned_get_object_test.go new file mode 100644 index 000000000..ebe5d5921 --- /dev/null +++ b/storage/minio/presigned_get_object_test.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func Test_PresignedGetObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "name": "test.xml", + }) + }) + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "bucketName": "foo", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err != nil { + t.Errorf("PresignedGetObject returned err: %v", err) + } + + // check if URL is valid + if url == "" { + t.Errorf("PresignedGetObject returned empty URL") + } +} + +func Test_PresignedGetObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err == nil { + t.Errorf("PresignedGetObject expected error but got none") + } + + if url != "" { + t.Errorf("PresignedGetObject returned URL when it should have failed") + } +} diff --git a/storage/minio/stat_object.go b/storage/minio/stat_object.go new file mode 100644 index 000000000..966a17699 --- /dev/null +++ b/storage/minio/stat_object.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + "github.com/minio/minio-go/v7" + + "github.com/go-vela/server/api/types" +) + +// StatObject retrieves the metadata of an object from the MinIO storage. +func (c *Client) StatObject(ctx context.Context, object *types.Object) (*types.Object, error) { + c.Logger.Tracef("retrieving metadata for object %s from bucket %s", object.ObjectName, object.Bucket.BucketName) + + // Get object info + info, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to get object info %s from bucket %s: %w", object.ObjectName, object.Bucket.BucketName, err) + } + + // Map MinIO object info to API object + return &types.Object{ + ObjectName: info.Key, + }, nil +} diff --git a/storage/minio/stat_object_test.go b/storage/minio/stat_object_test.go new file mode 100644 index 000000000..cd21b7dcd --- /dev/null +++ b/storage/minio/stat_object_test.go @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + + api "github.com/go-vela/server/api/types" +) + +func Test_StatObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "Buckets": []minio.BucketInfo{ + { + Name: "foo", + }, + }, + }) + }) + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "etag": "982beba05db8083656a03f544c8c7927", + "name": "test.xml", + "lastModified": "2025-03-20T19:01:40.968Z", + "size": 558677, + "contentType": "", + "expires": time.Now(), + "metadata": "null", + "UserTagCount": 0, + "Owner": gin.H{ + "owner": gin.H{ + "Space": "http://s3.amazonaws.com/doc/2006-03-01/", + "Local": "Owner", + }, + "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "id": "minio", + }, + "Grant": "null", + "storageClass": "STANDARD", + "IsLatest": false, + "IsDeleteMarker": false, + "VersionID": "", + "ReplicationStatus": "", + "ReplicationReady": false, + "Expiration": time.Now(), + "ExpirationRuleID": "", + "Restore": "null", + "ChecksumCRC32": "", + "ChecksumCRC32C": "", + "ChecksumSHA1": "", + "ChecksumSHA256": "", + "ChecksumCRC64NVME": "", + "Internal": "null", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + assert.NoError(t, err) + assert.Equal(t, "test.xml", result.ObjectName) +} + +func Test_StatObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + assert.Error(t, err) + assert.Nil(t, result) +} diff --git a/storage/minio/storage_enable.go b/storage/minio/storage_enable.go new file mode 100644 index 000000000..1b11692a4 --- /dev/null +++ b/storage/minio/storage_enable.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +func (c *Client) StorageEnable() bool { + // GetBucket returns the bucket name for the MinIO client. + return c.config.Enable +} diff --git a/storage/minio/test_data/create_bucket.json b/storage/minio/test_data/create_bucket.json new file mode 100644 index 000000000..b1e77da2f --- /dev/null +++ b/storage/minio/test_data/create_bucket.json @@ -0,0 +1,3 @@ +{ + "bucket_name": "foo" +} \ No newline at end of file diff --git a/storage/minio/test_data/test.xml b/storage/minio/test_data/test.xml new file mode 100644 index 000000000..66c653d49 --- /dev/null +++ b/storage/minio/test_data/test.xml @@ -0,0 +1,7 @@ + + + TEST + Upload + Reminder + Please upload me! + \ No newline at end of file diff --git a/storage/minio/upload.go b/storage/minio/upload.go new file mode 100644 index 000000000..6cb8293bc --- /dev/null +++ b/storage/minio/upload.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "io" + "mime" + "path/filepath" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// Upload uploads an object to a bucket in MinIO.ts. +func (c *Client) Upload(ctx context.Context, object *api.Object) error { + c.Logger.Tracef("uploading data to bucket %s", object.Bucket.BucketName) + info, err := c.client.FPutObject(ctx, object.Bucket.BucketName, object.ObjectName, object.FilePath, minio.PutObjectOptions{}) + + c.Logger.Infof("uploaded object %v with size %d", info, info.Size) + + return err +} + +// UploadObject uploads an object to a bucket in MinIO.ts. +func (c *Client) UploadObject(ctx context.Context, object *api.Object, reader io.Reader, size int64) error { + c.Logger.Infof("uploading data to bucket %s", object.Bucket.BucketName) + ext := filepath.Ext(object.FilePath) + contentType := mime.TypeByExtension(ext) + + c.Logger.Infof("uploading object %s with content type %s", object.ObjectName, contentType) + // TODO - better way to get bucket name + info, err := c.client.PutObject(ctx, object.Bucket.BucketName, object.ObjectName, reader, size, + minio.PutObjectOptions{ContentType: contentType}) + if err != nil { + c.Logger.Errorf("unable to upload object %s: %v", object.ObjectName, err) + return err + } + + c.Logger.Infof("uploaded object %v with size %d", info, info.Size) + + return nil +} diff --git a/storage/minio/upload_test.go b/storage/minio/upload_test.go new file mode 100644 index 000000000..f0569ff38 --- /dev/null +++ b/storage/minio/upload_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_Upload_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock upload call + engine.PUT("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("test_data/test.xml") + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + obj := new(api.Object) + obj.Bucket.BucketName = "foo" + obj.ObjectName = "test.xml" + obj.FilePath = "test_data/test.xml" + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + err = client.Upload(ctx, obj) + + if resp.Code != http.StatusOK { + t.Errorf("Upload returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("Upload returned err: %v", err) + } +} + +func TestMinioClient_Upload_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock bucket exists call + engine.PUT("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("test_data/test.xml") + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + ctx := context.TODO() + obj := new(api.Object) + obj.Bucket.BucketName = "foo" + obj.ObjectName = "test.xml" + obj.FilePath = "nonexist/test.xml" + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + err = client.Upload(ctx, obj) + + if resp.Code != http.StatusOK { + t.Errorf("Upload returned %v, want %v", resp.Code, http.StatusOK) + } + + if !os.IsNotExist(err) { + t.Errorf("Upload returned err: %v", err) + } +} diff --git a/storage/service.go b/storage/service.go new file mode 100644 index 000000000..dde86619d --- /dev/null +++ b/storage/service.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "io" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// Storage defines the service interface for object storage operations. +type Storage interface { + // Bucket Management + CreateBucket(ctx context.Context, bucket *api.Bucket) error + BucketExists(ctx context.Context, bucket *api.Bucket) (bool, error) + ListBuckets(ctx context.Context) ([]string, error) + GetBucket(ctx context.Context) string + // Object Operations + StatObject(ctx context.Context, object *api.Object) (*api.Object, error) + Upload(ctx context.Context, object *api.Object) error + UploadObject(ctx context.Context, object *api.Object, reader io.Reader, size int64) error + //Download(ctx context.Context, object *api.Object) error + ListObjects(ctx context.Context, bucket *api.Bucket) ([]minio.ObjectInfo, error) + ListObjectNames(ctx context.Context, bucket *api.Bucket) ([]string, error) + ListBuildObjectNames(context.Context, *api.Bucket, string, string, string) ([]string, error) + // Presigned URLs + PresignedGetObject(ctx context.Context, object *api.Object) (string, error) + // Storage info + StorageEnable() bool +} diff --git a/storage/setup.go b/storage/setup.go new file mode 100644 index 000000000..0dd873dcf --- /dev/null +++ b/storage/setup.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/constants" + "github.com/go-vela/server/storage/minio" +) + +// Setup represents the configuration necessary for +// creating a Vela service capable of integrating +// with a configured S3 environment. +type Setup struct { + Enable bool + Driver string + Endpoint string + AccessKey string + SecretKey string + Bucket string + Region string + Secure bool +} + +// Minio creates and returns a Vela service capable +// of integrating with an S3 environment. +func (s *Setup) Minio() (Storage, error) { + return minio.New( + s.Endpoint, + minio.WithEnable(s.Enable), + minio.WithAccessKey(s.AccessKey), + minio.WithSecretKey(s.SecretKey), + minio.WithSecure(s.Secure), + minio.WithBucket(s.Bucket), + ) +} + +// Validate verifies the necessary fields for the +// provided configuration are populated correctly. +func (s *Setup) Validate() error { + logrus.Trace("validating Storage setup for client") + + // storage disabled: nothing to validate + if s.Enable { + if s.Driver != "" && s.Driver != constants.DriverMinio { + return fmt.Errorf("storage driver should not be set (got %q)", s.Driver) + } + + if s.Bucket == "" { + return fmt.Errorf("storage is enabled but no bucket provided") + } + + if s.Endpoint == "" { + return fmt.Errorf("storage is enabled but no endpoint provided") + } + + if s.AccessKey == "" || s.SecretKey == "" { + return fmt.Errorf("storage is enabled but no access key or secret key provided") + } + + if _, err := url.ParseRequestURI(s.Endpoint); err != nil { + return fmt.Errorf("storage is enabled but endpoint is invalid") + } + } + + // setup is valid + return nil +} diff --git a/storage/setup_test.go b/storage/setup_test.go new file mode 100644 index 000000000..6b6cc1281 --- /dev/null +++ b/storage/setup_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/go-vela/server/constants" +) + +func TestSetup_Minio(t *testing.T) { + setup := &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://minio.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + } + + storageClient, err := setup.Minio() + assert.NoError(t, err) + assert.NotNil(t, storageClient) +} + +func TestSetup_Validate(t *testing.T) { + tests := []struct { + name string + setup *Setup + wantErr bool + }{ + { + name: "storage disabled", + setup: &Setup{ + Enable: false, + }, + wantErr: false, + }, + { + name: "valid config", + setup: &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: false, + }, + { + name: "missing bucket", + setup: &Setup{ + Enable: true, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + }, + wantErr: true, + }, + { + name: "driver set", + setup: &Setup{ + Enable: true, + Driver: constants.DriverMinio, + Endpoint: "http://example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: false, + }, + { + name: "missing endpoint", + setup: &Setup{ + Enable: true, + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: true, + }, + { + name: "missing credentials", + setup: &Setup{ + Enable: true, + Endpoint: "http://example.com", + Bucket: "bucket-name", + }, + wantErr: true, + }, + { + name: "invalid endpoint URL", + setup: &Setup{ + Enable: true, + Endpoint: "://bad-url", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + }, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.setup.Validate() + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 000000000..d567651f2 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + + "github.com/go-vela/server/constants" +) + +// FromCLICommand helper function to setup Minio Client from the CLI arguments. +func FromCLICommand(_ context.Context, c *cli.Command) (Storage, error) { + // S3 configuration + _setup := &Setup{ + Enable: c.Bool("storage.enable"), + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.addr"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + Secure: c.Bool("storage.use.ssl"), + } + + return New(_setup) +} + +// New creates and returns a Vela service capable of +// integrating with the configured storage environment. +// Currently, the following storages are supported: +// +// * minio +// . +func New(s *Setup) (Storage, error) { + // validate the setup being provided + // + // https://pkg.go.dev/github.com/go-vela/server/storage#Setup.Validate + if s.Enable { + err := s.Validate() + if err != nil { + return nil, fmt.Errorf("unable to validate storage setup: %w", err) + } + + logrus.Debug("creating storage client from setup") + // process the storage driver being provided + switch s.Driver { + case constants.DriverMinio: + // handle the Kafka queue driver being provided + // + // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#Setup.Kafka + return s.Minio() + default: + // handle an invalid queue driver being provided + return nil, fmt.Errorf("invalid storage driver provided: %s", s.Driver) + } + } + + return nil, nil +} diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 000000000..212fab3b0 --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/go-vela/server/constants" +) + +func TestStorage_New(t *testing.T) { + tests := []struct { + name string + failure bool + setup *Setup + }{ + { + name: "valid-minio-config", + failure: false, + setup: &Setup{ + Driver: constants.DriverMinio, + Enable: true, + Endpoint: "http://minio.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + { + name: "invalid-driver", + failure: true, + setup: &Setup{ + Driver: "invalid-driver", + Enable: true, + Endpoint: "http://invalid.example.com", + AccessKey: "storage-access-key", + SecretKey: "storage-secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := New(tt.setup) + + if tt.failure { + if err == nil { + t.Errorf("New() expected error, got nil") + } + + return + } + + // success case + if err != nil { + t.Errorf("New() unexpected error: %v", err) + } + }) + } +}