feat(artifacts): Object storage integration and artifacts key

.gitignore

@@ -38,6 +38,7 @@ secrets.env
 
 # Dotenv environment file
 .env
+.env_*
 .env.test
 
 # Files to be excluded.

DOCS.md

@@ -56,6 +56,12 @@ echo "VELA_SCM_CLIENT=<Github OAuth Client ID>" >> .env
 echo "VELA_SCM_SECRET=<Github OAuth Client Secret>" >> .env
 ```
 
+* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela:
+
+```bash
+ sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts'
+````
+
 ## Start
 
 **NOTE: Please review the [setup section](#setup) before moving forward.**

api/admin/storage.go

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/storage"
+	"github.com/go-vela/server/util"
+)
+
+// swagger:operation POST /api/v1/admin/storage/bucket admin CreateBucket
+//
+// Create a new bucket
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: body
+//   name: body
+//   description: The bucket name to be created
+//   required: true
+//   schema:
+//     type: object
+//     properties:
+//       bucketName:
+//         type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '201':
+//     description: Successfully created the bucket
+//   '400':
+//     description: Invalid request payload
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// CreateBucket represents the API handler to create a new bucket.
+func CreateBucket(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: creating bucket")
+
+	// capture body from API request
+	input := new(types.Bucket)
+
+	err := c.Bind(input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to decode JSON for bucket %s: %w", input.BucketName, err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	l.Debugf("bucket name: %s", input.BucketName)
+
+	err = storage.FromGinContext(c).CreateBucket(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to create bucket: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	c.Status(http.StatusCreated)
+}
+
+// swagger:operation GET /api/v1/admin/storage/presign admin GetPresignedURL
+//
+// # Generate a presigned URL for an object
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - in: query
+//     name: bucketName
+//     description: The name of the bucket
+//     required: true
+//     type: string
+//   - in: query
+//     name: objectName
+//     description: The name of the object
+//     required: true
+//     type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully generated the presigned URL
+//   '400':
+//     description: Invalid request payload
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// GetPresignedURL represents the API handler to generate a presigned URL for an object.
+func GetPresignedURL(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: generating presigned URL")
+
+	// capture query parameters from API request
+	bucketName := c.Query("bucketName")
+	objectName := c.Query("objectName")
+
+	if bucketName == "" || objectName == "" {
+		retErr := fmt.Errorf("bucketName and objectName are required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	input := &types.Object{
+		Bucket:     types.Bucket{BucketName: bucketName},
+		ObjectName: objectName,
+	}
+
+	url, err := storage.FromGinContext(c).PresignedGetObject(ctx, input)
+	if err != nil || url == "" {
+		retErr := fmt.Errorf("unable to generate presigned URL: %w", err)
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusOK, url)
+}

api/artifact/create.go

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package artifact
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/database"
+	"github.com/go-vela/server/util"
+)
+
+func CreateArtifact(c *gin.Context) {
+	// capture middleware values
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	// capture the artifact from the request body
+	input := new(types.Artifact)
+
+	err := c.Bind(input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to decode JSON for new Artifact: %w", err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	// ensure build_id is defined
+	if input.GetBuildID() <= 0 {
+		util.HandleError(c, http.StatusBadRequest, fmt.Errorf("build_id must set and greater than 0"))
+		return
+	}
+
+	input.SetCreatedAt(time.Now().UTC().Unix())
+
+	l.Debugf("creating new artifact")
+	// create the artifact in the database using the input from request
+	a, err := database.FromContext(c).CreateArtifact(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to create new artifact: %w", err)
+
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusCreated, a)
+}

api/artifact/get.go

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package artifact
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	artifact "github.com/go-vela/server/router/middleware/artifact"
+	"github.com/go-vela/server/router/middleware/build"
+)
+
+// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/artifact/{artifact} artifacts GetArtifact
+//
+// Get an artifact
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: path
+//   name: org
+//   description: Name of the organization
+//   required: true
+//   type: string
+// - in: path
+//   name: repo
+//   description: Name of the repository
+//   required: true
+//   type: string
+// - in: path
+//   name: build
+//   description: Build number
+//   required: true
+//   type: integer
+// - in: path
+//   name: artifact
+//   description: artifact ID
+//   required: true
+//   type: integer
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully retrieved the artifact
+//     type: json
+//     schema:
+//       "$ref": "#/definitions/Artifact"
+//   '400':
+//     description: Invalid request payload or path
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '401':
+//     description: Unauthorized
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '404':
+//     description: Not found
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// GetArtifact represents the API handler to get
+// an artifact for a build.
+func GetArtifact(c *gin.Context) {
+	// capture middleware values
+	l := c.MustGet("logger").(*logrus.Entry)
+	b := build.Retrieve(c)
+	a := artifact.Retrieve(c)
+
+	l.Debugf("getting artifact %d for build %d", a.GetID(), b.GetNumber())
+
+	// return the artifact with presigned URL
+	response := gin.H{
+		"id":            a.GetID(),
+		"build_id":      a.GetBuildID(),
+		"file_name":     a.GetFileName(),
+		"file_type":     a.GetFileType(),
+		"file_size":     a.GetFileSize(),
+		"object_path":   a.GetObjectPath(),
+		"presigned_url": a.GetPresignedURL(),
+		"created_at":    a.GetCreatedAt(),
+	}
+
+	c.JSON(http.StatusOK, response)
+}

api/artifact/list.go

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package artifact
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/database"
+	"github.com/go-vela/server/router/middleware/build"
+	"github.com/go-vela/server/util"
+)
+
+func ListArtifactsForBuild(c *gin.Context) {
+	// capture middleware values
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+	b := build.Retrieve(c)
+
+	l.Debugf("listing artifacts for build %d", b.GetNumber())
+
+	// retrieve artifacts from the database
+	artifacts, err := database.FromContext(c).ListArtifactsByBuildID(ctx, b.GetID())
+	if err != nil {
+		retErr := fmt.Errorf("unable to list artifacts for build %d: %w", b.GetNumber(), err)
+
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusOK, artifacts)
+}

api/storage/doc.go

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+
+// Package storage provides the storage handlers for the Vela API.
+//
+// Usage:
+//
+//	import "github.com/go-vela/server/api/storage"
+package storage