diff --git a/.github/workflows/test-s3.yml b/.github/workflows/test-s3.yml index 2cb3ffec..64c68fad 100644 --- a/.github/workflows/test-s3.yml +++ b/.github/workflows/test-s3.yml @@ -20,14 +20,7 @@ jobs: - 1.21.x steps: - name: Install MinIO - run: | - docker run -d -p 9000:9000 --name minio minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket + run: docker run -d -p 9000:9000 --name minio minio/minio server /data - name: Fetch Repository uses: actions/checkout@v4 - name: Install Go diff --git a/s3/README.md b/s3/README.md index 95bf6291..21a7280e 100644 --- a/s3/README.md +++ b/s3/README.md @@ -31,8 +31,15 @@ func (s *Storage) Delete(key string) error func (s *Storage) Reset() error func (s *Storage) Close() error func (s *Storage) Conn() *s3.Client + +// Additional useful methods. +func (s *Storage) CreateBucker(bucket string) error +func (s *Storage) DeleteBucket(bucket string) error +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.ChecksumAlgorithm][]byte) error ``` + ### Installation + S3 is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet: ```bash go mod init github.com// @@ -43,7 +50,9 @@ go get github.com/gofiber/storage/s3/v2 ``` ### Examples + Import the storage package. + ```go import "github.com/gofiber/storage/s3/v2" ``` @@ -62,6 +71,38 @@ store := s3.New(s3.Config{ }) ``` +Create an object with `Set()`: +```go +err := store.Set("my-key", []byte("my-value")) +``` + +Or, call `SetWithChecksum()` to create an object with checksum to +ask S3 server to verify data integrity on server side: + +> Currently 4 algorithms are supported: +> - types.ChecksumAlgorithmCrc32 (`CRC32`) +> - types.ChecksumAlgorithmCrc32c (`CRC32C`) +> - types.ChecksumAlgorithmSha1 (`SHA1`) +> - types.ChecksumAlgorithmSha256 (`SHA256`) +> +> For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). + +```go +key := "my-key" +val := []byte("my-value") + +hash := sha256.New() +hash.Write(val) +sha256sum := hash.Sum(nil) + +// import "github.com/aws/aws-sdk-go-v2/service/s3/types" +checksum = map[types.ChecksumAlgorithm][]byte{ + types.ChecksumAlgorithmSha256: sha256sum, +} + +err := store.SetWithChecksum(key, val, checksum) +``` + ### Config ```go // Config defines the config for storage. @@ -104,7 +145,9 @@ type Credentials struct { ``` ### Default Config + The default configuration lacks Bucket, Region, and Endpoint which are all required and must be overwritten: + ```go // ConfigDefault is the default config var ConfigDefault = Config{ diff --git a/s3/init_test.go b/s3/init_test.go new file mode 100644 index 00000000..7d8d0c21 --- /dev/null +++ b/s3/init_test.go @@ -0,0 +1,38 @@ +package s3 + +import ( + "os" + "testing" + "time" +) + +const ( + bucket = "testbucket" +) + +var testStore *Storage + +func TestMain(m *testing.M) { + testStore = New( + Config{ + Bucket: bucket, + Endpoint: "http://127.0.0.1:9000/", + Region: "us-east-1", + Credentials: Credentials{ + AccessKey: "minioadmin", + SecretAccessKey: "minioadmin", + }, + RequestTimeout: 3 * time.Second, + }, + ) + + // Create test bucket. + _ = testStore.CreateBucket(bucket) + + exitVal := m.Run() + + // Delete test bucket. + _ = testStore.DeleteBucket(bucket) + + os.Exit(exitVal) +} diff --git a/s3/s3.go b/s3/s3.go index 921b02a7..e206ab39 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -159,7 +159,7 @@ func (s *Storage) Close() error { return nil } -// Return database client +// Conn returns database client. func (s *Storage) Conn() *s3.Client { return s.svc } @@ -186,11 +186,11 @@ func returnAWSConfig(cfg Config) (aws.Config, error) { }) if cfg.Credentials != (Credentials{}) { - credentials := credentials.NewStaticCredentialsProvider(cfg.Credentials.AccessKey, cfg.Credentials.SecretAccessKey, "") + creds := credentials.NewStaticCredentialsProvider(cfg.Credentials.AccessKey, cfg.Credentials.SecretAccessKey, "") return awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithRegion(cfg.Region), awsconfig.WithEndpointResolverWithOptions(endpoint), - awsconfig.WithCredentialsProvider(credentials), + awsconfig.WithCredentialsProvider(creds), awsconfig.WithRetryer(func() aws.Retryer { return retry.AddWithMaxAttempts(retry.NewStandard(), cfg.MaxAttempts) }), diff --git a/s3/s3_methods.go b/s3/s3_methods.go new file mode 100644 index 00000000..d99a453b --- /dev/null +++ b/s3/s3_methods.go @@ -0,0 +1,83 @@ +package s3 + +import ( + "bytes" + "encoding/base64" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// Additional methods for S3, but not required by gofiber Storage interface. + +// CreateBucket creates a new bucket. +func (s *Storage) CreateBucket(bucket string) error { + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.svc.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + + return err +} + +// DeleteBucket deletes a bucket. +func (s *Storage) DeleteBucket(bucket string) error { + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.svc.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + + return err +} + +// SetWithChecksum sets key with value and checksum. +// +// Currently 4 algorithms are supported: +// - types.ChecksumAlgorithmCrc32 (`CRC32`) +// - types.ChecksumAlgorithmCrc32c (`CRC32C`) +// - types.ChecksumAlgorithmSha1 (`SHA1`) +// - types.ChecksumAlgorithmSha256 (`SHA256`) +// +// For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.ChecksumAlgorithm][]byte) error { + if len(key) <= 0 { + return nil + } + + poi := s3.PutObjectInput{ + Bucket: &s.bucket, + Key: aws.String(key), + Body: bytes.NewReader(val), + } + + for alg, sum := range checksum { + // S3 requires base64 encoded checksum. + b64str := base64.StdEncoding.EncodeToString(sum) + + switch alg { + case types.ChecksumAlgorithmCrc32: + poi.ChecksumCRC32 = aws.String(b64str) + case types.ChecksumAlgorithmCrc32c: + poi.ChecksumCRC32C = aws.String(b64str) + case types.ChecksumAlgorithmSha1: + poi.ChecksumSHA1 = aws.String(b64str) + case types.ChecksumAlgorithmSha256: + poi.ChecksumSHA256 = aws.String(b64str) + default: + return fmt.Errorf("invalid checksum algorithm: %s", alg) + } + } + + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.uploader.Upload(ctx, &poi) + + return err +} diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go new file mode 100644 index 00000000..afd665fc --- /dev/null +++ b/s3/s3_methods_test.go @@ -0,0 +1,50 @@ +package s3 + +import ( + "crypto/sha256" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/require" +) + +func Test_S3_CreateDeleteBucket(t *testing.T) { + bkt := "test-new-bucket" + + err := testStore.CreateBucket(bkt) + require.NoError(t, err) + + err = testStore.DeleteBucket(bkt) + require.NoError(t, err) +} + +func Test_S3_SetWithChecksum(t *testing.T) { + var ( + key = "set-with-checksum" + val = []byte("doe") + ) + + // Create SHA-256 hash and get checksum. + sha256Hash := sha256.New() + sha256Hash.Write(val) + sha256sum := sha256Hash.Sum(nil) + + checksum := map[types.ChecksumAlgorithm][]byte{ + types.ChecksumAlgorithmSha256: sha256sum, + } + + err := testStore.SetWithChecksum(key, val, checksum) + require.NoError(t, err) + + result, err := testStore.Get(key) + require.NoError(t, err) + + // Compare value. + require.Equal(t, result, val) + + // Compare checksum. + hash2 := sha256.New() + hash2.Write(result) + sha256sum2 := hash2.Sum(nil) + require.Equal(t, sha256sum, sha256sum2) +} diff --git a/s3/s3_test.go b/s3/s3_test.go index 599f8251..3a3e49e5 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -6,18 +6,6 @@ import ( "github.com/stretchr/testify/require" ) -var testStore = New( - Config{ - Bucket: "testbucket", - Endpoint: "http://127.0.0.1:9000/", - Region: "us-east-1", - Credentials: Credentials{ - AccessKey: "minioadmin", - SecretAccessKey: "minioadmin", - }, - }, -) - func Test_S3_Set(t *testing.T) { var ( key = "john"