From 26c755f312c3abad4f43b9b29661a2db15ede374 Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 16:01:32 +0800 Subject: [PATCH 1/9] New: SetWithChecksum. --- s3/s3_methods.go | 45 +++++++++++++++++++++++++++++++++++++++++++ s3/s3_methods_test.go | 23 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 s3/s3_methods.go create mode 100644 s3/s3_methods_test.go diff --git a/s3/s3_methods.go b/s3/s3_methods.go new file mode 100644 index 00000000..4372b95f --- /dev/null +++ b/s3/s3_methods.go @@ -0,0 +1,45 @@ +package s3 + +import ( + "bytes" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Additional methods for S3, but not required by gofiber Storage interface. + +// SetWithChecksum sets key with value and checksum. +// Key of `checksum` map is algorithm in upper cases like `CRC32`, `CRC32C`, +// `SHA1`, `SHA256`, value is the checksum. +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string]string) error { + if len(key) <= 0 { + return nil + } + + ctx, cancel := s.requestContext() + defer cancel() + + poi := &s3.PutObjectInput{ + Bucket: &s.bucket, + Key: aws.String(key), + Body: bytes.NewReader(val), + } + + for alg, sum := range checksum { + switch alg { + case "CRC32": + poi.ChecksumCRC32 = aws.String(sum) + case "CRC32C": + poi.ChecksumCRC32C = aws.String(sum) + case "SHA1": + poi.ChecksumSHA1 = aws.String(sum) + case "SHA256": + poi.ChecksumSHA256 = aws.String(sum) + } + } + + _, err := s.uploader.Upload(ctx, poi) + + return err +} diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go new file mode 100644 index 00000000..866acea1 --- /dev/null +++ b/s3/s3_methods_test.go @@ -0,0 +1,23 @@ +package s3 + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_S3_SetWithChecksum(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + sha256sum = sha256.New().Sum(val) + ) + + err := testStore.SetWithChecksum(key, val, map[string]string{"SHA256": string(sha256sum)}) + require.NoError(t, err) + + result, err := testStore.Get(key) + require.NoError(t, err) + require.Equal(t, sha256sum, sha256.New().Sum(result)) +} From 1ddcc7b3fcef4062fc08a656d865ddfc20ca1a5a Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 16:26:55 +0800 Subject: [PATCH 2/9] Add example in README. Use `map[string][]byte` instead of `map[string]string`. --- s3/README.md | 23 +++++++++++++++++++++++ s3/s3_methods.go | 19 ++++++++++++------- s3/s3_methods_test.go | 2 +- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/s3/README.md b/s3/README.md index 95bf6291..4134fcbb 100644 --- a/s3/README.md +++ b/s3/README.md @@ -32,7 +32,9 @@ func (s *Storage) Reset() error func (s *Storage) Close() error func (s *Storage) Conn() *s3.Client ``` + ### Installation + S3 is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet: ```bash go mod init github.com// @@ -43,7 +45,9 @@ go get github.com/gofiber/storage/s3/v2 ``` ### Examples + Import the storage package. + ```go import "github.com/gofiber/storage/s3/v2" ``` @@ -62,6 +66,25 @@ store := s3.New(s3.Config{ }) ``` +Create an object with `Set()`: +```go +err := store.Set("my-key", []byte("my-value")) +``` + +Or, call `SetWithChecksum()` to create an object with checksum to +ask S3 server to verify data integrity on server side: + +> Currently only 4 algorithm are supported: `CRC32`, `CRC32C`, `SHA1`, `SHA256`. +> For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). + +```go +val := []byte("my-value") +sha256sum := sha256.New().Sum256(val) +checksum := map[string][]byte{"SHA256": sha256sum} + +err := store.SetWithChecksum("my-key", []byte("my-value"), checksum) +``` + ### Config ```go // Config defines the config for storage. diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 4372b95f..6307ae4a 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -2,6 +2,7 @@ package s3 import ( "bytes" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -10,9 +11,11 @@ import ( // Additional methods for S3, but not required by gofiber Storage interface. // SetWithChecksum sets key with value and checksum. -// Key of `checksum` map is algorithm in upper cases like `CRC32`, `CRC32C`, -// `SHA1`, `SHA256`, value is the checksum. -func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string]string) error { +// Key of `checksum` map is algorithm in upper cases, value is the checksum. +// Currently only 4 algorithm are supported: `CRC32`, `CRC32C`, `SHA1`, `SHA256`. +// +// For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string][]byte) error { if len(key) <= 0 { return nil } @@ -29,13 +32,15 @@ func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string]st for alg, sum := range checksum { switch alg { case "CRC32": - poi.ChecksumCRC32 = aws.String(sum) + poi.ChecksumCRC32 = aws.String(fmt.Sprintf("%x", sum)) case "CRC32C": - poi.ChecksumCRC32C = aws.String(sum) + poi.ChecksumCRC32C = aws.String(fmt.Sprintf("%x", sum)) case "SHA1": - poi.ChecksumSHA1 = aws.String(sum) + poi.ChecksumSHA1 = aws.String(fmt.Sprintf("%x", sum)) case "SHA256": - poi.ChecksumSHA256 = aws.String(sum) + poi.ChecksumSHA256 = aws.String(fmt.Sprintf("%x", sum)) + default: + return fmt.Errorf("invalid checksum algorithm: %s", alg) } } diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index 866acea1..0a390225 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -14,7 +14,7 @@ func Test_S3_SetWithChecksum(t *testing.T) { sha256sum = sha256.New().Sum(val) ) - err := testStore.SetWithChecksum(key, val, map[string]string{"SHA256": string(sha256sum)}) + err := testStore.SetWithChecksum(key, val, map[string][]byte{"SHA256": sha256sum}) require.NoError(t, err) result, err := testStore.Get(key) From e09d83368c752c950b61485d7afd777ed9e1368d Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 16:39:19 +0800 Subject: [PATCH 3/9] Fix incorrect sum. --- s3/README.md | 13 +++++++++++-- s3/s3_methods.go | 26 ++++++++++++++++---------- s3/s3_methods_test.go | 6 +++++- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/s3/README.md b/s3/README.md index 4134fcbb..5dd02bfe 100644 --- a/s3/README.md +++ b/s3/README.md @@ -74,13 +74,22 @@ err := store.Set("my-key", []byte("my-value")) Or, call `SetWithChecksum()` to create an object with checksum to ask S3 server to verify data integrity on server side: -> Currently only 4 algorithm are supported: `CRC32`, `CRC32C`, `SHA1`, `SHA256`. +> Currently 4 algorithm are supported: +> - types.ChecksumAlgorithmCrc32 (`CRC32`) +> - types.ChecksumAlgorithmCrc32c (`CRC32C`) +> - types.ChecksumAlgorithmSha1 (`SHA1`) +> - types.ChecksumAlgorithmSha256 (`SHA256`) +> > For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). ```go val := []byte("my-value") sha256sum := sha256.New().Sum256(val) -checksum := map[string][]byte{"SHA256": sha256sum} + +// import "github.com/aws/aws-sdk-go-v2/service/s3/types" +checksum = map[types.ChecksumAlgorithm][]byte{ + types.ChecksumAlgorithmSha256: sha256sum, +} err := store.SetWithChecksum("my-key", []byte("my-value"), checksum) ``` diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 6307ae4a..65d96183 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -2,20 +2,26 @@ package s3 import ( "bytes" + "encoding/hex" "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) // Additional methods for S3, but not required by gofiber Storage interface. // SetWithChecksum sets key with value and checksum. // Key of `checksum` map is algorithm in upper cases, value is the checksum. -// Currently only 4 algorithm are supported: `CRC32`, `CRC32C`, `SHA1`, `SHA256`. +// Currently only 4 algorithm are supported: +// - types.ChecksumAlgorithmCrc32 (`CRC32`) +// - types.ChecksumAlgorithmCrc32c (`CRC32C`) +// - types.ChecksumAlgorithmSha1 (`SHA1`) +// - types.ChecksumAlgorithmSha256 (`SHA256`) // // For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). -func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string][]byte) error { +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.ChecksumAlgorithm][]byte) error { if len(key) <= 0 { return nil } @@ -31,14 +37,14 @@ func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[string][] for alg, sum := range checksum { switch alg { - case "CRC32": - poi.ChecksumCRC32 = aws.String(fmt.Sprintf("%x", sum)) - case "CRC32C": - poi.ChecksumCRC32C = aws.String(fmt.Sprintf("%x", sum)) - case "SHA1": - poi.ChecksumSHA1 = aws.String(fmt.Sprintf("%x", sum)) - case "SHA256": - poi.ChecksumSHA256 = aws.String(fmt.Sprintf("%x", sum)) + case types.ChecksumAlgorithmCrc32: + poi.ChecksumCRC32 = aws.String(hex.EncodeToString(sum)) + case types.ChecksumAlgorithmCrc32c: + poi.ChecksumCRC32C = aws.String(hex.EncodeToString(sum)) + case types.ChecksumAlgorithmSha1: + poi.ChecksumSHA1 = aws.String(hex.EncodeToString(sum)) + case types.ChecksumAlgorithmSha256: + poi.ChecksumSHA256 = aws.String(hex.EncodeToString(sum)) default: return fmt.Errorf("invalid checksum algorithm: %s", alg) } diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index 0a390225..a43aa04b 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "testing" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/require" ) @@ -12,9 +13,12 @@ func Test_S3_SetWithChecksum(t *testing.T) { key = "john" val = []byte("doe") sha256sum = sha256.New().Sum(val) + checksum = map[types.ChecksumAlgorithm][]byte{ + types.ChecksumAlgorithmSha256: sha256sum, + } ) - err := testStore.SetWithChecksum(key, val, map[string][]byte{"SHA256": sha256sum}) + err := testStore.SetWithChecksum(key, val, checksum) require.NoError(t, err) result, err := testStore.Get(key) From d06fbfc7f311dd04ee040f42a31478d67f98541a Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 16:47:36 +0800 Subject: [PATCH 4/9] Fix incorrect sum. --- s3/s3_methods.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 65d96183..746e701c 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -2,7 +2,6 @@ package s3 import ( "bytes" - "encoding/hex" "fmt" "github.com/aws/aws-sdk-go-v2/aws" @@ -38,13 +37,13 @@ func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.Che for alg, sum := range checksum { switch alg { case types.ChecksumAlgorithmCrc32: - poi.ChecksumCRC32 = aws.String(hex.EncodeToString(sum)) + poi.ChecksumCRC32 = aws.String(string(sum)) case types.ChecksumAlgorithmCrc32c: - poi.ChecksumCRC32C = aws.String(hex.EncodeToString(sum)) + poi.ChecksumCRC32C = aws.String(string(sum)) case types.ChecksumAlgorithmSha1: - poi.ChecksumSHA1 = aws.String(hex.EncodeToString(sum)) + poi.ChecksumSHA1 = aws.String(string(sum)) case types.ChecksumAlgorithmSha256: - poi.ChecksumSHA256 = aws.String(hex.EncodeToString(sum)) + poi.ChecksumSHA256 = aws.String(string(sum)) default: return fmt.Errorf("invalid checksum algorithm: %s", alg) } From dc00bab703fb4e0ee734e7a4d17ee49bfc76bbdd Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 17:07:43 +0800 Subject: [PATCH 5/9] Init testStore in first load test file. --- s3/s3_methods_test.go | 12 ++++++++++++ s3/s3_test.go | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index a43aa04b..25bf453d 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -8,6 +8,18 @@ import ( "github.com/stretchr/testify/require" ) +var testStore = New( + Config{ + Bucket: "testbucket", + Endpoint: "http://127.0.0.1:9000/", + Region: "us-east-1", + Credentials: Credentials{ + AccessKey: "minioadmin", + SecretAccessKey: "minioadmin", + }, + }, +) + func Test_S3_SetWithChecksum(t *testing.T) { var ( key = "john" diff --git a/s3/s3_test.go b/s3/s3_test.go index 599f8251..3a3e49e5 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -6,18 +6,6 @@ import ( "github.com/stretchr/testify/require" ) -var testStore = New( - Config{ - Bucket: "testbucket", - Endpoint: "http://127.0.0.1:9000/", - Region: "us-east-1", - Credentials: Credentials{ - AccessKey: "minioadmin", - SecretAccessKey: "minioadmin", - }, - }, -) - func Test_S3_Set(t *testing.T) { var ( key = "john" From ae41d2f5c16155cb083f48071d9a49a0ec6434ff Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 17:17:31 +0800 Subject: [PATCH 6/9] Init testStore in init(). --- s3/init_test.go | 18 ++++++++++++++++++ s3/s3_methods_test.go | 12 ------------ 2 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 s3/init_test.go diff --git a/s3/init_test.go b/s3/init_test.go new file mode 100644 index 00000000..99ce5c31 --- /dev/null +++ b/s3/init_test.go @@ -0,0 +1,18 @@ +package s3 + +var testStore *Storage + +func init() { + testStore = New( + Config{ + Bucket: "testbucket", + Endpoint: "http://127.0.0.1:9000/", + Region: "us-east-1", + Credentials: Credentials{ + AccessKey: "minioadmin", + SecretAccessKey: "minioadmin", + }, + }, + ) + +} diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index 25bf453d..a43aa04b 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -8,18 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -var testStore = New( - Config{ - Bucket: "testbucket", - Endpoint: "http://127.0.0.1:9000/", - Region: "us-east-1", - Credentials: Credentials{ - AccessKey: "minioadmin", - SecretAccessKey: "minioadmin", - }, - }, -) - func Test_S3_SetWithChecksum(t *testing.T) { var ( key = "john" From e735c69371bcaf2261864a2a5aa35cddbe936003 Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Tue, 5 Sep 2023 17:34:55 +0800 Subject: [PATCH 7/9] S3: Add RequestTimeout in unittest. --- s3/README.md | 2 +- s3/init_test.go | 6 +++++- s3/s3_methods.go | 4 ++-- s3/s3_methods_test.go | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/s3/README.md b/s3/README.md index 5dd02bfe..f2bb9259 100644 --- a/s3/README.md +++ b/s3/README.md @@ -74,7 +74,7 @@ err := store.Set("my-key", []byte("my-value")) Or, call `SetWithChecksum()` to create an object with checksum to ask S3 server to verify data integrity on server side: -> Currently 4 algorithm are supported: +> Currently 4 algorithms are supported: > - types.ChecksumAlgorithmCrc32 (`CRC32`) > - types.ChecksumAlgorithmCrc32c (`CRC32C`) > - types.ChecksumAlgorithmSha1 (`SHA1`) diff --git a/s3/init_test.go b/s3/init_test.go index 99ce5c31..e25002de 100644 --- a/s3/init_test.go +++ b/s3/init_test.go @@ -1,5 +1,9 @@ package s3 +import ( + "time" +) + var testStore *Storage func init() { @@ -12,7 +16,7 @@ func init() { AccessKey: "minioadmin", SecretAccessKey: "minioadmin", }, + RequestTimeout: 10 * time.Second, }, ) - } diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 746e701c..1f540a0d 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -12,8 +12,8 @@ import ( // Additional methods for S3, but not required by gofiber Storage interface. // SetWithChecksum sets key with value and checksum. -// Key of `checksum` map is algorithm in upper cases, value is the checksum. -// Currently only 4 algorithm are supported: +// +// Currently 4 algorithms are supported: // - types.ChecksumAlgorithmCrc32 (`CRC32`) // - types.ChecksumAlgorithmCrc32c (`CRC32C`) // - types.ChecksumAlgorithmSha1 (`SHA1`) diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index a43aa04b..6cacffce 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -10,7 +10,7 @@ import ( func Test_S3_SetWithChecksum(t *testing.T) { var ( - key = "john" + key = "set-with-checksum" val = []byte("doe") sha256sum = sha256.New().Sum(val) checksum = map[types.ChecksumAlgorithm][]byte{ From 4dfe51aa010ca52b2b2cad5f1da9c8efd4937503 Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Wed, 6 Sep 2023 08:45:46 +0800 Subject: [PATCH 8/9] Fix incorrect checksum (base64 encoded). Create bucket in unittest instead of GitHub Actions. --- .github/workflows/test-s3.yml | 9 +------ s3/init_test.go | 22 ++++++++++++++--- s3/s3.go | 6 ++--- s3/s3_methods.go | 45 ++++++++++++++++++++++++++++------- s3/s3_methods_test.go | 37 ++++++++++++++++++++++------ 5 files changed, 89 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test-s3.yml b/.github/workflows/test-s3.yml index 2cb3ffec..64c68fad 100644 --- a/.github/workflows/test-s3.yml +++ b/.github/workflows/test-s3.yml @@ -20,14 +20,7 @@ jobs: - 1.21.x steps: - name: Install MinIO - run: | - docker run -d -p 9000:9000 --name minio minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket + run: docker run -d -p 9000:9000 --name minio minio/minio server /data - name: Fetch Repository uses: actions/checkout@v4 - name: Install Go diff --git a/s3/init_test.go b/s3/init_test.go index e25002de..7d8d0c21 100644 --- a/s3/init_test.go +++ b/s3/init_test.go @@ -1,22 +1,38 @@ package s3 import ( + "os" + "testing" "time" ) +const ( + bucket = "testbucket" +) + var testStore *Storage -func init() { +func TestMain(m *testing.M) { testStore = New( Config{ - Bucket: "testbucket", + Bucket: bucket, Endpoint: "http://127.0.0.1:9000/", Region: "us-east-1", Credentials: Credentials{ AccessKey: "minioadmin", SecretAccessKey: "minioadmin", }, - RequestTimeout: 10 * time.Second, + RequestTimeout: 3 * time.Second, }, ) + + // Create test bucket. + _ = testStore.CreateBucket(bucket) + + exitVal := m.Run() + + // Delete test bucket. + _ = testStore.DeleteBucket(bucket) + + os.Exit(exitVal) } diff --git a/s3/s3.go b/s3/s3.go index 921b02a7..e206ab39 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -159,7 +159,7 @@ func (s *Storage) Close() error { return nil } -// Return database client +// Conn returns database client. func (s *Storage) Conn() *s3.Client { return s.svc } @@ -186,11 +186,11 @@ func returnAWSConfig(cfg Config) (aws.Config, error) { }) if cfg.Credentials != (Credentials{}) { - credentials := credentials.NewStaticCredentialsProvider(cfg.Credentials.AccessKey, cfg.Credentials.SecretAccessKey, "") + creds := credentials.NewStaticCredentialsProvider(cfg.Credentials.AccessKey, cfg.Credentials.SecretAccessKey, "") return awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithRegion(cfg.Region), awsconfig.WithEndpointResolverWithOptions(endpoint), - awsconfig.WithCredentialsProvider(credentials), + awsconfig.WithCredentialsProvider(creds), awsconfig.WithRetryer(func() aws.Retryer { return retry.AddWithMaxAttempts(retry.NewStandard(), cfg.MaxAttempts) }), diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 1f540a0d..207c6e1e 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -2,6 +2,7 @@ package s3 import ( "bytes" + "encoding/base64" "fmt" "github.com/aws/aws-sdk-go-v2/aws" @@ -11,6 +12,29 @@ import ( // Additional methods for S3, but not required by gofiber Storage interface. +// CreateBucket creates a new bucket. +func (s *Storage) CreateBucket(bucket string) error { + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.svc.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + + return err +} + +func (s *Storage) DeleteBucket(bucket string) error { + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.svc.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + + return err +} + // SetWithChecksum sets key with value and checksum. // // Currently 4 algorithms are supported: @@ -25,31 +49,34 @@ func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.Che return nil } - ctx, cancel := s.requestContext() - defer cancel() - - poi := &s3.PutObjectInput{ + poi := s3.PutObjectInput{ Bucket: &s.bucket, Key: aws.String(key), Body: bytes.NewReader(val), } for alg, sum := range checksum { + // S3 requires base64 encoded checksum. + b64str := base64.StdEncoding.EncodeToString(sum) + switch alg { case types.ChecksumAlgorithmCrc32: - poi.ChecksumCRC32 = aws.String(string(sum)) + poi.ChecksumCRC32 = aws.String(b64str) case types.ChecksumAlgorithmCrc32c: - poi.ChecksumCRC32C = aws.String(string(sum)) + poi.ChecksumCRC32C = aws.String(b64str) case types.ChecksumAlgorithmSha1: - poi.ChecksumSHA1 = aws.String(string(sum)) + poi.ChecksumSHA1 = aws.String(b64str) case types.ChecksumAlgorithmSha256: - poi.ChecksumSHA256 = aws.String(string(sum)) + poi.ChecksumSHA256 = aws.String(b64str) default: return fmt.Errorf("invalid checksum algorithm: %s", alg) } } - _, err := s.uploader.Upload(ctx, poi) + ctx, cancel := s.requestContext() + defer cancel() + + _, err := s.uploader.Upload(ctx, &poi) return err } diff --git a/s3/s3_methods_test.go b/s3/s3_methods_test.go index 6cacffce..afd665fc 100644 --- a/s3/s3_methods_test.go +++ b/s3/s3_methods_test.go @@ -8,20 +8,43 @@ import ( "github.com/stretchr/testify/require" ) +func Test_S3_CreateDeleteBucket(t *testing.T) { + bkt := "test-new-bucket" + + err := testStore.CreateBucket(bkt) + require.NoError(t, err) + + err = testStore.DeleteBucket(bkt) + require.NoError(t, err) +} + func Test_S3_SetWithChecksum(t *testing.T) { var ( - key = "set-with-checksum" - val = []byte("doe") - sha256sum = sha256.New().Sum(val) - checksum = map[types.ChecksumAlgorithm][]byte{ - types.ChecksumAlgorithmSha256: sha256sum, - } + key = "set-with-checksum" + val = []byte("doe") ) + // Create SHA-256 hash and get checksum. + sha256Hash := sha256.New() + sha256Hash.Write(val) + sha256sum := sha256Hash.Sum(nil) + + checksum := map[types.ChecksumAlgorithm][]byte{ + types.ChecksumAlgorithmSha256: sha256sum, + } + err := testStore.SetWithChecksum(key, val, checksum) require.NoError(t, err) result, err := testStore.Get(key) require.NoError(t, err) - require.Equal(t, sha256sum, sha256.New().Sum(result)) + + // Compare value. + require.Equal(t, result, val) + + // Compare checksum. + hash2 := sha256.New() + hash2.Write(result) + sha256sum2 := hash2.Sum(nil) + require.Equal(t, sha256sum, sha256sum2) } From bf6aa0132c49ed6b375c5bbec976d53ba84e715e Mon Sep 17 00:00:00 2001 From: Zhang Huangbin Date: Thu, 7 Sep 2023 10:17:38 +0800 Subject: [PATCH 9/9] Typo --- s3/README.md | 15 +++++++++++++-- s3/s3_methods.go | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/s3/README.md b/s3/README.md index f2bb9259..21a7280e 100644 --- a/s3/README.md +++ b/s3/README.md @@ -31,6 +31,11 @@ func (s *Storage) Delete(key string) error func (s *Storage) Reset() error func (s *Storage) Close() error func (s *Storage) Conn() *s3.Client + +// Additional useful methods. +func (s *Storage) CreateBucker(bucket string) error +func (s *Storage) DeleteBucket(bucket string) error +func (s *Storage) SetWithChecksum(key string, val []byte, checksum map[types.ChecksumAlgorithm][]byte) error ``` ### Installation @@ -83,15 +88,19 @@ ask S3 server to verify data integrity on server side: > For more information, see [PutObjectInput](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#PutObjectInput). ```go +key := "my-key" val := []byte("my-value") -sha256sum := sha256.New().Sum256(val) + +hash := sha256.New() +hash.Write(val) +sha256sum := hash.Sum(nil) // import "github.com/aws/aws-sdk-go-v2/service/s3/types" checksum = map[types.ChecksumAlgorithm][]byte{ types.ChecksumAlgorithmSha256: sha256sum, } -err := store.SetWithChecksum("my-key", []byte("my-value"), checksum) +err := store.SetWithChecksum(key, val, checksum) ``` ### Config @@ -136,7 +145,9 @@ type Credentials struct { ``` ### Default Config + The default configuration lacks Bucket, Region, and Endpoint which are all required and must be overwritten: + ```go // ConfigDefault is the default config var ConfigDefault = Config{ diff --git a/s3/s3_methods.go b/s3/s3_methods.go index 207c6e1e..d99a453b 100644 --- a/s3/s3_methods.go +++ b/s3/s3_methods.go @@ -24,6 +24,7 @@ func (s *Storage) CreateBucket(bucket string) error { return err } +// DeleteBucket deletes a bucket. func (s *Storage) DeleteBucket(bucket string) error { ctx, cancel := s.requestContext() defer cancel()