From 21eb34fec7854547af4fce9b77639c668c74b131 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 13:25:32 +0300 Subject: [PATCH 01/32] minio --- minio/README.md | 126 ++++++++++++++++++++++++++++++ minio/config.go | 81 +++++++++++++++++++ minio/go.mod | 24 ++++++ minio/go.sum | 49 ++++++++++++ minio/minio.go | 186 ++++++++++++++++++++++++++++++++++++++++++++ minio/minio_test.go | 172 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 638 insertions(+) create mode 100644 minio/README.md create mode 100644 minio/config.go create mode 100644 minio/go.mod create mode 100644 minio/go.sum create mode 100644 minio/minio.go create mode 100644 minio/minio_test.go diff --git a/minio/README.md b/minio/README.md new file mode 100644 index 00000000..5a58e2a9 --- /dev/null +++ b/minio/README.md @@ -0,0 +1,126 @@ +--- +id: minio +title: minio +--- + +![Release](https://img.shields.io/github/v/tag/gofiber/storage?filter=minio*) +[![Discord](https://img.shields.io/discord/704680098577514527?style=flat&label=%F0%9F%92%AC%20discord&color=00ACD7)](https://gofiber.io/discord) +![Test](https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-minio.yml?label=Tests) +![Security](https://img.shields.io/github/actions/workflow/status/gofiber/storage/gosec.yml?label=Security) +![Linter](https://img.shields.io/github/actions/workflow/status/gofiber/storage/linter.yml?label=Linter) + +A Minio storage driver using [minio/minio](https://github.com/minio/minio). + +### Table of Contents +- [Signatures](#signatures) +- [Installation](#installation) +- [Examples](#examples) +- [Config](#config) +- [Default Config](#default-config) + +### Signatures +```go +func New(config ...Config) Storage +func (s *Storage) Get(key string) ([]byte, error) +func (s *Storage) Set(key string, val []byte, exp time.Duration) error +func (s *Storage) Delete(key string) error +func (s *Storage) Reset() error +func (s *Storage) Close() error +func (s *Storage) Conn() *s3.Client +``` +### Installation +Install the Minio implementation: +```bash +go get github.com/gofiber/storage/minio +``` +And then run minio on Docker +```bash +docker run -d --restart always -p 9000:9000 -p 9001:9001 --name storage-minio --volume=minio:/var/lib/minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server --console-address ":9001" /var/lib/minio +``` + +### Examples +Import the storage package. +```go +import "github.com/gofiber/storage/minio" +``` + +You can use the following possibilities to create a storage: +```go +// Initialize default config +store := minio.New() + +// Initialize custom config +store := minio.New(minio.Config{ + Bucket: "test-bucket", + Endpoint: "storage-minio:9000", + Credentials: Credentials{ + accessKeyID: "minio-user", + secretAccessKey: "minio-password", + }, +}) +``` + +### Config +```go +// Config defines the config for storage. +type Config struct { + // Minio bucket name + Bucket string + + // Minio endpoint + Endpoint string + + // Minio region + Region string + + // Minio token + Token string + + // Minio secure + Secure bool + + // Reset clears any existing keys in existing Bucket + // Optional. Default is false + Reset bool + + // Credentials overrides Minio access key and Minio secret key. Not recommended. + // Optional. Default is Credentials{} + Credentials Credentials + + // Get object options + GetObjectOptions minio.GetObjectOptions + + // Put object options + PutObjectOptions minio.PutObjectOptions + + // List object options + ListObjectsOptions minio.ListObjectsOptions + + // Remove object options + RemoveObjectOptions minio.RemoveObjectOptions + +} +``` + +### Default Config +The default configuration lacks Bucket, Region, and Endpoint which are all required and must be overwritten: +```go +// ConfigDefault is the default config +var ConfigDefault = Config{ + Bucket: "", + Endpoint: "", + Region: "", + Token: "", + Secure: false, + Reset: false, + Credentials: Credentials{}, + GetObjectOptions: minio.GetObjectOptions{}, + PutObjectOptions: minio.PutObjectOptions{}, + ListObjectsOptions: minio.ListObjectsOptions{}, + RemoveObjectOptions: minio.RemoveObjectOptions{}, +} +type Credentials struct { + accessKeyID string + secretAccessKey string +} +``` diff --git a/minio/config.go b/minio/config.go new file mode 100644 index 00000000..ac27381f --- /dev/null +++ b/minio/config.go @@ -0,0 +1,81 @@ +package minio + +import ( + "github.com/minio/minio-go/v7" +) + +// Config defines the config for storage. +type Config struct { + // Minio bucket name + Bucket string + + // Minio endpoint + Endpoint string + + // Minio region + Region string + + // Minio token + Token string + + // Minio secure + Secure bool + + // Reset clears any existing keys in existing Bucket + // Optional. Default is false + Reset bool + + // Credentials overrides Minio access key and Minio secret key. Not recommended. + // Optional. Default is Credentials{} + Credentials Credentials + + // Get object options + GetObjectOptions minio.GetObjectOptions + + // Put object options + PutObjectOptions minio.PutObjectOptions + + // List object options + ListObjectsOptions minio.ListObjectsOptions + + // Remove object options + RemoveObjectOptions minio.RemoveObjectOptions +} + +type Credentials struct { + accessKeyID string + secretAccessKey string +} + +// ConfigDefault is the default config +var ConfigDefault = Config{ + Bucket: "", + Endpoint: "", + Region: "", + Token: "", + Secure: false, + Reset: false, + Credentials: Credentials{}, + GetObjectOptions: minio.GetObjectOptions{}, + PutObjectOptions: minio.PutObjectOptions{}, + ListObjectsOptions: minio.ListObjectsOptions{}, + RemoveObjectOptions: minio.RemoveObjectOptions{}, +} + +// Helper function to set default values +func configDefault(config ...Config) Config { + // Return default config if nothing provided + if len(config) < 1 { + return ConfigDefault + } + + // Override default config + cfg := config[0] + + // Set default values + if cfg.Bucket == "" { + cfg.Bucket = ConfigDefault.Bucket + } + + return cfg +} diff --git a/minio/go.mod b/minio/go.mod new file mode 100644 index 00000000..1d6e7353 --- /dev/null +++ b/minio/go.mod @@ -0,0 +1,24 @@ +module github.com/gofiber/storage/minio + +go 1.20 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gofiber/utils v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.62 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/minio/go.sum b/minio/go.sum new file mode 100644 index 00000000..0fa348a2 --- /dev/null +++ b/minio/go.sum @@ -0,0 +1,49 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= +github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.62 h1:qNYsFZHEzl+NfH8UxW4jpmlKav1qUAgfY30YNRneVhc= +github.com/minio/minio-go/v7 v7.0.62/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/minio/minio.go b/minio/minio.go new file mode 100644 index 00000000..81c5a1fc --- /dev/null +++ b/minio/minio.go @@ -0,0 +1,186 @@ +package minio + +import ( + "bytes" + "context" + "errors" + "log" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Storage interface that is implemented by storage providers +type Storage struct { + minio *minio.Client + cfg Config + ctx context.Context +} + +// New creates a new storage +func New(config ...Config) *Storage { + + // Set default config + cfg := configDefault(config...) + + // Minio instance + minioClient, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.Credentials.accessKeyID, cfg.Credentials.secretAccessKey, cfg.Token), + Secure: cfg.Secure, + Region: cfg.Region, + }) + if err != nil { + panic(err) + } + + storage := &Storage{minio: minioClient, cfg: cfg, ctx: context.Background()} + + // check bucket + err = storage.CheckBucket() + if err != nil { + // create bucket + err = storage.CreateBucket() + if err != nil { + panic(err) + } + } + + return storage +} + +// Get value by key +func (s *Storage) Get(key string) ([]byte, error) { + + if len(key) <= 0 { + return nil, errors.New("the key value is required") + } + + // check bucket + err := s.CheckBucket() + if err != nil { + return nil, err + } + + // get object + object, err := s.minio.GetObject(s.ctx, s.cfg.Bucket, key, s.cfg.GetObjectOptions) + if err != nil { + return nil, err + } + + // convert to byte + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(object) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// Set key with value +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { + + if len(key) <= 0 { + return errors.New("the key value is required") + } + + // check bucket + err := s.CheckBucket() + if err != nil { + // create bucket + err = s.CreateBucket() + if err != nil { + return err + } + } + + // create Reader + file := bytes.NewReader(val) + + // set content type + s.cfg.PutObjectOptions.ContentType = http.DetectContentType(val) + + // put object + _, err = s.minio.PutObject(s.ctx, s.cfg.Bucket, key, file, file.Size(), s.cfg.PutObjectOptions) + + return err +} + +// Delete entry by key +func (s *Storage) Delete(key string) error { + + if len(key) <= 0 { + return errors.New("the key value is required") + } + + // check bucket + err := s.CheckBucket() + if err != nil { + return err + } + + // remove + err = s.minio.RemoveObject(s.ctx, s.cfg.Bucket, key, s.cfg.RemoveObjectOptions) + + return err +} + +// Reset all entries, including unexpired +func (s *Storage) Reset() error { + + // check bucket + err := s.CheckBucket() + if err != nil { + return err + } + + objectsCh := make(chan minio.ObjectInfo) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range s.minio.ListObjects(s.ctx, s.cfg.Bucket, s.cfg.ListObjectsOptions) { + if object.Err != nil { + log.Println(object.Err) + } + objectsCh <- object + } + }() + + opts := minio.RemoveObjectsOptions{ + GovernanceBypass: true, + } + + for err := range s.minio.RemoveObjects(s.ctx, s.cfg.Bucket, objectsCh, opts) { + log.Println("Error detected during deletion: ", err) + } + + return nil +} + +// Close the storage +func (s *Storage) Close() error { + return nil +} + +// CheckBucket Check to see if already exist bucket +func (s *Storage) CheckBucket() error { + exists, err := s.minio.BucketExists(s.ctx, s.cfg.Bucket) + if !exists || err != nil { + return errors.New("the specified bucket does not exist") + } + return nil +} + +// CreateBucket Bucket not found so Make a new bucket +func (s *Storage) CreateBucket() error { + return s.minio.MakeBucket(s.ctx, s.cfg.Bucket, minio.MakeBucketOptions{Region: s.cfg.Region}) +} + +// RemoveBucket Bucket remove if bucket is empty +func (s *Storage) RemoveBucket() error { + return s.minio.RemoveBucket(s.ctx, s.cfg.Bucket) +} diff --git a/minio/minio_test.go b/minio/minio_test.go new file mode 100644 index 00000000..3b30efec --- /dev/null +++ b/minio/minio_test.go @@ -0,0 +1,172 @@ +package minio + +import ( + "strconv" + "testing" + "time" + + "github.com/gofiber/utils" +) + +var testStore = New( + Config{ + Bucket: "test-bucket", + Endpoint: "localhost:9000", + Credentials: Credentials{ + accessKeyID: "minio-user", + secretAccessKey: "minio-password", + }, + }, +) + +func Test_Get(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + result, err := testStore.Get(key) + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, val, result) + + result, err = testStore.Get("doe") + utils.AssertEqual(t, true, len(result) == 0) + utils.AssertEqual(t, true, len(err.Error()) > 0) +} + +func Test_Get_Empty_Key(t *testing.T) { + var ( + key = "" + ) + + result, err := testStore.Get(key) + utils.AssertEqual(t, true, len(result) == 0) + utils.AssertEqual(t, "the key value is required", err.Error()) + +} + +func Test_Get_Not_Exists_Bucket(t *testing.T) { + var ( + key = "john" + ) + + // random bucket name + testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) + + result, err := testStore.Get(key) + utils.AssertEqual(t, true, len(result) == 0) + utils.AssertEqual(t, "the specified bucket does not exist", err.Error()) + + testStore.cfg.Bucket = "test-bucket" + +} + +func Test_Set(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) +} + +func Test_Set_Empty_Key(t *testing.T) { + var ( + key = "" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, "the key value is required", err.Error()) + +} + +func Test_Set_Not_Exists_Bucket(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + // random bucket name + testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + // remove object + err = testStore.Delete(key) + utils.AssertEqual(t, nil, err) + + // remove bucket + err = testStore.RemoveBucket() + utils.AssertEqual(t, nil, err) + + testStore.cfg.Bucket = "test-bucket" +} + +func Test_Delete(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Delete(key) + utils.AssertEqual(t, nil, err) + +} + +func Test_Delete_Empty_Key(t *testing.T) { + var ( + key = "" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, "the key value is required", err.Error()) + +} + +func Test_Delete_Not_Exists_Bucket(t *testing.T) { + var ( + key = "john" + ) + + // random bucket name + testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) + + err := testStore.Delete(key) + utils.AssertEqual(t, "the specified bucket does not exist", err.Error()) + + testStore.cfg.Bucket = "test-bucket" + +} + +func Test_Reset(t *testing.T) { + var ( + val = []byte("doe") + ) + + err := testStore.Set("john1", val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Set("john2", val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Reset() + utils.AssertEqual(t, nil, err) + + result, err := testStore.Get("john1") + utils.AssertEqual(t, true, len(result) == 0) + +} + +func Test_Close(t *testing.T) { + utils.AssertEqual(t, nil, testStore.Close()) +} From 23de4dd79e746ca321965ab21e66d53847fb0f27 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 13:31:56 +0300 Subject: [PATCH 02/32] Fix README --- minio/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/README.md b/minio/README.md index 5a58e2a9..0579d30d 100644 --- a/minio/README.md +++ b/minio/README.md @@ -52,7 +52,7 @@ store := minio.New() // Initialize custom config store := minio.New(minio.Config{ Bucket: "test-bucket", - Endpoint: "storage-minio:9000", + Endpoint: "localhost:9000", Credentials: Credentials{ accessKeyID: "minio-user", secretAccessKey: "minio-password", From 580a711cb16591b93ba09c349c73391319a7dd63 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 14:17:00 +0300 Subject: [PATCH 03/32] Fix README --- minio/README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/minio/README.md b/minio/README.md index 0579d30d..89f349ca 100644 --- a/minio/README.md +++ b/minio/README.md @@ -1,13 +1,4 @@ ---- -id: minio -title: minio ---- - -![Release](https://img.shields.io/github/v/tag/gofiber/storage?filter=minio*) -[![Discord](https://img.shields.io/discord/704680098577514527?style=flat&label=%F0%9F%92%AC%20discord&color=00ACD7)](https://gofiber.io/discord) -![Test](https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-minio.yml?label=Tests) -![Security](https://img.shields.io/github/actions/workflow/status/gofiber/storage/gosec.yml?label=Security) -![Linter](https://img.shields.io/github/actions/workflow/status/gofiber/storage/linter.yml?label=Linter) +# Minio A Minio storage driver using [minio/minio](https://github.com/minio/minio). @@ -26,7 +17,9 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error func (s *Storage) Delete(key string) error func (s *Storage) Reset() error func (s *Storage) Close() error -func (s *Storage) Conn() *s3.Client +func (s *Storage) CheckBucket() error +func (s *Storage) CreateBucket() error +func (s *Storage) RemoveBucket() error ``` ### Installation Install the Minio implementation: From d3d3a58f6e143f72d52e7f247dff1c6c3d832ec4 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 14:29:49 +0300 Subject: [PATCH 04/32] Added Conn method to return the Minio instance --- minio/README.md | 1 + minio/minio.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/minio/README.md b/minio/README.md index 89f349ca..c251d2c8 100644 --- a/minio/README.md +++ b/minio/README.md @@ -20,6 +20,7 @@ func (s *Storage) Close() error func (s *Storage) CheckBucket() error func (s *Storage) CreateBucket() error func (s *Storage) RemoveBucket() error +func (s *Storage) Conn() *minio.Client ``` ### Installation Install the Minio implementation: diff --git a/minio/minio.go b/minio/minio.go index 81c5a1fc..56ce0af1 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -184,3 +184,8 @@ func (s *Storage) CreateBucket() error { func (s *Storage) RemoveBucket() error { return s.minio.RemoveBucket(s.ctx, s.cfg.Bucket) } + +// Conn Return minio client +func (s *Storage) Conn() *minio.Client { + return s.minio +} From 7f002b3158682a6f9639b973b0dfbb19a41a4cdd Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 14:33:10 +0300 Subject: [PATCH 05/32] go mod tidy --- minio/go.mod | 7 +++++-- minio/go.sum | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/minio/go.mod b/minio/go.mod index 1d6e7353..b5e5ccaf 100644 --- a/minio/go.mod +++ b/minio/go.mod @@ -2,15 +2,18 @@ module github.com/gofiber/storage/minio go 1.20 +require ( + github.com/gofiber/utils v1.1.0 + github.com/minio/minio-go/v7 v7.0.62 +) + require ( github.com/dustin/go-humanize v1.0.1 // indirect - github.com/gofiber/utils v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.62 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/minio/go.sum b/minio/go.sum index 0fa348a2..cfdbadfe 100644 --- a/minio/go.sum +++ b/minio/go.sum @@ -1,4 +1,5 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -25,6 +26,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -32,6 +34,7 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -46,4 +49,5 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 244330f96e5003a6f8c8909f05fd908bd469a300 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 14:44:24 +0300 Subject: [PATCH 06/32] Added to Minio storage --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 402621fd..803cf6b9 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ type Storage interface { - [Etcd](./etcd/README.md) - [Memcache](./memcache/README.md) - [Memory](./memory/README.md) +- [Minio](./minio/README.md) - [MongoDB](./mongodb/README.md) - [MSSQL](./mssql/README.md) - [MySQL](./mysql/README.md) From 5b49d3629c9c91a11d6ff9762d959bfb2ecf14f2 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 14:58:52 +0300 Subject: [PATCH 07/32] added to minio gosec and depeddabot and drafter --- .github/dependabot.yml | 6 +++ .github/release-drafter-minio.yml | 50 +++++++++++++++++++++ .github/workflows/gosec.yml | 6 ++- .github/workflows/release-drafter-minio.yml | 19 ++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .github/release-drafter-minio.yml create mode 100644 .github/workflows/release-drafter-minio.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index be5e9233..975c9586 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -68,6 +68,12 @@ updates: - "๐Ÿค– Dependencies" schedule: interval: "daily" + - package-ecosystem: "gomod" + directory: "/minio/" # Location of package manifests + labels: + - "๐Ÿค– Dependencies" + schedule: + interval: "daily" - package-ecosystem: "gomod" directory: "/mongodb/" # Location of package manifests labels: diff --git a/.github/release-drafter-minio.yml b/.github/release-drafter-minio.yml new file mode 100644 index 00000000..39f0e75e --- /dev/null +++ b/.github/release-drafter-minio.yml @@ -0,0 +1,50 @@ +name-template: 'Minio - v$RESOLVED_VERSION' +tag-template: 'minio/v$RESOLVED_VERSION' +tag-prefix: minio/v +include-paths: + - minio +categories: + - title: 'โ— Breaking Changes' + labels: + - 'โ— BreakingChange' + - title: '๐Ÿš€ New' + labels: + - 'โœ๏ธ Feature' + - title: '๐Ÿงน Updates' + labels: + - '๐Ÿงน Updates' + - '๐Ÿค– Dependencies' + - title: '๐Ÿ› Fixes' + labels: + - 'โ˜ข๏ธ Bug' + - title: '๐Ÿ“š Documentation' + labels: + - '๐Ÿ“’ Documentation' +change-template: '- $TITLE (#$NUMBER)' +change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. +exclude-contributors: + - dependabot + - dependabot[bot] +version-resolver: + major: + labels: + - 'major' + - 'โ— BreakingChange' + minor: + labels: + - 'minor' + - 'โœ๏ธ Feature' + patch: + labels: + - 'patch' + - '๐Ÿ“’ Documentation' + - 'โ˜ข๏ธ Bug' + - '๐Ÿค– Dependencies' + - '๐Ÿงน Updates' + default: patch +template: | + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...minio/v$RESOLVED_VERSION + + Thank you $CONTRIBUTORS for making this update possible. diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml index deaa8ddd..dbef6e79 100644 --- a/.github/workflows/gosec.yml +++ b/.github/workflows/gosec.yml @@ -38,7 +38,7 @@ jobs: - name: Run Gosec (root) working-directory: . run: | - gosec -exclude-dir=arangodb -exclude-dir=badger -exclude-dir=couchbase -exclude-dir=dynamodb -exclude-dir=etcd -exclude-dir=memcache -exclude-dir=memory -exclude-dir=mongodb -exclude-dir=mysql -exclude-dir=postgres -exclude-dir=redis -exclude-dir=ristretto -exclude-dir=sqlite3 -exclude-dir=s3 -exclude-dir=bbolt -exclude-dir=azureblob -exclude-dir=mssql -exclude-dir=pebble -exclude-dir=rueidis -exclude-dir=coherence ./.... + gosec -exclude-dir=arangodb -exclude-dir=badger -exclude-dir=couchbase -exclude-dir=dynamodb -exclude-dir=etcd -exclude-dir=memcache -exclude-dir=memory -exclude-dir=minio -exclude-dir=mongodb -exclude-dir=mysql -exclude-dir=postgres -exclude-dir=redis -exclude-dir=ristretto -exclude-dir=sqlite3 -exclude-dir=s3 -exclude-dir=bbolt -exclude-dir=azureblob -exclude-dir=mssql -exclude-dir=pebble -exclude-dir=rueidis -exclude-dir=coherence ./.... # ----- - name: Run Gosec (arangodb) working-directory: ./arangodb @@ -72,6 +72,10 @@ jobs: working-directory: ./memory run: gosec ./... # ----- + - name: Run Gosec (minio) + working-directory: ./minio + run: gosec ./... + # ----- - name: Run Gosec (mongodb) working-directory: ./mongodb run: gosec ./... diff --git a/.github/workflows/release-drafter-minio.yml b/.github/workflows/release-drafter-minio.yml new file mode 100644 index 00000000..6cddd599 --- /dev/null +++ b/.github/workflows/release-drafter-minio.yml @@ -0,0 +1,19 @@ +name: Release Drafter Minio +on: + push: + # branches to consider in the event; optional, defaults to all + branches: + - master + - main + paths: + - 'minio/**' +jobs: + draft_release_minio: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: release-drafter/release-drafter@v5 + with: + config-name: release-drafter-minio.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4493457cfa6e923c032969a15abf20390e73bc9e Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 15:59:24 +0300 Subject: [PATCH 08/32] minio test --- .github/workflows/release-drafter-minio.yml | 2 +- .github/workflows/test-minio.yml | 33 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-minio.yml diff --git a/.github/workflows/release-drafter-minio.yml b/.github/workflows/release-drafter-minio.yml index 6cddd599..039c0cee 100644 --- a/.github/workflows/release-drafter-minio.yml +++ b/.github/workflows/release-drafter-minio.yml @@ -14,6 +14,6 @@ jobs: steps: - uses: release-drafter/release-drafter@v5 with: - config-name: release-drafter-minio.yml + config-name: release-drafter-test-minio.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-minio.yml b/.github/workflows/test-minio.yml new file mode 100644 index 00000000..cefb87e4 --- /dev/null +++ b/.github/workflows/test-minio.yml @@ -0,0 +1,33 @@ +on: + push: + branches: + - master + - main + paths: + - 'minio/**' + pull_request: + paths: + - 'minio/**' +name: "Tests Minio" +jobs: + Tests: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - 1.19.x + - 1.20.x + - 1.21.x + steps: + - name: Install MinIO + run: | + run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data + + - name: Fetch Repository + uses: actions/checkout@v3 + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '${{ matrix.go-version }}' + - name: Run Test + run: cd ./minio && go test ./... -v -race From 68fe393de4a14245c0a27715a1b007121fb02667 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 16:01:10 +0300 Subject: [PATCH 09/32] minio drafter --- .github/workflows/release-drafter-minio.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter-minio.yml b/.github/workflows/release-drafter-minio.yml index 039c0cee..6cddd599 100644 --- a/.github/workflows/release-drafter-minio.yml +++ b/.github/workflows/release-drafter-minio.yml @@ -14,6 +14,6 @@ jobs: steps: - uses: release-drafter/release-drafter@v5 with: - config-name: release-drafter-test-minio.yml + config-name: release-drafter-minio.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dab578a0966efbdf68a6700362585e7d45714474 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 16:15:35 +0300 Subject: [PATCH 10/32] gosec --- .github/workflows/gosec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml index dbef6e79..db587765 100644 --- a/.github/workflows/gosec.yml +++ b/.github/workflows/gosec.yml @@ -38,7 +38,7 @@ jobs: - name: Run Gosec (root) working-directory: . run: | - gosec -exclude-dir=arangodb -exclude-dir=badger -exclude-dir=couchbase -exclude-dir=dynamodb -exclude-dir=etcd -exclude-dir=memcache -exclude-dir=memory -exclude-dir=minio -exclude-dir=mongodb -exclude-dir=mysql -exclude-dir=postgres -exclude-dir=redis -exclude-dir=ristretto -exclude-dir=sqlite3 -exclude-dir=s3 -exclude-dir=bbolt -exclude-dir=azureblob -exclude-dir=mssql -exclude-dir=pebble -exclude-dir=rueidis -exclude-dir=coherence ./.... + gosec . # ----- - name: Run Gosec (arangodb) working-directory: ./arangodb From b3fe37dd3d842f8e56ec1cbbb4706f98b3a9ac97 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 16:19:47 +0300 Subject: [PATCH 11/32] change go version --- minio/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/go.mod b/minio/go.mod index b5e5ccaf..1fff8f78 100644 --- a/minio/go.mod +++ b/minio/go.mod @@ -1,6 +1,6 @@ module github.com/gofiber/storage/minio -go 1.20 +go 1.19 require ( github.com/gofiber/utils v1.1.0 From b597a36454710b73752f4c8bd1c3968dc23fc9d1 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 16:56:36 +0300 Subject: [PATCH 12/32] The stretchr/testify package was used for testing --- minio/go.mod | 5 +++- minio/go.sum | 9 +++---- minio/minio_test.go | 58 +++++++++++++++++++++++++-------------------- 3 files changed, 41 insertions(+), 31 deletions(-) diff --git a/minio/go.mod b/minio/go.mod index 1fff8f78..d4a33c02 100644 --- a/minio/go.mod +++ b/minio/go.mod @@ -3,11 +3,12 @@ module github.com/gofiber/storage/minio go 1.19 require ( - github.com/gofiber/utils v1.1.0 github.com/minio/minio-go/v7 v7.0.62 + github.com/stretchr/testify v1.8.4 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -17,6 +18,7 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect golang.org/x/crypto v0.12.0 // indirect @@ -24,4 +26,5 @@ require ( golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/minio/go.sum b/minio/go.sum index cfdbadfe..085022c9 100644 --- a/minio/go.sum +++ b/minio/go.sum @@ -3,8 +3,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= -github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -34,8 +32,9 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= @@ -46,8 +45,10 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/minio/minio_test.go b/minio/minio_test.go index 3b30efec..1f74a73e 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/gofiber/utils" + "github.com/stretchr/testify/require" ) var testStore = New( @@ -26,15 +26,14 @@ func Test_Get(t *testing.T) { ) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) result, err := testStore.Get(key) - utils.AssertEqual(t, nil, err) - utils.AssertEqual(t, val, result) + require.NoError(t, err) + require.Equal(t, val, result) result, err = testStore.Get("doe") - utils.AssertEqual(t, true, len(result) == 0) - utils.AssertEqual(t, true, len(err.Error()) > 0) + require.Error(t, err) } func Test_Get_Empty_Key(t *testing.T) { @@ -42,9 +41,9 @@ func Test_Get_Empty_Key(t *testing.T) { key = "" ) - result, err := testStore.Get(key) - utils.AssertEqual(t, true, len(result) == 0) - utils.AssertEqual(t, "the key value is required", err.Error()) + _, err := testStore.Get(key) + require.Error(t, err) + require.EqualError(t, err, "the key value is required") } @@ -56,9 +55,9 @@ func Test_Get_Not_Exists_Bucket(t *testing.T) { // random bucket name testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) - result, err := testStore.Get(key) - utils.AssertEqual(t, true, len(result) == 0) - utils.AssertEqual(t, "the specified bucket does not exist", err.Error()) + _, err := testStore.Get(key) + require.Error(t, err) + require.EqualError(t, err, "the specified bucket does not exist") testStore.cfg.Bucket = "test-bucket" @@ -71,7 +70,7 @@ func Test_Set(t *testing.T) { ) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) } func Test_Set_Empty_Key(t *testing.T) { @@ -81,7 +80,9 @@ func Test_Set_Empty_Key(t *testing.T) { ) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, "the key value is required", err.Error()) + + require.Error(t, err) + require.EqualError(t, err, "the key value is required") } @@ -95,15 +96,15 @@ func Test_Set_Not_Exists_Bucket(t *testing.T) { testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) // remove object err = testStore.Delete(key) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) // remove bucket err = testStore.RemoveBucket() - utils.AssertEqual(t, nil, err) + require.NoError(t, err) testStore.cfg.Bucket = "test-bucket" } @@ -115,10 +116,10 @@ func Test_Delete(t *testing.T) { ) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) err = testStore.Delete(key) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) } @@ -129,7 +130,9 @@ func Test_Delete_Empty_Key(t *testing.T) { ) err := testStore.Set(key, val, 0) - utils.AssertEqual(t, "the key value is required", err.Error()) + + require.Error(t, err) + require.EqualError(t, err, "the key value is required") } @@ -142,7 +145,9 @@ func Test_Delete_Not_Exists_Bucket(t *testing.T) { testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) err := testStore.Delete(key) - utils.AssertEqual(t, "the specified bucket does not exist", err.Error()) + + require.Error(t, err) + require.EqualError(t, err, "the specified bucket does not exist") testStore.cfg.Bucket = "test-bucket" @@ -154,19 +159,20 @@ func Test_Reset(t *testing.T) { ) err := testStore.Set("john1", val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) err = testStore.Set("john2", val, 0) - utils.AssertEqual(t, nil, err) + require.NoError(t, err) err = testStore.Reset() - utils.AssertEqual(t, nil, err) + require.NoError(t, err) result, err := testStore.Get("john1") - utils.AssertEqual(t, true, len(result) == 0) + require.Error(t, err) + require.True(t, len(result) == 0) } func Test_Close(t *testing.T) { - utils.AssertEqual(t, nil, testStore.Close()) + require.NoError(t, testStore.Close()) } From dda7fcf987d74f19c19cfb43d55d535b9cf74430 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:01:51 +0300 Subject: [PATCH 13/32] The Reset flag added to New flag --- minio/minio.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/minio/minio.go b/minio/minio.go index 56ce0af1..79241237 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -37,6 +37,13 @@ func New(config ...Config) *Storage { storage := &Storage{minio: minioClient, cfg: cfg, ctx: context.Background()} + // Reset all entries if set to true + if cfg.Reset { + if err = storage.Reset(); err != nil { + panic(err) + } + } + // check bucket err = storage.CheckBucket() if err != nil { From fd631052827eb6c3e23ca07c66fc916f37157057 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:08:24 +0300 Subject: [PATCH 14/32] min go version supported added to README --- minio/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/minio/README.md b/minio/README.md index c251d2c8..da99d21e 100644 --- a/minio/README.md +++ b/minio/README.md @@ -2,6 +2,8 @@ A Minio storage driver using [minio/minio](https://github.com/minio/minio). +**Note: Requires Go 1.19 and above** + ### Table of Contents - [Signatures](#signatures) - [Installation](#installation) From cd33dc926f1cecde555111181d794cbbbb277b8c Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:13:06 +0300 Subject: [PATCH 15/32] test fixed --- minio/minio_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/minio/minio_test.go b/minio/minio_test.go index 1f74a73e..1eab1e18 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -34,6 +34,7 @@ func Test_Get(t *testing.T) { result, err = testStore.Get("doe") require.Error(t, err) + require.Zero(t, result) } func Test_Get_Empty_Key(t *testing.T) { @@ -169,7 +170,7 @@ func Test_Reset(t *testing.T) { result, err := testStore.Get("john1") require.Error(t, err) - require.True(t, len(result) == 0) + require.Zero(t, result) } From d64fa185db2983c12a92b525e2c9213cdff96194 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:18:11 +0300 Subject: [PATCH 16/32] fixed test --- minio/minio_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/minio/minio_test.go b/minio/minio_test.go index 1eab1e18..0a6896d5 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -34,7 +34,7 @@ func Test_Get(t *testing.T) { result, err = testStore.Get("doe") require.Error(t, err) - require.Zero(t, result) + require.Zero(t, len(result)) } func Test_Get_Empty_Key(t *testing.T) { @@ -56,8 +56,9 @@ func Test_Get_Not_Exists_Bucket(t *testing.T) { // random bucket name testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) - _, err := testStore.Get(key) + result, err := testStore.Get(key) require.Error(t, err) + require.Zero(t, len(result)) require.EqualError(t, err, "the specified bucket does not exist") testStore.cfg.Bucket = "test-bucket" @@ -170,7 +171,7 @@ func Test_Reset(t *testing.T) { result, err := testStore.Get("john1") require.Error(t, err) - require.Zero(t, result) + require.Zero(t, len(result)) } From 4c8f4dcdfc7c89ec95bf90963dd8701557538b44 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:19:40 +0300 Subject: [PATCH 17/32] fix minio test --- .github/workflows/test-minio.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-minio.yml b/.github/workflows/test-minio.yml index cefb87e4..dd3b03ed 100644 --- a/.github/workflows/test-minio.yml +++ b/.github/workflows/test-minio.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Install MinIO run: | - run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data + docker run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data - name: Fetch Repository uses: actions/checkout@v3 From 0ccc5d3faa0a179b13decbffd80d8d74d5ff3fb8 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Wed, 23 Aug 2023 17:25:02 +0300 Subject: [PATCH 18/32] UPDATE README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 803cf6b9..8416e603 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ type Storage interface { - [Etcd](./etcd/README.md) - [Memcache](./memcache/README.md) - [Memory](./memory/README.md) -- [Minio](./minio/README.md) +- [Minio](./minio/README.md) - [MongoDB](./mongodb/README.md) - [MSSQL](./mssql/README.md) - [MySQL](./mysql/README.md) From ca32702aa03ab0a33e5e1a142d1e0430a2ebfef8 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 25 Aug 2023 17:43:44 +0300 Subject: [PATCH 19/32] Update README --- minio/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/README.md b/minio/README.md index da99d21e..e4b7cd93 100644 --- a/minio/README.md +++ b/minio/README.md @@ -1,6 +1,6 @@ # Minio -A Minio storage driver using [minio/minio](https://github.com/minio/minio). +A Minio storage driver using [minio/minio-go](https://github.com/minio/minio-go). **Note: Requires Go 1.19 and above** From 949c597508ee36525943f66b84fac16094f87b27 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 25 Aug 2023 18:03:10 +0300 Subject: [PATCH 20/32] definitions added --- minio/config.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/minio/config.go b/minio/config.go index ac27381f..3b0417dc 100644 --- a/minio/config.go +++ b/minio/config.go @@ -6,50 +6,57 @@ import ( // Config defines the config for storage. type Config struct { - // Minio bucket name + // Bucket + // Default test-bucket Bucket string - // Minio endpoint + // Endpoint is a host name or an IP address Endpoint string - // Minio region + // Region Set this value to override region cache + // Optional Region string - // Minio token + // Token Set this value to provide x-amz-security-token (AWS S3 specific) + // Optional, Default is false Token string - // Minio secure + // Secure If set to true, https is used instead of http. + // Default is false Secure bool // Reset clears any existing keys in existing Bucket // Optional. Default is false Reset bool - // Credentials overrides Minio access key and Minio secret key. Not recommended. - // Optional. Default is Credentials{} + // Credentials Minio access key and Minio secret key. + // Need to be defined Credentials Credentials - // Get object options + // GetObjectOptions Options for GET requests specifying additional options like encryption, If-Match GetObjectOptions minio.GetObjectOptions - // Put object options + // PutObjectOptions + // Allows user to set optional custom metadata, content headers, encryption keys and number of threads for multipart upload operation. PutObjectOptions minio.PutObjectOptions - // List object options + // ListObjectsOptions Options per to list objects ListObjectsOptions minio.ListObjectsOptions - // Remove object options + // RemoveObjectOptions Allows user to set options RemoveObjectOptions minio.RemoveObjectOptions } type Credentials struct { - accessKeyID string + // accessKeyID is like user-id that uniquely identifies your account. + accessKeyID string + // secretAccessKey is the password to your account. secretAccessKey string } // ConfigDefault is the default config var ConfigDefault = Config{ - Bucket: "", + Bucket: "test-bucket", Endpoint: "", Region: "", Token: "", From 3d598445e335031f34003b856c97d9c24ef6139d Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 25 Aug 2023 19:57:39 +0300 Subject: [PATCH 21/32] updated test for key not in get requests --- minio/minio_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/minio/minio_test.go b/minio/minio_test.go index 0a6896d5..c5a00cb9 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -48,6 +48,17 @@ func Test_Get_Empty_Key(t *testing.T) { } +func Test_Get_Not_Exists_Key(t *testing.T) { + var ( + key = "not-exists" + ) + + _, err := testStore.Get(key) + require.Error(t, err) + require.EqualError(t, err, "The specified key does not exist.") + +} + func Test_Get_Not_Exists_Bucket(t *testing.T) { var ( key = "john" From c49bca76d753c514b9c6251ef632cab52e9b4a23 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 26 Aug 2023 12:16:49 +0300 Subject: [PATCH 22/32] change to default bucket name - fiber-bucket --- minio/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/config.go b/minio/config.go index 3b0417dc..bc57aa6a 100644 --- a/minio/config.go +++ b/minio/config.go @@ -56,7 +56,7 @@ type Credentials struct { // ConfigDefault is the default config var ConfigDefault = Config{ - Bucket: "test-bucket", + Bucket: "fiber-bucket", Endpoint: "", Region: "", Token: "", From e0523db53dbf3929fbc5259b6d1733940b047856 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 26 Aug 2023 12:18:56 +0300 Subject: [PATCH 23/32] update comment for CheckBucket --- minio/minio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/minio.go b/minio/minio.go index 79241237..b449e027 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -173,7 +173,7 @@ func (s *Storage) Close() error { return nil } -// CheckBucket Check to see if already exist bucket +// CheckBucket Check to see if bucket already exists func (s *Storage) CheckBucket() error { exists, err := s.minio.BucketExists(s.ctx, s.cfg.Bucket) if !exists || err != nil { From cb6a705473dbe702a3fdb0a632a1250a9561560e Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 26 Aug 2023 18:28:19 +0300 Subject: [PATCH 24/32] Update README --- minio/README.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/minio/README.md b/minio/README.md index e4b7cd93..75cd11b8 100644 --- a/minio/README.md +++ b/minio/README.md @@ -47,7 +47,7 @@ store := minio.New() // Initialize custom config store := minio.New(minio.Config{ - Bucket: "test-bucket", + Bucket: "fiber-bucket", Endpoint: "localhost:9000", Credentials: Credentials{ accessKeyID: "minio-user", @@ -60,41 +60,45 @@ store := minio.New(minio.Config{ ```go // Config defines the config for storage. type Config struct { - // Minio bucket name + // Bucket + // Default test-bucket Bucket string - // Minio endpoint + // Endpoint is a host name or an IP address Endpoint string - // Minio region + // Region Set this value to override region cache + // Optional Region string - // Minio token + // Token Set this value to provide x-amz-security-token (AWS S3 specific) + // Optional, Default is false Token string - // Minio secure + // Secure If set to true, https is used instead of http. + // Default is false Secure bool // Reset clears any existing keys in existing Bucket // Optional. Default is false Reset bool - // Credentials overrides Minio access key and Minio secret key. Not recommended. - // Optional. Default is Credentials{} + // Credentials Minio access key and Minio secret key. + // Need to be defined Credentials Credentials - // Get object options + // GetObjectOptions Options for GET requests specifying additional options like encryption, If-Match GetObjectOptions minio.GetObjectOptions - // Put object options + // PutObjectOptions + // Allows user to set optional custom metadata, content headers, encryption keys and number of threads for multipart upload operation. PutObjectOptions minio.PutObjectOptions - // List object options + // ListObjectsOptions Options per to list objects ListObjectsOptions minio.ListObjectsOptions - // Remove object options + // RemoveObjectOptions Allows user to set options RemoveObjectOptions minio.RemoveObjectOptions - } ``` @@ -103,7 +107,7 @@ The default configuration lacks Bucket, Region, and Endpoint which are all requi ```go // ConfigDefault is the default config var ConfigDefault = Config{ - Bucket: "", + Bucket: "fiber-bucket", Endpoint: "", Region: "", Token: "", From cdad22386e8a7154cf79d7e39554e46c0bb4f04d Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 26 Aug 2023 18:30:00 +0300 Subject: [PATCH 25/32] update comment --- minio/README.md | 2 +- minio/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/minio/README.md b/minio/README.md index 75cd11b8..4836b660 100644 --- a/minio/README.md +++ b/minio/README.md @@ -61,7 +61,7 @@ store := minio.New(minio.Config{ // Config defines the config for storage. type Config struct { // Bucket - // Default test-bucket + // Default fiber-bucket Bucket string // Endpoint is a host name or an IP address diff --git a/minio/config.go b/minio/config.go index bc57aa6a..c2def76c 100644 --- a/minio/config.go +++ b/minio/config.go @@ -7,7 +7,7 @@ import ( // Config defines the config for storage. type Config struct { // Bucket - // Default test-bucket + // Default fiber-bucket Bucket string // Endpoint is a host name or an IP address From 67218b0d19618f93e85bdaf8e40fa64e3cfff544 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 26 Aug 2023 18:49:16 +0300 Subject: [PATCH 26/32] bucket check control removed and update test --- minio/minio.go | 32 ++------------------------------ minio/minio_test.go | 23 ++++++++--------------- 2 files changed, 10 insertions(+), 45 deletions(-) diff --git a/minio/minio.go b/minio/minio.go index b449e027..ab7876f7 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -64,12 +64,6 @@ func (s *Storage) Get(key string) ([]byte, error) { return nil, errors.New("the key value is required") } - // check bucket - err := s.CheckBucket() - if err != nil { - return nil, err - } - // get object object, err := s.minio.GetObject(s.ctx, s.cfg.Bucket, key, s.cfg.GetObjectOptions) if err != nil { @@ -93,16 +87,6 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error { return errors.New("the key value is required") } - // check bucket - err := s.CheckBucket() - if err != nil { - // create bucket - err = s.CreateBucket() - if err != nil { - return err - } - } - // create Reader file := bytes.NewReader(val) @@ -110,7 +94,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error { s.cfg.PutObjectOptions.ContentType = http.DetectContentType(val) // put object - _, err = s.minio.PutObject(s.ctx, s.cfg.Bucket, key, file, file.Size(), s.cfg.PutObjectOptions) + _, err := s.minio.PutObject(s.ctx, s.cfg.Bucket, key, file, file.Size(), s.cfg.PutObjectOptions) return err } @@ -122,14 +106,8 @@ func (s *Storage) Delete(key string) error { return errors.New("the key value is required") } - // check bucket - err := s.CheckBucket() - if err != nil { - return err - } - // remove - err = s.minio.RemoveObject(s.ctx, s.cfg.Bucket, key, s.cfg.RemoveObjectOptions) + err := s.minio.RemoveObject(s.ctx, s.cfg.Bucket, key, s.cfg.RemoveObjectOptions) return err } @@ -137,12 +115,6 @@ func (s *Storage) Delete(key string) error { // Reset all entries, including unexpired func (s *Storage) Reset() error { - // check bucket - err := s.CheckBucket() - if err != nil { - return err - } - objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh diff --git a/minio/minio_test.go b/minio/minio_test.go index c5a00cb9..5f9876f5 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -10,7 +10,7 @@ import ( var testStore = New( Config{ - Bucket: "test-bucket", + Bucket: "fiber-bucket", Endpoint: "localhost:9000", Credentials: Credentials{ accessKeyID: "minio-user", @@ -70,9 +70,9 @@ func Test_Get_Not_Exists_Bucket(t *testing.T) { result, err := testStore.Get(key) require.Error(t, err) require.Zero(t, len(result)) - require.EqualError(t, err, "the specified bucket does not exist") + require.EqualError(t, err, "The specified bucket does not exist") - testStore.cfg.Bucket = "test-bucket" + testStore.cfg.Bucket = "fiber-bucket" } @@ -109,17 +109,10 @@ func Test_Set_Not_Exists_Bucket(t *testing.T) { testStore.cfg.Bucket = strconv.FormatInt(time.Now().UnixMicro(), 10) err := testStore.Set(key, val, 0) - require.NoError(t, err) - - // remove object - err = testStore.Delete(key) - require.NoError(t, err) - - // remove bucket - err = testStore.RemoveBucket() - require.NoError(t, err) + require.Error(t, err) + require.EqualError(t, err, "The specified bucket does not exist") - testStore.cfg.Bucket = "test-bucket" + testStore.cfg.Bucket = "fiber-bucket" } func Test_Delete(t *testing.T) { @@ -160,9 +153,9 @@ func Test_Delete_Not_Exists_Bucket(t *testing.T) { err := testStore.Delete(key) require.Error(t, err) - require.EqualError(t, err, "the specified bucket does not exist") + require.EqualError(t, err, "The specified bucket does not exist") - testStore.cfg.Bucket = "test-bucket" + testStore.cfg.Bucket = "fiber-bucket" } From 1887a32ca234fe75cb7964a4a0aa0076a3f1f4e0 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Tue, 29 Aug 2023 18:44:15 +0300 Subject: [PATCH 27/32] update credentials key name --- minio/README.md | 4 ++-- minio/config.go | 8 ++++---- minio/minio.go | 2 +- minio/minio_test.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/minio/README.md b/minio/README.md index 4836b660..7fd0adf8 100644 --- a/minio/README.md +++ b/minio/README.md @@ -120,7 +120,7 @@ var ConfigDefault = Config{ RemoveObjectOptions: minio.RemoveObjectOptions{}, } type Credentials struct { - accessKeyID string - secretAccessKey string + AccessKeyID string + SecretAccessKey string } ``` diff --git a/minio/config.go b/minio/config.go index c2def76c..d1976d7c 100644 --- a/minio/config.go +++ b/minio/config.go @@ -48,10 +48,10 @@ type Config struct { } type Credentials struct { - // accessKeyID is like user-id that uniquely identifies your account. - accessKeyID string - // secretAccessKey is the password to your account. - secretAccessKey string + // AccessKeyID is like user-id that uniquely identifies your account. + AccessKeyID string + // SecretAccessKey is the password to your account. + SecretAccessKey string } // ConfigDefault is the default config diff --git a/minio/minio.go b/minio/minio.go index ab7876f7..ab217525 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -27,7 +27,7 @@ func New(config ...Config) *Storage { // Minio instance minioClient, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.Credentials.accessKeyID, cfg.Credentials.secretAccessKey, cfg.Token), + Creds: credentials.NewStaticV4(cfg.Credentials.AccessKeyID, cfg.Credentials.SecretAccessKey, cfg.Token), Secure: cfg.Secure, Region: cfg.Region, }) diff --git a/minio/minio_test.go b/minio/minio_test.go index 5f9876f5..07fb0724 100644 --- a/minio/minio_test.go +++ b/minio/minio_test.go @@ -13,8 +13,8 @@ var testStore = New( Bucket: "fiber-bucket", Endpoint: "localhost:9000", Credentials: Credentials{ - accessKeyID: "minio-user", - secretAccessKey: "minio-password", + AccessKeyID: "minio-user", + SecretAccessKey: "minio-password", }, }, ) From edf7bf23e394ea81bf45345c90974919d859b638 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Tue, 29 Aug 2023 18:53:03 +0300 Subject: [PATCH 28/32] added yo bytebufferpool --- minio/go.mod | 5 +++-- minio/go.sum | 10 ++++++---- minio/minio.go | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/minio/go.mod b/minio/go.mod index d4a33c02..2e86fc8b 100644 --- a/minio/go.mod +++ b/minio/go.mod @@ -3,14 +3,15 @@ module github.com/gofiber/storage/minio go 1.19 require ( - github.com/minio/minio-go/v7 v7.0.62 + github.com/minio/minio-go/v7 v7.0.63 github.com/stretchr/testify v1.8.4 + github.com/valyala/bytebufferpool v1.0.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect diff --git a/minio/go.sum b/minio/go.sum index 085022c9..63183408 100644 --- a/minio/go.sum +++ b/minio/go.sum @@ -4,8 +4,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= @@ -15,8 +15,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.62 h1:qNYsFZHEzl+NfH8UxW4jpmlKav1qUAgfY30YNRneVhc= -github.com/minio/minio-go/v7 v7.0.62/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -35,6 +35,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= diff --git a/minio/minio.go b/minio/minio.go index ab217525..b9e7b053 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -10,6 +10,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/valyala/bytebufferpool" ) // Storage interface that is implemented by storage providers @@ -71,7 +72,7 @@ func (s *Storage) Get(key string) ([]byte, error) { } // convert to byte - buf := new(bytes.Buffer) + buf := new(bytebufferpool.ByteBuffer) _, err = buf.ReadFrom(object) if err != nil { return nil, err From 79fdc2ef783fb671bd6ffa0246c080c4bb3f5fab Mon Sep 17 00:00:00 2001 From: mstgnz Date: Thu, 31 Aug 2023 14:06:12 +0300 Subject: [PATCH 29/32] fix get --- minio/minio.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/minio/minio.go b/minio/minio.go index b9e7b053..22ff8143 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -72,13 +72,12 @@ func (s *Storage) Get(key string) ([]byte, error) { } // convert to byte - buf := new(bytebufferpool.ByteBuffer) - _, err = buf.ReadFrom(object) + bb := bytebufferpool.Get() + _, err = bb.ReadFrom(object) if err != nil { return nil, err } - - return buf.Bytes(), nil + return bb.Bytes(), nil } // Set key with value From 6eba76bd26f55ec361cbed746881d5be51e40046 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sun, 3 Sep 2023 11:09:44 +0300 Subject: [PATCH 30/32] fix bytebufferpool --- minio/minio.go | 1 + 1 file changed, 1 insertion(+) diff --git a/minio/minio.go b/minio/minio.go index 22ff8143..ebf65847 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -73,6 +73,7 @@ func (s *Storage) Get(key string) ([]byte, error) { // convert to byte bb := bytebufferpool.Get() + defer bytebufferpool.Put(bb) _, err = bb.ReadFrom(object) if err != nil { return nil, err From 8400db5161dbf673c1378db41e4857cad51fd7fc Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:34:35 -0400 Subject: [PATCH 31/32] Update test-minio.yml --- .github/workflows/test-minio.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-minio.yml b/.github/workflows/test-minio.yml index dd3b03ed..eb6df833 100644 --- a/.github/workflows/test-minio.yml +++ b/.github/workflows/test-minio.yml @@ -24,7 +24,7 @@ jobs: docker run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data - name: Fetch Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 with: From b6dbad7d88a40fe6ed7ade1dcf51cf1b0b2ed782 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:42:12 -0400 Subject: [PATCH 32/32] Update test-minio.yml --- .github/workflows/test-minio.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-minio.yml b/.github/workflows/test-minio.yml index eb6df833..dd3b03ed 100644 --- a/.github/workflows/test-minio.yml +++ b/.github/workflows/test-minio.yml @@ -24,7 +24,7 @@ jobs: docker run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data - name: Fetch Repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Install Go uses: actions/setup-go@v4 with: