From d3396f3f0be6c1712067918d5c453bdf5b987e1f Mon Sep 17 00:00:00 2001 From: Khanh Tran <32532742+khanhtc1202@users.noreply.github.com> Date: Mon, 29 Mar 2021 22:38:48 +0900 Subject: [PATCH] Enable supporting MongoDB (#1794) **What this PR does / why we need it**: This reverts commit 6c2b0bd0d568b06fd2bda3ec954e3a25165fbb59. **Which issue(s) this PR fixes**: Rel #1792 **Does this PR introduce a user-facing change?**: ```release-note NONE ``` This PR was merged by Kapetanios. --- cmd/pipecd/BUILD.bazel | 1 + cmd/pipecd/server.go | 16 +- go.mod | 3 + go.sum | 82 +++++++- pkg/config/control_plane.go | 12 ++ pkg/datastore/mongodb/BUILD.bazel | 21 ++ pkg/datastore/mongodb/iterator.go | 47 +++++ pkg/datastore/mongodb/model_wrapper.go | 193 +++++++++++++++++ pkg/datastore/mongodb/mongodb.go | 276 +++++++++++++++++++++++++ pkg/datastore/mongodb/operator.go | 40 ++++ repositories.bzl | 153 +++++++++++++- 11 files changed, 837 insertions(+), 7 deletions(-) create mode 100644 pkg/datastore/mongodb/BUILD.bazel create mode 100644 pkg/datastore/mongodb/iterator.go create mode 100644 pkg/datastore/mongodb/model_wrapper.go create mode 100644 pkg/datastore/mongodb/mongodb.go create mode 100644 pkg/datastore/mongodb/operator.go diff --git a/cmd/pipecd/BUILD.bazel b/cmd/pipecd/BUILD.bazel index 10df81764d..56708cb015 100644 --- a/cmd/pipecd/BUILD.bazel +++ b/cmd/pipecd/BUILD.bazel @@ -32,6 +32,7 @@ go_library( "//pkg/crypto:go_default_library", "//pkg/datastore:go_default_library", "//pkg/datastore/firestore:go_default_library", + "//pkg/datastore/mongodb:go_default_library", "//pkg/datastore/mysql:go_default_library", "//pkg/filestore:go_default_library", "//pkg/filestore/gcs:go_default_library", diff --git a/cmd/pipecd/server.go b/cmd/pipecd/server.go index cd8da086c8..2e609070fe 100644 --- a/cmd/pipecd/server.go +++ b/cmd/pipecd/server.go @@ -43,6 +43,7 @@ import ( "github.com/pipe-cd/pipe/pkg/crypto" "github.com/pipe-cd/pipe/pkg/datastore" "github.com/pipe-cd/pipe/pkg/datastore/firestore" + "github.com/pipe-cd/pipe/pkg/datastore/mongodb" "github.com/pipe-cd/pipe/pkg/datastore/mysql" "github.com/pipe-cd/pipe/pkg/filestore" "github.com/pipe-cd/pipe/pkg/filestore/gcs" @@ -412,8 +413,18 @@ func createDatastore(ctx context.Context, cfg *config.ControlPlaneSpec, logger * return nil, errors.New("dynamodb is unimplemented yet") case model.DataStoreMongoDB: - return nil, errors.New("mongodb is unimplemented yet") - + mdConfig := cfg.Datastore.MongoDBConfig + options := []mongodb.Option{ + mongodb.WithLogger(logger), + } + if mdConfig.UsernameFile != "" || mdConfig.PasswordFile != "" { + options = append(options, mongodb.WithAuthenticationFile(mdConfig.UsernameFile, mdConfig.PasswordFile)) + } + return mongodb.NewMongoDB( + ctx, + mdConfig.URL, + mdConfig.Database, + options...) case model.DataStoreMySQL: mqConfig := cfg.Datastore.MySQLConfig options := []mysql.Option{ @@ -423,7 +434,6 @@ func createDatastore(ctx context.Context, cfg *config.ControlPlaneSpec, logger * options = append(options, mysql.WithAuthenticationFile(mqConfig.UsernameFile, mqConfig.PasswordFile)) } return mysql.NewMySQL(mqConfig.URL, mqConfig.Database, options...) - default: return nil, fmt.Errorf("unknown datastore type %q", cfg.Datastore.Type) } diff --git a/go.mod b/go.mod index eeddb00595..310b43240c 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect github.com/DataDog/datadog-api-client-go v1.0.0-beta.16 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 + github.com/aws/aws-sdk-go v1.36.21 // indirect github.com/aws/aws-sdk-go-v2 v1.2.0 github.com/aws/aws-sdk-go-v2/config v1.1.1 github.com/aws/aws-sdk-go-v2/credentials v1.1.1 @@ -28,6 +29,7 @@ require ( github.com/google/uuid v1.2.0 github.com/googleapis/gnostic v0.2.2 // indirect github.com/hashicorp/golang-lru v0.5.3 + github.com/klauspost/compress v1.10.11 // indirect github.com/minio/minio-go/v7 v7.0.5 github.com/prometheus/client_golang v1.6.0 github.com/prometheus/client_model v0.2.0 @@ -36,6 +38,7 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 + go.mongodb.org/mongo-driver v1.4.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.2.0 // indirect go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0 diff --git a/go.sum b/go.sum index 5ecfce9793..a68ec0be3e 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,9 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IU github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aslakhellesoy/gox v1.0.100 h1:IP+x+v9Wya7OHP1OmaetTFZkL4OYY2/9t+7Ndc61mMo= github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= +github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/aws/aws-sdk-go v1.36.21 h1:7CcrvoPUYex6CVeEK7/UZlYx5/AcVCWrl1FsX7fVvFw= +github.com/aws/aws-sdk-go v1.36.21/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v1.2.0 h1:BS+UYpbsElC82gB+2E2jiCBg36i8HlubTB/dO/moQ9c= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= @@ -231,6 +234,42 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA= github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= @@ -271,6 +310,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -355,10 +396,13 @@ github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -373,15 +417,22 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk= +github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -399,6 +450,10 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a h1:TpvdAwDAt1K4ANVOfcihouRdvP+MgAfDWwBuct4l6ZY= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -425,6 +480,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= @@ -446,8 +503,9 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= @@ -484,6 +542,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= @@ -493,6 +553,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= @@ -510,6 +571,7 @@ github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= @@ -529,12 +591,18 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= @@ -546,6 +614,8 @@ github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.4.0 h1:C8rFn1VF4GVEM/rG+dSoMmlm2pyQ9cs2/oRtUATejRU= +go.mongodb.org/mongo-driver v1.4.0/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -564,7 +634,9 @@ go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0/go.mod h1:vwi/ZaCAaUcBkycH golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -654,6 +726,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -669,10 +742,13 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -724,9 +800,13 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/pkg/config/control_plane.go b/pkg/config/control_plane.go index f42e2e565b..c023ee4ab6 100644 --- a/pkg/config/control_plane.go +++ b/pkg/config/control_plane.go @@ -259,6 +259,18 @@ type DataStoreDynamoDBConfig struct { } type DataStoreMongoDBConfig struct { + // The url of MongoDB. All of credentials can be specified via this field. + URL string `json:"url"` + // The name of the database. + // Also set Database as 'AuthSource' (default as 'admin' or '$external') when UsernameFle || PasswordFile specify + // Ref: https://github.com/mongodb/mongo-go-driver/blob/9e2aca8afd8821e6b068cc2f25192bc640d90a0d/mongo/client.go#L390 + Database string `json:"database"` + // The path to the username file. + // For those who don't want to include the username in the URL. + UsernameFile string `json:"usernameFile"` + // The path to the password file. + // For those who don't want to include the password in the URL. + PasswordFile string `json:"passwordFile"` } type DataStoreMySQLConfig struct { diff --git a/pkg/datastore/mongodb/BUILD.bazel b/pkg/datastore/mongodb/BUILD.bazel new file mode 100644 index 0000000000..cb912a7a4a --- /dev/null +++ b/pkg/datastore/mongodb/BUILD.bazel @@ -0,0 +1,21 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "iterator.go", + "model_wrapper.go", + "mongodb.go", + "operator.go", + ], + importpath = "github.com/pipe-cd/pipe/pkg/datastore/mongodb", + visibility = ["//visibility:public"], + deps = [ + "//pkg/datastore:go_default_library", + "//pkg/model:go_default_library", + "@org_mongodb_go_mongo_driver//bson:go_default_library", + "@org_mongodb_go_mongo_driver//mongo:go_default_library", + "@org_mongodb_go_mongo_driver//mongo/options:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) diff --git a/pkg/datastore/mongodb/iterator.go b/pkg/datastore/mongodb/iterator.go new file mode 100644 index 0000000000..afbd95b71c --- /dev/null +++ b/pkg/datastore/mongodb/iterator.go @@ -0,0 +1,47 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/mongo" + + "github.com/pipe-cd/pipe/pkg/datastore" +) + +type Iterator struct { + ctx context.Context + cur *mongo.Cursor +} + +func (it *Iterator) Next(dst interface{}) error { + if !it.cur.Next(it.ctx) { + return datastore.ErrIteratorDone + } + wrapper, err := wrapModel(dst) + if err != nil { + return err + } + if err := it.cur.Decode(wrapper); err != nil { + return err + } + return extractModel(wrapper, dst) +} + +func (it *Iterator) Cursor() (string, error) { + // Note: Perhaps, not required. + return "", datastore.ErrUnimplemented +} diff --git a/pkg/datastore/mongodb/model_wrapper.go b/pkg/datastore/mongodb/model_wrapper.go new file mode 100644 index 0000000000..f73877d57c --- /dev/null +++ b/pkg/datastore/mongodb/model_wrapper.go @@ -0,0 +1,193 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongodb + +import ( + "fmt" + + "github.com/pipe-cd/pipe/pkg/model" +) + +// wrapModel returns a wrapper corresponding to the given entity. +// A wrapper wraps a model representing BSON a document so that the model comes with "_id". +func wrapModel(entity interface{}) (interface{}, error) { + switch e := entity.(type) { + case *model.Application: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &application{ + ID: e.GetId(), + Application: *e, + }, nil + case *model.Command: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &command{ + ID: e.GetId(), + Command: *e, + }, nil + case *model.Deployment: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &deployment{ + ID: e.GetId(), + Deployment: *e, + }, nil + case *model.Environment: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &environment{ + ID: e.GetId(), + Environment: *e, + }, nil + case *model.Piped: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &piped{ + ID: e.GetId(), + Piped: *e, + }, nil + case *model.Project: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &project{ + ID: e.GetId(), + Project: *e, + }, nil + case *model.APIKey: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &apiKey{ + ID: e.GetId(), + APIKey: *e, + }, nil + case *model.Event: + if e == nil { + return nil, fmt.Errorf("nil entity given") + } + return &event{ + ID: e.GetId(), + Event: *e, + }, nil + default: + return nil, fmt.Errorf("%T is not supported", e) + } +} + +// extractModel stores the unwrapped model in the value pointed to by e. +func extractModel(wrapper interface{}, e interface{}) error { + msg := "entity type doesn't correspond to the wrapper type (%T)" + + switch w := wrapper.(type) { + case *application: + e, ok := e.(*model.Application) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Application + case *command: + e, ok := e.(*model.Command) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Command + case *deployment: + e, ok := e.(*model.Deployment) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Deployment + case *environment: + e, ok := e.(*model.Environment) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Environment + case *piped: + e, ok := e.(*model.Piped) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Piped + case *project: + e, ok := e.(*model.Project) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Project + case *apiKey: + e, ok := e.(*model.APIKey) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.APIKey + case *event: + e, ok := e.(*model.Event) + if !ok { + return fmt.Errorf(msg, w) + } + *e = w.Event + default: + return fmt.Errorf("%T is not supported", w) + } + return nil +} + +type application struct { + model.Application `bson:",inline"` + ID string `bson:"_id"` +} + +type command struct { + model.Command `bson:",inline"` + ID string `bson:"_id"` +} + +type deployment struct { + model.Deployment `bson:",inline"` + ID string `bson:"_id"` +} + +type environment struct { + model.Environment `bson:",inline"` + ID string `bson:"_id"` +} + +type piped struct { + model.Piped `bson:",inline"` + ID string `bson:"_id"` +} + +type project struct { + model.Project `bson:",inline"` + ID string `bson:"_id"` +} + +type apiKey struct { + model.APIKey `bson:",inline"` + ID string `bson:"_id"` +} + +type event struct { + model.Event `bson:",inline"` + ID string `bson:"_id"` +} diff --git a/pkg/datastore/mongodb/mongodb.go b/pkg/datastore/mongodb/mongodb.go new file mode 100644 index 0000000000..99cb06c4ac --- /dev/null +++ b/pkg/datastore/mongodb/mongodb.go @@ -0,0 +1,276 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongodb + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "strings" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/datastore" +) + +const ( + // Scram type ref https://github.com/mongodb/mongo-go-driver/blob/9e2aca8afd8821e6b068cc2f25192bc640d90a0d/mongo/client_examples_test.go#L119 + // Default + scram string = "SCRAM" +) + +type MongoDB struct { + ctx context.Context + client *mongo.Client + database string + logger *zap.Logger + authMechanism string + usernameFile string + passwordFile string +} + +func NewMongoDB(ctx context.Context, url, database string, opts ...Option) (*MongoDB, error) { + m := &MongoDB{ + ctx: ctx, + database: database, + logger: zap.NewNop(), + } + for _, opt := range opts { + opt(m) + } + m.logger = m.logger.Named("mongodb") + + clientOpts := options.Client().ApplyURI(url) + + if m.authMechanism != "" { + credential, err := m.determineCredential() + if err != nil { + return nil, err + } + clientOpts.SetAuth(*credential) + } + + client, err := mongo.Connect(ctx, clientOpts) + if err != nil { + return nil, err + } + m.client = client + return m, nil +} + +type Option func(*MongoDB) + +func WithAuthenticationFile(usernameFile, passwordFile string) Option { + return func(m *MongoDB) { + m.usernameFile = usernameFile + m.passwordFile = passwordFile + m.authMechanism = scram + } +} + +func WithLogger(logger *zap.Logger) Option { + return func(s *MongoDB) { + s.logger = logger + } +} + +func (m *MongoDB) Find(ctx context.Context, kind string, opts datastore.ListOptions) (datastore.Iterator, error) { + col := m.client.Database(m.database).Collection(kind) + ops := make([]bson.M, len(opts.Filters)) + for i, f := range opts.Filters { + op, err := convertToMongoDBOperator(f.Operator) + if err != nil { + return nil, err + } + ops[i] = bson.M{ + // Note: The field name of protobuf is saved in lower case by default in mongodb. + // e.g. Name => name, ProjectId => projectid, CreatedAt => createdat + strings.ToLower(f.Field): bson.M{ + op: f.Value, + }, + } + } + query := bson.M{} + if len(ops) > 0 { + query["$and"] = ops + } + + findOpts := options.Find() + if opts.PageSize > 0 { + findOpts.SetLimit(int64(opts.PageSize)) + if opts.Page > 0 { + findOpts.SetSkip(int64(opts.PageSize * opts.Page)) + } + } + + cur, err := col.Find(ctx, query, findOpts) + if err != nil { + m.logger.Error("failed to get cursor", + zap.String("kind", kind), + zap.Error(err), + ) + return nil, err + } + return &Iterator{ + ctx: ctx, + cur: cur, + }, nil +} + +func (m *MongoDB) Get(ctx context.Context, kind, id string, v interface{}) error { + wrapper, err := wrapModel(v) + if err != nil { + return err + } + + col := m.client.Database(m.database).Collection(kind) + err = col.FindOne(ctx, makePrimaryKeyFilter(id)).Decode(wrapper) + if errors.Is(err, mongo.ErrNoDocuments) { + return datastore.ErrNotFound + } + if err != nil { + m.logger.Error("failed to retrieve entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + + return extractModel(wrapper, v) +} + +func (m *MongoDB) Create(ctx context.Context, kind, id string, entity interface{}) error { + wrapper, err := wrapModel(entity) + if err != nil { + return err + } + + // TODO: Support updating process with using transaction on mongoDB cluster + // err := m.client.UseSession(ctx, func(sessCtx mongo.SessionContext) error { } + // See the example at: https://godoc.org/go.mongodb.org/mongo-driver/mongo#Client.UseSessionWithOptions + // NOTE: + // - Multi-document transactions are only available in version 4.0 or later. + // - Also available for replica set deployments only. + // - Available even on a standalone server but need to configure it as a replica set (with just one node) + col := m.client.Database(m.database).Collection(kind) + err = col.FindOne(ctx, makePrimaryKeyFilter(id), options.FindOne()).Err() + if err == nil { + return datastore.ErrAlreadyExists + } + if !errors.Is(err, mongo.ErrNoDocuments) { + m.logger.Error("failed to retrieve entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + + if _, err := col.InsertOne(ctx, wrapper); err != nil { + m.logger.Error("failed to insert entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + return nil +} + +func (m *MongoDB) Put(ctx context.Context, kind, id string, entity interface{}) error { + wrapper, err := wrapModel(entity) + if err != nil { + return err + } + col := m.client.Database(m.database).Collection(kind) + if _, err := col.UpdateOne(ctx, makePrimaryKeyFilter(id), wrapper); err != nil { + m.logger.Error("failed to put entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + return nil +} + +func (m *MongoDB) Update(ctx context.Context, kind, id string, factory datastore.Factory, updater datastore.Updater) error { + col := m.client.Database(m.database).Collection(kind) + entity := factory() + err := col.FindOne(ctx, makePrimaryKeyFilter(id)).Decode(entity) + if errors.Is(err, mongo.ErrNoDocuments) { + return datastore.ErrNotFound + } + if err != nil { + return err + } + if err := updater(entity); err != nil { + m.logger.Error("failed to run updater to update entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + wrapper, err := wrapModel(entity) + if err != nil { + return err + } + update := bson.D{{"$set", wrapper}} + if _, err := col.UpdateOne(ctx, makePrimaryKeyFilter(id), update); err != nil { + m.logger.Error("failed to update entity", + zap.String("id", id), + zap.String("kind", kind), + zap.Error(err), + ) + return err + } + return nil +} + +func (m *MongoDB) Close() error { + return m.client.Disconnect(m.ctx) +} + +func makePrimaryKeyFilter(id string) bson.D { + return bson.D{{"_id", id}} +} + +func (m *MongoDB) determineCredential() (*options.Credential, error) { + switch m.authMechanism { + case scram: + username, err := ioutil.ReadFile(m.usernameFile) + if err != nil { + return nil, fmt.Errorf("failed to read username file: %w", err) + } + password, err := ioutil.ReadFile(m.passwordFile) + if err != nil { + return nil, fmt.Errorf("failed to read password file: %w", err) + } + + return &options.Credential{ + Username: strings.TrimRight(string(username), "\n"), + Password: strings.TrimRight(string(password), "\n"), + AuthSource: m.database, + }, nil + default: + return nil, fmt.Errorf("unsupported %q authMechanism credential", m.authMechanism) + } +} diff --git a/pkg/datastore/mongodb/operator.go b/pkg/datastore/mongodb/operator.go new file mode 100644 index 0000000000..be2a3a983b --- /dev/null +++ b/pkg/datastore/mongodb/operator.go @@ -0,0 +1,40 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongodb + +import ( + "fmt" +) + +func convertToMongoDBOperator(op string) (string, error) { + switch op { + case "==": + return "$eq", nil + case "!=": + return "$ne", nil + case ">": + return "$gt", nil + case ">=": + return "$gte", nil + case "in": + return "$in", nil + case "<": + return "$lt", nil + case "<=": + return "$lte", nil + default: + return "", fmt.Errorf("unacceptable operator for mongodb: %s", op) + } +} diff --git a/repositories.bzl b/repositories.bzl index a95df7d17f..f0e0676c9e 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -33,6 +33,12 @@ def go_repositories(): version = "v1.0.100", ) + go_repository( + name = "com_github_aws_aws_sdk_go", + importpath = "github.com/aws/aws-sdk-go", + sum = "h1:7CcrvoPUYex6CVeEK7/UZlYx5/AcVCWrl1FsX7fVvFw=", + version = "v1.36.21", + ) go_repository( name = "com_github_aws_aws_sdk_go_v2", importpath = "github.com/aws/aws-sdk-go-v2", @@ -512,6 +518,79 @@ def go_repositories(): version = "v1.8.0", ) + go_repository( + name = "com_github_gobuffalo_attrs", + importpath = "github.com/gobuffalo/attrs", + sum = "h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4=", + version = "v0.0.0-20190224210810-a9411de4debd", + ) + go_repository( + name = "com_github_gobuffalo_depgen", + importpath = "github.com/gobuffalo/depgen", + sum = "h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4=", + version = "v0.1.0", + ) + go_repository( + name = "com_github_gobuffalo_envy", + importpath = "github.com/gobuffalo/envy", + sum = "h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=", + version = "v1.7.0", + ) + go_repository( + name = "com_github_gobuffalo_flect", + importpath = "github.com/gobuffalo/flect", + sum = "h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU=", + version = "v0.1.3", + ) + go_repository( + name = "com_github_gobuffalo_genny", + importpath = "github.com/gobuffalo/genny", + sum = "h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0=", + version = "v0.1.1", + ) + go_repository( + name = "com_github_gobuffalo_gitgen", + importpath = "github.com/gobuffalo/gitgen", + sum = "h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8=", + version = "v0.0.0-20190315122116-cc086187d211", + ) + go_repository( + name = "com_github_gobuffalo_gogen", + importpath = "github.com/gobuffalo/gogen", + sum = "h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI=", + version = "v0.1.1", + ) + go_repository( + name = "com_github_gobuffalo_logger", + importpath = "github.com/gobuffalo/logger", + sum = "h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs=", + version = "v0.0.0-20190315122211-86e12af44bc2", + ) + go_repository( + name = "com_github_gobuffalo_mapi", + importpath = "github.com/gobuffalo/mapi", + sum = "h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk=", + version = "v1.0.2", + ) + go_repository( + name = "com_github_gobuffalo_packd", + importpath = "github.com/gobuffalo/packd", + sum = "h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU=", + version = "v0.1.0", + ) + go_repository( + name = "com_github_gobuffalo_packr_v2", + importpath = "github.com/gobuffalo/packr/v2", + sum = "h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A=", + version = "v2.2.0", + ) + go_repository( + name = "com_github_gobuffalo_syncx", + importpath = "github.com/gobuffalo/syncx", + sum = "h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4=", + version = "v0.0.0-20190224160051-33c29581e754", + ) + go_repository( name = "com_github_goccy_go_yaml", importpath = "github.com/goccy/go-yaml", @@ -556,6 +635,13 @@ def go_repositories(): version = "v1.4.2", ) + go_repository( + name = "com_github_golang_snappy", + importpath = "github.com/golang/snappy", + sum = "h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=", + version = "v0.0.1", + ) + go_repository( name = "com_github_gomodule_redigo", importpath = "github.com/gomodule/redigo", @@ -762,6 +848,12 @@ def go_repositories(): version = "v1.5.1", ) + go_repository( + name = "com_github_joho_godotenv", + importpath = "github.com/joho/godotenv", + sum = "h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=", + version = "v1.3.0", + ) go_repository( name = "com_github_jonboulle_clockwork", importpath = "github.com/jonboulle/clockwork", @@ -795,6 +887,12 @@ def go_repositories(): version = "v1.2.0", ) + go_repository( + name = "com_github_karrick_godirwalk", + importpath = "github.com/karrick/godirwalk", + sum = "h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY=", + version = "v1.10.3", + ) go_repository( name = "com_github_kisielk_errcheck", importpath = "github.com/kisielk/errcheck", @@ -807,7 +905,12 @@ def go_repositories(): sum = "h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=", version = "v1.0.0", ) - + go_repository( + name = "com_github_klauspost_compress", + importpath = "github.com/klauspost/compress", + sum = "h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=", + version = "v1.10.11", + ) go_repository( name = "com_github_klauspost_cpuid", importpath = "github.com/klauspost/cpuid", @@ -865,6 +968,19 @@ def go_repositories(): version = "v0.0.0-20160728113105-d5b7844b561a", ) + go_repository( + name = "com_github_markbates_oncer", + importpath = "github.com/markbates/oncer", + sum = "h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k=", + version = "v0.0.0-20181203154359-bf2de49a0be2", + ) + go_repository( + name = "com_github_markbates_safe", + importpath = "github.com/markbates/safe", + sum = "h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=", + version = "v1.0.1", + ) + go_repository( name = "com_github_mattn_go_colorable", importpath = "github.com/mattn/go-colorable", @@ -947,6 +1063,13 @@ def go_repositories(): version = "v1.0.1", ) + go_repository( + name = "com_github_montanaflynn_stats", + importpath = "github.com/montanaflynn/stats", + sum = "h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=", + version = "v0.0.0-20171201202039-1bf9dbcd8cbe", + ) + go_repository( name = "com_github_munnerz_goautoneg", importpath = "github.com/munnerz/goautoneg", @@ -1022,8 +1145,8 @@ def go_repositories(): go_repository( name = "com_github_pelletier_go_toml", importpath = "github.com/pelletier/go-toml", - sum = "h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=", - version = "v1.2.0", + sum = "h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=", + version = "v1.4.0", ) go_repository( name = "com_github_peterbourgon_diskv", @@ -1220,6 +1343,12 @@ def go_repositories(): version = "v1.7.0", ) + go_repository( + name = "com_github_tidwall_pretty", + importpath = "github.com/tidwall/pretty", + sum = "h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=", + version = "v1.0.0", + ) go_repository( name = "com_github_tinylib_msgp", importpath = "github.com/tinylib/msgp", @@ -1240,6 +1369,18 @@ def go_repositories(): version = "v1.1.4", ) + go_repository( + name = "com_github_xdg_scram", + importpath = "github.com/xdg/scram", + sum = "h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=", + version = "v0.0.0-20180814205039-7eeb5667e42c", + ) + go_repository( + name = "com_github_xdg_stringprep", + importpath = "github.com/xdg/stringprep", + sum = "h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=", + version = "v0.0.0-20180714160509-73f8eece6fdc", + ) go_repository( name = "com_github_xiang90_probing", importpath = "github.com/xiang90/probing", @@ -1608,6 +1749,12 @@ def go_repositories(): version = "v0.0.0-20200804184101-5ec99f83aff1", ) + go_repository( + name = "org_mongodb_go_mongo_driver", + importpath = "go.mongodb.org/mongo-driver", + sum = "h1:C8rFn1VF4GVEM/rG+dSoMmlm2pyQ9cs2/oRtUATejRU=", + version = "v1.4.0", + ) go_repository( name = "org_uber_go_atomic", importpath = "go.uber.org/atomic",