Skip to content

Commit

Permalink
Enable supporting MongoDB (#1794)
Browse files Browse the repository at this point in the history
**What this PR does / why we need it**:

This reverts commit 6c2b0bd.

**Which issue(s) this PR fixes**:

Rel #1792 

**Does this PR introduce a user-facing change?**:
<!--
If no, just write "NONE" in the release-note block below.
-->
```release-note
NONE
```

This PR was merged by Kapetanios.
  • Loading branch information
khanhtc1202 authored Mar 29, 2021
1 parent 5a1e38a commit d3396f3
Show file tree
Hide file tree
Showing 11 changed files with 837 additions and 7 deletions.
1 change: 1 addition & 0 deletions cmd/pipecd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ go_library(
"//pkg/crypto:go_default_library",
"//pkg/datastore:go_default_library",
"//pkg/datastore/firestore:go_default_library",
"//pkg/datastore/mongodb:go_default_library",
"//pkg/datastore/mysql:go_default_library",
"//pkg/filestore:go_default_library",
"//pkg/filestore/gcs:go_default_library",
Expand Down
16 changes: 13 additions & 3 deletions cmd/pipecd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"github.com/pipe-cd/pipe/pkg/crypto"
"github.com/pipe-cd/pipe/pkg/datastore"
"github.com/pipe-cd/pipe/pkg/datastore/firestore"
"github.com/pipe-cd/pipe/pkg/datastore/mongodb"
"github.com/pipe-cd/pipe/pkg/datastore/mysql"
"github.com/pipe-cd/pipe/pkg/filestore"
"github.com/pipe-cd/pipe/pkg/filestore/gcs"
Expand Down Expand Up @@ -412,8 +413,18 @@ func createDatastore(ctx context.Context, cfg *config.ControlPlaneSpec, logger *
return nil, errors.New("dynamodb is unimplemented yet")

case model.DataStoreMongoDB:
return nil, errors.New("mongodb is unimplemented yet")

mdConfig := cfg.Datastore.MongoDBConfig
options := []mongodb.Option{
mongodb.WithLogger(logger),
}
if mdConfig.UsernameFile != "" || mdConfig.PasswordFile != "" {
options = append(options, mongodb.WithAuthenticationFile(mdConfig.UsernameFile, mdConfig.PasswordFile))
}
return mongodb.NewMongoDB(
ctx,
mdConfig.URL,
mdConfig.Database,
options...)
case model.DataStoreMySQL:
mqConfig := cfg.Datastore.MySQLConfig
options := []mysql.Option{
Expand All @@ -423,7 +434,6 @@ func createDatastore(ctx context.Context, cfg *config.ControlPlaneSpec, logger *
options = append(options, mysql.WithAuthenticationFile(mqConfig.UsernameFile, mqConfig.PasswordFile))
}
return mysql.NewMySQL(mqConfig.URL, mqConfig.Database, options...)

default:
return nil, fmt.Errorf("unknown datastore type %q", cfg.Datastore.Type)
}
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
github.com/DataDog/datadog-api-client-go v1.0.0-beta.16
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
github.com/aws/aws-sdk-go v1.36.21 // indirect
github.com/aws/aws-sdk-go-v2 v1.2.0
github.com/aws/aws-sdk-go-v2/config v1.1.1
github.com/aws/aws-sdk-go-v2/credentials v1.1.1
Expand All @@ -28,6 +29,7 @@ require (
github.com/google/uuid v1.2.0
github.com/googleapis/gnostic v0.2.2 // indirect
github.com/hashicorp/golang-lru v0.5.3
github.com/klauspost/compress v1.10.11 // indirect
github.com/minio/minio-go/v7 v7.0.5
github.com/prometheus/client_golang v1.6.0
github.com/prometheus/client_model v0.2.0
Expand All @@ -36,6 +38,7 @@ require (
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
go.mongodb.org/mongo-driver v1.4.0
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.2.0 // indirect
go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0
Expand Down
82 changes: 81 additions & 1 deletion go.sum

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions pkg/config/control_plane.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,18 @@ type DataStoreDynamoDBConfig struct {
}

type DataStoreMongoDBConfig struct {
// The url of MongoDB. All of credentials can be specified via this field.
URL string `json:"url"`
// The name of the database.
// Also set Database as 'AuthSource' (default as 'admin' or '$external') when UsernameFle || PasswordFile specify
// Ref: https://github.com/mongodb/mongo-go-driver/blob/9e2aca8afd8821e6b068cc2f25192bc640d90a0d/mongo/client.go#L390
Database string `json:"database"`
// The path to the username file.
// For those who don't want to include the username in the URL.
UsernameFile string `json:"usernameFile"`
// The path to the password file.
// For those who don't want to include the password in the URL.
PasswordFile string `json:"passwordFile"`
}

type DataStoreMySQLConfig struct {
Expand Down
21 changes: 21 additions & 0 deletions pkg/datastore/mongodb/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = [
"iterator.go",
"model_wrapper.go",
"mongodb.go",
"operator.go",
],
importpath = "github.com/pipe-cd/pipe/pkg/datastore/mongodb",
visibility = ["//visibility:public"],
deps = [
"//pkg/datastore:go_default_library",
"//pkg/model:go_default_library",
"@org_mongodb_go_mongo_driver//bson:go_default_library",
"@org_mongodb_go_mongo_driver//mongo:go_default_library",
"@org_mongodb_go_mongo_driver//mongo/options:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
47 changes: 47 additions & 0 deletions pkg/datastore/mongodb/iterator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2020 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mongodb

import (
"context"

"go.mongodb.org/mongo-driver/mongo"

"github.com/pipe-cd/pipe/pkg/datastore"
)

type Iterator struct {
ctx context.Context
cur *mongo.Cursor
}

func (it *Iterator) Next(dst interface{}) error {
if !it.cur.Next(it.ctx) {
return datastore.ErrIteratorDone
}
wrapper, err := wrapModel(dst)
if err != nil {
return err
}
if err := it.cur.Decode(wrapper); err != nil {
return err
}
return extractModel(wrapper, dst)
}

func (it *Iterator) Cursor() (string, error) {
// Note: Perhaps, not required.
return "", datastore.ErrUnimplemented
}
193 changes: 193 additions & 0 deletions pkg/datastore/mongodb/model_wrapper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
// Copyright 2020 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mongodb

import (
"fmt"

"github.com/pipe-cd/pipe/pkg/model"
)

// wrapModel returns a wrapper corresponding to the given entity.
// A wrapper wraps a model representing BSON a document so that the model comes with "_id".
func wrapModel(entity interface{}) (interface{}, error) {
switch e := entity.(type) {
case *model.Application:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &application{
ID: e.GetId(),
Application: *e,
}, nil
case *model.Command:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &command{
ID: e.GetId(),
Command: *e,
}, nil
case *model.Deployment:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &deployment{
ID: e.GetId(),
Deployment: *e,
}, nil
case *model.Environment:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &environment{
ID: e.GetId(),
Environment: *e,
}, nil
case *model.Piped:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &piped{
ID: e.GetId(),
Piped: *e,
}, nil
case *model.Project:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &project{
ID: e.GetId(),
Project: *e,
}, nil
case *model.APIKey:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &apiKey{
ID: e.GetId(),
APIKey: *e,
}, nil
case *model.Event:
if e == nil {
return nil, fmt.Errorf("nil entity given")
}
return &event{
ID: e.GetId(),
Event: *e,
}, nil
default:
return nil, fmt.Errorf("%T is not supported", e)
}
}

// extractModel stores the unwrapped model in the value pointed to by e.
func extractModel(wrapper interface{}, e interface{}) error {
msg := "entity type doesn't correspond to the wrapper type (%T)"

switch w := wrapper.(type) {
case *application:
e, ok := e.(*model.Application)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Application
case *command:
e, ok := e.(*model.Command)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Command
case *deployment:
e, ok := e.(*model.Deployment)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Deployment
case *environment:
e, ok := e.(*model.Environment)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Environment
case *piped:
e, ok := e.(*model.Piped)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Piped
case *project:
e, ok := e.(*model.Project)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Project
case *apiKey:
e, ok := e.(*model.APIKey)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.APIKey
case *event:
e, ok := e.(*model.Event)
if !ok {
return fmt.Errorf(msg, w)
}
*e = w.Event
default:
return fmt.Errorf("%T is not supported", w)
}
return nil
}

type application struct {
model.Application `bson:",inline"`
ID string `bson:"_id"`
}

type command struct {
model.Command `bson:",inline"`
ID string `bson:"_id"`
}

type deployment struct {
model.Deployment `bson:",inline"`
ID string `bson:"_id"`
}

type environment struct {
model.Environment `bson:",inline"`
ID string `bson:"_id"`
}

type piped struct {
model.Piped `bson:",inline"`
ID string `bson:"_id"`
}

type project struct {
model.Project `bson:",inline"`
ID string `bson:"_id"`
}

type apiKey struct {
model.APIKey `bson:",inline"`
ID string `bson:"_id"`
}

type event struct {
model.Event `bson:",inline"`
ID string `bson:"_id"`
}
Loading

0 comments on commit d3396f3

Please sign in to comment.