Merge pull request #403 from pingcap-inc/sprint/20211231/sprint6

Sprint/20211231/sprint6

.gitignore

@@ -16,4 +16,7 @@ etcd/
 library/client/cluster
 library/client/metadb
 library/framework/logs
+library/secondparty/logs
 test/
+/template
+/nohup.out

Makefile

@@ -44,6 +44,7 @@ prepare:
 	@if [ -f "$(CURDIR)/$(PROTOC_PKG)" ]; then unzip -o $(PROTOC_PKG) -d $(GOPATH) bin/protoc; fi
 	@if [ -f "$(CURDIR)/$(PROTOC_PKG)" ]; then unzip -o $(PROTOC_PKG) -d $(GOPATH) 'include/*'; fi
 	@if [ -f "$(CURDIR)/$(PROTOC_PKG)" ]; then rm -rf $(PROTOC_PKG); fi
+	@cp -r micro-cluster/cluster/management/handler/template ./
 
 # generate protobuf files
 proto:
@@ -200,8 +201,9 @@ clean:
 	@if [ -f ${TIEM_BINARY_DIR}/vfsgendev ] ; then rm ${TIEM_BINARY_DIR}/vfsgendev; fi
 	@if [ -f ${TIEM_BINARY_DIR}/golangci-lint ] ; then rm ${TIEM_BINARY_DIR}/golangci-lint; fi
 	@if [ -f ${TIEM_BINARY_DIR}/errdoc-gen ] ; then rm ${TIEM_BINARY_DIR}/errdoc-gen; fi
-	@if [ -f ${GENERATE_TARGET_DIR}/cluster ] ; then rm -rf ${GENERATE_TARGET_DIR}/cluster; fi
-	@if [ -f ${GENERATE_TARGET_DIR}/metadb ] ; then rm -rf ${GENERATE_TARGET_DIR}/metadb; fi
+	@if [ -d ${GENERATE_TARGET_DIR}/cluster ] ; then rm -rf ${GENERATE_TARGET_DIR}/cluster; fi
+	@if [ -d ${GENERATE_TARGET_DIR}/metadb ] ; then rm -rf ${GENERATE_TARGET_DIR}/metadb; fi
+	@if [ -d ${CURDIR}/test ] ; then rm -rf ${CURDIR}/test; fi
 
 help:
 	@echo "make build, build binary for all servers"
@@ -271,7 +273,26 @@ add_test_file:
 	build_helper/add_test_file.sh
 
 mock:
-	go get github.com/golang/mock/mockgen
+	$(GO) install github.com/golang/mock/mockgen
 	mockgen -destination ./test/mocksecondparty/mock_second_party_manager.go -package mocksecondparty -source ./library/secondparty/second_party_manager.go
 	mockgen -destination ./test/mockdb/mock_db.pb.micro.go -package mockdb -source ./library/client/metadb/dbpb/db.pb.micro.go
 	mockgen -destination ./test/mockcluster/mock_cluster.pb.micro.go -package mockcluster -source ./library/client/cluster/clusterpb/cluster.pb.micro.go
+
+	mockgen -destination ./test/mockmodels/mockmanagement/mock_management_interface.go -package mockmanagement -source ./models/cluster/management/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockworkflow/mock_workflow_interface.go -package mockworkflow -source ./models/workflow/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockbr/mock_br_interface.go -package mockbr -source ./models/cluster/backuprestore/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockimportexport/mock_importexport_interface.go -package mockimportexport -source ./models/datatransfer/importexport/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockresource/mock_resource_interface.go -package mockresource -source ./models/resource/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockconfig/mock_config_interface.go -package mockconfig -source ./models/platform/config/readerwriter.go
+	mockgen -destination ./test/mockmodels/mocksecondparty/mock_secondparty_interface.go -package mocksecondparty -source ./models/workflow/secondparty/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockparametergroup/mock_parametergroup_interface.go -package mockparametergroup -source ./models/parametergroup/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockclusterparameter/mock_clusterparameter_interface.go -package mockclusterparameter -source ./models/cluster/parameter/readerwriter.go
+	mockgen -destination ./test/mockmodels/mockclustermanagement/mock_cluster_management_interface.go -package mockclustermanagement -source ./models/cluster/management/readerwriter.go
+
+	mockgen -destination ./test/mockworkflow/mock_workflow.go -package mock_workflow_service -source ./workflow/workflow.go
+	mockgen -destination ./test/mockbr/mock_br.go -package mock_br_service -source ./micro-cluster/cluster/backuprestore/service.go
+	mockgen -destination ./test/mocksecondparty_v2/mock_secondparty.go -package mock_secondparty -source ./library/secondparty/second_party_manager_v2.go
+
+swag:
+	$(GO) install github.com/swaggo/swag/cmd/[email protected]
+	swag init -g micro-api/main.go

Makefile.common

@@ -67,7 +67,7 @@ endif
 
 LINUX     := "Linux"
 MAC       := "Darwin"
-PACKAGE_LIST  := go list ./...| grep -vE "docs|proto|library/util|micro-api|micro-metadb/service|micro-cluster/service/cluster/adapt|micro-cluster/service/tenant/adapt|^github.com/pingcap-inc/tiem/micro-cluster/service$$|clusterpb|dbpb|adapt|test/mock"
+PACKAGE_LIST  := go list ./...| grep -vE "docs|proto|library/util|library/client|library/copywriting|micro-api|file-server|micro-metadb/service|micro-cluster/datatransfer|micro-cluster/cluster|micro-cluster/service/cluster/adapt|micro-cluster/service/tenant/adapt|^github.com/pingcap-inc/tiem/micro-cluster/service$$|clusterpb|dbpb|adapt|test/mock|deprecated"
 PACKAGES  ?= $$($(PACKAGE_LIST))
 #PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)||'
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST)

README.md

@@ -5,7 +5,6 @@
 - library - Common components and resources.
 - micro-api - OpenAPI http handler.
 - micro-cluster - Core service.
-- micro-metadb - Client of go-micro service.
 - docs - Documentation.
 
 ## Build and Run TiEM
@@ -23,20 +22,14 @@ make test
 ### Run Service
 If you want to run the service locally, try the following commands, enjoy it!
 
-start metadb-server
-
-```shell
-$ ./bin/metadb-server --host=127.0.0.1 --registry-address=127.0.0.1:4101
-```
-
 start cluster-server
 ```shell
-$ ./bin/cluster-server --host=127.0.0.1 --metrics-port=4122 --registry-address=127.0.0.1:4101
+$ ./bin/cluster-server --host=127.0.0.1 --port=4101 --metrics-port=4104 --registry-address=127.0.0.1:4106 --elasticsearch-address=127.0.0.1:4108
 ```
 
 start openapi-server
 ```shell
-$ ./bin/openapi-server --host=127.0.0.1 --metrics-port=4123 --registry-address=127.0.0.1:4101 --elasticsearch-address=127.0.0.1:9200
+$ ./bin/openapi-server --host=127.0.0.1 --port=4100 --metrics-port=4103 --registry-address=127.0.0.1:4106
 ```
 
 ### Try it out

common/constants/changefeed.go

@@ -0,0 +1,65 @@
+/******************************************************************************
+ * Copyright (c)  2021 PingCAP, Inc.                                          *
+ * Licensed under the Apache License, Version 2.0 (the "License");            *
+ * you may not use this file except in compliance with the License.           *
+ * You may obtain a copy of the License at                                    *
+ *                                                                            *
+ * http://www.apache.org/licenses/LICENSE-2.0                                 *
+ *                                                                            *
+ * Unless required by applicable law or agreed to in writing, software        *
+ * distributed under the License is distributed on an "AS IS" BASIS,          *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
+ * See the License for the specific language governing permissions and        *
+ * limitations under the License.                                             *
+ ******************************************************************************/
+
+package constants
+
+import (
+	"github.com/pingcap-inc/tiem/common/errors"
+)
+
+type ChangeFeedStatus string
+
+const (
+	Initial  ChangeFeedStatus = "Initial"
+	Normal   ChangeFeedStatus = "Normal"
+	Stopped  ChangeFeedStatus = "Stopped"
+	Finished ChangeFeedStatus = "Finished"
+	Error    ChangeFeedStatus = "Error"
+	Failed   ChangeFeedStatus = "Failed"
+	Unknown  ChangeFeedStatus = "Unknown"
+)
+
+func (s ChangeFeedStatus) IsFinal() bool {
+	return Finished == s || Failed == s
+}
+
+func (s ChangeFeedStatus) ToString() string {
+	return string(s)
+}
+
+func IsValidStatus(s string) bool {
+	return Initial.ToString() == s ||
+		Normal.ToString() == s ||
+		Stopped.ToString() == s ||
+		Finished.ToString() == s ||
+		Error.ToString() == s ||
+		Failed.ToString() == s
+}
+
+func ConvertStatus(s string) (status ChangeFeedStatus, err error) {
+	if IsValidStatus(s) {
+		return ChangeFeedStatus(s), nil
+	} else {
+		return Unknown, errors.NewError(errors.TIEM_PARAMETER_INVALID, "unexpected change feed status")
+	}
+}
+
+type DownstreamType string
+
+const (
+	DownstreamTypeTiDB  DownstreamType = "tidb"
+	DownstreamTypeKafka DownstreamType = "kafka"
+	DownstreamTypeMysql DownstreamType = "mysql"
+)

common/constants/cluster.go

@@ -0,0 +1,160 @@
+/******************************************************************************
+ * Copyright (c)  2021 PingCAP, Inc.                                          *
+ * Licensed under the Apache License, Version 2.0 (the "License");            *
+ * you may not use this file except in compliance with the License.           *
+ * You may obtain a copy of the License at                                    *
+ *                                                                            *
+ * http://www.apache.org/licenses/LICENSE-2.0                                 *
+ *                                                                            *
+ * Unless required by applicable law or agreed to in writing, software        *
+ * distributed under the License is distributed on an "AS IS" BASIS,          *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
+ * See the License for the specific language governing permissions and        *
+ * limitations under the License.                                             *
+ ******************************************************************************/
+
+/*******************************************************************************
+ * @File: cluster.go
+ * @Description:
+ * @Author: [email protected]
+ * @Version: 1.0.0
+ * @Date: 2021/12/4
+*******************************************************************************/
+
+package constants
+
+type ClusterRunningStatus string
+
+//Definition of cluster running status information
+const (
+	ClusterInitializing ClusterRunningStatus = "Initializing"
+	ClusterStopped      ClusterRunningStatus = "Stopped"
+	ClusterRunning      ClusterRunningStatus = "Running"
+	ClusterRecovering   ClusterRunningStatus = "Recovering"
+	ClusterFailure      ClusterRunningStatus = "Failure"
+)
+
+type ClusterMaintenanceStatus string
+
+// Definition cluster maintenance status information
+const (
+	ClusterMaintenanceCreating                     ClusterMaintenanceStatus = "Creating"
+	ClusterMaintenanceCloning                      ClusterMaintenanceStatus = "Cloning"
+	ClusterMaintenanceDeleting                     ClusterMaintenanceStatus = "Deleting"
+	ClusterMaintenanceStopping                     ClusterMaintenanceStatus = "Stopping"
+	ClusterMaintenanceRestarting                   ClusterMaintenanceStatus = "Restarting"
+	ClusterMaintenanceBackUp                       ClusterMaintenanceStatus = "BackUp"
+	ClusterMaintenanceRestore                      ClusterMaintenanceStatus = "Restore"
+	ClusterMaintenanceScaleIn                      ClusterMaintenanceStatus = "ScaleIn"
+	ClusterMaintenanceScaleOut                     ClusterMaintenanceStatus = "ScaleOut"
+	ClusterMaintenanceUpgrading                    ClusterMaintenanceStatus = "Upgrading"
+	ClusterMaintenanceSwitching                    ClusterMaintenanceStatus = "Switching"
+	ClusterMaintenanceModifyParameterAndRestarting ClusterMaintenanceStatus = "ModifyParameterRestarting"
+	ClusterMaintenanceTakeover                     ClusterMaintenanceStatus = "Takeover"
+	ClusterMaintenanceNone                         ClusterMaintenanceStatus = ""
+)
+
+const (
+	FlowCreateCluster       = "CreateCluster"
+	FlowDeleteCluster       = "DeleteCluster"
+	FlowBackupCluster       = "BackupCluster"
+	FlowRestoreNewCluster   = "RestoreNewCluster"
+	FlowRestoreExistCluster = "RestoreExistCluster"
+	FlowModifyParameters    = "ModifyParameters"
+	FlowExportData          = "ExportData"
+	FlowImportData          = "ImportData"
+	FlowRestartCluster      = "RestartCluster"
+	FlowStopCluster         = "StopCluster"
+	FlowTakeoverCluster     = "BuildForTakeover"
+	FlowBuildLogConfig      = "BuildLogConfig"
+	FlowScaleOutCluster     = "ScaleOutCluster"
+	FlowScaleInCluster      = "ScaleInCluster"
+	FlowCloneCluster        = "CloneCluster"
+)
+
+type ClusterInstanceRunningStatus string
+
+//Definition of cluster instance running status information
+const (
+	ClusterInstanceInitializing ClusterInstanceRunningStatus = "Initializing"
+	ClusterInstanceStopped      ClusterInstanceRunningStatus = "Stopped"
+	ClusterInstanceRunning      ClusterInstanceRunningStatus = "Running"
+	ClusterInstanceRecovering   ClusterInstanceRunningStatus = "Recovering"
+	ClusterInstanceFailure      ClusterInstanceRunningStatus = "Failure"
+)
+
+type ClusterInstanceMaintenanceStatus string
+
+// Definition cluster instance maintenance status information
+const (
+	ClusterInstanceMaintenanceCreating                     ClusterInstanceMaintenanceStatus = "Creating"
+	ClusterInstanceMaintenanceDeleting                     ClusterInstanceMaintenanceStatus = "Deleting"
+	ClusterInstanceMaintenanceStopping                     ClusterInstanceMaintenanceStatus = "Stopping"
+	ClusterInstanceMaintenanceRestarting                   ClusterInstanceMaintenanceStatus = "Restarting"
+	ClusterInstanceMaintenanceUpgrading                    ClusterInstanceMaintenanceStatus = "Upgrading"
+	ClusterInstanceMaintenanceModifyParameterAndRestarting ClusterInstanceMaintenanceStatus = "ModifyParameterRestarting"
+)
+
+type ClusterBackupStatus string
+
+//Definition of cluster backup status information
+const (
+	ClusterBackupInitializing ClusterBackupStatus = "Initializing"
+	ClusterBackupProcessing   ClusterBackupStatus = "Processing"
+	ClusterBackupFinished     ClusterBackupStatus = "Finished"
+	ClusterBackupFailed       ClusterBackupStatus = "Failed"
+)
+
+type ClusterRelationType string
+
+//Constants for the relationships between clusters
+const (
+	ClusterRelationSlaveTo     ClusterRelationType = "SlaveTo"
+	ClusterRelationStandBy     ClusterRelationType = "StandBy"
+	ClusterRelationCloneFrom   ClusterRelationType = "CloneFrom"
+	ClusterRelationRecoverFrom ClusterRelationType = "RecoverFrom"
+)
+
+type ClusterCloneStrategy string
+
+// Definition cluster clone strategy
+const (
+	ClusterTopologyClone ClusterCloneStrategy = "TopologyClone"
+	SnapShotClone        ClusterCloneStrategy = "Snapshot"
+	TiCDCSyncClone       ClusterCloneStrategy = "TiCDCSync"
+)
+
+type BackupType string
+type BackupMethod string
+
+//Definition backup data method and type
+const (
+	BackupTypeFull      BackupType   = "full"
+	BackupTypeIncrement BackupType   = "incr"
+	BackupMethodLogic   BackupMethod = "logical"
+	BackupMethodPhysics BackupMethod = "physical"
+)
+
+type BackupMode string
+
+//Definition backup data mode
+const (
+	BackupModeAuto   BackupMode = "auto"
+	BackupModeManual BackupMode = "manual"
+)
+
+type StorageType string
+
+//Definition backup data storage type
+const (
+	StorageTypeLocal StorageType = "local"
+	StorageTypeS3    StorageType = "s3"
+	StorageTypeNFS   StorageType = "nfs"
+)
+
+const (
+	DefaultBackupStoragePath       string = "nfs/em/backup"
+	DefaultBackupS3AccessKey       string = "minioadmin"
+	DefaultBackupS3SecretAccessKey string = "minioadmin"
+	DefaultBackupS3Endpoint        string = "http://minio.pingcap.net:9000"
+)