From 2fa78535c1998fded5d025293ad5c07a5771d6b1 Mon Sep 17 00:00:00 2001 From: calvin Date: Mon, 29 Jan 2024 11:43:30 +0800 Subject: [PATCH] support split table for internal storage Signed-off-by: calvin --- examples/pediacluster.yaml | 8 +-- internalstorage-config.yaml | 13 ++++ pkg/storage/internalstorage/register.go | 7 +- .../internalstorage/resource_storage.go | 47 ++++++++------ pkg/storage/internalstorage/storage.go | 64 ++++++++++++++----- 5 files changed, 100 insertions(+), 39 deletions(-) create mode 100644 internalstorage-config.yaml diff --git a/examples/pediacluster.yaml b/examples/pediacluster.yaml index 87f385d23..070236401 100644 --- a/examples/pediacluster.yaml +++ b/examples/pediacluster.yaml @@ -3,11 +3,11 @@ kind: PediaCluster metadata: name: cluster-example spec: - apiserver: "https://10.30.43.43:6443" - caData: + apiserver: "https://10.6.212.13:6443" + caData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJUFZqajZCRkt2Nmt3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TXpFeU1qZ3dPRFF5TURSYUZ3MHpNekV5TWpVd09EUTNNRFJhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUM3VksrangrL2RxbDNKNW45N0VLSUYzdG0zT0FsUC9OeHBuL1F1NDkxdmxtWFRRQ2xHQ2NSOG1JaXcKVDZvODVDTm5YZG53NUEvdGRSM3l5Q0RlNlNwTm00NENMRWJxaDJmUXE2RXJoMGNBNFJVeWlwVCtXZVgxVHBSTQp6MDU5Y3pwT0twMllrbTZINmM0UWwxSktSb3F0OHpHelRCMHdDOG1IcUtMRlpqWFBZckYrRXdBWi9XaGlQaVQ3CjFRc3cyRjQyR2cvYk5ZVHAxclFVRU40NmxCZE1PNVF2Ykg0RlNGekY4OENhZUJOTlJBdUczZWp5L0doRWVUb0wKVGQ3RWgvS3REZWptK01mK1N4SWNMdzdOZTA2UnJSSThReWkrRUlBUEJiN0F4Sk9KTlphZWpsWG9PdGk2SEZaSApMV0xXY0I1VzlCa0lZWms2UmVHc2J2OWxYdzNWQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSUFE0eDQxQnZUOG9ra3lld1hMVVorV0Fzc2xqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQllTNVBrYmtmOAp1Z3Y4eE1aRk1QZExWY3AzV3hWQllSQ1RaTDdZR2tQUTlFMHNsdHROWDF1bG9UektuNzF2ZlphMDRJdW03UHZyCmhOTW9yajQzZE9lSTkrYWxaRTJDQmJPRHg3djNZS2JSQVdNODdIMUt2S0E3Q3NCaCszV1lad1BrQlNNOEZVRmgKVmZLREFxMjF2SU1XakZNVVpUV3VSUDZjYTNPTUhCT0dxOWRQRTdGMXBNMnNOV0lCN2wwQlpKQjM5SEhOK09Wbwp1MURnRkw2a0hLakR5bkZTcFJuc0lhVzd4ejFhZjlocEJkUWNYZTU3RGtEUUh5QkFSNDh0a0k1bUt0OTM1VENnCldFdzhpbklxRHFsaDVUMkZ3UERCRHdLeTBGMDREdFpCWnk1UlhIaWp3YTFYYk94eUc5Sk8zU1dvNXIwLzFOblAKZFJKdlVFYUtyQ1BTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K tokenData: - certData: - keyData: + certData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURJVENDQWdtZ0F3SUJBZ0lJRjRsd2Y0dlQvN0l3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TXpFeU1qZ3dPRFF5TURSYUZ3MHlOREV5TWpjd09EUTNNRGRhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXoxQlNrcyt1VmhoWTByU2sKS1o5SU5sazBqd1VQWGpXbGVGZGVCeHBJd29ETWFBSlAzckFUeGNJa3FvRmlDWXJPNDJSUWgrSzdzWkdYTUhJbQptYXUwZUxmVVdNdCsvY3Bma0NtYTFiVnBsWm5nVjZNYjlwU2N4cmp6ZFpmWU9pMVlwU0hGN1dhOFZ2Z1Q1VFJwCjV4ZlFJK2F4Y21RY0h6QW42Sm1rbnpIZ1crbXI4WmUvRXRHbUlheTREbCtnQ2lTLzVsOXU1QVdtb2p5RHpSU2IKQnVtVVNEeEpoR0JYSG40cGFPNXYwVFpQRzV1THZDR2dGR2ZXbk92TzQ5WUJ4U29ZYWxwbm9PTHl4WVBQMDI5cQoxSE1FVEJsejhEUTE0a1EwYUl2SWxiWVdubTQyKytMQy9UcTd3M1JnbW8rRzJtaVNuTFRSM3dHc2dOVlJSRUdFCjJNOGtGUUlEQVFBQm8xWXdWREFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JSUFE0eDQxQnZUOG9ra3lld1hMVVorV0FzcwpsakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBQ3hrN2xJejkwUnp5L01QSitFWDdPaU1XTjZKZmZJcHBvbjFKCmNtOGxoK015cEZNWnU4NWEzNTA3SWF3NlBXKzExQk1nbU4wc1BCL1MzS014U05LVUp1cFp5VlNCK1lud2VTTVMKekxTbnF6MzhDeVQrODUrb2lEQUt0SFZDZnltR3hPSHNXQ2FGVE5UdHJhYTFnNUI3MCtrNXpvakpMWnV3L3ZMeQpJRGxvM3U1elRPczJEV1hMQ3gxdXI1WWxSYnVybGh4Z0VGUXJncDUvOW9Mc1dNNGl3c2R0NW94Q0tTTVg4YXBuCjVrZmVtQ3BQenFrcXV2Rmt6WVVaczJZNGRCVGhKSCs4QVBJd0lFOW1FTG9KOTBZQWpLVGtwcGV6Ymhtc3RNZGIKTjBZZEZ6UEd2cXpoQzlpM3J4ZmR5RnQ1QTZrcG9GNnQrdTNwWkNKa0Q1SG84RGpla0E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + keyData: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBejFCU2tzK3VWaGhZMHJTa0taOUlObGswandVUFhqV2xlRmRlQnhwSXdvRE1hQUpQCjNyQVR4Y0lrcW9GaUNZck80MlJRaCtLN3NaR1hNSEltbWF1MGVMZlVXTXQrL2NwZmtDbWExYlZwbFpuZ1Y2TWIKOXBTY3hyanpkWmZZT2kxWXBTSEY3V2E4VnZnVDVUUnA1eGZRSStheGNtUWNIekFuNkpta256SGdXK21yOFplLwpFdEdtSWF5NERsK2dDaVMvNWw5dTVBV21vanlEelJTYkJ1bVVTRHhKaEdCWEhuNHBhTzV2MFRaUEc1dUx2Q0dnCkZHZlduT3ZPNDlZQnhTb1lhbHBub09MeXhZUFAwMjlxMUhNRVRCbHo4RFExNGtRMGFJdklsYllXbm00MisrTEMKL1RxN3czUmdtbytHMm1pU25MVFIzd0dzZ05WUlJFR0UyTThrRlFJREFRQUJBb0lCQUN1dEFXaDZwb0R1NmkraAo3UC96MWJQVmdWUXNQYnVjVTdzUnRQYU1IQTRXTW5vckdvYVEwK21TNkVTZnMzYk5McDhMY0VmZ2hhaDRRc0RyCkc2UytYdXdiMW5uS2pYdU1XaXUyQ0MvRDQzV29FSEtHSTRzRWJpOFRvZjhCT1lXd0xmL0VLTDNHS2lHc2JBeWEKeVVEdGJHVlQ0Y2gydk5INVJEOGx0MS91a3hUcHVjRXdZYTRhUGRyNXBTTitSWk5SS3NmWlJQandDQi9YMFNuUwpJYXJuYjVIdXkvemlNZ3ZGM1VJWlVLclVUT2hNOWUxdjVlM3h0c29JSWlreEtSUnRITjJEMDFYUFpmMnVRSkY2ClNvRGUxaWZwS0tFVFRXR3dxbnAvTitKMzdhY1BRV0FVU0g4Nnd2UGxYd0F2QTRrTVRtSWhtQ2xsRGVUZENzSlAKZ1lZNUk3VUNnWUVBNm5BdFhBckNnQXkwb2xTb3VhcEc0cE5IRHdQcmhIRUx1OWRDRGpYZmtQQ0ZORmRrWjcyZwphL0VIUHI4UTRLQ0s3Y3ZhK2t4ZVpDcnhybUlVc0xkMW1kTHVZZzZOdjNEVjhpK0lNSk14NW9reDZaYnZrdUFPClBaVlpDOGVmK0VibDR3ME5TWmZWT3pFRm92elBNWENya01wbHFQWkR2TXRGR25rRXc5dTNnNk1DZ1lFQTRtR0EKNFVCMm5DK2tDUEtxY0ZrYWxlY3p6T20rVlpkTWRwSVFqODBSd1NsdDRoQXZvU3Z6YnRFWVFhQW5uOU9hSnZoWQpQSXMxbmc4aWZPUm9WTyt6bmg1ZkZTdnZIWjRQcXFnWW56cFp4cFd4NCsrZTlpcW1CbGRQaXl6Ny9kYld4S094Ck54bnljekVIYml6SFhiTXFOUjREaXZseGsrNWt4MXFoaDJ4QTlPY0NnWUJzSlVOSGFrOCs4cDJCSkx5UXFWUDUKd1NiT2JtaCt1QWRKcDRyNU16RG1rNDFmRDlGcGFiaFBFdWpjc1JjQUNBOE9rek40c3pIMzB6ZVowZlRwY0RmMwp0YTN4M2ZUVmt4K2VrUDZxS2J5ak5EYXFJczV4V3BvM0htWHRZVEVTUHVKZFV2QUdPdHhLYWRySHpRZG5MeXAxCnFHZ3RBUzFPTWhhMmw0enpMTEFETVFLQmdGNDRzTmlQNlE0V3NoSmJWZDhPNXdiUFJQc2JxbkdiV0dMZFpuRkQKckFrV2dqcWQvQVQ3OHdVRG1ESHdRNjI4OTZNU3FEOEN2ZDA4ZFdFLys4Q21SNzhlNnZvMzMvMFd4WFNGSkVKWgptZllTWmdqMXVkYkJaM3FxS2x4RjY5ZjU3RWZKemIrZVdBL2pyekVIQ0MrU201V2lHVjFsZHdWWUtxL2lvK3BtCmZ4NnZBb0dCQU9TeWpOWkw5eHhJVGx6bDBzMFRDb1R1YTIzSlBORjlTakNuVElRak9NUlhhNnUxdnBocXNKTFcKSEhyNy84NUFZQWZ1clp2VitzVzcvcnZmSmFaMStVeW4yeUF1THJFeE5CWmNJVC94WWZweFlVNDhnQ3dvZW8zQgorZ083NXdFUDRuUHEvNC81cWF3VnRkWkhveXhYWk9KTkFYUkl3eUdPc211Y0wrN0Q3WGd4Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== syncResources: - group: apps resources: diff --git a/internalstorage-config.yaml b/internalstorage-config.yaml new file mode 100644 index 000000000..7d545973b --- /dev/null +++ b/internalstorage-config.yaml @@ -0,0 +1,13 @@ +type: "mysql" +dsn: "" +host: "10.6.212.13" +port: 30870 +password: dangerous0 +user: root +database: clusterpedia +autoMigration: true +divisionPolicy: GroupResource +connPool: + maxIdleConns: 5 + maxOpenConns: 40 + connMaxLifetime: 60m diff --git a/pkg/storage/internalstorage/register.go b/pkg/storage/internalstorage/register.go index 42061499b..0b5d19894 100644 --- a/pkg/storage/internalstorage/register.go +++ b/pkg/storage/internalstorage/register.go @@ -93,7 +93,12 @@ func NewStorageFactory(configPath string) (storage.StorageFactory, error) { sqlDB.SetMaxOpenConns(connPool.MaxOpenConns) sqlDB.SetConnMaxLifetime(connPool.ConnMaxLifetime) - return &StorageFactory{db}, nil + return &StorageFactory{ + db: db, + AutoMigration: cfg.AutoMigration, + DivisionPolicy: cfg.DivisionPolicy, + Mapper: cfg.Mapper, + }, nil } func newLogger(cfg *Config) (logger.Interface, error) { diff --git a/pkg/storage/internalstorage/resource_storage.go b/pkg/storage/internalstorage/resource_storage.go index c306ac9b6..4c887a7da 100644 --- a/pkg/storage/internalstorage/resource_storage.go +++ b/pkg/storage/internalstorage/resource_storage.go @@ -29,6 +29,7 @@ import ( type ResourceStorage struct { db *gorm.DB codec runtime.Codec + table string storageGroupResource schema.GroupResource storageVersion schema.GroupVersion @@ -83,7 +84,7 @@ func (s *ResourceStorage) Create(ctx context.Context, cluster string, obj runtim resource.DeletedAt = sql.NullTime{Time: deletedAt.Time, Valid: true} } - result := s.db.WithContext(ctx).Create(&resource) + result := s.db.WithContext(ctx).Table(s.table).Create(&resource) return InterpretResourceDBError(cluster, metaobj.GetName(), result.Error) } @@ -116,14 +117,19 @@ func (s *ResourceStorage) Update(ctx context.Context, cluster string, obj runtim updatedResource["deleted_at"] = sql.NullTime{Time: deletedAt.Time, Valid: true} } - result := s.db.WithContext(ctx).Model(&Resource{}).Where(map[string]interface{}{ - "cluster": cluster, - "group": s.storageGroupResource.Group, - "version": s.storageVersion.Version, - "resource": s.storageGroupResource.Resource, - "namespace": metaobj.GetNamespace(), - "name": metaobj.GetName(), - }).Updates(updatedResource) + result := s.db.WithContext(ctx). + Model(&Resource{}). + Where(map[string]interface{}{ + "cluster": cluster, + "group": s.storageGroupResource.Group, + "version": s.storageVersion.Version, + "resource": s.storageGroupResource.Resource, + "namespace": metaobj.GetNamespace(), + "name": metaobj.GetName(), + }). + Table(s.table). + Updates(updatedResource) + return InterpretResourceDBError(cluster, metaobj.GetName(), result.Error) } @@ -144,14 +150,17 @@ func (c *ResourceStorage) ConvertDeletedObject(obj interface{}) (runtime.Object, } func (s *ResourceStorage) deleteObject(cluster, namespace, name string) *gorm.DB { - return s.db.Model(&Resource{}).Where(map[string]interface{}{ - "cluster": cluster, - "group": s.storageGroupResource.Group, - "version": s.storageVersion.Version, - "resource": s.storageGroupResource.Resource, - "namespace": namespace, - "name": name, - }).Delete(&Resource{}) + return s.db.Model(&Resource{}). + Where(map[string]interface{}{ + "cluster": cluster, + "group": s.storageGroupResource.Group, + "version": s.storageVersion.Version, + "resource": s.storageGroupResource.Resource, + "namespace": namespace, + "name": name, + }). + Table(s.table). + Delete(&Resource{}) } func (s *ResourceStorage) Delete(ctx context.Context, cluster string, obj runtime.Object) error { @@ -167,7 +176,7 @@ func (s *ResourceStorage) Delete(ctx context.Context, cluster string, obj runtim } func (s *ResourceStorage) genGetObjectQuery(ctx context.Context, cluster, namespace, name string) *gorm.DB { - return s.db.WithContext(ctx).Model(&Resource{}).Select("object").Where(map[string]interface{}{ + return s.db.WithContext(ctx).Model(&Resource{}).Table(s.table).Select("object").Where(map[string]interface{}{ "cluster": cluster, "group": s.storageGroupResource.Group, "version": s.storageVersion.Version, @@ -199,7 +208,7 @@ func (s *ResourceStorage) genListObjectsQuery(ctx context.Context, opts *interna result = &ResourceMetadataList{} } - query := s.db.WithContext(ctx).Model(&Resource{}) + query := s.db.WithContext(ctx).Model(&Resource{}).Table(s.table) query = query.Where(map[string]interface{}{ "group": s.storageGroupResource.Group, "version": s.storageVersion.Version, diff --git a/pkg/storage/internalstorage/storage.go b/pkg/storage/internalstorage/storage.go index 25530b283..ed9dfa687 100644 --- a/pkg/storage/internalstorage/storage.go +++ b/pkg/storage/internalstorage/storage.go @@ -3,6 +3,8 @@ package internalstorage import ( "context" "fmt" + "strings" + "sync" "gorm.io/gorm" "k8s.io/apimachinery/pkg/runtime/schema" @@ -11,6 +13,8 @@ import ( "github.com/clusterpedia-io/clusterpedia/pkg/storage" ) +var mutex sync.Mutex + type StorageFactory struct { db *gorm.DB AutoMigration *bool @@ -19,34 +23,54 @@ type StorageFactory struct { } func (s *StorageFactory) AutoMigrate() error { + return nil +} + +func (s *StorageFactory) GetSupportedRequestVerbs() []string { + return []string{"get", "list"} +} + +func (s *StorageFactory) NewResourceStorage(config *storage.ResourceStorageConfig) (storage.ResourceStorage, error) { + mutex.Lock() + defer mutex.Unlock() + + var table string if s.AutoMigration != nil && *s.AutoMigration { switch s.DivisionPolicy { - if err := s.db.AutoMigrate(&Resource{}); err != nil { - return err - } case "", DivisionPolicyNone: + table = "resources" + + if exist := s.db.Migrator().HasTable(table); !exist { + if err := s.db.AutoMigrate(&Resource{}); err != nil { + return nil, err + } + } case DivisionPolicyGroupResource: + gvr := schema.GroupVersionResource{ + Group: config.StorageGroupResource.Group, + Version: config.StorageVersion.Version, + Resource: config.StorageGroupResource.Resource, + } - } + table = GenerateTableFor(gvr) + + if exist := s.db.Migrator().HasTable(table); !exist { + if err := s.db.AutoMigrate(&Resource{}); err != nil { + return nil, err + } - if s.DivisionPolicy == "" || s.DivisionPolicy == DivisionPolicyNone { - if err := s.db.AutoMigrate(&Resource{}); err != nil { - return err + err := s.db.Migrator().RenameTable("resources", table) + if err != nil { + return nil, err + } } } } - return nil -} - -func (s *StorageFactory) GetSupportedRequestVerbs() []string { - return []string{"get", "list"} -} - -func (s *StorageFactory) NewResourceStorage(config *storage.ResourceStorageConfig) (storage.ResourceStorage, error) { return &ResourceStorage{ db: s.db, codec: config.Codec, + table: table, storageGroupResource: config.StorageGroupResource, storageVersion: config.StorageVersion, @@ -116,3 +140,13 @@ func (s *StorageFactory) GetCollectionResources(ctx context.Context) ([]*interna func (s *StorageFactory) PrepareCluster(cluster string) error { return nil } + +// GenerateTableFor return table name using gvr string +func GenerateTableFor(gvr schema.GroupVersionResource) string { + if gvr.Group == "" { + return fmt.Sprintf("%s_%s", gvr.Version, gvr.Resource) + } + + group := strings.ReplaceAll(gvr.Group, ".", "_") + return fmt.Sprintf("%s_%s_%s", group, gvr.Version, gvr.Resource) +}