diff --git a/api/v1alpha1/lvm_volume_group_set.go b/api/v1alpha1/lvm_volume_group_set.go new file mode 100644 index 00000000..aee052a0 --- /dev/null +++ b/api/v1alpha1/lvm_volume_group_set.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type LVMVolumeGroupSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []LVMVolumeGroupSet `json:"items"` +} + +type LVMVolumeGroupSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LVMVolumeGroupSetSpec `json:"spec"` + Status LVMVolumeGroupSetStatus `json:"status,omitempty"` +} + +type LVMVolumeGroupSetSpec struct { + NodeSelector *metav1.LabelSelector `json:"nodeSelector"` + LVGTemplate LVMVolumeGroupTemplate `json:"lvmVolumeGroupTemplate"` + Strategy string `json:"strategy"` +} +type LVMVolumeGroupTemplate struct { + Metadata LVMVolumeGroupTemplateMeta `json:"metadata"` + BlockDeviceSelector *metav1.LabelSelector `json:"blockDeviceSelector"` + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + ThinPools []LVMVolumeGroupThinPoolSpec `json:"thinPools"` + Type string `json:"type"` +} + +type LVMVolumeGroupTemplateMeta struct { + Labels map[string]string `json:"labels"` +} + +type LVMVolumeGroupSetStatus struct { + CreatedLVGs []LVMVolumeGroupSetStatusLVG `json:"createdLVMVolumeGroups"` + CurrentLVMVolumeGroupsCount int `json:"currentLVMVolumeGroupsCount"` + DesiredLVMVolumeGroupsCount int `json:"desiredLVMVolumeGroupsCount"` + Phase string `json:"phase"` + Reason string `json:"reason"` +} + +type LVMVolumeGroupSetStatusLVG struct { + LVMVolumeGroupName string `json:"lvmVolumeGroupName"` + NodeName string `json:"nodeName"` +} diff --git a/api/v1alpha1/register.go b/api/v1alpha1/register.go index 55f47986..618aabbf 100644 --- a/api/v1alpha1/register.go +++ b/api/v1alpha1/register.go @@ -46,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &LVMVolumeGroupList{}, &LVMLogicalVolume{}, &LVMLogicalVolumeList{}, + &LVMVolumeGroupSet{}, + &LVMVolumeGroupSetList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 18da16ae..7bb10b24 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -188,3 +188,60 @@ func (in *LVMLogicalVolumeList) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroupSet) DeepCopyInto(out *LVMVolumeGroupSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyBlockDevice. +func (in *LVMVolumeGroupSet) DeepCopy() *LVMVolumeGroupSet { + if in == nil { + return nil + } + out := new(LVMVolumeGroupSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMVolumeGroupSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroupSetList) DeepCopyInto(out *LVMVolumeGroupSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LVMVolumeGroupSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuestbookList. +func (in *LVMVolumeGroupSetList) DeepCopy() *LVMVolumeGroupSetList { + if in == nil { + return nil + } + out := new(LVMVolumeGroupSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMVolumeGroupSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/crds/doc-ru-lvmvolumegroup.yaml b/crds/doc-ru-lvmvolumegroup.yaml index 23c0b779..b9bc54be 100644 --- a/crds/doc-ru-lvmvolumegroup.yaml +++ b/crds/doc-ru-lvmvolumegroup.yaml @@ -47,6 +47,9 @@ spec: Желаемый размер thin pool. Может быть указан как в численном, так и процентном отношении к общему размеру VG. > Обратите внимание, что при указании размера в процентах thin pool будет автоматически расширен при расширении VG. + allocationLimit: + description: | + Максимальная степень расширения thin pool-а. По умолчанию 150%. status: properties: phase: diff --git a/crds/doc-ru-lvmvolumegroupset.yaml b/crds/doc-ru-lvmvolumegroupset.yaml new file mode 100644 index 00000000..3e2b497f --- /dev/null +++ b/crds/doc-ru-lvmvolumegroupset.yaml @@ -0,0 +1,92 @@ +spec: + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: | + Интерфейс для одновременного создания нескольких LVMVolumeGroup ресурсов по общему шаблону. + properties: + spec: + properties: + strategy: + description: | + Стратегия (правила) создания LVMVolumeGroup ресурсов по текущему ресурсу. + nodeSelector: + description: | + Желаемый selector для узлов, которые будут использованы LVMVolumeGroup ресурсами. + properties: + matchLabels: + description: | + Желаемые метки. + matchExpressions: + description: | + Желаемые выражения. + lvmVolumeGroupTemplate: + description: | + Общий шаблон для LVMVolumeGroup ресурсов, созданных с помощью данного ресурса. + properties: + blockDeviceSelector: + description: | + Желаемый селектор для BlockDevice ресурсов, используемый в LVMVolumeGroup ресурсах. + properties: + matchLabels: + description: | + Желаемые метки. + matchExpressions: + description: | + Желаемые выражения. + metadata: + description: | + Метаинформация для LVMVolumeGroup ресурсов. + properties: + labels: + description: | + Обязательные метки для LVMVolumeGroup ресурсов. + type: + description: | + Тип Volume Group. Может быть: + - Local, то есть локальным, если используемые девайсы не являются распределенными (не Shared LUN). + actualVGNameOnTheNode: + description: | + Желаемое имя для Volume Group. Должно быть уникальным в рамках узла, на котором будет располагаться. + + > Неизменяемое поле. + > Обратите внимание, что указанное имя Volume Group будет одинаковым для каждого LVMVolumeGroup ресурса. + thinPools: + description: | + Желаемая конфигурация для Thin-pool'ов текущей Volume Group. + + > Обратите внимание, что данная конфигурация будет одинаковой для каждого LVMVolumeGroup ресурса. + items: + properties: + name: + description: | + Желаемое имя thin pool. + + > Неизменяемое поле. + size: + description: | + Желаемый размер thin pool. Может быть указан как в численном, так и процентном отношении к общему размеру VG. + + > Обратите внимание, что при указании размера в процентах thin pool будет автоматически расширен при расширении VG. + allocationLimit: + description: | + Максимальная степень расширения thin pool-а. По умолчанию 150%. + status: + type: object + properties: + createdLVMVolumeGroups: + description: | + Короткая информация о LVMVolumeGroup ресурсах, созданных по текущему ресурсу. + currentLVMVolumeGroupsCount: + description: | + Текущее количество созданных LVMVolumeGroup ресурсов. + desiredLVMVolumeGroupsCount: + description: | + Желаемое количество созданных LVMVolumeGroup ресурсов. + phase: + description: | + Показывает статус создания LVMVolumeGroup ресурсов. + reason: + description: | + Показывает причину текущего статуса. diff --git a/crds/lvmvolumegroupset.yaml b/crds/lvmvolumegroupset.yaml new file mode 100644 index 00000000..f973664b --- /dev/null +++ b/crds/lvmvolumegroupset.yaml @@ -0,0 +1,246 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: lvmvolumegroupsets.storage.deckhouse.io + labels: + heritage: deckhouse + module: storage +spec: + group: storage.deckhouse.io + scope: Cluster + names: + kind: LVMVolumeGroupSet + plural: lvmvolumegroupsets + singular: lvmvolumegroupset + shortNames: + - lvgset + preserveUnknownFields: false + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + description: | + An interface for creating several LVMVolumeGroups by common template at once. + required: + - spec + properties: + spec: + type: object + required: + - strategy + - lvmVolumeGroupTemplate + - nodeSelector + properties: + strategy: + type: string + description: | + The strategy (rule) to provide LVMVolumeGroup resources by the set. + x-kubernetes-validations: + - rule: self == oldSelf + message: "The strategy field is immutable." + enum: + - PerNode + nodeSelector: + type: object + description: | + The desired node selector for nodes which will be used by LVMVolumeGroups. + properties: + matchLabels: + type: object + description: | + The desired node selector labels. + additionalProperties: + type: string + matchExpressions: + type: array + description: | + The desired node selector expressions. + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + values: + type: array + items: + type: string + lvmVolumeGroupTemplate: + type: object + description: | + The common template for LVMVolumeGroup resources provided by the set. + required: + - type + - actualVGNameOnTheNode + - blockDeviceSelector + properties: + blockDeviceSelector: + type: object + description: | + The desired BlockDevice resources selector to provide in LVMVolumeGroups configurations. + properties: + matchLabels: + type: object + description: | + The desired block device selector labels. + additionalProperties: + type: string + matchExpressions: + type: array + description: | + The desired block device selector expressions. + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + values: + type: array + items: + type: string + metadata: + type: object + description: | + Metadata of LVMVolumeGroup resources. + properties: + labels: + type: object + additionalProperties: + type: string + description: | + Must-have labels for LVMVolumeGroup resources. + type: + type: string + description: | + The type of a VolumeGroup in LVMVolumeGroups. Might be: + - Local, that is, local if the devices used are not distributed (not Shared LUN). + enum: + - Local + x-kubernetes-validations: + - rule: self == oldSelf + message: "The type field is immutable." + actualVGNameOnTheNode: + type: string + description: | + The desired name of a Volume Group in LVMVolumeGroups. Must be unique for the node it is on. + + > This field is immutable. + > Note, that this Volume Group name will be common for every LVMVolumeGroup created by the set. + x-kubernetes-validations: + - rule: self == oldSelf + message: "The actualVGNameOnTheNode field is immutable." + thinPools: + type: array + description: | + The desired Thin-pool configuration in LVMVolumeGroups. + + > Note, that the configuration will be common for every LVMVolumeGroup created by the set. + items: + type: object + properties: + name: + type: string + description: | + The desired thin pool name. + + > This field is immutable. + size: + x-kubernetes-int-or-string: true + pattern: '^[0-9]+(\.[0-9]+)?(E|P|T|G|M|k|Ei|Pi|Ti|Gi|Mi|Ki)?$|^[1-9][0-9]?%$|100%' + description: | + The desired thin pool size. Might be specified as number or percent size of total VG space. + + > Note, that if you specify the percent size, the thin pool will be automatically extended when VG is extended. + allocationLimit: + type: string + pattern: '^[1-9][0-9]{2,3}%$' + default: "150%" + description: | + Thin pool oversize limit. Default is 150%. + required: + - name + - size + status: + type: object + properties: + createdLVMVolumeGroups: + type: array + description: | + Short information about LVMVolumeGroups created by the set. + items: + type: object + properties: + lvmVolumeGroupName: + type: string + nodeName: + type: string + currentLVMVolumeGroupsCount: + type: integer + description: | + Shows the current LVMVolumeGroup count created by the set. + desiredLVMVolumeGroupsCount: + type: integer + description: | + Shows the desired LVMVolumeGroup count created by the set. + phase: + type: string + description: | + Shows the LVMVolumeGroups creation phase. + enum: + - Created + - Pending + - NotCreated + - "" + reason: + type: string + description: | + Shows the reason of the phase. + subresources: + status: { } + additionalPrinterColumns: + - jsonPath: .spec.strategy + name: Strategy + type: string + description: The specified LVMVolumeGroups strategy creation. + - jsonPath: .spec.lvmVolumeGroupTemplate.actualVGNameOnTheNode + name: VG + type: string + description: Actual VG name. + - jsonPath: .spec.lvmVolumeGroupTemplate.type + name: type + type: string + description: Volume Group type. + priority: 1 + - jsonPath: .status.currentLVMVolumeGroupsCount + name: Current LVG Count + type: integer + description: Current LVMVolumeGroups count created by the set. + - jsonPath: .status.desiredLVMVolumeGroupsCount + name: Desired LVG Count + type: integer + description: Desired LVMVolumeGroups count created by the set. + - jsonPath: .status.phase + name: Phase + type: string + description: Resource phase. + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + description: The age of this resource diff --git a/images/agent/src/go.mod b/images/agent/src/go.mod index 94f92ebe..43018e69 100644 --- a/images/agent/src/go.mod +++ b/images/agent/src/go.mod @@ -3,7 +3,7 @@ module agent go 1.22.2 require ( - github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea + github.com/deckhouse/sds-node-configurator/api v0.0.0-20241103120640-3e41c48c48fa github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.19.0 diff --git a/images/agent/src/internal/const.go b/images/agent/src/internal/const.go index 5796f36e..7cd7d2a8 100644 --- a/images/agent/src/internal/const.go +++ b/images/agent/src/internal/const.go @@ -52,6 +52,7 @@ const ( ReasonTerminating = "Terminating" ReasonScanFailed = "ScanFailed" ReasonUpdated = "Updated" + ReasonApplied = "Applied" BlockDeviceLabelPrefix = "status.blockdevice.storage.deckhouse.io" diff --git a/images/agent/src/pkg/controller/lvm_volume_group_discover.go b/images/agent/src/pkg/controller/lvm_volume_group_discover.go index fa00a3db..a79d002f 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_discover.go +++ b/images/agent/src/pkg/controller/lvm_volume_group_discover.go @@ -418,7 +418,7 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m for _, vg := range vgWithTag { allocateSize := getVGAllocatedSize(vg) - health, message := checkVGHealth(sortedBDs, vgIssues, pvIssues, lvIssues, vg) + health, message := checkVGHealth(vgIssues, pvIssues, lvIssues, vg) candidate := internal.LVMVolumeGroupCandidate{ LVMVGName: generateLVMVGName(), @@ -434,7 +434,7 @@ func GetLVMVolumeGroupCandidates(log logger.Logger, sdsCache *cache.Cache, bds m VGSize: *resource.NewQuantity(vg.VGSize.Value(), resource.BinarySI), VGFree: *resource.NewQuantity(vg.VGFree.Value(), resource.BinarySI), VGUUID: vg.VGUUID, - Nodes: configureCandidateNodeDevices(sortedPVs, sortedBDs, vg, currentNode), + Nodes: configureCandidateNodeDevices(log, sortedPVs, sortedBDs, vg, currentNode), } candidates = append(candidates, candidate) @@ -449,13 +449,9 @@ func getVGAllocatedSize(vg internal.VGData) resource.Quantity { return allocatedSize } -func checkVGHealth(blockDevices map[string][]v1alpha1.BlockDevice, vgIssues map[string]string, pvIssues map[string][]string, lvIssues map[string]map[string]string, vg internal.VGData) (health, message string) { +func checkVGHealth(vgIssues map[string]string, pvIssues map[string][]string, lvIssues map[string]map[string]string, vg internal.VGData) (health, message string) { issues := make([]string, 0, len(vgIssues)+len(pvIssues)+len(lvIssues)+1) - if bds, exist := blockDevices[vg.VGName+vg.VGUUID]; !exist || len(bds) == 0 { - issues = append(issues, fmt.Sprintf("[ERROR] Unable to get BlockDevice resources for VG, name: %s ; uuid: %s", vg.VGName, vg.VGUUID)) - } - if vgIssue, exist := vgIssues[vg.VGName+vg.VGUUID]; exist { issues = append(issues, vgIssue) } @@ -617,7 +613,7 @@ func sortBlockDevicesByVG(bds map[string]v1alpha1.BlockDevice, vgs []internal.VG return result } -func configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) map[string][]internal.LVMVGDevice { +func configureCandidateNodeDevices(log logger.Logger, pvs map[string][]internal.PVData, bds map[string][]v1alpha1.BlockDevice, vg internal.VGData, currentNode string) map[string][]internal.LVMVGDevice { filteredPV := pvs[vg.VGName+vg.VGUUID] filteredBds := bds[vg.VGName+vg.VGUUID] bdPathStatus := make(map[string]v1alpha1.BlockDevice, len(bds)) @@ -628,16 +624,23 @@ func configureCandidateNodeDevices(pvs map[string][]internal.PVData, bds map[str } for _, pv := range filteredPV { + bd, exist := bdPathStatus[pv.PVName] + // this is very rare case which might occurred while VG extend operation goes. In this case, in the cache the controller + // sees a new PV included in the VG, but BlockDeviceDiscover did not update the corresponding BlockDevice resource on time, + // so the BlockDevice resource does not have any info, that it is in the VG. + if !exist { + log.Warning(fmt.Sprintf("[configureCandidateNodeDevices] no BlockDevice resource is yet configured for PV %s in VG %s, retry on the next iteration", pv.PVName, vg.VGName)) + continue + } + device := internal.LVMVGDevice{ Path: pv.PVName, PVSize: *resource.NewQuantity(pv.PVSize.Value(), resource.BinarySI), PVUUID: pv.PVUuid, } - if bd, exist := bdPathStatus[pv.PVName]; exist { - device.DevSize = *resource.NewQuantity(bd.Status.Size.Value(), resource.BinarySI) - device.BlockDevice = bd.Name - } + device.DevSize = *resource.NewQuantity(bd.Status.Size.Value(), resource.BinarySI) + device.BlockDevice = bd.Name result[currentNode] = append(result[currentNode], device) } diff --git a/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go b/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go index 07e419a4..19e1022b 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go +++ b/images/agent/src/pkg/controller/lvm_volume_group_discover_test.go @@ -76,15 +76,12 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { vgName = "testVg" vgUUID = "testUuid" ) - bds := map[string][]v1alpha1.BlockDevice{ - vgName + vgUUID: {{}}, - } vgIssues := map[string]string{} pvIssues := map[string][]string{} lvIssues := map[string]map[string]string{} vg := internal.VGData{VGName: vgName, VGUUID: vgUUID} - health, _ := checkVGHealth(bds, vgIssues, pvIssues, lvIssues, vg) + health, _ := checkVGHealth(vgIssues, pvIssues, lvIssues, vg) assert.Equal(t, health, internal.LVMVGHealthOperational) }) @@ -93,15 +90,14 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { vgName = "testVg" vgUUID = "testUuid" ) - bds := map[string][]v1alpha1.BlockDevice{ - vgName + vgUUID: {}, + vgIssues := map[string]string{ + vgName + vgUUID: "some-issue", } - vgIssues := map[string]string{} pvIssues := map[string][]string{} lvIssues := map[string]map[string]string{} vg := internal.VGData{VGName: vgName, VGUUID: vgUUID} - health, _ := checkVGHealth(bds, vgIssues, pvIssues, lvIssues, vg) + health, _ := checkVGHealth(vgIssues, pvIssues, lvIssues, vg) assert.Equal(t, health, internal.LVMVGHealthNonOperational) }) @@ -325,7 +321,7 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { mp := map[string][]v1alpha1.BlockDevice{vgName + vgUUID: bds} ar := map[string][]internal.PVData{vgName + vgUUID: pvs} - actual := configureCandidateNodeDevices(ar, mp, vg, nodeName) + actual := configureCandidateNodeDevices(log, ar, mp, vg, nodeName) assert.Equal(t, expected, actual) }) diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher.go b/images/agent/src/pkg/controller/lvm_volume_group_watcher.go index f1bc33e4..8baf7fb6 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher.go +++ b/images/agent/src/pkg/controller/lvm_volume_group_watcher.go @@ -118,7 +118,7 @@ func RunLVMVolumeGroupWatcherController( log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully removed the label %s from the LVMVolumeGroup %s", internal.LVGUpdateTriggerLabel, lvg.Name)) } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector.MatchLabels)) + log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] tries to get block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) blockDevices, err := GetAPIBlockDevices(ctx, cl, metrics, lvg.Spec.BlockDeviceSelector) if err != nil { log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to get BlockDevices. Retry in %s", cfg.BlockDeviceScanIntervalSec.String())) @@ -129,8 +129,9 @@ func RunLVMVolumeGroupWatcherController( return reconcile.Result{RequeueAfter: cfg.BlockDeviceScanIntervalSec}, nil } - log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector.MatchLabels)) + log.Debug(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] successfully got block device resources for the LVMVolumeGroup %s by the selector %v", lvg.Name, lvg.Spec.BlockDeviceSelector)) + blockDevices = filterBlockDevicesByNodeName(blockDevices, lvg.Spec.Local.NodeName) valid, reason := validateSpecBlockDevices(lvg, blockDevices) if !valid { log.Warning(fmt.Sprintf("[RunLVMVolumeGroupController] validation failed for the LVMVolumeGroup %s, reason: %s", lvg.Name, reason)) @@ -429,7 +430,7 @@ func reconcileLVGUpdateFunc(ctx context.Context, cl client.Client, log logger.Lo } log.Debug(fmt.Sprintf("[reconcileLVGUpdateFunc] tries to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, "Applied", "configuration has been applied") + err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "configuration has been applied") if err != nil { log.Error(err, fmt.Sprintf("[reconcileLVGUpdateFunc] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) return true, err @@ -518,7 +519,7 @@ func reconcileLVGCreateFunc(ctx context.Context, cl client.Client, log logger.Lo log.Debug(fmt.Sprintf("[reconcileLVGCreateFunc] successfully created thin-pools for the LVMVolumeGroup %s", lvg.Name)) } - err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, "Success", "all configuration has been applied") + err = updateLVGConditionIfNeeded(ctx, cl, log, lvg, v1.ConditionTrue, internal.TypeVGConfigurationApplied, internal.ReasonApplied, "all configuration has been applied") if err != nil { log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupWatcherController] unable to add a condition %s to the LVMVolumeGroup %s", internal.TypeVGConfigurationApplied, lvg.Name)) return true, err diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go b/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go index 85751f06..cde50ad1 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go +++ b/images/agent/src/pkg/controller/lvm_volume_group_watcher_func.go @@ -101,6 +101,15 @@ func shouldLVGWatcherReconcileUpdateEvent(log logger.Logger, oldLVG, newLVG *v1a return true } + for _, c := range newLVG.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied { + if c.Reason == internal.ReasonUpdating || c.Reason == internal.ReasonCreating { + log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should not be reconciled as the LVMVolumeGroup %s reconciliation still in progress", newLVG.Name)) + return false + } + } + } + if _, exist := newLVG.Labels[internal.LVGUpdateTriggerLabel]; exist { log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should be reconciled as the LVMVolumeGroup %s has the label %s", newLVG.Name, internal.LVGUpdateTriggerLabel)) return true @@ -116,15 +125,6 @@ func shouldLVGWatcherReconcileUpdateEvent(log logger.Logger, oldLVG, newLVG *v1a return true } - for _, c := range newLVG.Status.Conditions { - if c.Type == internal.TypeVGConfigurationApplied { - if c.Reason == internal.ReasonUpdating || c.Reason == internal.ReasonCreating { - log.Debug(fmt.Sprintf("[shouldLVGWatcherReconcileUpdateEvent] update event should not be reconciled as the LVMVolumeGroup %s reconciliation still in progress", newLVG.Name)) - return false - } - } - } - for _, n := range newLVG.Status.Nodes { for _, d := range n.Devices { if !utils.AreSizesEqualWithinDelta(d.PVSize, d.DevSize, internal.ResizeDelta) { @@ -294,18 +294,18 @@ func validateSpecBlockDevices(lvg *v1alpha1.LVMVolumeGroup, blockDevices map[str } } - bdFromOtherNode := make([]string, 0, len(blockDevices)) + return true, "" +} + +func filterBlockDevicesByNodeName(blockDevices map[string]v1alpha1.BlockDevice, nodeName string) map[string]v1alpha1.BlockDevice { + bdsForUsage := make(map[string]v1alpha1.BlockDevice, len(blockDevices)) for _, bd := range blockDevices { - if bd.Status.NodeName != lvg.Spec.Local.NodeName { - bdFromOtherNode = append(bdFromOtherNode, bd.Name) + if bd.Status.NodeName == nodeName { + bdsForUsage[bd.Name] = bd } } - if len(bdFromOtherNode) != 0 { - return false, fmt.Sprintf("block devices %s have different node names from LVMVolumeGroup Local.NodeName", strings.Join(bdFromOtherNode, ",")) - } - - return true, "" + return bdsForUsage } func deleteLVGIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, cfg config.Options, sdsCache *cache.Cache, lvg *v1alpha1.LVMVolumeGroup) (bool, error) { diff --git a/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go b/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go index 9d62a24a..bbb1290e 100644 --- a/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go +++ b/images/agent/src/pkg/controller/lvm_volume_group_watcher_test.go @@ -754,51 +754,6 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { assert.Equal(t, "these BlockDevices no longer match the blockDeviceSelector: first", reason) }) - t.Run("validation_fails_due_to_bd_has_dif_node", func(t *testing.T) { - const ( - nodeName = "nodeName" - ) - lvg := &v1alpha1.LVMVolumeGroup{ - Spec: v1alpha1.LVMVolumeGroupSpec{ - Local: v1alpha1.LVMVolumeGroupLocalSpec{ - NodeName: nodeName, - }, - BlockDeviceSelector: &v1.LabelSelector{ - MatchExpressions: []v1.LabelSelectorRequirement{ - { - Key: internal.MetadataNameLabelKey, - Operator: v1.LabelSelectorOpIn, - Values: []string{"first", "second"}, - }, - }, - }, - }, - } - - bds := map[string]v1alpha1.BlockDevice{ - "first": { - ObjectMeta: v1.ObjectMeta{ - Name: "first", - }, - Status: v1alpha1.BlockDeviceStatus{ - NodeName: nodeName, - }, - }, - "second": { - ObjectMeta: v1.ObjectMeta{ - Name: "second", - }, - Status: v1alpha1.BlockDeviceStatus{ - NodeName: "another-node", - }, - }, - } - - valid, reason := validateSpecBlockDevices(lvg, bds) - assert.False(t, valid) - assert.Equal(t, "block devices second have different node names from LVMVolumeGroup Local.NodeName", reason) - }) - t.Run("validation_fails_due_to_no_block_devices_were_found", func(t *testing.T) { const ( nodeName = "nodeName" @@ -1131,7 +1086,7 @@ func TestLVMVolumeGroupWatcherCtrl(t *testing.T) { newLVG.Status.Conditions = []v1.Condition{ { Type: internal.TypeVGConfigurationApplied, - Reason: internal.ReasonCreating, + Reason: internal.ReasonApplied, }, } newLVG.Labels = map[string]string{LVGMetadateNameLabelKey: "some-other-name"} diff --git a/images/agent/werf.inc.yaml b/images/agent/werf.inc.yaml index ea3a4be1..0128eabe 100644 --- a/images/agent/werf.inc.yaml +++ b/images/agent/werf.inc.yaml @@ -1,4 +1,4 @@ -{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.6-bullseye@sha256:260918a3795372a6d33225d361fe5349723be9667de865a23411b50fbcc76c5a" }} +{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.8-alpine@sha256:54bb7313917c733191a079ccae2e52bd3b80664e46c7879efa06513d4221d804" }} {{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.ru/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc" }} {{- $_ := set . "BASE_ALPINE_DEV" "registry.deckhouse.ru/base_images/dev-alpine:3.16.3@sha256:c706fa83cc129079e430480369a3f062b8178cac9ec89266ebab753a574aca8e" }} {{- $_ := set . "BASE_ALT_DEV" "registry.deckhouse.ru/base_images/dev-alt:p10@sha256:76e6e163fa982f03468166203488b569e6d9fc10855d6a259c662706436cdcad" }} diff --git a/images/sds-health-watcher-controller/src/cmd/main.go b/images/sds-health-watcher-controller/src/cmd/main.go index 708bc53d..fe8d30ce 100644 --- a/images/sds-health-watcher-controller/src/cmd/main.go +++ b/images/sds-health-watcher-controller/src/cmd/main.go @@ -130,6 +130,12 @@ func main() { os.Exit(1) } + err = controller.RunLVMVolumeGroupSetWatcher(mgr, *log, *cfgParams, metrics) + if err != nil { + log.Error(err, "[main] unable to run RunLVMVolumeGroupSetWatcher controller") + os.Exit(1) + } + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Error(err, "[main] unable to mgr.AddHealthzCheck") os.Exit(1) diff --git a/images/sds-health-watcher-controller/src/go.mod b/images/sds-health-watcher-controller/src/go.mod index d7f98e41..5eb1b4ad 100644 --- a/images/sds-health-watcher-controller/src/go.mod +++ b/images/sds-health-watcher-controller/src/go.mod @@ -4,7 +4,7 @@ go 1.22.3 require ( github.com/cloudflare/cfssl v1.5.0 - github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea + github.com/deckhouse/sds-node-configurator/api v0.0.0-20241103120640-3e41c48c48fa github.com/go-logr/logr v1.4.2 github.com/prometheus/client_golang v1.19.1 github.com/stretchr/testify v1.9.0 diff --git a/images/sds-health-watcher-controller/src/go.sum b/images/sds-health-watcher-controller/src/go.sum index 4d0847f3..6b82a2d4 100644 --- a/images/sds-health-watcher-controller/src/go.sum +++ b/images/sds-health-watcher-controller/src/go.sum @@ -18,10 +18,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckhouse/sds-node-configurator/api v0.0.0-20240805103635-969dc811217b h1:EYmHWTWcWMpyxJGZK05ZxlIFnh9s66DRrxLw/LNb/xw= -github.com/deckhouse/sds-node-configurator/api v0.0.0-20240805103635-969dc811217b/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0= -github.com/deckhouse/sds-node-configurator/api v0.0.0-20240905123334-64f17b70f035 h1:2kluZX0T5gk8YgNRk2bzd+m/mSkNmcKKaDHd6sVHP8I= -github.com/deckhouse/sds-node-configurator/api v0.0.0-20240905123334-64f17b70f035/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= diff --git a/images/sds-health-watcher-controller/src/pkg/controller/block_device_labels_watcher.go b/images/sds-health-watcher-controller/src/pkg/controller/block_device_labels_watcher.go index 8fc811a8..bafc6120 100644 --- a/images/sds-health-watcher-controller/src/pkg/controller/block_device_labels_watcher.go +++ b/images/sds-health-watcher-controller/src/pkg/controller/block_device_labels_watcher.go @@ -119,6 +119,12 @@ func reconcileBlockDeviceLabels(ctx context.Context, cl client.Client, log logge continue } + if checkIfLVGInProgress(&lvg) { + log.Warning(fmt.Sprintf("[reconcileBlockDeviceLabels] the LVMVolumeGroup %s is in a progress, retry later...", lvg.Name)) + shouldRetry = true + continue + } + log.Debug(fmt.Sprintf("[reconcileBlockDeviceLabels] tries to configure a selector from blockDeviceSelector of the LVMVolumeGroup %s", lvg.Name)) selector, err := metav1.LabelSelectorAsSelector(lvg.Spec.BlockDeviceSelector) if err != nil { @@ -157,6 +163,18 @@ func reconcileBlockDeviceLabels(ctx context.Context, cl client.Client, log logge return shouldRetry, nil } +func checkIfLVGInProgress(newLVG *v1alpha1.LVMVolumeGroup) bool { + for _, c := range newLVG.Status.Conditions { + if c.Type == internal.TypeVGConfigurationApplied { + if c.Reason == internal.ReasonUpdating || c.Reason == internal.ReasonCreating { + return true + } + } + } + + return false +} + func shouldTriggerLVGUpdateIfMatches(log logger.Logger, lvg *v1alpha1.LVMVolumeGroup, blockDevice *v1alpha1.BlockDevice, usedBdNames map[string]struct{}) bool { log.Debug(fmt.Sprintf("[reconcileBlockDeviceLabels] BlockDevice %s matches a blockDeviceSelector of the LVMVolumeGroup %s", blockDevice.Name, lvg.Name)) if _, used := usedBdNames[blockDevice.Name]; !used { diff --git a/images/sds-health-watcher-controller/src/pkg/controller/lvm_volume_group_set_watcher.go b/images/sds-health-watcher-controller/src/pkg/controller/lvm_volume_group_set_watcher.go new file mode 100644 index 00000000..64bfca4a --- /dev/null +++ b/images/sds-health-watcher-controller/src/pkg/controller/lvm_volume_group_set_watcher.go @@ -0,0 +1,344 @@ +package controller + +import ( + "context" + "fmt" + "reflect" + "time" + + "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "sds-health-watcher-controller/config" + "sds-health-watcher-controller/internal" + "sds-health-watcher-controller/pkg/logger" + "sds-health-watcher-controller/pkg/monitoring" +) + +const ( + LVMVolumeGroupSetCtrlName = "lvm-volume-group-set-watcher-controller" + + phaseCreated = "Created" + phaseNotCreated = "NotCreated" + + reasonWorkInProgress = "WorkInProgress" + strategyPerNode = "PerNode" +) + +func RunLVMVolumeGroupSetWatcher( + mgr manager.Manager, + log logger.Logger, + cfg config.Options, + metrics monitoring.Metrics, +) error { + cl := mgr.GetClient() + + c, err := controller.New(LVMVolumeGroupSetCtrlName, mgr, controller.Options{ + Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] tries to reconcile the request of the LVMVolumeGroupSet %s", request.Name)) + lvgSet := &v1alpha1.LVMVolumeGroupSet{} + err := cl.Get(ctx, request.NamespacedName, lvgSet) + if err != nil { + if errors.IsNotFound(err) { + log.Warning(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] seems like the LVMVolumeGroupSet %s has been deleted. Stop the reconcile", lvgSet.Name)) + return reconcile.Result{}, nil + } + + log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] unable to get the LVMVolumeGroupSet %s", request.Name)) + return reconcile.Result{}, err + } + + shouldRequeue, err := reconcileLVMVolumeGroupSet(ctx, cl, log, metrics, lvgSet) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] unable to reconcile the LVMVolumeGroupSet %s", lvgSet.Name)) + return reconcile.Result{}, err + } + + if shouldRequeue { + log.Warning(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] the LVMVolumeGroupSet %s request should be requeued in %s", lvgSet.Name, cfg.ScanIntervalSec.String())) + return reconcile.Result{RequeueAfter: cfg.ScanIntervalSec}, nil + } + + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] successfully reconciled the request of the LVMVolumeGroupSet %s", request.Name)) + return reconcile.Result{}, nil + }), + }) + if err != nil { + log.Error(err, "[RunLVMVolumeGroupSetWatcher] unable to create the controller") + return err + } + + err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.LVMVolumeGroupSet{}, handler.TypedFuncs[*v1alpha1.LVMVolumeGroupSet, reconcile.Request]{ + CreateFunc: func(_ context.Context, e event.TypedCreateEvent[*v1alpha1.LVMVolumeGroupSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] createFunc got a create event for the LVMVolumeGroupSet, name: %s", e.Object.GetName())) + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} + q.Add(request) + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] createFunc added a request for the LVMVolumeGroupSet %s to the Reconcilers queue", e.Object.GetName())) + }, + UpdateFunc: func(_ context.Context, e event.TypedUpdateEvent[*v1alpha1.LVMVolumeGroupSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] UpdateFunc got a update event for the LVMVolumeGroupSet %s", e.ObjectNew.GetName())) + if !shouldLVGSetWatcherReconcileUpdateEvent(e.ObjectOld, e.ObjectNew) { + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] update event for the LVMVolumeGroupSet %s should not be reconciled as not target changed were made", e.ObjectNew.Name)) + return + } + + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.ObjectNew.GetNamespace(), Name: e.ObjectNew.GetName()}} + q.Add(request) + log.Info(fmt.Sprintf("[RunLVMVolumeGroupSetWatcher] updateFunc added a request for the LVMVolumeGroupSet %s to the Reconcilers queue", e.ObjectNew.Name)) + }, + })) + if err != nil { + log.Error(err, "[RunLVMVolumeGroupSetWatcher] the controller is unable to watch the LVMVolumeGroupSet resources") + return err + } + + return nil +} + +func shouldLVGSetWatcherReconcileUpdateEvent(old, new *v1alpha1.LVMVolumeGroupSet) bool { + return !reflect.DeepEqual(old.Spec, new.Spec) +} + +func reconcileLVMVolumeGroupSet(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvgSet *v1alpha1.LVMVolumeGroupSet) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] starts the reconciliation of the LVMVolumeGroupSet %s", lvgSet.Name)) + err := updateLVMVolumeGroupSetPhaseIfNeeded(ctx, cl, log, lvgSet, internal.PhasePending, reasonWorkInProgress) + if err != nil { + return false, err + } + + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] tries to get nodes by the LVMVolumeGroupSet %s nodeSelector", lvgSet.Name)) + nodes, err := GetNodes(ctx, cl, metrics, lvgSet.Spec.NodeSelector) + if err != nil { + return false, err + } + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] successfully got nodes by the LVMVolumeGroupSet %s nodeSelector", lvgSet.Name)) + log.Trace(fmt.Sprintf("[reconcileLVMVolumeGroupSet] nodes: %+v", nodes)) + + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] starts to validate the LVMVolumeGroupSet %s nodes", lvgSet.Name)) + valid, reason := validateLVMVolumeGroupSetNodes(nodes) + if !valid { + log.Warning(fmt.Sprintf("[reconcileLVMVolumeGroupSet] the LVMVolumeGroupSet %s nodes are invalid: %s", lvgSet.Name, reason)) + err = updateLVMVolumeGroupSetPhaseIfNeeded(ctx, cl, log, lvgSet, phaseNotCreated, reason) + if err != nil { + return false, err + } + + return true, nil + } + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] the LVMVolumeGroupSet %s nodes are valid", lvgSet.Name)) + + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] tries to provide LVMVolumeGroups by the LVMVolumeGroupSet %s", lvgSet.Name)) + err = provideLVMVolumeGroupsBySet(ctx, cl, log, metrics, lvgSet, nodes) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLVMVolumeGroupSet] unable to provide LVMVolumeGroups by LVMVolumeGroupSet %s", lvgSet.Name)) + updErr := updateLVMVolumeGroupSetPhaseIfNeeded(ctx, cl, log, lvgSet, phaseNotCreated, err.Error()) + if updErr != nil { + return false, updErr + } + return false, err + } + log.Debug(fmt.Sprintf("[reconcileLVMVolumeGroupSet] successfully provided LVMVolumeGroups by the LVMVolumeGroupSet %s", lvgSet.Name)) + + err = updateLVMVolumeGroupSetPhaseIfNeeded(ctx, cl, log, lvgSet, phaseCreated, "") + if err != nil { + return false, err + } + + return false, nil +} + +func provideLVMVolumeGroupsBySet(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvgSet *v1alpha1.LVMVolumeGroupSet, nodes map[string]v1.Node) error { + //nolint:gocritic + switch lvgSet.Spec.Strategy { + case strategyPerNode: + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsBySet] the LVMVolumeGroupSet %s has strategy %s, tries to provide the LVMVolumeGroups", lvgSet.Name, strategyPerNode)) + err := provideLVMVolumeGroupsPerNode(ctx, cl, log, metrics, lvgSet, nodes) + if err != nil { + log.Error(err, fmt.Sprintf("[provideLVMVolumeGroupsBySet] unable to provide LVMVolumeGroups by the LVMVolumeGroupSet %s with strategy %s", lvgSet.Name, strategyPerNode)) + return err + } + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsBySet] successfully provided LVMVolumeGroups by the LVMVolumeGroupSet %s with strategy %s", lvgSet.Name, strategyPerNode)) + default: + return fmt.Errorf("LVMVolumeGroupSet %s strategy %s is not implemented", lvgSet.Name, lvgSet.Spec.Strategy) + } + + return nil +} + +func provideLVMVolumeGroupsPerNode(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, lvgSet *v1alpha1.LVMVolumeGroupSet, nodes map[string]v1.Node) error { + log.Debug("[provideLVMVolumeGroupsPerNode] tries to get LVMVolumeGroups") + currentLVGs, err := GetLVMVolumeGroups(ctx, cl, metrics) + if err != nil { + log.Error(err, "[provideLVMVolumeGroupsPerNode] unable to get LVMVolumeGroups") + return err + } + log.Debug("[provideLVMVolumeGroupsPerNode] successfully got LVMVolumeGroups") + log.Trace(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] current LVMVolumeGroups: %+v", currentLVGs)) + + for _, node := range nodes { + configuredLVG := configureLVGBySet(lvgSet, node) + log.Trace(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] configurated LVMVolumeGroup: %+v", configuredLVG)) + + currentLVG := matchConfiguredLVGWithExistingOne(configuredLVG, currentLVGs) + if currentLVG != nil { + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] tries to update the LVMVolumeGroup %s", currentLVG.Name)) + err = updateLVMVolumeGroupByConfiguredFromSet(ctx, cl, currentLVG, configuredLVG) + if err != nil { + log.Error(err, fmt.Sprintf("[provideLVMVolumeGroupsPerNode] unable to update the LVMVolumeGroup %s", currentLVG.Name)) + return err + } + log.Info(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] LVMVolumeGroup %s has been successfully updated", currentLVG.Name)) + } else { + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] tries to create the LVMVolumeGroup %s", configuredLVG.Name)) + err = createLVMVolumeGroup(ctx, cl, configuredLVG) + if err != nil { + log.Error(err, fmt.Sprintf("[provideLVMVolumeGroupsPerNode] unable to create the LVMVolumeGroup %s", configuredLVG.Name)) + return err + } + + log.Info(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] the LVMVolumeGroup %s has been created", configuredLVG.Name)) + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] tries to update the LVMVolumeGroupSet %s status by the created LVMVolumeGroup %s", lvgSet.Name, configuredLVG.Name)) + err = updateLVMVolumeGroupSetStatusByLVGIfNeeded(ctx, cl, log, lvgSet, configuredLVG, nodes) + if err != nil { + log.Error(err, fmt.Sprintf("[provideLVMVolumeGroupsPerNode] unable to update the LVMVolumeGroupSet %s", lvgSet.Name)) + return err + } + log.Debug(fmt.Sprintf("[provideLVMVolumeGroupsPerNode] successfully updated the LVMVolumeGroupSet %s status by the created LVMVolumeGroup %s", lvgSet.Name, configuredLVG.Name)) + } + } + + return nil +} + +func matchConfiguredLVGWithExistingOne(lvg *v1alpha1.LVMVolumeGroup, lvgs map[string]v1alpha1.LVMVolumeGroup) *v1alpha1.LVMVolumeGroup { + for _, l := range lvgs { + if l.Spec.Local.NodeName == lvg.Spec.Local.NodeName && l.Spec.ActualVGNameOnTheNode == lvg.Spec.ActualVGNameOnTheNode { + return &l + } + } + + return nil +} + +func updateLVMVolumeGroupByConfiguredFromSet(ctx context.Context, cl client.Client, existing, configured *v1alpha1.LVMVolumeGroup) error { + existing.Spec.ThinPools = configured.Spec.ThinPools + existing.Spec.BlockDeviceSelector = configured.Spec.BlockDeviceSelector + + return cl.Update(ctx, existing) +} + +func updateLVMVolumeGroupSetStatusByLVGIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, lvgSet *v1alpha1.LVMVolumeGroupSet, lvg *v1alpha1.LVMVolumeGroup, nodes map[string]v1.Node) error { + for _, createdLVG := range lvgSet.Status.CreatedLVGs { + if createdLVG.LVMVolumeGroupName == lvg.Name { + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetStatusByLVGIfNeeded] no need to update the LVMVolumeGroupSet status %s by the LVMVolumeGroup %s", lvgSet.Name, lvg.Name)) + return nil + } + } + + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetStatusByLVGIfNeeded] the LVMVolumeGroupSet status %s should be updated by the LVMVolumeGroup %s", lvgSet.Name, lvg.Name)) + if cap(lvgSet.Status.CreatedLVGs) == 0 { + lvgSet.Status.CreatedLVGs = make([]v1alpha1.LVMVolumeGroupSetStatusLVG, 0, len(nodes)) + } + + lvgSet.Status.CreatedLVGs = append(lvgSet.Status.CreatedLVGs, v1alpha1.LVMVolumeGroupSetStatusLVG{ + LVMVolumeGroupName: lvg.Name, + NodeName: lvg.Spec.Local.NodeName, + }) + + lvgSet.Status.CurrentLVMVolumeGroupsCount = len(lvgSet.Status.CreatedLVGs) + lvgSet.Status.DesiredLVMVolumeGroupsCount = len(nodes) + + return cl.Status().Update(ctx, lvgSet) +} + +func createLVMVolumeGroup(ctx context.Context, cl client.Client, lvg *v1alpha1.LVMVolumeGroup) error { + return cl.Create(ctx, lvg) +} + +func configureLVGBySet(lvgSet *v1alpha1.LVMVolumeGroupSet, node v1.Node) *v1alpha1.LVMVolumeGroup { + return &v1alpha1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: configureLVGNameFromSet(lvgSet), + Labels: lvgSet.Spec.LVGTemplate.Metadata.Labels, + }, + Spec: v1alpha1.LVMVolumeGroupSpec{ + ActualVGNameOnTheNode: lvgSet.Spec.LVGTemplate.ActualVGNameOnTheNode, + BlockDeviceSelector: lvgSet.Spec.LVGTemplate.BlockDeviceSelector, + ThinPools: lvgSet.Spec.LVGTemplate.ThinPools, + Type: lvgSet.Spec.LVGTemplate.Type, + Local: v1alpha1.LVMVolumeGroupLocalSpec{ + NodeName: node.Name, + }, + }, + } +} + +func configureLVGNameFromSet(lvgSet *v1alpha1.LVMVolumeGroupSet) string { + return fmt.Sprintf("%s-%d", lvgSet.Name, len(lvgSet.Status.CreatedLVGs)) +} + +func GetNodes(ctx context.Context, cl client.Client, metrics monitoring.Metrics, selector *metav1.LabelSelector) (map[string]v1.Node, error) { + list := &v1.NodeList{} + s, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, err + } + if s == labels.Nothing() { + s = nil + } + start := time.Now() + err = cl.List(ctx, list, &client.ListOptions{LabelSelector: s}) + metrics.APIMethodsDuration(LVMVolumeGroupSetCtrlName, "list").Observe(metrics.GetEstimatedTimeInSeconds(start)) + metrics.APIMethodsExecutionCount(LVMVolumeGroupSetCtrlName, "list").Inc() + if err != nil { + metrics.APIMethodsErrors(LVMVolumeGroupSetCtrlName, "list").Inc() + return nil, err + } + + result := make(map[string]v1.Node, len(list.Items)) + for _, item := range list.Items { + result[item.Name] = item + } + + return result, nil +} + +func updateLVMVolumeGroupSetPhaseIfNeeded(ctx context.Context, cl client.Client, log logger.Logger, lvgSet *v1alpha1.LVMVolumeGroupSet, phase, reason string) error { + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetPhaseIfNeeded] tries to update the LVMVolumeGroupSet %s status phase to %s and reason to %s", lvgSet.Name, phase, reason)) + if lvgSet.Status.Phase == phase && lvgSet.Status.Reason == reason { + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetPhaseIfNeeded] no need to update phase or reason of the LVMVolumeGroupSet %s as they are same", lvgSet.Name)) + return nil + } + + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetPhaseIfNeeded] the LVMVolumeGroupSet %s status phase %s and reason %s should be updated to the phase %s and reason %s", lvgSet.Name, lvgSet.Status.Phase, lvgSet.Status.Reason, phase, reason)) + lvgSet.Status.Phase = phase + lvgSet.Status.Reason = reason + err := cl.Status().Update(ctx, lvgSet) + if err != nil { + log.Error(err, fmt.Sprintf("[updateLVMVolumeGroupSetPhaseIfNeeded] unable to update the LVMVolumeGroupSet %s", lvgSet.Name)) + return err + } + + log.Debug(fmt.Sprintf("[updateLVMVolumeGroupSetPhaseIfNeeded] successfully updated the LVMVolumeGroupSet %s to phase %s and reason %s", lvgSet.Name, phase, reason)) + return nil +} + +func validateLVMVolumeGroupSetNodes(nodes map[string]v1.Node) (bool, string) { + if len(nodes) == 0 { + return false, "no nodes found by specified nodeSelector" + } + + return true, "" +} diff --git a/images/sds-health-watcher-controller/werf.inc.yaml b/images/sds-health-watcher-controller/werf.inc.yaml index 882bbae1..bea9b7be 100644 --- a/images/sds-health-watcher-controller/werf.inc.yaml +++ b/images/sds-health-watcher-controller/werf.inc.yaml @@ -1,4 +1,4 @@ -{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.6-bullseye@sha256:260918a3795372a6d33225d361fe5349723be9667de865a23411b50fbcc76c5a" }} +{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.8-alpine@sha256:54bb7313917c733191a079ccae2e52bd3b80664e46c7879efa06513d4221d804" }} {{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc" }} --- diff --git a/images/sds-utils-installer/werf.inc.yaml b/images/sds-utils-installer/werf.inc.yaml index e105e6aa..fcf6039c 100644 --- a/images/sds-utils-installer/werf.inc.yaml +++ b/images/sds-utils-installer/werf.inc.yaml @@ -1,4 +1,4 @@ -{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.6-bullseye@sha256:260918a3795372a6d33225d361fe5349723be9667de865a23411b50fbcc76c5a" }} +{{- $_ := set . "BASE_GOLANG" "registry.deckhouse.io/base_images/golang:1.22.8-alpine@sha256:54bb7313917c733191a079ccae2e52bd3b80664e46c7879efa06513d4221d804" }} {{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.ru/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc" }} {{- $_ := set . "BASE_ALPINE_DEV" "registry.deckhouse.ru/base_images/dev-alpine:3.16.3@sha256:c706fa83cc129079e430480369a3f062b8178cac9ec89266ebab753a574aca8e" }} {{- $_ := set . "BASE_ALT_DEV" "registry.deckhouse.ru/base_images/dev-alt:p10@sha256:76e6e163fa982f03468166203488b569e6d9fc10855d6a259c662706436cdcad" }} diff --git a/templates/agent/daemonset.yaml b/templates/agent/daemonset.yaml index 8f8af27a..a3dcc0e6 100644 --- a/templates/agent/daemonset.yaml +++ b/templates/agent/daemonset.yaml @@ -97,7 +97,7 @@ spec: requests: {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "static_utils_copier_resources" . | nindent 14 }} + {{- include "sds_utils_installer_resources" . | nindent 14 }} {{- end }} {{- end }} containers: diff --git a/templates/sds-health-watcher-controller/rbac-for-us.yaml b/templates/sds-health-watcher-controller/rbac-for-us.yaml index 7ce3e114..7f759dfa 100644 --- a/templates/sds-health-watcher-controller/rbac-for-us.yaml +++ b/templates/sds-health-watcher-controller/rbac-for-us.yaml @@ -25,14 +25,32 @@ rules: - apiGroups: - "storage.deckhouse.io" resources: - - lvmvolumegroups - - lvmvolumegroups/status - blockdevices verbs: - get - list + - watch + - apiGroups: + - "storage.deckhouse.io" + resources: + - lvmvolumegroupsets + - lvmvolumegroupsets/status + verbs: + - get + - list + - watch - update + - apiGroups: + - "storage.deckhouse.io" + resources: + - lvmvolumegroups + - lvmvolumegroups/status + verbs: + - get + - list - watch + - update + - create - apiGroups: - "" resources: @@ -48,21 +66,21 @@ rules: - leases verbs: - get - - watch - list - - delete + - watch - update - create - - verbs: + - delete + - apiGroups: + - "deckhouse.io" + resources: + - "moduleconfigs" + verbs: - get - list + - watch - update - patch - - watch - apiGroups: - - "deckhouse.io" - resources: - - "moduleconfigs" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding