From 908d5920ad001da67fd60c6022deae8d1feae74c Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Wed, 17 Jan 2024 19:33:12 +0300 Subject: [PATCH 01/15] first-draft Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/cmd/bc/main.go | 17 ++++-- .../controller/lvm_logical_volume_watcher.go | 55 ++++++++++++++++--- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/images/agent/cmd/bc/main.go b/images/agent/cmd/bc/main.go index 739e992b..c302b5f7 100644 --- a/images/agent/cmd/bc/main.go +++ b/images/agent/cmd/bc/main.go @@ -110,28 +110,33 @@ func main() { } log.Info("[main] ReTag ends") - if _, err := controller.RunBlockDeviceController(ctx, mgr, *cfgParams, *log, metrics); err != nil { + if _, err = controller.RunBlockDeviceController(ctx, mgr, *cfgParams, *log, metrics); err != nil { log.Error(err, "[main] unable to controller.RunBlockDeviceController") os.Exit(1) } - if _, err := controller.RunLVMVolumeGroupWatcherController(mgr, *cfgParams, *log, metrics); err != nil { - log.Error(err, "[main] error Run RunLVMVolumeGroupController") + if _, err = controller.RunLVMVolumeGroupWatcherController(mgr, *cfgParams, *log, metrics); err != nil { + log.Error(err, "[main] unable to controller.RunLVMVolumeGroupWatcherController") os.Exit(1) } - if _, err := controller.RunLVMVolumeGroupDiscoverController(ctx, mgr, *cfgParams, *log, metrics); err != nil { + if _, err = controller.RunLVMVolumeGroupDiscoverController(ctx, mgr, *cfgParams, *log, metrics); err != nil { log.Error(err, "[main] unable to controller.RunLVMVolumeGroupDiscoverController") os.Exit(1) } - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + if _, err = controller.RunLVMLogicalVolumeWatcherController(mgr, *cfgParams, *log, metrics); err != nil { + log.Error(err, "[main] unable to controller.RunLVMLogicalVolumeWatcherController") + os.Exit(1) + } + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Error(err, "[main] unable to mgr.AddHealthzCheck") os.Exit(1) } log.Info("[main] successfully AddHealthzCheck") - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { log.Error(err, "[main] unable to mgr.AddReadyzCheck") os.Exit(1) } diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index e0195854..dba6257f 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -2,22 +2,26 @@ package controller import ( "context" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/config" "sds-node-configurator/pkg/logger" "sds-node-configurator/pkg/monitoring" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + "time" ) const ( LVMLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" ) -func RunLVMLogicalVolumeController( +func RunLVMLogicalVolumeWatcherController( mgr manager.Manager, cfg config.Options, log logger.Logger, @@ -28,24 +32,59 @@ func RunLVMLogicalVolumeController( c, err := controller.New(LVMLogicalVolumeWatcherCtrlName, mgr, controller.Options{ Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil + log.Info("[RunLVMLogicalVolumeWatcherController] HELLO FROM RECONCILER") + return reconcile.Result{ + RequeueAfter: 5 * time.Second, + }, nil }), }) if err != nil { - log.Error(err, "[RunLVMLogicalVolumeController] unable to create controller") + log.Error(err, "[RunLVMLogicalVolumeWatcherController] unable to create controller") return nil, err } + // lvremove + createFunc := func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info("[RunLVMLogicalVolumeWatcherController] Hello from CreateFunc") + request := reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: e.Object.GetNamespace(), + Name: e.Object.GetName(), + }} + log.Info("NAME:" + request.Name) + log.Info("NAMESPACE:" + request.Namespace) + // Если создается ресурс llv, я + // - должен получить информацию о том, где я должен создавать PV + // - я проверяю возможность создания PV в указанном месте + // - пытаюсь создать PV нужного размера + // - если создаю, то обновляю статус ресурса успехом, проставляю актуальные данные + // - если нет, кидаю ошибку и сообщение + } + err = c.Watch(source.Kind(cache, &v1alpha1.LvmLogicalVolume{}), handler.Funcs{ - CreateFunc: nil, - UpdateFunc: nil, - DeleteFunc: nil, - GenericFunc: nil, + CreateFunc: createFunc, }) if err != nil { - log.Error(err, "[RunLVMLogicalVolumeController] the controller is unable to watch") + log.Error(err, "[RunLVMLogicalVolumeWatcherController] the controller is unable to watch") return nil, err } return c, err } + +func updateFunc() { + // Если обновляется спека llv, я + // - проверяю текущий статус ресурса + // - если статус соответствует спеке, ничего не делаю + // - если статус не соответствует спеке, тогда + // - я проверяю возможность создания PV в указанном месте + // - пытаюсь создать PV нужного размера + // - если создаю, то обновляю статус ресурса успехом, проставляю актуальные данные + // - если нет, кидаю ошибку и сообщение +} + +func deleteFunc() { + // Если удаляется ресурс llv, я + // - получаю данные для удаления из спеки ресурса + // - выполняю удаления + // - если удачно, то +} From d9c2b166c3d6f66ea3186e6ec36754cb94c10ec6 Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Fri, 19 Jan 2024 13:32:52 +0300 Subject: [PATCH 02/15] first version of createFunc Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- .../controller/lvm_logical_volume_watcher.go | 222 +++++++++++++++--- .../controller/lvm_volume_group_watcher.go | 8 +- .../lvm_volume_group_watcher_func.go | 5 +- images/agent/pkg/utils/commands.go | 39 ++- 4 files changed, 235 insertions(+), 39 deletions(-) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index dba6257f..5d90a4d5 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -2,40 +2,48 @@ package controller import ( "context" + "fmt" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/config" "sds-node-configurator/pkg/logger" "sds-node-configurator/pkg/monitoring" + "sds-node-configurator/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "time" ) const ( - LVMLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" + LVMLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" + CreatedStatusPhase = "Created" + PendingStatusPhase = "Pending" + ResizingStatusPhase = "Resizing" + FailedStatusPhase = "Failed" + Thick DeviceType = "Thick" + Thin DeviceType = "Thin" ) +type DeviceType string + func RunLVMLogicalVolumeWatcherController( mgr manager.Manager, cfg config.Options, log logger.Logger, metrics monitoring.Metrics, ) (controller.Controller, error) { - //cl := mgr.GetClient() + cl := mgr.GetClient() cache := mgr.GetCache() c, err := controller.New(LVMLogicalVolumeWatcherCtrlName, mgr, controller.Options{ Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log.Info("[RunLVMLogicalVolumeWatcherController] HELLO FROM RECONCILER") - return reconcile.Result{ - RequeueAfter: 5 * time.Second, - }, nil + return reconcile.Result{}, nil }), }) if err != nil { @@ -43,25 +51,17 @@ func RunLVMLogicalVolumeWatcherController( return nil, err } - // lvremove - createFunc := func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { - log.Info("[RunLVMLogicalVolumeWatcherController] Hello from CreateFunc") - request := reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: e.Object.GetNamespace(), - Name: e.Object.GetName(), - }} - log.Info("NAME:" + request.Name) - log.Info("NAMESPACE:" + request.Namespace) - // Если создается ресурс llv, я - // - должен получить информацию о том, где я должен создавать PV - // - я проверяю возможность создания PV в указанном месте - // - пытаюсь создать PV нужного размера - // - если создаю, то обновляю статус ресурса успехом, проставляю актуальные данные - // - если нет, кидаю ошибку и сообщение - } - err = c.Watch(source.Kind(cache, &v1alpha1.LvmLogicalVolume{}), handler.Funcs{ - CreateFunc: createFunc, + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info("[RunLVMLogicalVolumeWatcherController] Watcher.CreateFunc starts reconciliation") + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: e.Object.GetNamespace(), + Name: e.Object.GetName(), + }} + + reconcileFunc(ctx, cl, log, metrics, req, cfg.NodeName) + log.Info("[RunLVMLogicalVolumeWatcherController] Watcher.CreateFunc ends reconciliation") + }, }) if err != nil { log.Error(err, "[RunLVMLogicalVolumeWatcherController] the controller is unable to watch") @@ -71,6 +71,174 @@ func RunLVMLogicalVolumeWatcherController( return c, err } +func reconcileFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, req reconcile.Request, nodeName string) { + // Получаю ресурс + log.Info("[reconcileFunc] starts reconciliation") + log.Debug(fmt.Sprintf("[reconcileFunc] reconciles LVMLogicalVolume, name: %s", req.Name)) + llv, err := getLVMLogicalVolume(ctx, metrics, cl, req) + if err != nil { + log.Error(err, "[reconcileFunc] unable to getLVMLogicalVolume") + return + } + + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, req.Namespace, llv.Spec.LvmVolumeGroup) + if err != nil { + log.Error(err, "[reconcileFunc] unable to getLVMVolumeGroup") + return + } + + if !belongsToNode(lvg, nodeName) { + log.Debug(fmt.Sprintf("[reconcileFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) + return + } + log.Debug(fmt.Sprintf("[reconcileFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) + + err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, PendingStatusPhase, "") + if err != nil { + log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + } + log.Debug(fmt.Sprintf("[reconcileFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, PendingStatusPhase)) + + log.Debug(fmt.Sprintf("[reconcileFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) + // добавить проверки на наличие нужного кол-ва места + switch getLVMLogicalVolumeType(llv) { + case Thick: + freeSpace, err := getFreeVGSpace(log, lvg) + if err != nil { + log.Error(err, "[reconcileFunc] unable to getFreeVGSpace") + } + + if freeSpace.Value() < llv.Spec.Size.Value() { + log.Warning(fmt.Sprintf("[reconcileFunc] not enough space left on the VG %s for the LVMLogicalVolume %s", lvg.Spec.ActualVGNameOnTheNode, llv.Name)) + return + } + + cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.String()) + log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") + err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, FailedStatusPhase, "Unable to create Thick LV") + if err != nil { + log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + case Thin: + cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.String()) + log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") + err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, FailedStatusPhase, "Unable to create Thin LV") + if err != nil { + log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + } + + //if llv.Spec.Thin.PoolName == "" { + // cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.String()) + // log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + // if err != nil { + // log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") + // llv.Status.Phase = FailedStatusPhase + // llv.Status.Reason = err.Error() + // return + // } + //} else { + // cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.String()) + // log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + // if err != nil { + // log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") + // llv.Status.Phase = FailedStatusPhase + // llv.Status.Reason = err.Error() + // return + // } + //} + log.Debug(fmt.Sprintf("[reconcileFunc] successfully created Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) + + llv.Status.Phase = CreatedStatusPhase + llv.Status.ActualSize = llv.Spec.Size + err = updateLVMLogicalVolume(ctx, metrics, cl, llv) + if err != nil { + log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolume") + return + } + + log.Info("[reconcileFunc] ends reconciliation") +} + +func getFreeVGSpace(log logger.Logger, lvg *v1alpha1.LvmVolumeGroup) (resource.Quantity, error) { + vgs, cmd, _, err := utils.GetAllVGs() + log.Debug(fmt.Sprintf("[getFreeVGSpace] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[getFreeVGSpace] unable to run cmd") + return resource.Quantity{}, err + } + + for _, vg := range vgs { + if vg.VGName == lvg.Spec.ActualVGNameOnTheNode { + return vg.VGFree, nil + } + } + + return resource.Quantity{}, nil +} + +func getLVMLogicalVolumeType(llv *v1alpha1.LvmLogicalVolume) DeviceType { + if llv.Spec.Thin.PoolName == "" { + return Thick + } + + return Thin +} + +func belongsToNode(lvg *v1alpha1.LvmVolumeGroup, nodeName string) bool { + var belongs bool + for _, node := range lvg.Status.Nodes { + if node.Name == nodeName { + belongs = true + } + } + + return belongs +} + +func updateLVMLogicalVolumePhase(ctx context.Context, metrics monitoring.Metrics, cl client.Client, llv *v1alpha1.LvmLogicalVolume, phase, reason string) error { + llv.Status.Phase = phase + llv.Status.Reason = reason + + err := updateLVMLogicalVolume(ctx, metrics, cl, llv) + if err != nil { + return err + } + + return nil +} + +func updateLVMLogicalVolume(ctx context.Context, metrics monitoring.Metrics, cl client.Client, llv *v1alpha1.LvmLogicalVolume) error { + err := cl.Update(ctx, llv) + if err != nil { + return err + } + + return nil +} + +func getLVMLogicalVolume(ctx context.Context, metrics monitoring.Metrics, cl client.Client, req reconcile.Request) (*v1alpha1.LvmLogicalVolume, error) { + llv := &v1alpha1.LvmLogicalVolume{} + + err := cl.Get(ctx, client.ObjectKey{ + Namespace: req.Namespace, + Name: req.Name, + }, llv) + if err != nil { + return nil, err + } + + return llv, err +} + func updateFunc() { // Если обновляется спека llv, я // - проверяю текущий статус ресурса @@ -84,7 +252,5 @@ func updateFunc() { func deleteFunc() { // Если удаляется ресурс llv, я - // - получаю данные для удаления из спеки ресурса - // - выполняю удаления - // - если удачно, то + // - удаляю llv } diff --git a/images/agent/pkg/controller/lvm_volume_group_watcher.go b/images/agent/pkg/controller/lvm_volume_group_watcher.go index 5ec29ec0..72f12f2d 100644 --- a/images/agent/pkg/controller/lvm_volume_group_watcher.go +++ b/images/agent/pkg/controller/lvm_volume_group_watcher.go @@ -404,12 +404,12 @@ func ReconcileLVMVG( log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateEventLVMVolumeGroup, resource name: %s", group.Name)) } start := time.Now() - command, err := utils.CreateLV(pool, group.Spec.ActualVGNameOnTheNode) + command, err := utils.CreateThinPool(pool, group.Spec.ActualVGNameOnTheNode) metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "lvcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() log.Debug(command) if err != nil { - log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateLV, thin pool: %s", pool.Name)) + log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateThinPool, thin pool: %s", pool.Name)) metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() if err = updateLVMVolumeGroupHealthStatus(ctx, cl, metrics, group.Name, group.Namespace, err.Error(), NonOperational); err != nil { log.Error(err, fmt.Sprintf("[ReconcileLVMVG] unable to update LVMVolumeGroupStatus, resource name: %s", group.Name)) @@ -496,13 +496,13 @@ func ReconcileLVMVG( if len(group.Spec.ThinPools) != 0 { for _, v := range group.Spec.ThinPools { start := time.Now() - command, err := utils.CreateLV(v, group.Spec.ActualVGNameOnTheNode) + command, err := utils.CreateThinPool(v, group.Spec.ActualVGNameOnTheNode) metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "lvcreate").Observe(metrics.GetEstimatedTimeInSeconds(start)) metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() log.Debug(command) if err != nil { metrics.UtilsCommandsErrorsCount(LVMVolumeGroupWatcherCtrlName, "lvcreate").Inc() - log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateLV, thin pool: %s", v.Name)) + log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateThinPool, thin pool: %s", v.Name)) if err = updateLVMVolumeGroupHealthStatus(ctx, cl, metrics, group.Name, group.Namespace, err.Error(), NonOperational); err != nil { log.Error(err, fmt.Sprintf("[ReconcileLVMVG] unable to update LVMVolumeGroupStatus, resource name: %s", group.Name)) } diff --git a/images/agent/pkg/controller/lvm_volume_group_watcher_func.go b/images/agent/pkg/controller/lvm_volume_group_watcher_func.go index bd8b38f1..92597f0d 100644 --- a/images/agent/pkg/controller/lvm_volume_group_watcher_func.go +++ b/images/agent/pkg/controller/lvm_volume_group_watcher_func.go @@ -135,13 +135,10 @@ func ValidateLVMGroup(ctx context.Context, cl client.Client, metrics monitoring. status.Health = NonOperational status.Phase = Failed status.Message = "selected block devices are from different nodes for local LVMVolumeGroup" - return false, &status, nil + return false, &status, errors.New("wrong block devices selected") } if membership == 0 { - status.Health = NonOperational - status.Phase = Failed - status.Message = "selected block devices not affiliated to current Watcher's node" return false, &status, nil } } diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index 47d0f991..de2c8814 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -172,8 +172,7 @@ func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { return cmd.String(), nil } -func CreateLV(thinPool v1alpha1.SpecThinPool, VGName string) (string, error) { - +func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, error) { cmd := exec.Command( "lvcreate", "-L", thinPool.Size.String(), "-T", fmt.Sprintf("%s/%s", VGName, thinPool.Name)) @@ -181,11 +180,45 @@ func CreateLV(thinPool v1alpha1.SpecThinPool, VGName string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to CreateLV, err: %w tderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to CreateThinPool, err: %w tderr = %s", err, stderr.String()) } return cmd.String(), nil } +func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error) { + args := []string{"-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", size} + cmd := exec.Command("lvcreate", args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + var stdout bytes.Buffer + cmd.Stdout = &stdout + + fmt.Println("RUNS COMMAND:" + cmd.String()) + fmt.Println("CMD IS ABOUT TO RUN") + err := cmd.Run() + fmt.Println(stderr.String()) + fmt.Println(stdout.String()) + if err != nil { + fmt.Println("CMD ERROR ALERT") + return cmd.String(), err + } + + fmt.Println("CMD ALL GOOD") + return cmd.String(), nil +} + +func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { + args := []string{"-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", size, "-W", "y", "-y"} + cmd := exec.Command("lvcreate", args...) + + if err := cmd.Run(); err != nil { + return cmd.String(), err + } + + return cmd.String(), nil +} + func ExtendVG(vgName string, paths []string) (string, error) { var arg []string arg = append(arg, vgName) From 3dc111e999f69600406b36d631d499c45b5a2813 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Fri, 19 Jan 2024 13:39:04 +0300 Subject: [PATCH 03/15] Add hostPID true and remove volume mounts Signed-off-by: Aleksandr Zimin Signed-off-by: Viktor Kramarenko --- templates/agent/daemonset.yaml | 51 +--------------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/templates/agent/daemonset.yaml b/templates/agent/daemonset.yaml index db681713..a6565449 100644 --- a/templates/agent/daemonset.yaml +++ b/templates/agent/daemonset.yaml @@ -49,54 +49,5 @@ spec: {{- else if eq .Values.sdsNodeConfigurator.logLevel "TRACE" }} value: "4" {{- end }} - volumeMounts: - - mountPath: /dev/ - name: host-device-dir - - mountPath: /lib/modules - name: host-modules-dir - mountPropagation: Bidirectional - - mountPath: /sys/ - name: host-sys-dir - - mountPath: /run/udev/ - name: host-run-udev-dir - - mountPath: /run/lvm/ - name: host-run-lvm-dir - - mountPath: /run/dmeventd-client - name: host-run-dmeventd-client - - mountPath: /run/dmeventd-server - name: host-run-dmeventd-server - - mountPath: /host-root/etc/machine-id - name: host-machine-id - readOnly: true - volumes: - - hostPath: - path: /dev/ - type: "" - name: host-device-dir - - hostPath: - path: /sys/ - type: Directory - name: host-sys-dir - - hostPath: - path: /run/udev/ - type: Directory - name: host-run-udev-dir - - hostPath: - path: /run/lvm - type: Directory - name: host-run-lvm-dir - - hostPath: - path: /run/dmeventd-client - name: host-run-dmeventd-client - - hostPath: - path: /run/dmeventd-server - name: host-run-dmeventd-server - - hostPath: - path: /etc/machine-id - type: File - name: host-machine-id - - hostPath: - path: /lib/modules/ - type: DirectoryOrCreate - name: host-modules-dir + hostPID: true {{- end }} From ca15eafc3935c11f1263c810f30a38f5f016b641 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Fri, 19 Jan 2024 13:45:17 +0300 Subject: [PATCH 04/15] Fix indentation Signed-off-by: Aleksandr Zimin Signed-off-by: Viktor Kramarenko --- templates/agent/daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/agent/daemonset.yaml b/templates/agent/daemonset.yaml index a6565449..67b8dd02 100644 --- a/templates/agent/daemonset.yaml +++ b/templates/agent/daemonset.yaml @@ -23,6 +23,7 @@ spec: imagePullSecrets: - name: {{ .Chart.Name }}-module-registry serviceAccountName: sds-node-configurator + hostPID: true containers: - name: sds-node-configurator-agent image: {{ include "helm_lib_module_image" (list . "agent") }} @@ -49,5 +50,4 @@ spec: {{- else if eq .Values.sdsNodeConfigurator.logLevel "TRACE" }} value: "4" {{- end }} - hostPID: true {{- end }} From 2138e3a6255b1d1370877a90614061744eca3778 Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Fri, 19 Jan 2024 13:54:11 +0300 Subject: [PATCH 05/15] fixed machine-id taking Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/config/config.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/images/agent/config/config.go b/images/agent/config/config.go index f2c804eb..4252dd33 100644 --- a/images/agent/config/config.go +++ b/images/agent/config/config.go @@ -17,8 +17,10 @@ limitations under the License. package config import ( + "bytes" "fmt" "os" + "os/exec" "sds-node-configurator/pkg/logger" "time" ) @@ -75,12 +77,17 @@ func NewConfig() (*Options, error) { func getMachineId() (string, error) { id := os.Getenv(MachineID) if id == "" { - byteId, err := os.ReadFile("/host-root/etc/machine-id") + args := []string{"-m", "-u", "-i", "-n", "-p", "-t", "1", "cat", "/etc/machine-id"} + + var stdout *bytes.Buffer + cmd := exec.Command("/usr/bin/nsenter", args...) + cmd.Stdout = stdout + err := cmd.Run() if err != nil { return "", err } - id = string(byteId) + id = stdout.String() } return id, nil From 8141f827f473fb1f2cfa2170c1a43dcf7d44cc72 Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Tue, 23 Jan 2024 14:59:45 +0300 Subject: [PATCH 06/15] added a LVMLogicalVolumeWatcher controller Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- crds/lvmlogicalvolume.yaml | 10 + images/agent/config/config.go | 8 +- images/agent/internal/const.go | 18 +- .../controller/lvm_logical_volume_watcher.go | 580 +++++++++++++++--- .../lvm_logical_volume_watcher_test.go | 18 + .../controller/lvm_volume_group_discover.go | 13 +- .../lvm_volume_group_discover_test.go | 4 +- .../controller/lvm_volume_group_watcher.go | 1 + images/agent/pkg/utils/commands.go | 106 ++-- 9 files changed, 590 insertions(+), 168 deletions(-) create mode 100644 images/agent/pkg/controller/lvm_logical_volume_watcher_test.go diff --git a/crds/lvmlogicalvolume.yaml b/crds/lvmlogicalvolume.yaml index 6d07087e..9d969a53 100644 --- a/crds/lvmlogicalvolume.yaml +++ b/crds/lvmlogicalvolume.yaml @@ -33,13 +33,19 @@ spec: description: | properties: type: + x-kubernetes-validations: + - rule: self == oldSelf type: string enum: [Thick, Thin] size: type: string lvmVolumeGroup: + x-kubernetes-validations: + - rule: self == oldSelf type: string thin: + x-kubernetes-validations: + - rule: self == oldSelf type: object properties: poolName: @@ -56,6 +62,10 @@ spec: actualSize: type: string additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + description: The current resource status. - jsonPath: .spec.lvmVolumeGroup name: LVMVolumeGroup type: string diff --git a/images/agent/config/config.go b/images/agent/config/config.go index 4252dd33..db4383a9 100644 --- a/images/agent/config/config.go +++ b/images/agent/config/config.go @@ -77,17 +77,19 @@ func NewConfig() (*Options, error) { func getMachineId() (string, error) { id := os.Getenv(MachineID) if id == "" { - args := []string{"-m", "-u", "-i", "-n", "-p", "-t", "1", "cat", "/etc/machine-id"} + args := []string{"-m", "-u", "-i", "-n", "-p", "-t", "1", "cat", "./etc/machine-id"} - var stdout *bytes.Buffer + var stdout bytes.Buffer cmd := exec.Command("/usr/bin/nsenter", args...) - cmd.Stdout = stdout + cmd.Stdout = &stdout err := cmd.Run() if err != nil { return "", err } id = stdout.String() + fmt.Println("MACHINE ID " + id) + } return id, nil diff --git a/images/agent/internal/const.go b/images/agent/internal/const.go index 4cdf0748..687d5ec1 100644 --- a/images/agent/internal/const.go +++ b/images/agent/internal/const.go @@ -17,14 +17,14 @@ limitations under the License. package internal const ( - TypePart = "part" - DRBDName = "/dev/drbd" - LoopDeviceType = "loop" - LVMDeviceType = "lvm" - LVMFSType = "LVM2_member" - SdsNodeConfigurator = "storage.deckhouse.io/sds-node-configurator" - LVMVGHealthOperational = "Operational" - LVMVGHealthNonOperational = "NonOperational" + TypePart = "part" + DRBDName = "/dev/drbd" + LoopDeviceType = "loop" + LVMDeviceType = "lvm" + LVMFSType = "LVM2_member" + SdsNodeConfiguratorFinalizer = "storage.deckhouse.io/sds-node-configurator" + LVMVGHealthOperational = "Operational" + LVMVGHealthNonOperational = "NonOperational" ) var ( @@ -32,6 +32,6 @@ var ( InvalidDeviceTypes = [...]string{LoopDeviceType, LVMDeviceType} BlockDeviceValidSize = "1G" ResizeDelta = "32Mi" - Finalizers = []string{SdsNodeConfigurator} + Finalizers = []string{SdsNodeConfiguratorFinalizer} LVMTags = []string{"storage.deckhouse.io/enabled=true", "linstor-"} ) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index 5d90a4d5..d84e45b9 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -2,12 +2,13 @@ package controller import ( "context" + "errors" "fmt" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/config" + "sds-node-configurator/internal" "sds-node-configurator/pkg/logger" "sds-node-configurator/pkg/monitoring" "sds-node-configurator/pkg/utils" @@ -21,16 +22,20 @@ import ( ) const ( - LVMLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" - CreatedStatusPhase = "Created" - PendingStatusPhase = "Pending" - ResizingStatusPhase = "Resizing" - FailedStatusPhase = "Failed" - Thick DeviceType = "Thick" - Thin DeviceType = "Thin" + Thick deviceType = "Thick" + Thin deviceType = "Thin" + + lvmLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" + + createdStatusPhase = "Created" + pendingStatusPhase = "Pending" + resizingStatusPhase = "Resizing" + failedStatusPhase = "Failed" ) -type DeviceType string +type ( + deviceType string +) func RunLVMLogicalVolumeWatcherController( mgr manager.Manager, @@ -41,7 +46,7 @@ func RunLVMLogicalVolumeWatcherController( cl := mgr.GetClient() cache := mgr.GetCache() - c, err := controller.New(LVMLogicalVolumeWatcherCtrlName, mgr, controller.Options{ + c, err := controller.New(lvmLogicalVolumeWatcherCtrlName, mgr, controller.Options{ Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { return reconcile.Result{}, nil }), @@ -53,14 +58,49 @@ func RunLVMLogicalVolumeWatcherController( err = c.Watch(source.Kind(cache, &v1alpha1.LvmLogicalVolume{}), handler.Funcs{ CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { - log.Info("[RunLVMLogicalVolumeWatcherController] Watcher.CreateFunc starts reconciliation") - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: e.Object.GetNamespace(), - Name: e.Object.GetName(), - }} - - reconcileFunc(ctx, cl, log, metrics, req, cfg.NodeName) - log.Info("[RunLVMLogicalVolumeWatcherController] Watcher.CreateFunc ends reconciliation") + log.Info("[RunLVMLogicalVolumeWatcherController] CreateFunc starts reconciliation") + + llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) + if !ok { + err := errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling create event") + return + } + + reconcileLLVCreateFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + log.Info("[RunLVMLogicalVolumeWatcherController] CreateFunc ends reconciliation") + }, + + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc starts reconciliation") + + newLlv, ok := e.ObjectNew.(*v1alpha1.LvmLogicalVolume) + if !ok { + err := errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling update event") + return + } + + reconcileLLVUpdateFunc(ctx, cl, log, metrics, newLlv, cfg.NodeName) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, newLlv, cfg.NodeName) + log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc ends reconciliation") + }, + + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc starts reconciliation") + + llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) + if !ok { + err := errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling delete event") + return + } + + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + + log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc ends reconciliation") }, }) if err != nil { @@ -71,104 +111,470 @@ func RunLVMLogicalVolumeWatcherController( return c, err } -func reconcileFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, req reconcile.Request, nodeName string) { - // Получаю ресурс - log.Info("[reconcileFunc] starts reconciliation") - log.Debug(fmt.Sprintf("[reconcileFunc] reconciles LVMLogicalVolume, name: %s", req.Name)) - llv, err := getLVMLogicalVolume(ctx, metrics, cl, req) +func reconcileLLVDeleteFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { + log.Info("[reconcileLLVDeleteFunc] starts reconciliation") + + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) if err != nil { - log.Error(err, "[reconcileFunc] unable to getLVMLogicalVolume") + log.Error(err, "[reconcileLLVDeleteFunc] unable to getLVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + if err != nil { + log.Error(err, "[reconcileLLVDeleteFunc] unable to updateLVMLogicalVolumePhase") + } return } - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, req.Namespace, llv.Spec.LvmVolumeGroup) + if !belongsToNode(lvg, nodeName) { + log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s does not belongs to the current node: %s", llv.Name, nodeName)) + return + } + log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s belongs to the current node: %s", llv.Name, nodeName)) + + shouldReconcile := shouldReconcileByDeleteFunc(llv) + if !shouldReconcile { + log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) + return + } + + err = deleteLVifExisted(log, llv) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete LV %s", llv.Name)) + return + } + + log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully deleted LV %s", llv.Name)) + + err = removeLLVFinalizers(ctx, cl, metrics, log, llv) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to remove finalizers from the LVMVolumeGroup %s", llv.Name)) + } + + log.Info("[reconcileLLVDeleteFunc] ends reconciliation") +} + +func shouldReconcileByDeleteFunc(llv *v1alpha1.LvmLogicalVolume) bool { + if llv.DeletionTimestamp == nil { + return false + } + + return true +} + +func removeLLVFinalizers(ctx context.Context, cl client.Client, metrics monitoring.Metrics, log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error { + var removed bool + for i, f := range llv.Finalizers { + if f == internal.SdsNodeConfiguratorFinalizer { + llv.Finalizers = append(llv.Finalizers[:i], llv.Finalizers[i+1:]...) + removed = true + log.Debug(fmt.Sprintf("[removeLLVFinalizers] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + } + } + + if removed { + err := updateLVMLogicalVolume(ctx, metrics, cl, llv) + if err != nil { + log.Error(err, fmt.Sprintf("[updateLVMLogicalVolume] unable to update the LVMVolumeGroup %s", llv.Name)) + return err + } + } + + return nil +} + +func deleteLVifExisted(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error { + lvs, cmd, _, err := utils.GetAllLVs() + log.Debug(fmt.Sprintf("[deleteLVifExisted] runs cmd: %s", cmd)) + if err != nil { + return err + } + + var ( + lv *internal.LVData + ) + for _, l := range lvs { + if l.LVName == llv.Name { + lv = &l + break + } + } + + if lv == nil { + log.Debug(fmt.Sprintf("[deleteLVifExisted] did not find LV %s", lv.LVName)) + return errors.New("lv does not exist") + } + + cmd, err = utils.RemoveLV(lv.VGName, lv.LVName) + log.Debug("[deleteLVifExisted] runs cmd: %s", cmd) + if err != nil { + log.Error(err, "[deleteLVifExisted] unable to RemoveLV") + return err + } + + return nil +} + +func getExtendingSize(log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume) (resource.Quantity, error) { + lvs, cmd, _, err := utils.GetAllLVs() + log.Debug(fmt.Sprintf("[getExtendingSize] runs cmd %s", cmd)) + if err != nil { + log.Error(err, "[getExtendingSize] unable to GetAllLVs") + return resource.Quantity{}, err + } + + var ( + oldSize resource.Quantity + newSize = llv.Spec.Size + ) + for _, lv := range lvs { + if lv.LVName == llv.Name { + oldSize = lv.LVSize + break + } + } + + if newSize.Value() < oldSize.Value() { + return resource.Quantity{}, fmt.Errorf("selected size is less than exising one") + } + + return subtractQuantity(newSize, oldSize), nil +} + +func reconcileLLVUpdateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { + log.Info("[reconcileLLVUpdateFunc] starts reconciliation") + + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) if err != nil { - log.Error(err, "[reconcileFunc] unable to getLVMVolumeGroup") + log.Error(err, "[reconcileLLVUpdateFunc] unable to getLVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } return } if !belongsToNode(lvg, nodeName) { - log.Debug(fmt.Sprintf("[reconcileFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) return } - log.Debug(fmt.Sprintf("[reconcileFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) - err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, PendingStatusPhase, "") + shouldReconcile, err := shouldReconcileByUpdateFunc(log, llv) if err != nil { - log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to check if LV %s should be reconciled", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to check LV state") + return + } + + if !shouldReconcile { + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) + return + } + + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, resizingStatusPhase, "") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) + + extendingSize, err := getExtendingSize(log, metrics, llv) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] error occurs while getting extending size for the LVMLogicalVolume %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to count extending LV size") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return } - log.Debug(fmt.Sprintf("[reconcileFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, PendingStatusPhase)) - log.Debug(fmt.Sprintf("[reconcileFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) - // добавить проверки на наличие нужного кол-ва места switch getLVMLogicalVolumeType(llv) { case Thick: - freeSpace, err := getFreeVGSpace(log, lvg) + freeSpace, err := getFreeVGSpace(log, lvg.Spec.ActualVGNameOnTheNode) + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] free VG space %d", freeSpace.Value())) if err != nil { - log.Error(err, "[reconcileFunc] unable to getFreeVGSpace") + log.Error(err, "[reconcileLLVUpdateFunc] unable to getFreeVGSpace") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get free VG space") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if freeSpace.Value() < extendingSize.Value() { + err = errors.New("not enough space") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s requested size is more than the VG %s free space", llv.Name, lvg.Spec.ActualVGNameOnTheNode)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on VG") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + cmd, err := utils.ExtendLV(llv.Spec.Size.String(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to ExtendLV") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to extend Thick LV") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + case Thin: + freeSpace, err := getFreeLVSpace(log, llv.Spec.Thin.PoolName) + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] free Thin-pool space %d", freeSpace.Value())) + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to getFreeVGSpace") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on LV") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if freeSpace.Value() < extendingSize.Value() { + err = errors.New("not enough space") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s requested size is more than the Thin-pool %s free space", llv.Name, llv.Spec.Thin.PoolName)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on Thin pool") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + cmd, err := utils.ExtendLV(llv.Spec.Size.String(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to ExtendLV") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to extend Thin pool") + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + } + + log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully extended Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) + + llv.Status.Phase = createdStatusPhase + llv.Status.ActualSize = llv.Spec.Size + err = updateLVMLogicalVolume(ctx, metrics, cl, llv) + if err != nil { + log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolume") + return + } + + log.Info("[reconcileLLVUpdateFunc] ends reconciliation") +} + +func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (bool, error) { + if llv.DeletionTimestamp != nil { + return false, nil + } + + should, err := shouldReconcileByCreateFunc(log, llv) + if err != nil { + return false, err + } + + if should { + return false, nil + } + + if llv.Status.ActualSize.Value() == 0 { + return false, nil + } + + if llv.Spec.Size.Value() == llv.Status.ActualSize.Value() { + return false, nil + } + + return true, nil +} + +func reconcileLLVCreateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { + log.Info("[reconcileLLVCreateFunc] starts reconciliation") + + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to getLVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if !belongsToNode(lvg, nodeName) { + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) + return + } + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) + + shouldReconcile, err := shouldReconcileByCreateFunc(log, llv) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to check if LV %s should be reconciled", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to check LV state") + return + } + + if !shouldReconcile { + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) + return + } + + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, pendingStatusPhase, "") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) + + switch getLVMLogicalVolumeType(llv) { + case Thick: + freeSpace, err := getFreeVGSpace(log, lvg.Spec.ActualVGNameOnTheNode) + log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] free VG space %d", freeSpace.Value())) + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to getFreeVGSpace") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get free VG space") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } + return } if freeSpace.Value() < llv.Spec.Size.Value() { - log.Warning(fmt.Sprintf("[reconcileFunc] not enough space left on the VG %s for the LVMLogicalVolume %s", lvg.Spec.ActualVGNameOnTheNode, llv.Name)) + err = errors.New("not enough space") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size is more than the VG %s free space", llv.Name, lvg.Spec.ActualVGNameOnTheNode)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on VG") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } return } cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.String()) - log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] runs cmd: %s", cmd)) if err != nil { - log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") - err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, FailedStatusPhase, "Unable to create Thick LV") + log.Error(err, "[reconcileLLVCreateFunc] unable to CreateThickLogicalVolume") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thick LV") if err != nil { - log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") } return } case Thin: + freeSpace, err := getFreeLVSpace(log, llv.Spec.Thin.PoolName) + log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] free Thin-pool space %d", freeSpace.Value())) + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to getFreeVGSpace") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on LV") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if freeSpace.Value() < llv.Spec.Size.Value() { + err = errors.New("not enough space") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size is more than the Thin-pool %s free space", llv.Name, llv.Spec.Thin.PoolName)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thin LV") + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.String()) - log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] runs cmd: %s", cmd)) if err != nil { - log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") - err = updateLVMLogicalVolumePhase(ctx, metrics, cl, llv, FailedStatusPhase, "Unable to create Thin LV") + log.Error(err, "[reconcileLLVCreateFunc] unable to CreateThickLogicalVolume") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thin LV") if err != nil { - log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") } return } } - //if llv.Spec.Thin.PoolName == "" { - // cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.String()) - // log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) - // if err != nil { - // log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") - // llv.Status.Phase = FailedStatusPhase - // llv.Status.Reason = err.Error() - // return - // } - //} else { - // cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.String()) - // log.Debug(fmt.Sprintf("[reconcileFunc] runs cmd: %s", cmd)) - // if err != nil { - // log.Error(err, "[reconcileFunc] unable to CreateThickLogicalVolume") - // llv.Status.Phase = FailedStatusPhase - // llv.Status.Reason = err.Error() - // return - // } - //} - log.Debug(fmt.Sprintf("[reconcileFunc] successfully created Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) - - llv.Status.Phase = CreatedStatusPhase + log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully created Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) + + llv.Status.Phase = createdStatusPhase llv.Status.ActualSize = llv.Spec.Size err = updateLVMLogicalVolume(ctx, metrics, cl, llv) if err != nil { - log.Error(err, "[reconcileFunc] unable to updateLVMLogicalVolume") + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolume") return } - log.Info("[reconcileFunc] ends reconciliation") + log.Info("[reconcileLLVCreateFunc] ends reconciliation") +} + +func shouldReconcileByCreateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (bool, error) { + lvs, cmd, _, err := utils.GetAllLVs() + log.Debug(fmt.Sprintf("[shouldReconcileByCreateFunc] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[shouldReconcileByCreateFunc] unable to GetAllLVs") + return false, err + } + + for _, lv := range lvs { + if lv.LVName == llv.Name { + return false, nil + } + } + + return true, nil +} + +func getFreeLVSpace(log logger.Logger, thinPoolName string) (resource.Quantity, error) { + lvs, cmd, _, err := utils.GetAllLVs() + log.Debug(fmt.Sprintf("[getFreeLVSpace] runs cmd: %s", cmd)) + if err != nil { + log.Error(err, "[getFreeVGSpace] unable to GetAllLVs") + return resource.Quantity{}, err + } + + for _, lv := range lvs { + if lv.LVName == thinPoolName { + used, err := getLVUsedSize(lv) + vlsSize := getVirtualLVSize(lv.LVName, lvs) + + if err != nil { + log.Error(err, "[getFreeLVSpace] unable to getLVUsedSize") + return resource.Quantity{}, err + } + + free := subtractQuantity(lv.LVSize, *used) + free = subtractQuantity(free, vlsSize) + + return free, nil + } + } + + return resource.Quantity{}, nil } -func getFreeVGSpace(log logger.Logger, lvg *v1alpha1.LvmVolumeGroup) (resource.Quantity, error) { +func getVirtualLVSize(thinPool string, lvs []internal.LVData) resource.Quantity { + sum := int64(0) + + for _, lv := range lvs { + if lv.PoolLv == thinPool { + sum += lv.LVSize.Value() + } + } + + return *resource.NewQuantity(sum, resource.BinarySI) +} + +func subtractQuantity(min, sub resource.Quantity) resource.Quantity { + val := min + val.Sub(sub) + return val +} + +func getFreeVGSpace(log logger.Logger, vgName string) (resource.Quantity, error) { vgs, cmd, _, err := utils.GetAllVGs() log.Debug(fmt.Sprintf("[getFreeVGSpace] runs cmd: %s", cmd)) if err != nil { @@ -177,7 +583,7 @@ func getFreeVGSpace(log logger.Logger, lvg *v1alpha1.LvmVolumeGroup) (resource.Q } for _, vg := range vgs { - if vg.VGName == lvg.Spec.ActualVGNameOnTheNode { + if vg.VGName == vgName { return vg.VGFree, nil } } @@ -185,7 +591,7 @@ func getFreeVGSpace(log logger.Logger, lvg *v1alpha1.LvmVolumeGroup) (resource.Q return resource.Quantity{}, nil } -func getLVMLogicalVolumeType(llv *v1alpha1.LvmLogicalVolume) DeviceType { +func getLVMLogicalVolumeType(llv *v1alpha1.LvmLogicalVolume) deviceType { if llv.Spec.Thin.PoolName == "" { return Thick } @@ -204,7 +610,7 @@ func belongsToNode(lvg *v1alpha1.LvmVolumeGroup, nodeName string) bool { return belongs } -func updateLVMLogicalVolumePhase(ctx context.Context, metrics monitoring.Metrics, cl client.Client, llv *v1alpha1.LvmLogicalVolume, phase, reason string) error { +func updateLVMLogicalVolumePhase(ctx context.Context, cl client.Client, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, phase, reason string) error { llv.Status.Phase = phase llv.Status.Reason = reason @@ -224,33 +630,3 @@ func updateLVMLogicalVolume(ctx context.Context, metrics monitoring.Metrics, cl return nil } - -func getLVMLogicalVolume(ctx context.Context, metrics monitoring.Metrics, cl client.Client, req reconcile.Request) (*v1alpha1.LvmLogicalVolume, error) { - llv := &v1alpha1.LvmLogicalVolume{} - - err := cl.Get(ctx, client.ObjectKey{ - Namespace: req.Namespace, - Name: req.Name, - }, llv) - if err != nil { - return nil, err - } - - return llv, err -} - -func updateFunc() { - // Если обновляется спека llv, я - // - проверяю текущий статус ресурса - // - если статус соответствует спеке, ничего не делаю - // - если статус не соответствует спеке, тогда - // - я проверяю возможность создания PV в указанном месте - // - пытаюсь создать PV нужного размера - // - если создаю, то обновляю статус ресурса успехом, проставляю актуальные данные - // - если нет, кидаю ошибку и сообщение -} - -func deleteFunc() { - // Если удаляется ресурс llv, я - // - удаляю llv -} diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher_test.go b/images/agent/pkg/controller/lvm_logical_volume_watcher_test.go new file mode 100644 index 00000000..cdcc0792 --- /dev/null +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher_test.go @@ -0,0 +1,18 @@ +package controller + +import ( + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/resource" + "testing" +) + +func TestLVMLogicaVolumeWatcher(t *testing.T) { + t.Run("subtractQuantity_returns_correct_value", func(t *testing.T) { + min := resource.NewQuantity(1000, resource.BinarySI) + sub := resource.NewQuantity(300, resource.BinarySI) + expected := resource.NewQuantity(700, resource.BinarySI) + + actual := subtractQuantity(*min, *sub) + assert.Equal(t, expected, &actual) + }) +} diff --git a/images/agent/pkg/controller/lvm_volume_group_discover.go b/images/agent/pkg/controller/lvm_volume_group_discover.go index ceee1dea..34ba046f 100644 --- a/images/agent/pkg/controller/lvm_volume_group_discover.go +++ b/images/agent/pkg/controller/lvm_volume_group_discover.go @@ -669,20 +669,20 @@ func getStatusThinPools(log logger.Logger, thinPools map[string][]internal.LVDat tps := make([]internal.LVMVGStatusThinPool, 0, len(filtered)) for _, lv := range filtered { - usedSize, err := getUsedSizeMiB(lv) //todo rename + usedSize, err := getLVUsedSize(lv) if err != nil { - log.Error(err, "[getStatusThinPools] unable to getUsedSizeMiB") + log.Error(err, "[getStatusThinPools] unable to getLVUsedSize") } tps = append(tps, internal.LVMVGStatusThinPool{ Name: lv.LVName, ActualSize: lv.LVSize, - UsedSize: usedSize, + UsedSize: usedSize.String(), }) } return tps } -func getUsedSizeMiB(lv internal.LVData) (string, error) { +func getLVUsedSize(lv internal.LVData) (*resource.Quantity, error) { var ( err error dataPercent float64 @@ -693,12 +693,13 @@ func getUsedSizeMiB(lv internal.LVData) (string, error) { } else { dataPercent, err = strconv.ParseFloat(lv.DataPercent, 64) if err != nil { - return "", err + return nil, err } } tmp := float64(lv.LVSize.Value()) * dataPercent - return utils.BytesToQuantity(int64(tmp)), nil + + return resource.NewQuantity(int64(tmp), resource.BinarySI), nil } func isThinPool(lv internal.LVData) bool { diff --git a/images/agent/pkg/controller/lvm_volume_group_discover_test.go b/images/agent/pkg/controller/lvm_volume_group_discover_test.go index 25a78344..58ce78c8 100644 --- a/images/agent/pkg/controller/lvm_volume_group_discover_test.go +++ b/images/agent/pkg/controller/lvm_volume_group_discover_test.go @@ -112,10 +112,10 @@ func TestLVMVolumeGroupDiscover(t *testing.T) { DataPercent: "50", } expected := "97656250Ki" - actual, err := getUsedSizeMiB(lv) + actual, err := getLVUsedSize(lv) if assert.NoError(t, err) { - assert.Equal(t, expected, actual) + assert.Equal(t, expected, actual.String()) } }) diff --git a/images/agent/pkg/controller/lvm_volume_group_watcher.go b/images/agent/pkg/controller/lvm_volume_group_watcher.go index 72f12f2d..766af400 100644 --- a/images/agent/pkg/controller/lvm_volume_group_watcher.go +++ b/images/agent/pkg/controller/lvm_volume_group_watcher.go @@ -90,6 +90,7 @@ func RunLVMVolumeGroupWatcherController( log.Warning(fmt.Sprintf(`Added request, namespace: "%s" name: "%s", to requeue`, request.Namespace, request.Name)) } } + updateFunc := func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { log.Info(fmt.Sprintf("[RunLVMVolumeGroupWatcherController] update LVMVolumeGroupn, name: %s", e.ObjectNew.GetName())) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index de2c8814..f4852bf0 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -23,11 +23,17 @@ import ( "os/exec" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/internal" + "strings" +) + +const ( + nsenter = "/usr/bin/nsenter" ) func GetBlockDevices() ([]internal.Device, string, error) { var outs bytes.Buffer - cmd := exec.Command("lsblk", "-J", "-lpfb", "-no", "name,MOUNTPOINT,PARTUUID,HOTPLUG,MODEL,SERIAL,SIZE,FSTYPE,TYPE,WWN,KNAME,PKNAME,ROTA") + args := []string{"lsblk", "-J", "-lpfb", "-no", "name,MOUNTPOINT,PARTUUID,HOTPLUG,MODEL,SERIAL,SIZE,FSTYPE,TYPE,WWN,KNAME,PKNAME,ROTA"} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs err := cmd.Run() @@ -45,7 +51,8 @@ func GetBlockDevices() ([]internal.Device, string, error) { func GetAllVGs() (data []internal.VGData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer - cmd := exec.Command("vgs", "-o", "+uuid,tags,shared", "--units", "B", "--nosuffix", "--reportformat", "json") + args := []string{"vgs", "-o", "+uuid,tags,shared", "--units", "B", "--nosuffix", "--reportformat", "json"} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -63,7 +70,8 @@ func GetAllVGs() (data []internal.VGData, command string, stdErr bytes.Buffer, e func GetAllLVs() (data []internal.LVData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer - cmd := exec.Command("lvs", "-o", "+vg_uuid,tags", "--units", "B", "--nosuffix", "--reportformat", "json") + args := []string{"lvs", "-o", "+vg_uuid,tags", "--units", "B", "--nosuffix", "--reportformat", "json"} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -81,7 +89,8 @@ func GetAllLVs() (data []internal.LVData, command string, stdErr bytes.Buffer, e func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer - cmd := exec.Command("pvs", "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--units", "B", "--nosuffix", "--reportformat", "json") + args := []string{"pvs", "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--units", "B", "--nosuffix", "--reportformat", "json"} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -99,7 +108,8 @@ func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, e func GetSinglePV(pVname string) (*internal.PVData, string, error) { var outs bytes.Buffer - cmd := exec.Command("pvs", pVname, "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--reportformat", "json") + args := []string{"pvs", pVname, "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--reportformat", "json"} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs if err := cmd.Run(); err != nil { @@ -122,7 +132,8 @@ func GetSinglePV(pVname string) (*internal.PVData, string, error) { } func CreatePV(path string) (string, error) { - cmd := exec.Command("pvcreate", path) + args := []string{"pvcreate", path} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -135,14 +146,10 @@ func CreatePV(path string) (string, error) { } func CreateVGLocal(vgName, lvmName string, pvNames []string) (string, error) { - tmpStr := fmt.Sprintf("storage.deckhouse.io/lvmVolumeGroupName=%s", lvmName) - var arg []string - arg = append(arg, vgName) - arg = append(arg, pvNames...) - arg = append(arg, "--addtag", "storage.deckhouse.io/enabled=true", "--addtag", tmpStr) + args := []string{"vgcreate", vgName, strings.Join(pvNames, " "), "--addtag", "storage.deckhouse.io/enabled=true", "--addtag", tmpStr} - cmd := exec.Command("vgcreate", arg...) + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -155,15 +162,8 @@ func CreateVGLocal(vgName, lvmName string, pvNames []string) (string, error) { } func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { - var arg []string - arg = append(arg, "--shared") - arg = append(arg, vgName) - arg = append(arg, pvNames...) - arg = append(arg, "--addtag", - "storage.deckhouse.io/enabled=true", - "--addtag", fmt.Sprintf("storage.deckhouse.io/lvmVolumeGroupName=%s", lvmName)) - - cmd := exec.Command("vgcreate", arg...) + args := []string{"vgcreate", "--shared", vgName, strings.Join(pvNames, " "), "--addtag", "storage.deckhouse.io/enabled=true", "--addtag", fmt.Sprintf("storage.deckhouse.io/lvmVolumeGroupName=%s", lvmName)} + cmd := exec.Command(nsenter, args...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to CreateVGShared, err: %w", err) @@ -173,8 +173,8 @@ func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { } func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, error) { - cmd := exec.Command( - "lvcreate", "-L", thinPool.Size.String(), "-T", fmt.Sprintf("%s/%s", VGName, thinPool.Name)) + args := []string{"lvcreate", "-L", thinPool.Size.String(), "-T", fmt.Sprintf("%s/%s", VGName, thinPool.Name)} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -186,31 +186,25 @@ func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, erro } func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error) { - args := []string{"-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", size} - cmd := exec.Command("lvcreate", args...) + args := []string{"lvcreate", "-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", size, "-W", "y", "-y"} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr var stdout bytes.Buffer cmd.Stdout = &stdout - fmt.Println("RUNS COMMAND:" + cmd.String()) - fmt.Println("CMD IS ABOUT TO RUN") err := cmd.Run() - fmt.Println(stderr.String()) - fmt.Println(stdout.String()) if err != nil { - fmt.Println("CMD ERROR ALERT") return cmd.String(), err } - fmt.Println("CMD ALL GOOD") return cmd.String(), nil } func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { - args := []string{"-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", size, "-W", "y", "-y"} - cmd := exec.Command("lvcreate", args...) + args := []string{"lvcreate", "-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", size, "-W", "y", "-y"} + cmd := exec.Command(nsenter, args...) if err := cmd.Run(); err != nil { return cmd.String(), err @@ -220,10 +214,8 @@ func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { } func ExtendVG(vgName string, paths []string) (string, error) { - var arg []string - arg = append(arg, vgName) - arg = append(arg, paths...) - cmd := exec.Command("vgextend", arg...) + args := []string{"vgextend", vgName, strings.Join(paths, " ")} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -236,7 +228,8 @@ func ExtendVG(vgName string, paths []string) (string, error) { } func ExtendLV(size, vgName, lvName string) (string, error) { - cmd := exec.Command("lvextend", "-L", size, fmt.Sprintf("/dev/%s/%s", vgName, lvName)) + args := []string{"lvextend", "-L", size, fmt.Sprintf("/dev/%s/%s", vgName, lvName)} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -249,7 +242,8 @@ func ExtendLV(size, vgName, lvName string) (string, error) { } func ResizePV(pvName string) (string, error) { - cmd := exec.Command("pvresize", pvName) + args := []string{"pvresize", pvName} + cmd := exec.Command(nsenter, args...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to ResizePV, err: %w", err) } @@ -295,7 +289,8 @@ func RemovePVFromVG(pvName, vgName string) (string, error) { return cmdStr, fmt.Errorf(`unable to RemovePVFromVG, err: can't move LV segments from PVData, pv name: "%s"`, pvName) } - cmd := exec.Command("vgreduce", vgName, pvName) + args := []string{"vgreduce", vgName, pvName} + cmd := exec.Command(nsenter, args...) if err = cmd.Run(); err != nil { return cmd.String(), fmt.Errorf(`unable to RemovePVFromVG with vgName: "%s", pvName: "%s", err: %w`, vgName, pvName, err) @@ -314,7 +309,8 @@ func CheckPVHasNoData(pvName string) (bool, string, error) { } func MovePV(pvName string) (string, error) { - cmd := exec.Command("pvmove", pvName) + args := []string{"pvmove", pvName} + cmd := exec.Command(nsenter, args...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to MovePV, err: %w", err) @@ -324,7 +320,8 @@ func MovePV(pvName string) (string, error) { } func RemoveVG(vgName string) (string, error) { - cmd := exec.Command("vgremove", vgName) + args := []string{"vgremove", vgName} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -337,7 +334,8 @@ func RemoveVG(vgName string) (string, error) { } func RemovePV(pvNames []string) (string, error) { - cmd := exec.Command("pvremove", pvNames...) + args := []string{"pvremove", strings.Join(pvNames, " ")} + cmd := exec.Command(nsenter, args...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -348,6 +346,19 @@ func RemovePV(pvNames []string) (string, error) { return cmd.String(), nil } +func RemoveLV(vgName, lvName string) (string, error) { + args := []string{"lvremove", fmt.Sprintf("/dev/%s/%s", vgName, lvName), "-y"} + cmd := exec.Command(nsenter, args...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to RemoveLV, err: %w stderr = %s", err, stderr.String()) + } + return cmd.String(), nil +} + func unmarshalDevices(out []byte) ([]internal.Device, error) { var devices internal.Devices if err := json.Unmarshal(out, &devices); err != nil { @@ -413,7 +424,8 @@ func unmarshalLVs(out []byte) ([]internal.LVData, error) { func VGChangeAddTag(vGName, tag string) (string, error) { var outs, stdErr bytes.Buffer - cmd := exec.Command("vgchange", vGName, "--addtag", tag) + args := []string{"vgchange", vGName, "--addtag", tag} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -425,7 +437,8 @@ func VGChangeAddTag(vGName, tag string) (string, error) { func VGChangeDelTag(vGName, tag string) (string, error) { var outs, stdErr bytes.Buffer - cmd := exec.Command("vgchange", vGName, "--deltag", tag) + args := []string{"vgchange", vGName, "--deltag", tag} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -438,7 +451,8 @@ func VGChangeDelTag(vGName, tag string) (string, error) { func LVChangeDelTag(lv internal.LVData, tag string) (string, error) { tmpStr := fmt.Sprintf("/dev/%s/%s", lv.VGName, lv.LVName) var outs, stdErr bytes.Buffer - cmd := exec.Command("lvchange", tmpStr, "--deltag", tag) + args := []string{"lvchange", tmpStr, "--deltag", tag} + cmd := exec.Command(nsenter, args...) cmd.Stdout = &outs cmd.Stderr = &stdErr From b70b24975a7af9d82e1aa2fd3dce85c1a6c94d69 Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Fri, 26 Jan 2024 21:23:04 +0300 Subject: [PATCH 07/15] removed deleteFunc reconciliation Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- .../controller/lvm_logical_volume_watcher.go | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index d84e45b9..cf0b4be8 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -88,20 +88,20 @@ func RunLVMLogicalVolumeWatcherController( log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc ends reconciliation") }, - DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { - log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc starts reconciliation") - - llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) - if !ok { - err := errors.New("unable to cast event object to a given type") - log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling delete event") - return - } - - reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) - - log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc ends reconciliation") - }, + //DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + // log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc starts reconciliation") + // + // llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) + // if !ok { + // err := errors.New("unable to cast event object to a given type") + // log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling delete event") + // return + // } + // + // reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + // + // log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc ends reconciliation") + //}, }) if err != nil { log.Error(err, "[RunLVMLogicalVolumeWatcherController] the controller is unable to watch") @@ -167,6 +167,7 @@ func removeLLVFinalizers(ctx context.Context, cl client.Client, metrics monitori llv.Finalizers = append(llv.Finalizers[:i], llv.Finalizers[i+1:]...) removed = true log.Debug(fmt.Sprintf("[removeLLVFinalizers] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + break } } From eb2ee10d9e041e2654c82570c10313b500e25ed8 Mon Sep 17 00:00:00 2001 From: "viktor.kramarenko" Date: Tue, 30 Jan 2024 18:38:00 +0300 Subject: [PATCH 08/15] fixed nsenter command Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/pkg/utils/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index f4852bf0..ad97f073 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -27,7 +27,7 @@ import ( ) const ( - nsenter = "/usr/bin/nsenter" + nsenter = "/usr/bin/nsenter -m -u -i -n -p -t 1" ) func GetBlockDevices() ([]internal.Device, string, error) { From 06c592065cfc30285059d3e40f4f285ead9fb20d Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Tue, 30 Jan 2024 20:16:10 +0300 Subject: [PATCH 09/15] Fix nsenter args Signed-off-by: Aleksandr Zimin Signed-off-by: Viktor Kramarenko --- images/agent/pkg/utils/commands.go | 73 ++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index ad97f073..4de0a900 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -27,13 +27,14 @@ import ( ) const ( - nsenter = "/usr/bin/nsenter -m -u -i -n -p -t 1" + nsenter = "/usr/bin/nsenter" ) func GetBlockDevices() ([]internal.Device, string, error) { var outs bytes.Buffer args := []string{"lsblk", "-J", "-lpfb", "-no", "name,MOUNTPOINT,PARTUUID,HOTPLUG,MODEL,SERIAL,SIZE,FSTYPE,TYPE,WWN,KNAME,PKNAME,ROTA"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs err := cmd.Run() @@ -52,7 +53,8 @@ func GetBlockDevices() ([]internal.Device, string, error) { func GetAllVGs() (data []internal.VGData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer args := []string{"vgs", "-o", "+uuid,tags,shared", "--units", "B", "--nosuffix", "--reportformat", "json"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -71,7 +73,8 @@ func GetAllVGs() (data []internal.VGData, command string, stdErr bytes.Buffer, e func GetAllLVs() (data []internal.LVData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer args := []string{"lvs", "-o", "+vg_uuid,tags", "--units", "B", "--nosuffix", "--reportformat", "json"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -90,7 +93,8 @@ func GetAllLVs() (data []internal.LVData, command string, stdErr bytes.Buffer, e func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer args := []string{"pvs", "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--units", "B", "--nosuffix", "--reportformat", "json"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -109,7 +113,8 @@ func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, e func GetSinglePV(pVname string) (*internal.PVData, string, error) { var outs bytes.Buffer args := []string{"pvs", pVname, "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--reportformat", "json"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs if err := cmd.Run(); err != nil { @@ -133,7 +138,8 @@ func GetSinglePV(pVname string) (*internal.PVData, string, error) { func CreatePV(path string) (string, error) { args := []string{"pvcreate", path} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -149,7 +155,8 @@ func CreateVGLocal(vgName, lvmName string, pvNames []string) (string, error) { tmpStr := fmt.Sprintf("storage.deckhouse.io/lvmVolumeGroupName=%s", lvmName) args := []string{"vgcreate", vgName, strings.Join(pvNames, " "), "--addtag", "storage.deckhouse.io/enabled=true", "--addtag", tmpStr} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -163,7 +170,8 @@ func CreateVGLocal(vgName, lvmName string, pvNames []string) (string, error) { func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { args := []string{"vgcreate", "--shared", vgName, strings.Join(pvNames, " "), "--addtag", "storage.deckhouse.io/enabled=true", "--addtag", fmt.Sprintf("storage.deckhouse.io/lvmVolumeGroupName=%s", lvmName)} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to CreateVGShared, err: %w", err) @@ -174,7 +182,8 @@ func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, error) { args := []string{"lvcreate", "-L", thinPool.Size.String(), "-T", fmt.Sprintf("%s/%s", VGName, thinPool.Name)} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -187,7 +196,8 @@ func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, erro func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error) { args := []string{"lvcreate", "-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", size, "-W", "y", "-y"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -204,7 +214,8 @@ func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { args := []string{"lvcreate", "-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", size, "-W", "y", "-y"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) if err := cmd.Run(); err != nil { return cmd.String(), err @@ -215,7 +226,8 @@ func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { func ExtendVG(vgName string, paths []string) (string, error) { args := []string{"vgextend", vgName, strings.Join(paths, " ")} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -229,7 +241,8 @@ func ExtendVG(vgName string, paths []string) (string, error) { func ExtendLV(size, vgName, lvName string) (string, error) { args := []string{"lvextend", "-L", size, fmt.Sprintf("/dev/%s/%s", vgName, lvName)} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -243,7 +256,8 @@ func ExtendLV(size, vgName, lvName string) (string, error) { func ResizePV(pvName string) (string, error) { args := []string{"pvresize", pvName} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to ResizePV, err: %w", err) } @@ -290,7 +304,8 @@ func RemovePVFromVG(pvName, vgName string) (string, error) { } args := []string{"vgreduce", vgName, pvName} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) if err = cmd.Run(); err != nil { return cmd.String(), fmt.Errorf(`unable to RemovePVFromVG with vgName: "%s", pvName: "%s", err: %w`, vgName, pvName, err) @@ -310,7 +325,8 @@ func CheckPVHasNoData(pvName string) (bool, string, error) { func MovePV(pvName string) (string, error) { args := []string{"pvmove", pvName} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) if err := cmd.Run(); err != nil { return cmd.String(), fmt.Errorf("unable to MovePV, err: %w", err) @@ -321,7 +337,8 @@ func MovePV(pvName string) (string, error) { func RemoveVG(vgName string) (string, error) { args := []string{"vgremove", vgName} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -335,7 +352,8 @@ func RemoveVG(vgName string) (string, error) { func RemovePV(pvNames []string) (string, error) { args := []string{"pvremove", strings.Join(pvNames, " ")} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -348,7 +366,8 @@ func RemovePV(pvNames []string) (string, error) { func RemoveLV(vgName, lvName string) (string, error) { args := []string{"lvremove", fmt.Sprintf("/dev/%s/%s", vgName, lvName), "-y"} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) var stderr bytes.Buffer cmd.Stderr = &stderr @@ -425,7 +444,8 @@ func unmarshalLVs(out []byte) ([]internal.LVData, error) { func VGChangeAddTag(vGName, tag string) (string, error) { var outs, stdErr bytes.Buffer args := []string{"vgchange", vGName, "--addtag", tag} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -438,7 +458,8 @@ func VGChangeAddTag(vGName, tag string) (string, error) { func VGChangeDelTag(vGName, tag string) (string, error) { var outs, stdErr bytes.Buffer args := []string{"vgchange", vGName, "--deltag", tag} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -452,7 +473,8 @@ func LVChangeDelTag(lv internal.LVData, tag string) (string, error) { tmpStr := fmt.Sprintf("/dev/%s/%s", lv.VGName, lv.LVName) var outs, stdErr bytes.Buffer args := []string{"lvchange", tmpStr, "--deltag", tag} - cmd := exec.Command(nsenter, args...) + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs cmd.Stderr = &stdErr @@ -461,3 +483,8 @@ func LVChangeDelTag(lv internal.LVData, tag string) (string, error) { } return cmd.String(), nil } + +func extendArgs(args []string) []string { + nsenterArgs := []string{"-t", "1", "-m", "-u", "-i", "-n", "-p"} + return append(nsenterArgs, args...) +} From eaa1498c20628b7296990d6bbf8f695eab6064ce Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Mon, 5 Feb 2024 12:08:24 +0300 Subject: [PATCH 10/15] node attachment check was moved to the events layer, finalizers attachment in create event was added Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- .../controller/lvm_logical_volume_watcher.go | 185 ++++++++++-------- 1 file changed, 103 insertions(+), 82 deletions(-) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index cf0b4be8..f4093e37 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -6,6 +6,7 @@ import ( "fmt" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/strings/slices" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/config" "sds-node-configurator/internal" @@ -62,46 +63,65 @@ func RunLVMLogicalVolumeWatcherController( llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) if !ok { - err := errors.New("unable to cast event object to a given type") - log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling create event") + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[CreateFunc] an error occurs while handling create event") return } - reconcileLLVCreateFunc(ctx, cl, log, metrics, llv, cfg.NodeName) - reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, cfg.NodeName) - reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) + if err != nil { + log.Error(err, "[CreateFunc] unable to getLVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + if err != nil { + log.Error(err, "[CreateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if !belongsToNode(lvg, cfg.NodeName) { + log.Debug(fmt.Sprintf("[CreateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s. Reconciliation stopped", lvg.Name, cfg.NodeName)) + return + } + log.Debug(fmt.Sprintf("[CreateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, cfg.NodeName)) + + reconcileLLVCreateFunc(ctx, cl, log, metrics, llv, lvg) + reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) + log.Info("[RunLVMLogicalVolumeWatcherController] CreateFunc ends reconciliation") }, UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc starts reconciliation") - newLlv, ok := e.ObjectNew.(*v1alpha1.LvmLogicalVolume) + llv, ok := e.ObjectNew.(*v1alpha1.LvmLogicalVolume) if !ok { - err := errors.New("unable to cast event object to a given type") - log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling update event") + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[UpdateFunc] an error occurs while handling update event") + return + } + + lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) + if err != nil { + log.Error(err, "[UpdateFunc] unable to getLVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + if err != nil { + log.Error(err, "[UpdateFunc] unable to updateLVMLogicalVolumePhase") + } + return + } + + if !belongsToNode(lvg, cfg.NodeName) { + log.Debug(fmt.Sprintf("[UpdateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s. Reconciliation stopped", lvg.Name, cfg.NodeName)) return } + log.Debug(fmt.Sprintf("[UpdateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, cfg.NodeName)) + + reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) - reconcileLLVUpdateFunc(ctx, cl, log, metrics, newLlv, cfg.NodeName) - reconcileLLVDeleteFunc(ctx, cl, log, metrics, newLlv, cfg.NodeName) log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc ends reconciliation") }, - - //DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { - // log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc starts reconciliation") - // - // llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) - // if !ok { - // err := errors.New("unable to cast event object to a given type") - // log.Error(err, "[RunLVMLogicalVolumeWatcherController] an error occurs while handling delete event") - // return - // } - // - // reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv, cfg.NodeName) - // - // log.Info("[RunLVMLogicalVolumeWatcherController] DeleteFunc ends reconciliation") - //}, }) if err != nil { log.Error(err, "[RunLVMLogicalVolumeWatcherController] the controller is unable to watch") @@ -111,32 +131,22 @@ func RunLVMLogicalVolumeWatcherController( return c, err } -func reconcileLLVDeleteFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { +func reconcileLLVDeleteFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + llv *v1alpha1.LvmLogicalVolume, +) { log.Info("[reconcileLLVDeleteFunc] starts reconciliation") - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) - if err != nil { - log.Error(err, "[reconcileLLVDeleteFunc] unable to getLVMVolumeGroup") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") - if err != nil { - log.Error(err, "[reconcileLLVDeleteFunc] unable to updateLVMLogicalVolumePhase") - } - return - } - - if !belongsToNode(lvg, nodeName) { - log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s does not belongs to the current node: %s", llv.Name, nodeName)) - return - } - log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s belongs to the current node: %s", llv.Name, nodeName)) - shouldReconcile := shouldReconcileByDeleteFunc(llv) if !shouldReconcile { log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) return } - err = deleteLVifExisted(log, llv) + err := deleteLVifExisted(log, llv) if err != nil { log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete LV %s", llv.Name)) return @@ -160,7 +170,13 @@ func shouldReconcileByDeleteFunc(llv *v1alpha1.LvmLogicalVolume) bool { return true } -func removeLLVFinalizers(ctx context.Context, cl client.Client, metrics monitoring.Metrics, log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error { +func removeLLVFinalizers( + ctx context.Context, + cl client.Client, + metrics monitoring.Metrics, + log logger.Logger, + llv *v1alpha1.LvmLogicalVolume, +) error { var removed bool for i, f := range llv.Finalizers { if f == internal.SdsNodeConfiguratorFinalizer { @@ -240,25 +256,16 @@ func getExtendingSize(log logger.Logger, metrics monitoring.Metrics, llv *v1alph return subtractQuantity(newSize, oldSize), nil } -func reconcileLLVUpdateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { +func reconcileLLVUpdateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + llv *v1alpha1.LvmLogicalVolume, + lvg *v1alpha1.LvmVolumeGroup, +) { log.Info("[reconcileLLVUpdateFunc] starts reconciliation") - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) - if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to getLVMVolumeGroup") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") - if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") - } - return - } - - if !belongsToNode(lvg, nodeName) { - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) - return - } - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) - shouldReconcile, err := shouldReconcileByUpdateFunc(log, llv) if err != nil { log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to check if LV %s should be reconciled", llv.Name)) @@ -374,6 +381,10 @@ func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolu return false, nil } + if llv.Status.Phase == pendingStatusPhase { + return false, nil + } + should, err := shouldReconcileByCreateFunc(log, llv) if err != nil { return false, err @@ -383,10 +394,6 @@ func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolu return false, nil } - if llv.Status.ActualSize.Value() == 0 { - return false, nil - } - if llv.Spec.Size.Value() == llv.Status.ActualSize.Value() { return false, nil } @@ -394,25 +401,16 @@ func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolu return true, nil } -func reconcileLLVCreateFunc(ctx context.Context, cl client.Client, log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume, nodeName string) { +func reconcileLLVCreateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + metrics monitoring.Metrics, + llv *v1alpha1.LvmLogicalVolume, + lvg *v1alpha1.LvmVolumeGroup, +) { log.Info("[reconcileLLVCreateFunc] starts reconciliation") - lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) - if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to getLVMVolumeGroup") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") - if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") - } - return - } - - if !belongsToNode(lvg, nodeName) { - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMVolumeGroup %s does not belongs to the current node: %s", lvg.Name, nodeName)) - return - } - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, nodeName)) - shouldReconcile, err := shouldReconcileByCreateFunc(log, llv) if err != nil { log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to check if LV %s should be reconciled", llv.Name)) @@ -428,10 +426,19 @@ func reconcileLLVCreateFunc(ctx context.Context, cl client.Client, log logger.Lo err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, pendingStatusPhase, "") if err != nil { log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + return } log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) + added, err := addLLVFinalizerIfNotExist(ctx, cl, metrics, llv) + if err != nil { + log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + return + } + + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] a finalizer to the LVMLogicalVolume %s was added: %t", llv.Name, added)) + switch getLVMLogicalVolumeType(llv) { case Thick: freeSpace, err := getFreeVGSpace(log, lvg.Spec.ActualVGNameOnTheNode) @@ -512,6 +519,20 @@ func reconcileLLVCreateFunc(ctx context.Context, cl client.Client, log logger.Lo log.Info("[reconcileLLVCreateFunc] ends reconciliation") } +func addLLVFinalizerIfNotExist(ctx context.Context, cl client.Client, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume) (bool, error) { + if slices.Contains(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) { + return false, nil + } + + llv.Finalizers = append(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) + err := cl.Update(ctx, llv) + if err != nil { + return false, err + } + + return true, nil +} + func shouldReconcileByCreateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (bool, error) { lvs, cmd, _, err := utils.GetAllLVs() log.Debug(fmt.Sprintf("[shouldReconcileByCreateFunc] runs cmd: %s", cmd)) From 1c0402a5593bfcdc6d8f3dd10180d1da2d6ff23e Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Mon, 5 Feb 2024 12:53:41 +0300 Subject: [PATCH 11/15] more info for lvcreate error Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/pkg/utils/commands.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index 4de0a900..f4dc03ef 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -217,8 +217,10 @@ func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) + var stderr bytes.Buffer + cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), err + return cmd.String(), fmt.Errorf("unable to lvcreate, err: %w stdert: %s", err, stderr.String()) } return cmd.String(), nil From 64b8bed69ddc8577beffd338dd2e3c887502431d Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Mon, 5 Feb 2024 13:11:31 +0300 Subject: [PATCH 12/15] more info for lvcreate error Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/pkg/utils/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index f4dc03ef..ca3e81d6 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -220,7 +220,7 @@ func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to lvcreate, err: %w stdert: %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to lvcreate, cmd: %s ,err: %w stdert: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil From 6acc51d4b0706025cc33ce5170a9494b8725751c Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 6 Feb 2024 19:22:21 +0300 Subject: [PATCH 13/15] added stdErr out to error logs for utils commands Signed-off-by: viktor.kramarenko Signed-off-by: Viktor Kramarenko --- images/agent/pkg/utils/commands.go | 223 +++++++++-------------------- 1 file changed, 69 insertions(+), 154 deletions(-) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index ca3e81d6..e2695ff2 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -37,9 +37,12 @@ func GetBlockDevices() ([]internal.Device, string, error) { cmd := exec.Command(nsenter, extendedArgs...) cmd.Stdout = &outs + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() if err != nil { - return nil, cmd.String(), fmt.Errorf("unable to GetBlockDevices, err: %w", err) + return nil, cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } devices, err := unmarshalDevices(outs.Bytes()) @@ -58,8 +61,8 @@ func GetAllVGs() (data []internal.VGData, command string, stdErr bytes.Buffer, e cmd.Stdout = &outs cmd.Stderr = &stdErr - if err := cmd.Run(); err != nil { - return nil, cmd.String(), stdErr, fmt.Errorf("unable to GetAllVGs, err: %w", err) + if err = cmd.Run(); err != nil { + return nil, cmd.String(), stdErr, fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stdErr.String()) } data, err = unmarshalVGs(outs.Bytes()) @@ -78,8 +81,8 @@ func GetAllLVs() (data []internal.LVData, command string, stdErr bytes.Buffer, e cmd.Stdout = &outs cmd.Stderr = &stdErr - if err := cmd.Run(); err != nil { - return nil, cmd.String(), stdErr, fmt.Errorf("unable to GetAllLVs, err: %w", err) + if err = cmd.Run(); err != nil { + return nil, cmd.String(), stdErr, fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stdErr.String()) } lvs, err := unmarshalLVs(outs.Bytes()) @@ -98,8 +101,8 @@ func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, e cmd.Stdout = &outs cmd.Stderr = &stdErr - if err := cmd.Run(); err != nil { - return nil, cmd.String(), stdErr, fmt.Errorf("unable to GetAllPVs, err: %w", err) + if err = cmd.Run(); err != nil { + return nil, cmd.String(), stdErr, fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stdErr.String()) } data, err = unmarshalPVs(outs.Bytes()) @@ -110,32 +113,6 @@ func GetAllPVs() (data []internal.PVData, command string, stdErr bytes.Buffer, e return data, cmd.String(), stdErr, nil } -func GetSinglePV(pVname string) (*internal.PVData, string, error) { - var outs bytes.Buffer - args := []string{"pvs", pVname, "-o", "+pv_used,pv_uuid,vg_tags,vg_uuid", "--reportformat", "json"} - extendedArgs := extendArgs(args) - cmd := exec.Command(nsenter, extendedArgs...) - cmd.Stdout = &outs - - if err := cmd.Run(); err != nil { - return nil, cmd.String(), fmt.Errorf("unable to GetSinglePV, err: %w", err) - } - - pvs, err := unmarshalPVs(outs.Bytes()) - if err != nil { - return nil, cmd.String(), fmt.Errorf("unable to GetSinglePV, err: %w", err) - } - - singlePv := pvs[0] - - if len(pvs) != 1 || - singlePv.PVName != pVname { - return nil, cmd.String(), fmt.Errorf(`unable to GetSinglePV by name: "%s"`, pVname) - } - - return &singlePv, cmd.String(), nil -} - func CreatePV(path string) (string, error) { args := []string{"pvcreate", path} extendedArgs := extendArgs(args) @@ -145,7 +122,7 @@ func CreatePV(path string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to CreatePV, err: %w , stderror = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderror = %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -162,7 +139,7 @@ func CreateVGLocal(vgName, lvmName string, pvNames []string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to CreateVGLocal, err: %w , stderror = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderror: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -173,8 +150,11 @@ func CreateVGShared(vgName, lvmName string, pvNames []string) (string, error) { extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to CreateVGShared, err: %w", err) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -189,7 +169,7 @@ func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, erro cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to CreateThinPool, err: %w tderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil } @@ -206,7 +186,7 @@ func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error err := cmd.Run() if err != nil { - return cmd.String(), err + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -219,8 +199,9 @@ func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { var stderr bytes.Buffer cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to lvcreate, cmd: %s ,err: %w stdert: %s", cmd.String(), err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -235,7 +216,7 @@ func ExtendVG(vgName string, paths []string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to ExtendVG, err: %w stderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -250,7 +231,7 @@ func ExtendLV(size, vgName, lvName string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to ExtendLV, err: %w stderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil @@ -260,85 +241,48 @@ func ResizePV(pvName string) (string, error) { args := []string{"pvresize", pvName} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to ResizePV, err: %w", err) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } return cmd.String(), nil } -func RemovePVFromVG(pvName, vgName string) (string, error) { - pvs, cmdStr, _, err := GetAllPVs() - if err != nil { - return cmdStr, fmt.Errorf("unable to RemovePVFromVG, err: %w", err) - } - - if len(pvs) == 1 { - pv := pvs[0] - - if pv.PVName != pvName || - pv.VGName != vgName { - return cmdStr, - fmt.Errorf( - `unable to RemovePVFromVG, err: unexpected pv gotten, no pv with pvName: "%s", vgName: "%s"`, - pvName, vgName) - } - - if pv.PVUsed != "0 " { - return cmdStr, fmt.Errorf("unable to RemovePVFromVG, err: single PVData has data") - } - - cmdStr, err = RemoveVG(pv.VGName) - return cmdStr, err - } - - if cmdStr, err = MovePV(pvName); err != nil { - return cmdStr, err - } - - clr, cmdStr, err := CheckPVHasNoData(pvName) - if err != nil { - return cmdStr, err - } - - if !clr { - return cmdStr, fmt.Errorf(`unable to RemovePVFromVG, err: can't move LV segments from PVData, pv name: "%s"`, pvName) - } - - args := []string{"vgreduce", vgName, pvName} +func RemoveVG(vgName string) (string, error) { + args := []string{"vgremove", vgName} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) - if err = cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf(`unable to RemovePVFromVG with vgName: "%s", pvName: "%s", err: %w`, - vgName, pvName, err) - } - return cmd.String(), nil -} + var stderr bytes.Buffer + cmd.Stderr = &stderr -func CheckPVHasNoData(pvName string) (bool, string, error) { - pv, cmdStr, err := GetSinglePV(pvName) - if err != nil { - return true, cmdStr, err + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } - return pv.PVUsed == "0 ", cmdStr, nil + return cmd.String(), nil } -func MovePV(pvName string) (string, error) { - args := []string{"pvmove", pvName} +func RemovePV(pvNames []string) (string, error) { + args := []string{"pvremove", strings.Join(pvNames, " ")} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to MovePV, err: %w", err) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr, %s", cmd.String(), err, stderr.String()) } - return cmd.String(), nil } -func RemoveVG(vgName string) (string, error) { - args := []string{"vgremove", vgName} +func RemoveLV(vgName, lvName string) (string, error) { + args := []string{"lvremove", fmt.Sprintf("/dev/%s/%s", vgName, lvName), "-y"} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) @@ -346,36 +290,50 @@ func RemoveVG(vgName string) (string, error) { cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to RemoveVG, err: %w stderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stderr: %s", cmd.String(), err, stderr.String()) } - return cmd.String(), nil } -func RemovePV(pvNames []string) (string, error) { - args := []string{"pvremove", strings.Join(pvNames, " ")} +func VGChangeAddTag(vGName, tag string) (string, error) { + var outs, stdErr bytes.Buffer + args := []string{"vgchange", vGName, "--addtag", tag} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) - - var stderr bytes.Buffer - cmd.Stderr = &stderr + cmd.Stdout = &outs + cmd.Stderr = &stdErr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to RemovePV, err: %w stderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stdErr: %s", cmd.String(), err, stdErr.String()) } return cmd.String(), nil } -func RemoveLV(vgName, lvName string) (string, error) { - args := []string{"lvremove", fmt.Sprintf("/dev/%s/%s", vgName, lvName), "-y"} +func VGChangeDelTag(vGName, tag string) (string, error) { + var outs, stdErr bytes.Buffer + args := []string{"vgchange", vGName, "--deltag", tag} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) + cmd.Stdout = &outs + cmd.Stderr = &stdErr - var stderr bytes.Buffer - cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stdErr: %s", cmd.String(), err, stdErr.String()) + } + return cmd.String(), nil +} + +func LVChangeDelTag(lv internal.LVData, tag string) (string, error) { + tmpStr := fmt.Sprintf("/dev/%s/%s", lv.VGName, lv.LVName) + var outs, stdErr bytes.Buffer + args := []string{"lvchange", tmpStr, "--deltag", tag} + extendedArgs := extendArgs(args) + cmd := exec.Command(nsenter, extendedArgs...) + cmd.Stdout = &outs + cmd.Stderr = &stdErr if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to RemoveLV, err: %w stderr = %s", err, stderr.String()) + return cmd.String(), fmt.Errorf("unable to run cmd: %s, err: %w, stdErr: %s", cmd.String(), err, stdErr.String()) } return cmd.String(), nil } @@ -443,49 +401,6 @@ func unmarshalLVs(out []byte) ([]internal.LVData, error) { return lvs, nil } -func VGChangeAddTag(vGName, tag string) (string, error) { - var outs, stdErr bytes.Buffer - args := []string{"vgchange", vGName, "--addtag", tag} - extendedArgs := extendArgs(args) - cmd := exec.Command(nsenter, extendedArgs...) - cmd.Stdout = &outs - cmd.Stderr = &stdErr - - if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to VGChangeAddTag, err: %w , stdErr: %s", err, stdErr.String()) - } - return cmd.String(), nil -} - -func VGChangeDelTag(vGName, tag string) (string, error) { - var outs, stdErr bytes.Buffer - args := []string{"vgchange", vGName, "--deltag", tag} - extendedArgs := extendArgs(args) - cmd := exec.Command(nsenter, extendedArgs...) - cmd.Stdout = &outs - cmd.Stderr = &stdErr - - if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to VGChangeDelTag, err: %w , stdErr: %s", err, stdErr.String()) - } - return cmd.String(), nil -} - -func LVChangeDelTag(lv internal.LVData, tag string) (string, error) { - tmpStr := fmt.Sprintf("/dev/%s/%s", lv.VGName, lv.LVName) - var outs, stdErr bytes.Buffer - args := []string{"lvchange", tmpStr, "--deltag", tag} - extendedArgs := extendArgs(args) - cmd := exec.Command(nsenter, extendedArgs...) - cmd.Stdout = &outs - cmd.Stderr = &stdErr - - if err := cmd.Run(); err != nil { - return cmd.String(), fmt.Errorf("unable to LVChangeDelTag, err: %w , stdErr: %s", err, stdErr.String()) - } - return cmd.String(), nil -} - func extendArgs(args []string) []string { nsenterArgs := []string{"-t", "1", "-m", "-u", "-i", "-n", "-p"} return append(nsenterArgs, args...) From c9833ff3cdcaa6c92e3d84e8d6c411376d9a7de4 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Wed, 7 Feb 2024 14:15:43 +0300 Subject: [PATCH 14/15] little fixes for update reconciliation Signed-off-by: Viktor Kramarenko --- images/agent/pkg/controller/lvm_logical_volume_watcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index f4093e37..0e098777 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -282,7 +282,7 @@ func reconcileLLVUpdateFunc( if err != nil { log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") } - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, resizingStatusPhase)) log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) extendingSize, err := getExtendingSize(log, metrics, llv) @@ -381,7 +381,7 @@ func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolu return false, nil } - if llv.Status.Phase == pendingStatusPhase { + if llv.Status.Phase == pendingStatusPhase || llv.Status.Phase == resizingStatusPhase { return false, nil } From 095910b6610718b0afb88a6062b73f995cd227d9 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Wed, 14 Feb 2024 12:39:33 +0300 Subject: [PATCH 15/15] added more logs, optimized reconciliation func runs, little refactoring Signed-off-by: Viktor Kramarenko Signed-off-by: Viktor Kramarenko --- .../controller/lvm_logical_volume_watcher.go | 360 +++++++++++------- .../controller/lvm_volume_group_watcher.go | 4 +- images/agent/pkg/utils/commands.go | 12 +- 3 files changed, 221 insertions(+), 155 deletions(-) diff --git a/images/agent/pkg/controller/lvm_logical_volume_watcher.go b/images/agent/pkg/controller/lvm_logical_volume_watcher.go index 0e098777..2853bb5e 100644 --- a/images/agent/pkg/controller/lvm_logical_volume_watcher.go +++ b/images/agent/pkg/controller/lvm_logical_volume_watcher.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/client-go/util/workqueue" "k8s.io/utils/strings/slices" + "math" "sds-node-configurator/api/v1alpha1" "sds-node-configurator/config" "sds-node-configurator/internal" @@ -26,6 +27,10 @@ const ( Thick deviceType = "Thick" Thin deviceType = "Thin" + CreateReconcile reconcileType = "Create" + UpdateReconcile reconcileType = "Update" + DeleteReconcile reconcileType = "Delete" + lvmLogicalVolumeWatcherCtrlName = "lvm-logical-volume-watcher-controller" createdStatusPhase = "Created" @@ -35,7 +40,8 @@ const ( ) type ( - deviceType string + deviceType string + reconcileType string ) func RunLVMLogicalVolumeWatcherController( @@ -64,16 +70,16 @@ func RunLVMLogicalVolumeWatcherController( llv, ok := e.Object.(*v1alpha1.LvmLogicalVolume) if !ok { err = errors.New("unable to cast event object to a given type") - log.Error(err, "[CreateFunc] an error occurs while handling create event") + log.Error(err, "[CreateFunc] an error occurred while handling create event") return } lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) if err != nil { - log.Error(err, "[CreateFunc] unable to getLVMVolumeGroup") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + log.Error(err, "[CreateFunc] unable to get a LVMVolumeGroup") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get selected LVMVolumeGroup, err: %s", err.Error())) if err != nil { - log.Error(err, "[CreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, "[CreateFunc] unable to update a LVMLogicalVolume Phase") } return } @@ -84,9 +90,28 @@ func RunLVMLogicalVolumeWatcherController( } log.Debug(fmt.Sprintf("[CreateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, cfg.NodeName)) - reconcileLLVCreateFunc(ctx, cl, log, metrics, llv, lvg) - reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) - reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) + recType, err := identifyReconcileFunc(log, llv) + if err != nil { + log.Error(err, "[CreateFunc] an error occurs while identify reconcile func") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to identify reconcile func, err: %s", err.Error())) + if err != nil { + log.Error(err, "[CreateFunc] unable to update a LVMLogicalVolume Phase") + } + return + } + switch recType { + case CreateReconcile: + log.Debug(fmt.Sprintf("[CreateFunc] CreateReconcile starts reconciliation for the LVMLogicalVolume: %s", llv.Name)) + reconcileLLVCreateFunc(ctx, cl, log, metrics, llv, lvg) + case UpdateReconcile: + log.Debug(fmt.Sprintf("[CreateFunc] UpdateReconcile starts reconciliation for the LVMLogicalVolume: %s", llv.Name)) + reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[CreateFunc] DeleteReconcile starts reconciliation for the LVMLogicalVolume: %s", llv.Name)) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) + default: + log.Debug(fmt.Sprintf("[CreateFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) + } log.Info("[RunLVMLogicalVolumeWatcherController] CreateFunc ends reconciliation") }, @@ -103,8 +128,8 @@ func RunLVMLogicalVolumeWatcherController( lvg, err := getLVMVolumeGroup(ctx, cl, metrics, "", llv.Spec.LvmVolumeGroup) if err != nil { - log.Error(err, "[UpdateFunc] unable to getLVMVolumeGroup") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get selected LVMVolumeGroup") + log.Error(err, fmt.Sprintf("[UpdateFunc] unable to get the LVMVolumeGroup, name: %s", llv.Spec.LvmVolumeGroup)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get selected LVMVolumeGroup, err: %s", err.Error())) if err != nil { log.Error(err, "[UpdateFunc] unable to updateLVMLogicalVolumePhase") } @@ -117,8 +142,25 @@ func RunLVMLogicalVolumeWatcherController( } log.Debug(fmt.Sprintf("[UpdateFunc] the LVMVolumeGroup %s belongs to the current node: %s", lvg.Name, cfg.NodeName)) - reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) - reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) + recType, err := identifyReconcileFunc(log, llv) + if err != nil { + log.Error(err, "[UpdateFunc] an error occurs while identify reconcile func") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to identify reconcile func, err: %s", err.Error())) + if err != nil { + log.Error(err, "[UpdateFunc] unable to update a LVMLogicalVolume Phase") + } + return + } + switch recType { + case UpdateReconcile: + log.Debug(fmt.Sprintf("[UpdateFunc] UpdateReconcile starts reconciliation for the LVMLogicalVolume: %s", llv.Name)) + reconcileLLVUpdateFunc(ctx, cl, log, metrics, llv, lvg) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[UpdateFunc] DeleteReconcile starts reconciliation for the LVMLogicalVolume: %s", llv.Name)) + reconcileLLVDeleteFunc(ctx, cl, log, metrics, llv) + default: + log.Debug(fmt.Sprintf("[UpdateFunc] should not reconcile the LVMLogicalVolume %s", llv.Name)) + } log.Info("[RunLVMLogicalVolumeWatcherController] UpdateFunc ends reconciliation") }, @@ -131,6 +173,31 @@ func RunLVMLogicalVolumeWatcherController( return c, err } +func identifyReconcileFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (reconcileType, error) { + should, err := shouldReconcileByCreateFunc(log, llv) + if err != nil { + return "", err + } + if should { + return CreateReconcile, nil + } + + should, err = shouldReconcileByUpdateFunc(llv) + if err != nil { + return "", err + } + if should { + return UpdateReconcile, nil + } + + should = shouldReconcileByDeleteFunc(llv) + if should { + return DeleteReconcile, nil + } + + return "", nil +} + func reconcileLLVDeleteFunc( ctx context.Context, cl client.Client, @@ -140,23 +207,25 @@ func reconcileLLVDeleteFunc( ) { log.Info("[reconcileLLVDeleteFunc] starts reconciliation") - shouldReconcile := shouldReconcileByDeleteFunc(llv) - if !shouldReconcile { - log.Debug(fmt.Sprintf("[reconcileLLVDeleteFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) - return - } - - err := deleteLVifExisted(log, llv) + err := deleteLVIfExists(log, llv) if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete LV %s", llv.Name)) + log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to delete the LV %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to delete the LV, name %s, err: %s", llv.Name, err.Error())) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + } return } - log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully deleted LV %s", llv.Name)) + log.Info(fmt.Sprintf("[reconcileLLVDeleteFunc] successfully deleted the LV %s", llv.Name)) - err = removeLLVFinalizers(ctx, cl, metrics, log, llv) + err = removeLLVFinalizersIfExist(ctx, cl, metrics, log, llv) if err != nil { log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to remove finalizers from the LVMVolumeGroup %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to remove finalizer %s, err: %s", internal.SdsNodeConfiguratorFinalizer, err.Error())) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVDeleteFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + } } log.Info("[reconcileLLVDeleteFunc] ends reconciliation") @@ -170,7 +239,7 @@ func shouldReconcileByDeleteFunc(llv *v1alpha1.LvmLogicalVolume) bool { return true } -func removeLLVFinalizers( +func removeLLVFinalizersIfExist( ctx context.Context, cl client.Client, metrics monitoring.Metrics, @@ -182,7 +251,7 @@ func removeLLVFinalizers( if f == internal.SdsNodeConfiguratorFinalizer { llv.Finalizers = append(llv.Finalizers[:i], llv.Finalizers[i+1:]...) removed = true - log.Debug(fmt.Sprintf("[removeLLVFinalizers] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) + log.Debug(fmt.Sprintf("[removeLLVFinalizersIfExist] removed finalizer %s from the LVMLogicalVolume %s", internal.SdsNodeConfiguratorFinalizer, llv.Name)) break } } @@ -198,9 +267,9 @@ func removeLLVFinalizers( return nil } -func deleteLVifExisted(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error { +func deleteLVIfExists(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error { lvs, cmd, _, err := utils.GetAllLVs() - log.Debug(fmt.Sprintf("[deleteLVifExisted] runs cmd: %s", cmd)) + log.Debug(fmt.Sprintf("[deleteLVIfExists] runs cmd: %s", cmd)) if err != nil { return err } @@ -216,44 +285,22 @@ func deleteLVifExisted(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) error } if lv == nil { - log.Debug(fmt.Sprintf("[deleteLVifExisted] did not find LV %s", lv.LVName)) + log.Debug(fmt.Sprintf("[deleteLVIfExists] did not find LV %s", lv.LVName)) return errors.New("lv does not exist") } cmd, err = utils.RemoveLV(lv.VGName, lv.LVName) - log.Debug("[deleteLVifExisted] runs cmd: %s", cmd) + log.Debug("[deleteLVIfExists] runs cmd: %s", cmd) if err != nil { - log.Error(err, "[deleteLVifExisted] unable to RemoveLV") + log.Error(err, "[deleteLVIfExists] unable to RemoveLV") return err } return nil } -func getExtendingSize(log logger.Logger, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume) (resource.Quantity, error) { - lvs, cmd, _, err := utils.GetAllLVs() - log.Debug(fmt.Sprintf("[getExtendingSize] runs cmd %s", cmd)) - if err != nil { - log.Error(err, "[getExtendingSize] unable to GetAllLVs") - return resource.Quantity{}, err - } - - var ( - oldSize resource.Quantity - newSize = llv.Spec.Size - ) - for _, lv := range lvs { - if lv.LVName == llv.Name { - oldSize = lv.LVSize - break - } - } - - if newSize.Value() < oldSize.Value() { - return resource.Quantity{}, fmt.Errorf("selected size is less than exising one") - } - - return subtractQuantity(newSize, oldSize), nil +func getExtendingSize(llv *v1alpha1.LvmLogicalVolume) (resource.Quantity, error) { + return subtractQuantity(llv.Spec.Size, llv.Status.ActualSize), nil } func reconcileLLVUpdateFunc( @@ -266,97 +313,87 @@ func reconcileLLVUpdateFunc( ) { log.Info("[reconcileLLVUpdateFunc] starts reconciliation") - shouldReconcile, err := shouldReconcileByUpdateFunc(log, llv) + err := updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, resizingStatusPhase, "") if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to check if LV %s should be reconciled", llv.Name)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to check LV state") - return - } - - if !shouldReconcile { - log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) - return - } - - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, resizingStatusPhase, "") - if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, resizingStatusPhase)) log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) - extendingSize, err := getExtendingSize(log, metrics, llv) + extendingSize, err := getExtendingSize(llv) if err != nil { log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] error occurs while getting extending size for the LVMLogicalVolume %s", llv.Name)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to count extending LV size") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, err.Error()) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } return } + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s has extending size %d", llv.Name, extendingSize.Value())) switch getLVMLogicalVolumeType(llv) { case Thick: - freeSpace, err := getFreeVGSpace(log, lvg.Spec.ActualVGNameOnTheNode) - log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] free VG space %d", freeSpace.Value())) + freeSpace, err := getFreeVGSpace(lvg) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to getFreeVGSpace") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get free VG space") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to count free space in VG, name: %s", lvg.Spec.ActualVGNameOnTheNode)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to count free VG space, VG name %s, err: %s", lvg.Spec.ActualVGNameOnTheNode, err.Error())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } - + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s requested size: %d, free size: %d", llv.Name, llv.Spec.Size.Value(), freeSpace.Value())) if freeSpace.Value() < extendingSize.Value() { err = errors.New("not enough space") log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s requested size is more than the VG %s free space", llv.Name, lvg.Spec.ActualVGNameOnTheNode)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on VG") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Not enough space on VG, requested: %d, free: %d", llv.Spec.Size.Value(), freeSpace.Value())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } - cmd, err := utils.ExtendLV(llv.Spec.Size.String(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s will be extended in VG %s with Quantity value: %d", llv.Name, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Size.Value())) + cmd, err := utils.ExtendLV(llv.Spec.Size.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to ExtendLV") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to extend Thick LV") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to extend LV, name: %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to extend Thick LV, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } case Thin: freeSpace, err := getFreeLVSpace(log, llv.Spec.Thin.PoolName) - log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] free Thin-pool space %d", freeSpace.Value())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to getFreeVGSpace") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on LV") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to count free space in Thin-pool, name: %s", llv.Spec.Thin.PoolName)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to count free Thin-pool space, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s extending size: %d, free size: %d", llv.Name, extendingSize.Value(), freeSpace.Value())) if freeSpace.Value() < extendingSize.Value() { err = errors.New("not enough space") log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume %s requested size is more than the Thin-pool %s free space", llv.Name, llv.Spec.Thin.PoolName)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on Thin pool") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space in a Thin-pool") if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } - cmd, err := utils.ExtendLV(llv.Spec.Size.String(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) + log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] LV %s will be extended in Thin-pool %s with Quantity value: %d", llv.Name, llv.Spec.Thin.PoolName, llv.Spec.Size.Value())) + cmd, err := utils.ExtendLV(llv.Spec.Size.Value(), lvg.Spec.ActualVGNameOnTheNode, llv.Name) log.Debug(fmt.Sprintf("[reconcileLLVUpdateFunc] runs cmd: %s", cmd)) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to ExtendLV") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to extend Thin pool") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to ExtendLV, name: %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to extend a Thin-pool, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } @@ -364,9 +401,19 @@ func reconcileLLVUpdateFunc( } log.Info(fmt.Sprintf("[reconcileLLVUpdateFunc] successfully extended Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) + actualSize, err := getLVActualSize(log, llv.Name) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to get actual size for LV %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get LV actual size, err: %s", err.Error())) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVUpdateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) + } + return + } + log.Trace(fmt.Sprintf("[reconcileLLVUpdateFunc] the LVMLogicalVolume, name %s actual size %d", llv.Name, actualSize.Value())) llv.Status.Phase = createdStatusPhase - llv.Status.ActualSize = llv.Spec.Size + llv.Status.ActualSize = actualSize err = updateLVMLogicalVolume(ctx, metrics, cl, llv) if err != nil { log.Error(err, "[reconcileLLVUpdateFunc] unable to updateLVMLogicalVolume") @@ -376,25 +423,26 @@ func reconcileLLVUpdateFunc( log.Info("[reconcileLLVUpdateFunc] ends reconciliation") } -func shouldReconcileByUpdateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (bool, error) { +func shouldReconcileByUpdateFunc(llv *v1alpha1.LvmLogicalVolume) (bool, error) { if llv.DeletionTimestamp != nil { return false, nil } - if llv.Status.Phase == pendingStatusPhase || llv.Status.Phase == resizingStatusPhase { + if llv.Status.Phase == pendingStatusPhase || + llv.Status.Phase == resizingStatusPhase { return false, nil } - should, err := shouldReconcileByCreateFunc(log, llv) + delta, err := resource.ParseQuantity(internal.ResizeDelta) if err != nil { return false, err } - if should { - return false, nil + if llv.Spec.Size.Value()+delta.Value() < llv.Status.ActualSize.Value() { + return false, fmt.Errorf("requested size %d is less than actual %d", llv.Spec.Size.Value(), llv.Status.ActualSize.Value()) } - if llv.Spec.Size.Value() == llv.Status.ActualSize.Value() { + if math.Abs(float64(llv.Spec.Size.Value()-llv.Status.ActualSize.Value())) < float64(delta.Value()) { return false, nil } @@ -411,96 +459,85 @@ func reconcileLLVCreateFunc( ) { log.Info("[reconcileLLVCreateFunc] starts reconciliation") - shouldReconcile, err := shouldReconcileByCreateFunc(log, llv) - if err != nil { - log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to check if LV %s should be reconciled", llv.Name)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to check LV state") - return - } - - if !shouldReconcile { - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s should not be reconciled", llv.Name)) - return - } - - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, pendingStatusPhase, "") + err := updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, pendingStatusPhase, "") if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) return } - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] updated LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] updated the LVMLogicaVolume %s status.phase to %s", llv.Name, pendingStatusPhase)) log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s spec.thin.poolname: \"%s\"", llv.Name, llv.Spec.Thin.PoolName)) added, err := addLLVFinalizerIfNotExist(ctx, cl, metrics, llv) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) return } - log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] a finalizer to the LVMLogicalVolume %s was added: %t", llv.Name, added)) switch getLVMLogicalVolumeType(llv) { case Thick: - freeSpace, err := getFreeVGSpace(log, lvg.Spec.ActualVGNameOnTheNode) - log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] free VG space %d", freeSpace.Value())) + freeSpace, err := getFreeVGSpace(lvg) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to getFreeVGSpace") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to get free VG space") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to count free space in VG, name: %s", lvg.Spec.ActualVGNameOnTheNode)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get free VG space, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase for LVMLogicalVolume %s", llv.Name)) } return } + log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size: %d, free size: %d", llv.Name, llv.Spec.Size.Value(), freeSpace.Value())) if freeSpace.Value() < llv.Spec.Size.Value() { err = errors.New("not enough space") log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size is more than the VG %s free space", llv.Name, lvg.Spec.ActualVGNameOnTheNode)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on VG") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space in VG") if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume %s", llv.Name)) } return } - cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.String()) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s will be create in VG %s with Quantity value: %d", llv.Name, lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Size.Value())) + cmd, err := utils.CreateThickLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Name, llv.Spec.Size.Value()) log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] runs cmd: %s", cmd)) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to CreateThickLogicalVolume") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thick LV") + log.Error(err, "[reconcileLLVCreateFunc] unable to create a thick LogicalVolume") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to create Thick LV, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } return } case Thin: freeSpace, err := getFreeLVSpace(log, llv.Spec.Thin.PoolName) - log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] free Thin-pool space %d", freeSpace.Value())) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to getFreeVGSpace") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space on LV") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to count free space in LV, name: %s", llv.Spec.Thin.PoolName)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get free LV space, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } return } + log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size: %d, free size: %d", llv.Name, llv.Spec.Size.Value(), freeSpace.Value())) if freeSpace.Value() < llv.Spec.Size.Value() { err = errors.New("not enough space") log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] the LVMLogicalVolume %s requested size is more than the Thin-pool %s free space", llv.Name, llv.Spec.Thin.PoolName)) - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thin LV") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Not enough space in Thin-pool") if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } return } - cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.String()) + log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] LV %s will be create in Thin-pool %s with size %s", llv.Name, llv.Spec.Thin.PoolName, llv.Spec.Size.String())) + cmd, err := utils.CreateThinLogicalVolume(lvg.Spec.ActualVGNameOnTheNode, llv.Spec.Thin.PoolName, llv.Name, llv.Spec.Size.Value()) log.Debug(fmt.Sprintf("[reconcileLLVCreateFunc] runs cmd: %s", cmd)) if err != nil { log.Error(err, "[reconcileLLVCreateFunc] unable to CreateThickLogicalVolume") - err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, "Unable to create Thin LV") + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to create Thin LV, err: %s", err.Error())) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolumePhase") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) } return } @@ -508,17 +545,44 @@ func reconcileLLVCreateFunc( log.Info(fmt.Sprintf("[reconcileLLVCreateFunc] successfully created Logical Volume for LVMLogicalVolume, name: %s", llv.Name)) + actualSize, err := getLVActualSize(log, llv.Name) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to get actual size for LV %s", llv.Name)) + err = updateLVMLogicalVolumePhase(ctx, cl, metrics, llv, failedStatusPhase, fmt.Sprintf("Unable to get actual LV size, LV name: %s, err: %s", llv.Name, err.Error())) + if err != nil { + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) + } + return + } + log.Trace(fmt.Sprintf("[reconcileLLVCreateFunc] the LV, name: %s has actual size: %d", llv.Name, actualSize.Value())) + llv.Status.Phase = createdStatusPhase - llv.Status.ActualSize = llv.Spec.Size + llv.Status.ActualSize = actualSize err = updateLVMLogicalVolume(ctx, metrics, cl, llv) if err != nil { - log.Error(err, "[reconcileLLVCreateFunc] unable to updateLVMLogicalVolume") + log.Error(err, fmt.Sprintf("[reconcileLLVCreateFunc] unable to update the LVMLogicalVolume, name: %s", llv.Name)) return } log.Info("[reconcileLLVCreateFunc] ends reconciliation") } +func getLVActualSize(log logger.Logger, lvName string) (resource.Quantity, error) { + lvs, cmd, _, err := utils.GetAllLVs() + log.Debug(fmt.Sprintf("[getActualSize] runs cmd: %s", cmd)) + if err != nil { + return resource.Quantity{}, err + } + + for _, lv := range lvs { + if lv.LVName == lvName { + return *resource.NewQuantity(lv.LVSize.Value(), resource.BinarySI), nil + } + } + + return resource.Quantity{}, nil +} + func addLLVFinalizerIfNotExist(ctx context.Context, cl client.Client, metrics monitoring.Metrics, llv *v1alpha1.LvmLogicalVolume) (bool, error) { if slices.Contains(llv.Finalizers, internal.SdsNodeConfiguratorFinalizer) { return false, nil @@ -534,6 +598,11 @@ func addLLVFinalizerIfNotExist(ctx context.Context, cl client.Client, metrics mo } func shouldReconcileByCreateFunc(log logger.Logger, llv *v1alpha1.LvmLogicalVolume) (bool, error) { + if llv.Status.Phase == createdStatusPhase || + llv.Status.Phase == resizingStatusPhase { + return false, nil + } + lvs, cmd, _, err := utils.GetAllLVs() log.Debug(fmt.Sprintf("[shouldReconcileByCreateFunc] runs cmd: %s", cmd)) if err != nil { @@ -596,21 +665,18 @@ func subtractQuantity(min, sub resource.Quantity) resource.Quantity { return val } -func getFreeVGSpace(log logger.Logger, vgName string) (resource.Quantity, error) { - vgs, cmd, _, err := utils.GetAllVGs() - log.Debug(fmt.Sprintf("[getFreeVGSpace] runs cmd: %s", cmd)) +func getFreeVGSpace(lvg *v1alpha1.LvmVolumeGroup) (resource.Quantity, error) { + total, err := resource.ParseQuantity(lvg.Status.VGSize) if err != nil { - log.Error(err, "[getFreeVGSpace] unable to run cmd") return resource.Quantity{}, err } - for _, vg := range vgs { - if vg.VGName == vgName { - return vg.VGFree, nil - } + allocated, err := resource.ParseQuantity(lvg.Status.AllocatedSize) + if err != nil { + return resource.Quantity{}, err } - return resource.Quantity{}, nil + return subtractQuantity(total, allocated), nil } func getLVMLogicalVolumeType(llv *v1alpha1.LvmLogicalVolume) deviceType { diff --git a/images/agent/pkg/controller/lvm_volume_group_watcher.go b/images/agent/pkg/controller/lvm_volume_group_watcher.go index 766af400..d45061cc 100644 --- a/images/agent/pkg/controller/lvm_volume_group_watcher.go +++ b/images/agent/pkg/controller/lvm_volume_group_watcher.go @@ -454,9 +454,9 @@ func ReconcileLVMVG( if err != nil { log.Error(err, fmt.Sprintf("[ReconcileLVMVG] error CreateEventLVMVolumeGroup, resource name: %s", group.Name)) } - newLVSizeStr := strconv.FormatInt(pool.Size.Value()/1024, 10) + //newLVSizeStr := strconv.FormatInt(pool.Size.Value()/1024, 10) start := time.Now() - cmd, err := utils.ExtendLV(newLVSizeStr+"K", group.Spec.ActualVGNameOnTheNode, pool.Name) + cmd, err := utils.ExtendLV(pool.Size.Value(), group.Spec.ActualVGNameOnTheNode, pool.Name) metrics.UtilsCommandsDuration(LVMVolumeGroupWatcherCtrlName, "lvextend").Observe(metrics.GetEstimatedTimeInSeconds(start)) metrics.UtilsCommandsExecutionCount(LVMVolumeGroupWatcherCtrlName, "lvextend").Inc() log.Debug(cmd) diff --git a/images/agent/pkg/utils/commands.go b/images/agent/pkg/utils/commands.go index e2695ff2..839ba852 100644 --- a/images/agent/pkg/utils/commands.go +++ b/images/agent/pkg/utils/commands.go @@ -174,8 +174,8 @@ func CreateThinPool(thinPool v1alpha1.SpecThinPool, VGName string) (string, erro return cmd.String(), nil } -func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error) { - args := []string{"lvcreate", "-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", size, "-W", "y", "-y"} +func CreateThinLogicalVolume(vgName, tpName, lvName string, size int64) (string, error) { + args := []string{"lvcreate", "-T", fmt.Sprintf("%s/%s", vgName, tpName), "-n", lvName, "-V", fmt.Sprintf("%dk", size/1024), "-W", "y", "-y"} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) @@ -192,8 +192,8 @@ func CreateThinLogicalVolume(vgName, tpName, lvName, size string) (string, error return cmd.String(), nil } -func CreateThickLogicalVolume(vgName, lvName, size string) (string, error) { - args := []string{"lvcreate", "-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", size, "-W", "y", "-y"} +func CreateThickLogicalVolume(vgName, lvName string, size int64) (string, error) { + args := []string{"lvcreate", "-n", fmt.Sprintf("%s/%s", vgName, lvName), "-L", fmt.Sprintf("%dk", size/1024), "-W", "y", "-y"} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...) @@ -222,8 +222,8 @@ func ExtendVG(vgName string, paths []string) (string, error) { return cmd.String(), nil } -func ExtendLV(size, vgName, lvName string) (string, error) { - args := []string{"lvextend", "-L", size, fmt.Sprintf("/dev/%s/%s", vgName, lvName)} +func ExtendLV(size int64, vgName, lvName string) (string, error) { + args := []string{"lvextend", "-L", fmt.Sprintf("%dk", size/1024), fmt.Sprintf("/dev/%s/%s", vgName, lvName)} extendedArgs := extendArgs(args) cmd := exec.Command(nsenter, extendedArgs...)