From e0e96eea9b7b4a9d271fdb7e8821a7769cf010f0 Mon Sep 17 00:00:00 2001 From: ChenYong Date: Mon, 19 Dec 2022 13:38:09 +0800 Subject: [PATCH 1/2] throttle not happen when cpu-usage exceed nodeqos threshhold --- pkg/ensurance/executor/watermark.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/ensurance/executor/watermark.go b/pkg/ensurance/executor/watermark.go index 5200fb8ba..e155566c6 100644 --- a/pkg/ensurance/executor/watermark.go +++ b/pkg/ensurance/executor/watermark.go @@ -215,21 +215,21 @@ func calculateGaps(stateMap map[string][]common.TimeSeries, throttleDownWatermark, throttleDownExist := throttleExecutor.ThrottleDownWatermark[m.Name] throttleUpWatermark, throttleUpExist := throttleExecutor.ThrottleUpWatermark[m.Name] - // If a metric does not exist in ThrottleDownWatermark, throttleDownGapToWatermarks of this metric will can't be calculated - if !throttleDownExist { - delete(result, m.Name) - } else { - klog.V(6).Infof("BuildThrottleDownWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) + if throttleDownExist && throttleUpExist { + klog.V(6).Infof("BuildThrottleDownWatermarkGap & BuildThrottleUpWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWatermark.PopSmallest().Value())) - } - // If metric not exist in ThrottleUpWatermark, throttleUpGapToWatermarks of metric will can't be calculated - if !throttleUpExist { - delete(result, m.Name) - } else { + // Attention: different with throttleDown and evict, use watermark - used + result[m.Name] = (1 + executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) + } else if throttleDownExist { + klog.V(6).Infof("BuildThrottleDownWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) + result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWatermark.PopSmallest().Value())) + } else if throttleUpExist { klog.V(6).Infof("BuildThrottleUpWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleUpWatermark.PopSmallest().Value())) // Attention: different with throttleDown and evict, use watermark - used result[m.Name] = (1 + executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) + } else { + delete(result, m.Name) } } } From 4902c979e27d9e35956ffee393eb759e44f11911 Mon Sep 17 00:00:00 2001 From: ChenYong Date: Mon, 27 Mar 2023 15:04:30 +0800 Subject: [PATCH 2/2] fix accoring to comments --- pkg/ensurance/analyzer/analyzer.go | 2 +- pkg/ensurance/executor/watermark.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 64fed0781..f36892e35 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -291,7 +291,7 @@ func (s *AnomalyAnalyzer) computeActionContext(aboveThreshold bool, key string, s.restored[key] = uint64(object.RestoreThreshold) } // only do restore action after trigger x times and restore y times - if s.triggered[key] >= uint64(object.AvoidanceThreshold) && + if (s.triggered[key] >= uint64(object.AvoidanceThreshold) || s.triggered[key] == 0) && s.restored[key] >= uint64(object.RestoreThreshold) { // reset trigger count when restored s.triggered[key] = 0 diff --git a/pkg/ensurance/executor/watermark.go b/pkg/ensurance/executor/watermark.go index e155566c6..1ba887165 100644 --- a/pkg/ensurance/executor/watermark.go +++ b/pkg/ensurance/executor/watermark.go @@ -216,18 +216,16 @@ func calculateGaps(stateMap map[string][]common.TimeSeries, throttleUpWatermark, throttleUpExist := throttleExecutor.ThrottleUpWatermark[m.Name] if throttleDownExist && throttleUpExist { - klog.V(6).Infof("BuildThrottleDownWatermarkGap & BuildThrottleUpWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) - result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWatermark.PopSmallest().Value())) - - // Attention: different with throttleDown and evict, use watermark - used - result[m.Name] = (1 + executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) + // Wrong case: throttleUpExist && throttleDownExist won't co-exist + klog.Warningf("throttleUpExist && throttleDownExist won't co-exist: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) + continue } else if throttleDownExist { klog.V(6).Infof("BuildThrottleDownWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleDownWatermark.PopSmallest().Value())) result[m.Name] = (1 + executeExcessPercent) * (maxUsed - float64(throttleDownWatermark.PopSmallest().Value())) } else if throttleUpExist { klog.V(6).Infof("BuildThrottleUpWatermarkGap: For metrics %s, maxUsed is %f, watermark is %f", m.Name, maxUsed, float64(throttleUpWatermark.PopSmallest().Value())) // Attention: different with throttleDown and evict, use watermark - used - result[m.Name] = (1 + executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) + result[m.Name] = (1 - executeExcessPercent) * (float64(throttleUpWatermark.PopSmallest().Value()) - maxUsed) } else { delete(result, m.Name) }