OM-209 - latencies command is not working

internal/pkg/statprocessors/sp_latency.go

@@ -49,28 +49,23 @@ func (lw *LatencyStatsProcessor) getLatenciesCommands(rawMetrics map[string]stri
 
 	// below latency-command are added to the auto-enabled list, i.e. latencies: command
 	// re-repl is auto-enabled, but not coming as part of latencies: list, hence we are adding it explicitly
-	//
-	// Hashmap content format := namespace-<histogram-key> = <0/1>
-	for latencyHistName := range LatencyBenchmarks {
-		histTokens := strings.Split(latencyHistName, "-")
-
-		histCommand := "latencies:hist="
-
-		// service-enable-benchmarks-fabric or ns-enable-benchmarks-ops-sub or service-enable-hist-info
-		if histTokens[0] != "service" {
-			histCommand = histCommand + "{" + histTokens[0] + "}-"
-		}
-
-		if strings.Contains(latencyHistName, "enable-benchmarks-") {
-			histCommand = histCommand + strings.Join(histTokens[2:], "-")
-		} else {
-			histCommand = histCommand + strings.Join(histTokens[3:], "-")
+	// command will be like
+	//      latencies:hist={NAMESPACE}-proxy / latencies:hist={NAMESPACE}-benchmarks-read
+	//      latencies:hist=info
+
+	for nsName := range NamespaceLatencyBenchmarks {
+		for _, latencyCommand := range NamespaceLatencyBenchmarks[nsName] {
+			histCommand := "latencies:hist=" + latencyCommand
+			commands = append(commands, histCommand)
 		}
+	}
 
+	for _, latencyCommand := range ServiceLatencyBenchmarks {
+		histCommand := "latencies:hist=" + latencyCommand
 		commands = append(commands, histCommand)
 	}
 
-	log.Tracef("latency-passtwokeys:%s", commands)
+	log.Tracef("latency-getLatenciesCommands:%s", commands)
 
 	return commands
 }
@@ -126,7 +121,6 @@ func parseSingleLatenciesKey(singleLatencyKey string, rawMetrics map[string]stri
 
 	// log.Tracef("latency-stats:%+v", latencyStats)
 	log.Tracef("latencies-stats:%+v:%+v", singleLatencyKey, rawMetrics[singleLatencyKey])
-
 	var latencyMetricsToSend = []AerospikeStat{}
 
 	for namespaceName, nsLatencyStats := range latencyStats {

internal/pkg/statprocessors/sp_namespaces.go

@@ -50,14 +50,17 @@ func (nw *NamespaceStatsProcessor) PassOneKeys() []string {
 
 func (nw *NamespaceStatsProcessor) PassTwoKeys(rawMetrics map[string]string) []string {
 	s := rawMetrics[KEY_NS_METADATA]
-	list := strings.Split(s, ";")
+	nsList := strings.Split(s, ";")
 
 	log.Tracef("namespaces:%s", s)
 
 	var infoKeys []string
-	for _, k := range list {
+	for _, ns := range nsList {
 		// infoKey ==> namespace/test, namespace/bar
-		infoKeys = append(infoKeys, KEY_NS_NAMESPACE+"/"+k)
+		infoKeys = append(infoKeys, KEY_NS_NAMESPACE+"/"+ns)
+		if NamespaceLatencyBenchmarks[ns] == nil {
+			NamespaceLatencyBenchmarks[ns] = make(map[string]string)
+		}
 	}
 
 	if nw.canSendIndexPressureInfoKey() {
@@ -228,32 +231,28 @@ func (nw *NamespaceStatsProcessor) refreshNamespaceStats(singleInfoKey string, i
 			nsMetricsToSend = append(nsMetricsToSend, asMetric)
 		}
 
-		// below code section is to ensure ns+latencies combination is handled during LatencyWatcher
-		//
-		// check and if latency benchmarks stat - is it enabled (bool true==1 and false==0 after conversion)
-		if isStatLatencyHistRelated(stat) {
-			// remove old value as microbenchmark may get enabled / disable on-the-fly at server so we cannot rely on value
-			delete(LatencyBenchmarks, nsName+"-"+stat)
-
-			if pv == 1 {
-				LatencyBenchmarks[nsName+"-"+stat] = stat
-			}
-		}
-
-		// below code section is to ensure ns+latencies combination is handled during LatencyWatcher
-		//
-		// check and if latency benchmarks stat - is it enabled (bool true==1 and false==0 after conversion)
+		// below code section is to ensure latencies combination is handled during LatencyWatcher
 		if isStatLatencyHistRelated(stat) {
-			delete(LatencyBenchmarks, nsName+"-"+stat)
-
+			// pv==1 means histogram is enabled
 			if pv == 1 {
-				LatencyBenchmarks[nsName+"-"+stat] = stat
+				latencySubcommand := "{" + nsName + "}-" + stat
+				if strings.Contains(latencySubcommand, "enable-") {
+					latencySubcommand = strings.ReplaceAll(latencySubcommand, "enable-", "")
+				}
+				// some histogram stats has 'hist-' in the config, but the latency command does not expect hist- when issue the command
+				if strings.Contains(latencySubcommand, "hist-") {
+					latencySubcommand = strings.ReplaceAll(latencySubcommand, "hist-", "")
+				}
+				NamespaceLatencyBenchmarks[nsName][stat] = latencySubcommand
+			} else {
+				// pv==0 means histogram is disabled
+				delete(NamespaceLatencyBenchmarks[nsName], stat)
 			}
 		}
 
 	}
 	// append default re-repl, as this auto-enabled, but not coming as part of latencies, we need this as namespace is available only here
-	LatencyBenchmarks[nsName+"-latency-hist-re-repl"] = "{" + nsName + "}-re-repl"
+	NamespaceLatencyBenchmarks[nsName]["re-repl"] = "{" + nsName + "}-" + "re-repl"
 
 	return nsMetricsToSend
 }

internal/pkg/statprocessors/sp_node_stats.go

@@ -1,6 +1,8 @@
 package statprocessors
 
 import (
+	"strings"
+
 	"github.com/aerospike/aerospike-prometheus-exporter/internal/pkg/commons"
 
 	log "github.com/sirupsen/logrus"
@@ -87,11 +89,21 @@ func (sw *NodeStatsProcessor) handleRefresh(nodeRawMetrics string) []AerospikeSt
 		// check and if latency benchmarks stat, is it enabled (bool true==1 and false==0 after conversion)
 		if isStatLatencyHistRelated(stat) {
 
-			// remove old value as microbenchmark may get enabled / disable on-the-fly at server so we cannot rely on value
-			delete(LatencyBenchmarks, "service-"+stat)
-
+			// pv==1 means histogram is enabled
 			if pv == 1 {
-				LatencyBenchmarks["service-"+stat] = stat
+				latencySubcommand := stat
+				if strings.Contains(latencySubcommand, "enable-") {
+					latencySubcommand = strings.ReplaceAll(latencySubcommand, "enable-", "")
+				}
+				// some histogram stats has 'hist-' in the config, but the latency command does not expect hist- when issue the command
+				if strings.Contains(latencySubcommand, "hist-") {
+					latencySubcommand = strings.ReplaceAll(latencySubcommand, "hist-", "")
+				}
+
+				ServiceLatencyBenchmarks[stat] = latencySubcommand
+			} else {
+				// pv==0 means histogram is disabled
+				delete(ServiceLatencyBenchmarks, stat)
 			}
 		}
 	}

internal/pkg/statprocessors/statsprocessor.go

@@ -5,7 +5,8 @@ var (
 	Service, ClusterName, Build string
 )
 
-var LatencyBenchmarks = make(map[string]string)
+var ServiceLatencyBenchmarks = make(map[string]string)
+var NamespaceLatencyBenchmarks = make(map[string]map[string]string)
 
 type StatProcessor interface {
 	PassOneKeys() []string