OM-211 - user-metrics not coming when using PKI authentication (#131)

fixed bug - xdr-metric is wrongly validated against user-allow list
updated test-case metadata

.gitignore

@@ -1,2 +1 @@
 aerospike-prometheus-exporter
-

internal/pkg/commons/testutils.go

@@ -14,6 +14,7 @@ Constants, common variables and helper functions
 const (
 	TESTS_DEFAULT_CONFIG_FILE     = "tests_data/default_ape.toml"
 	TESTS_USERS_CONFIG_FILE       = "tests_data/users_ape.toml"
+	TESTS_USERS_PKI_CONFIG_FILE   = "tests_data/users_pki_ape.toml"
 	TESTS_MOCK_CONFIG_FILE        = "tests_data/mock_ape.toml"
 	TESTS_DEFAULT_GAUGE_LIST_FILE = "configs/gauge_stats_list.toml"
 )

internal/pkg/config/config.go

@@ -157,6 +157,7 @@ func (c *Config) ValidateAndUpdate(md toml.MetaData) {
 	if c.Aerospike.AuthMode == "" {
 		c.Aerospike.AuthMode = "internal"
 	}
+	c.Aerospike.AuthMode = strings.ToLower(strings.TrimSpace(c.Aerospike.AuthMode))
 
 	if c.Aerospike.Timeout == 0 {
 		c.Aerospike.Timeout = 5

internal/pkg/config/tests/gauge_stats_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	GAUGES_NAMESPACES_COUNT = 99
+	GAUGES_NAMESPACES_COUNT = 110
 	GAUGES_NODE_STATS_COUNT = 69
 	GAUGES_SETS_COUNT       = 9
 	GAUGES_SINDEX_COUNT     = 13

internal/pkg/dataprovider/aero_server.go

@@ -68,9 +68,7 @@ func initializeAndConnectAerospikeServer() (*aero.Connection, error) {
 	clientPolicy.User = string(username)
 	clientPolicy.Password = string(password)
 
-	authMode := strings.ToLower(strings.TrimSpace(config.Cfg.Aerospike.AuthMode))
-
-	switch authMode {
+	switch config.Cfg.Aerospike.AuthMode {
 	case "internal", "":
 		clientPolicy.AuthMode = aero.AuthModeInternal
 	case "external":

internal/pkg/statprocessors/sp_users.go

@@ -31,7 +31,8 @@ func (uw *UserStatsProcessor) PassTwoKeys(rawMetrics map[string]string) []string
 func (uw *UserStatsProcessor) Refresh(infoKeys []string, rawMetrics map[string]string) ([]AerospikeStat, error) {
 
 	// check if security configurations are enabled
-	if config.Cfg.Aerospike.User == "" && config.Cfg.Aerospike.Password == "" {
+	if config.Cfg.Aerospike.AuthMode != "pki" &&
+		(config.Cfg.Aerospike.User == "" && config.Cfg.Aerospike.Password == "") {
 		return nil, nil
 	}
 

internal/pkg/statprocessors/sp_xdr.go

@@ -120,7 +120,7 @@ func (xw *XdrStatsProcessor) handleRefresh(infoKeyToProcess string, xdrRawMetric
 		dynamicStatname := metricPrefix + stat
 
 		if !exists {
-			allowed := isMetricAllowed(commons.CTX_USERS, stat)
+			allowed := isMetricAllowed(commons.CTX_XDR, stat)
 			asMetric = NewAerospikeStat(commons.CTX_XDR, dynamicStatname, allowed)
 			xw.xdrMetrics[stat] = asMetric
 		}

internal/pkg/statprocessors/tests/sp_latency_test.go

@@ -84,6 +84,10 @@ func latency_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while latencyWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while latencyWatcher.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with setsWatcher
 	latencyMetrics, err := latencyWatcher.Refresh(passTwoOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while latencyWatcher.Refresh with passTwoOutputs ")

internal/pkg/statprocessors/tests/sp_namespaces_test.go

@@ -93,6 +93,10 @@ func namespace_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while NamespaceWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while NamespaceWatcher.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with NamespaceWatcher
 	nsMetrics, err := nsWatcher.Refresh(passTwokeyOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while NamespaceWatcher.Refresh with passTwokeyOutputs ")

internal/pkg/statprocessors/tests/sp_node_stats_test.go

@@ -84,6 +84,10 @@ func node_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while NodeStatsWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while NamespaceWatcher.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with NodeStatsWatcher
 	nodeMetrics, err := nodeWatcher.Refresh(passTwoOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while NodeStatsWatcher.Refresh with passTwoOutputs ")

internal/pkg/statprocessors/tests/sp_sets_test.go

@@ -84,6 +84,10 @@ func sets_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while setsWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while setsWatcher.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with setsWatcher
 	setsMetrics, err := setsWatcher.Refresh(passTwoOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while setsWatcher.Refresh with passTwoOutputs ")

internal/pkg/statprocessors/tests/sp_sindex_test.go

@@ -89,6 +89,10 @@ func sindex_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while sindexMetrics.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while sindexMetrics.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with sindexWatcher
 	sindexMetrics, err := sindexWatcher.Refresh(passTwoOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while sindexMetrics.Refresh with passTwoOutputs ")

internal/pkg/statprocessors/tests/sp_users_test.go

@@ -79,6 +79,10 @@ func users_runTestcase(t *testing.T) {
 	assert.Nil(t, err, "Error while usersWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while usersWatcher.PassTwokeys, RawMetrics is EMPTY ")
 
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
 	// check the output with usersWatcher
 	usersMetrics, err := usersWatcher.Refresh(passTwoOutputs, arrRawMetrics)
 	assert.Nil(t, err, "Error while usersWatcher.Refresh with passTwoOutputs ")
@@ -96,3 +100,46 @@ func users_runTestcase(t *testing.T) {
 	}
 
 }
+
+func Test_Users_TestPKI(t *testing.T) {
+
+	fmt.Println("initializing config ... Test_Users_TestPKI")
+
+	// initialize config and gauge-lists
+	commons.InitConfigurations(commons.GetWatchersConfigFile(commons.TESTS_USERS_PKI_CONFIG_FILE))
+
+	users_runTestcase(t)
+}
+
+func Test_Users_Not_Configured(t *testing.T) {
+
+	fmt.Println("initializing config ... Test_Users_Not_Configured")
+
+	// initialize config and gauge-lists
+	commons.InitConfigurations(commons.GetWatchersConfigFile(commons.TESTS_DEFAULT_CONFIG_FILE))
+
+	// Check passoneKeys
+	usersWatcher := &statprocessors.UserStatsProcessor{}
+	nwPassOneKeys := usersWatcher.PassOneKeys()
+	passOneOutput, _ := dataprovider.GetProvider().RequestInfo(nwPassOneKeys)
+	fmt.Println("users_runTestcase: passOneOutput: ", passOneOutput)
+	passTwoOutputs := usersWatcher.PassTwoKeys(passOneOutput)
+
+	// append common keys
+	infoKeys := []string{statprocessors.Infokey_ClusterName, statprocessors.Infokey_Service, statprocessors.Infokey_Build}
+	passTwoOutputs = append(passTwoOutputs, infoKeys...)
+
+	arrRawMetrics, err := dataprovider.GetProvider().RequestInfo(passTwoOutputs)
+	assert.Nil(t, err, "Error while usersWatcher.PassTwokeys ")
+	assert.NotEmpty(t, arrRawMetrics, "Error while usersWatcher.PassTwokeys, RawMetrics is EMPTY ")
+
+	statprocessors.ClusterName = arrRawMetrics[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = arrRawMetrics[statprocessors.Infokey_Build]
+	statprocessors.Service = arrRawMetrics[statprocessors.Infokey_Service]
+
+	// check the output with usersWatcher
+	usersMetrics, err := usersWatcher.Refresh(passTwoOutputs, arrRawMetrics)
+	assert.Nil(t, err, "Error while usersWatcher.Refresh with passTwoOutputs ")
+	assert.Empty(t, usersMetrics, "Error while usersWatcher.Refresh, usersWatcher is EMPTY ")
+
+}

internal/pkg/statprocessors/tests/sp_xdr_test.go

@@ -90,6 +90,10 @@ func xdr_runTestcase(t *testing.T) {
 	passTwoOutputs = append(passTwoOutputs, infoKeys...)
 	arrRawMetrics, err := dataprovider.GetProvider().RequestInfo(passTwoOutputs)
 
+	statprocessors.ClusterName = passOneOutput[statprocessors.Infokey_ClusterName]
+	statprocessors.Build = passOneOutput[statprocessors.Infokey_Build]
+	statprocessors.Service = passOneOutput[statprocessors.Infokey_Service]
+
 	assert.Nil(t, err, "Error while XdrWatcher.PassTwokeys ")
 	assert.NotEmpty(t, arrRawMetrics, "Error while XdrWatcher.PassTwokeys, RawMetrics is EMPTY ")
 

internal/pkg/statprocessors/tests/tests_data/users_pki_ape.toml

@@ -0,0 +1,153 @@
+[Agent]
+# Exporter HTTPS (TLS) configuration
+# HTTPS between Prometheus and Exporter
+
+# TLS certificates.
+# Supports below formats,
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
+
+# Server certificate
+cert_file = ""
+
+# Private key associated with server certificate
+key_file = ""
+
+# Root CA to validate client certificates (for mutual TLS)
+root_ca = ""
+
+# Passphrase for encrypted key_file. Supports below formats,
+# 1. Passphrase directly                                                      - "<passphrase>"
+# 2. Passphrase via file                                                      - "file:<file-that-contains-passphrase>"
+# 3. Passphrase via environment variable                                      - "env:<environment-variable-that-holds-passphrase>"
+# 4. Passphrase via environment variable containing base64 encoded passphrase - "env-b64:<environment-variable-that-contains-base64-encoded-passphrase>"
+# 5. Passphrase in base64 encoded form                                        - "b64:<base64-encoded-passphrase>"
+key_file_passphrase = ""
+
+# labels to add to the prometheus metrics for e.g. labels={zone="asia-south1-a", platform="google compute engine"}
+labels = {}
+
+bind = ":9145"
+
+# is exporter running in mock test mode
+use_mock_datasource = true
+
+# metrics server timeout in seconds
+timeout = 10
+
+# Exporter logging configuration
+# Log file path (optional, logs to console by default)
+# Level can be info|warning,warn|error,err|debug|trace ('info' by default)
+log_file = ""
+log_level = ""
+
+# Basic HTTP authentication for '/metrics'.
+# Supports below formats,
+# 1. Credential directly                                                      - "<credential>"
+# 2. Credential via file                                                      - "file:<file-that-contains-credential>"
+# 3. Credential via environment variable                                      - "env:<environment-variable-that-contains-credential>"
+# 4. Credential via environment variable containing base64 encoded credential - "env-b64:<environment-variable-that-contains-base64-encoded-credential>"
+# 5. Credential in base64 encoded form                                        - "b64:<base64-encoded-credential>"
+basic_auth_username = ""
+basic_auth_password = ""
+
+[Aerospike]
+db_host = "localhost"
+db_port = 4000
+
+# TLS certificates.
+# Supports below formats,
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
+
+# root certificate file
+root_ca = "/etc/aerospike/certs/pki-ca.crt"
+
+# certificate file
+cert_file = "/etc/aerospike/certs/exporter-client.crt"
+
+# key file
+key_file = "/etc/aerospike/certs/exporter-client.key"
+
+# Passphrase for encrypted key_file. Supports below formats,
+# 1. Passphrase directly                                                      - "<passphrase>"
+# 2. Passphrase via file                                                      - "file:<file-that-contains-passphrase>"
+# 3. Passphrase via environment variable                                      - "env:<environment-variable-that-holds-passphrase>"
+# 4. Passphrase via environment variable containing base64 encoded passphrase - "env-b64:<environment-variable-that-contains-base64-encoded-passphrase>"
+# 5. Passphrase in base64 encoded form                                        - "b64:<base64-encoded-passphrase>"
+key_file_passphrase = ""
+
+# node TLS name for authentication
+node_tls_name = "exporter"
+
+# Aerospike cluster security credentials.
+# Supports below formats,
+# 1. Credential directly                                                      - "<credential>"
+# 2. Credential via file                                                      - "file:<file-that-contains-credential>"
+# 3. Credential via environment variable                                      - "env:<environment-variable-that-contains-credential>"
+# 4. Credential via environment variable containing base64 encoded credential - "env-b64:<environment-variable-that-contains-base64-encoded-credential>"
+# 5. Credential in base64 encoded form                                        - "b64:<base64-encoded-credential>"
+# Applicable to 'user' and 'password' configurations.
+
+# database user
+user = ""
+
+# database password
+password = ""
+
+# authentication mode: internal (server authentication) [default], external (e.g., LDAP), pki. 
+auth_mode = "pki"
+
+# timeout for sending commands to the server node in seconds
+timeout = 5
+
+# Number of histogram buckets to export for latency metrics. Bucket thresholds range from 2^0 to 2^16 (17 buckets).
+# e.g. latency_buckets_count=5 will export first five buckets i.e. <=1ms, <=2ms, <=4ms, <=8ms and <=16ms.
+# Default: 0 (export all threshold buckets).
+latency_buckets_count = 0
+
+# Order of context - namespace, set, latencies, node-stats, xdr, user, jobs, sindex
+
+# Metrics Allowlist - If specified, only these metrics will be scraped. An empty list will exclude all metrics.
+# Commenting out the below allowlist configs will disable metrics filtering (i.e. all metrics will be scraped).
+
+# Metrics Blocklist - If specified, these metrics will be NOT be scraped. An empty list will include all metrics.
+# Commenting out the below blocklist configs will disable metrics filtering (i.e. no metrics will be blocked/filtered).
+
+# globbing pattern (wildcard) are allowd for allowlist and blocklist
+# for example "batch_index_*_buffers"
+
+# Namespace metrics allowlist, to control which Namespace metrics should be collected.
+# namespace_metrics_allowlist = []
+# namespace_metrics_blocklist = []
+
+# set_metrics_allowlist = []
+# set_metrics_blocklist = []
+
+# Latencies histogram allowlist, to control which Histogram-name metrics should be collected. 
+# Note: globbing patterns (wildcard) are not supported for this latency metric configuration. 
+# latencies_metrics_allowlist = []
+# latencies_metrics_blocklist = []
+
+# node_metrics_allowlist = []
+# node_metrics_blocklist = []
+
+# Support only from Aerospike versions 5.0 and above
+# xdr_metrics_allowlist = []
+# xdr_metrics_blocklist = []
+
+# User statistics are available in Aerospike 5.6+
+# Note globbing patterns (wildcard) are not supported for this User configuration.
+# user_metrics_users_allowlist = []
+# user_metrics_users_blocklist = []
+
+# job_metrics_allowlist = []
+# job_metrics_blocklist = []
+
+# sindex_metrics_allowlist = []
+# sindex_metrics_blocklist = []
+