From 078c612ea193273855267098047a4f31bad17b63 Mon Sep 17 00:00:00 2001 From: SP Kesan Date: Mon, 16 Nov 2020 18:38:54 +0530 Subject: [PATCH] Dashboard improvements (#23) - Dashboard improvements - Port Dashboards to Grafana 7.2 - Make datasource configurable through a dashboard variable (Co-authored-by: realmgic ) - Alert when 'close to' stop writes, when node is proxying - Add docker swarm service discovery to the example --- config/grafana/dashboards/alerts.json | 116 +- config/grafana/dashboards/cluster.json | 1727 +++++++--- config/grafana/dashboards/latency.json | 149 +- config/grafana/dashboards/namespace.json | 3798 +++++++++++----------- config/grafana/dashboards/node.json | 2168 +++++------- config/grafana/dashboards/xdr.json | 611 +++- config/prometheus/aerospike_rules.yml | 71 +- examples/swarm/aerospike.conf | 2 +- examples/swarm/docker-stack.yaml | 8 +- examples/swarm/prometheus.yaml | 15 +- 10 files changed, 4673 insertions(+), 3992 deletions(-) diff --git a/config/grafana/dashboards/alerts.json b/config/grafana/dashboards/alerts.json index 2e6c0bb..80d2ba3 100644 --- a/config/grafana/dashboards/alerts.json +++ b/config/grafana/dashboards/alerts.json @@ -1,4 +1,24 @@ { + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table-old", + "name": "Table (old)", + "version": "" + } + ], "annotations": { "list": [ { @@ -15,12 +35,19 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1565277670876, + "id": null, + "iteration": 1602891686933, "links": [], "panels": [ { "columns": [], - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fontSize": "100%", "gridPos": { "h": 24, @@ -30,7 +57,6 @@ }, "id": 85, "links": [], - "options": {}, "pageSize": null, "scroll": true, "showHeader": true, @@ -41,12 +67,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "date" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -63,6 +91,7 @@ }, { "alias": "Alert Name", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -77,6 +106,7 @@ }, { "alias": "Alert State", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -97,6 +127,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -112,7 +143,8 @@ "unit": "short" }, { - "alias": "Cluster", + "alias": "Cluster Name", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -128,7 +160,8 @@ "unit": "short" }, { - "alias": "", + "alias": "Exporter Instance", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -145,6 +178,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -160,7 +194,8 @@ "unit": "short" }, { - "alias": "Host", + "alias": "Aerospike Instance", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -177,6 +212,7 @@ }, { "alias": "Severity", + "align": "auto", "colorMode": "row", "colors": [ "rgba(245, 54, 54, 0.9)", @@ -214,6 +250,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -227,6 +264,23 @@ "thresholds": [], "type": "hidden", "unit": "short" + }, + { + "alias": "Namespace", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ns", + "thresholds": [], + "type": "string", + "unit": "short" } ], "targets": [ @@ -242,22 +296,19 @@ "timeShift": null, "title": "Alerts", "transform": "table", - "type": "table" + "type": "table-old" } ], "refresh": "1m", - "schemaVersion": 19, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "text": "null", - "value": "null" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": true, @@ -278,11 +329,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -304,8 +352,11 @@ { "allValue": null, "current": { + "selected": true, "tags": [], - "text": "All", + "text": [ + "All" + ], "value": [ "$__all" ] @@ -343,13 +394,18 @@ } ], "query": "1,2,3,4", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": null, "current": { - "text": "All", + "selected": true, + "tags": [], + "text": [ + "All" + ], "value": [ "$__all" ] @@ -377,8 +433,28 @@ } ], "query": "firing,pending", + "queryValue": "", "skipUrlSync": false, "type": "custom" + }, + { + "current": { + "selected": true, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" } ] }, diff --git a/config/grafana/dashboards/cluster.json b/config/grafana/dashboards/cluster.json index 105e599..0c3496d 100644 --- a/config/grafana/dashboards/cluster.json +++ b/config/grafana/dashboards/cluster.json @@ -1,4 +1,36 @@ { + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "7.1.0" + } + ], "annotations": { "list": [ { @@ -18,11 +50,13 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1573694138467, + "id": null, + "iteration": 1603366846449, "links": [], "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -37,8 +71,14 @@ { "cacheTimeout": null, "content": "

$cluster

", - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Cluster name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "gridPos": { "h": 3, "w": 6, @@ -48,7 +88,11 @@ "id": 35, "links": [], "mode": "html", - "options": {}, + "options": { + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "7.1.0", "targets": [ { "expr": "min(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", @@ -66,194 +110,322 @@ }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Build version", - "decimals": 0, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Cluster size", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { "h": 3, - "w": 3, + "w": 6, "x": 6, "y": 1 }, - "id": 36, + "id": 4, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "build", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", - "format": "table", + "expr": "sum(aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster\"})", + "format": "time_series", "instant": true, "intervalFactor": 1, - "legendFormat": "", "refId": "A" } ], - "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Aerospike Version", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [], - "valueName": "current" + "title": "Cluster Size", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Cluster size", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace stop_writes flag", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { "h": 3, - "w": 3, - "x": 9, + "w": 6, + "x": 12, "y": 1 }, - "id": 4, + "id": 14, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ { - "name": "range to text", - "value": 2 + "expr": "sum(aerospike_namespace_stop_writes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" } ], + "timeFrom": null, + "timeShift": null, + "title": "Stop Writes", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Total migration partitions remaining (transmit + receive)", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 16, + "interval": null, + "links": [], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "sum(aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster\"})", + "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) + sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A" } ], - "thresholds": "1000,1000", "timeFrom": null, "timeShift": null, - "title": "Cluster Size", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ + "title": "Migrations Remaining", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Build version", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 36, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^build$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" } ], - "valueName": "current" + "timeFrom": null, + "timeShift": null, + "title": "Aerospike Version", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Alert count", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { "h": 3, - "w": 3, - "x": 12, - "y": 1 + "w": 6, + "x": 6, + "y": 4 }, "id": 5, "interval": null, @@ -264,39 +436,22 @@ "url": "/d/0bMepuvZz/alerts" } ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { "expr": "count(ALERTS{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", @@ -306,314 +461,757 @@ "refId": "A" } ], - "thresholds": "1,1", "timeFrom": null, "timeShift": null, "title": "Alerts", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Namespace stop_writes flag", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace clock_skew_stop_writes flag", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { "h": 3, - "w": 3, - "x": 15, - "y": 1 + "w": 6, + "x": 12, + "y": 4 }, - "id": 14, + "id": 15, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "sum(aerospike_namespace_stop_writes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "sum(aerospike_namespace_clock_skew_stop_writes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A" } ], - "thresholds": "1,1", "timeFrom": null, "timeShift": null, - "title": "Stop Writes", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Clock Skew Stop Writes", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Namespace clock_skew_stop_writes flag", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Dead Partitions in an strong consistent namespace", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { "h": 3, - "w": 3, + "w": 6, "x": 18, - "y": 1 + "y": 4 }, - "id": 15, + "id": 65, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "sum(aerospike_namespace_clock_skew_stop_writes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "sum(aerospike_namespace_unavailable_partitions{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Unavailable Partitions", + "refId": "B" + }, + { + "expr": "sum(aerospike_namespace_dead_partitions{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "Dead Paritions", "refId": "A" } ], - "thresholds": "1,1", "timeFrom": null, "timeShift": null, - "title": "Clock Skew Stop Writes", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Unavailable / Dead Partitions", + "type": "stat" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total migration partitions remaining (transmit + receive)", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 38, + "panels": [], + "title": "Node Statstics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Client connection count", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 1 + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 33, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Client Connections", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Cluster size (cluster_size)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 41, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "aerospike_node_stats_cluster_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Size", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 62, + "panels": [], + "title": "Resources (Memory)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Amount of memory in in-use pages (heap_active_kbytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 66, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "aerospike_node_stats_heap_active_kbytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Heap Memory", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "kbytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Heap allocated memory (heap_allocated_kbytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 43, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "aerospike_node_stats_heap_allocated_kbytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocated Heap Memory", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "kbytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Heap mapped memory (heap_mapped_kbytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 17 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 44, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true }, - "id": 16, - "interval": null, + "lines": true, + "linewidth": 1, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "nullPointMode": "null", + "options": { + "alertThreshold": true }, - "tableColumn": "", + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) + sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_node_stats_heap_mapped_kbytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", - "instant": true, + "instant": false, + "interval": "", "intervalFactor": 1, + "legendFormat": "{{service}}", "refId": "A" } ], - "thresholds": "1,1", + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Migrations Remaining", - "type": "singlestat", - "valueFontSize": "110%", - "valueMaps": [ + "title": "Mapped Memory", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "kbytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, { - "op": "=", - "text": "N/A", - "value": "null" + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true } ], - "valueName": "current" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 38, - "panels": [], - "title": "Node Statstics", - "type": "row" + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Client connection count", - "decimals": 0, - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Heap efficiency % (heap_efficiency_pct)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 5 + "y": 25 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 33, + "id": 45, "interval": "", "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, - "max": true, - "min": true, + "max": false, + "min": false, "rightSide": true, "show": true, "total": false, @@ -624,9 +1222,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -636,7 +1235,7 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_node_stats_heap_efficiency_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", "instant": false, "interval": "", @@ -649,7 +1248,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Client Connections", + "title": "Heap Efficiency", "tooltip": { "shared": true, "sort": 1, @@ -666,7 +1265,7 @@ "yaxes": [ { "decimals": 0, - "format": "short", + "format": "percent", "label": "", "logBase": 1, "max": null, @@ -693,26 +1292,33 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Cluster size (cluster_size)", - "decimals": 0, - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of free system memory available", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 5 + "w": 8, + "x": 8, + "y": 25 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 41, + "id": 70, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, - "min": true, + "max": false, + "min": false, "rightSide": true, "show": true, "total": false, @@ -723,9 +1329,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -735,7 +1342,7 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_cluster_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_node_stats_system_free_mem_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", "instant": false, "interval": "", @@ -748,7 +1355,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Cluster Size", + "title": "Free System Memory", "tooltip": { "shared": true, "sort": 1, @@ -765,7 +1372,7 @@ "yaxes": [ { "decimals": 0, - "format": "short", + "format": "percent", "label": "", "logBase": 1, "max": null, @@ -789,15 +1396,16 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 33 }, - "id": 62, + "id": 69, "panels": [], - "title": "Resources", + "title": "Resources (CPU)", "type": "row" }, { @@ -805,18 +1413,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Heap allocated memory (heap_allocated_kbytes)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of CPU usage by the asd process.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, "x": 0, - "y": 14 + "y": 34 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 43, + "id": 71, "interval": "", "legend": { "alignAsTable": true, @@ -834,9 +1450,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -846,7 +1463,7 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_heap_allocated_kbytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_node_stats_process_cpu_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", "instant": false, "interval": "", @@ -859,7 +1476,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Allocated Heap Memory", + "title": "ASD CPU usage", "tooltip": { "shared": true, "sort": 1, @@ -876,7 +1493,7 @@ "yaxes": [ { "decimals": 0, - "format": "kbytes", + "format": "percent", "label": "", "logBase": 1, "max": null, @@ -903,18 +1520,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Heap mapped memory (heap_mapped_kbytes)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of CPU usage by all running processes.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, "x": 8, - "y": 14 + "y": 34 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 44, + "id": 73, "interval": "", "legend": { "alignAsTable": true, @@ -932,9 +1557,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -944,20 +1570,20 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_heap_mapped_kbytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_node_stats_system_total_cpu_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{service}}", - "refId": "A" + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mapped Memory", + "title": "Total CPU usage", "tooltip": { "shared": true, "sort": 1, @@ -974,7 +1600,7 @@ "yaxes": [ { "decimals": 0, - "format": "kbytes", + "format": "percent", "label": "", "logBase": 1, "max": null, @@ -1001,18 +1627,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Heap efficiency % (heap_efficiency_pct)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of CPU usage by processes running in kernel mode.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, "x": 16, - "y": 14 + "y": 34 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 45, + "id": 72, "interval": "", "legend": { "alignAsTable": true, @@ -1030,9 +1664,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1042,7 +1677,7 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_heap_efficiency_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_node_stats_system_kernel_cpu_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", "instant": false, "interval": "", @@ -1055,7 +1690,114 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Heap Efficiency", + "title": "Kernel mode CPU usage", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of CPU usage by processes running in user mode.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 74, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "aerospike_node_stats_system_user_cpu_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "User mode CPU usage", "tooltip": { "shared": true, "sort": 1, @@ -1096,11 +1838,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 50 }, "id": 28, "panels": [], @@ -1108,13 +1851,20 @@ "type": "row" }, { - "content": " \n\n### For namespace specific throughput please see the [Namespace View dashboard](/d/zGcUKcDZz/namespace-view).", + "content": " \n\n### For namespace specific throughput please see the [Namespace View dashboard](/d/zGcUKcDZz/namespace-view).", + "datasource": null, "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "gridPos": { - "h": 3, + "h": 2, "w": 24, "x": 0, - "y": 23 + "y": 51 }, "id": 64, "links": [ @@ -1124,7 +1874,11 @@ } ], "mode": "markdown", - "options": {}, + "options": { + "content": "
\n

See Namespace View dashboard for namespace and transaction type specific TPS and Latency

", + "mode": "html" + }, + "pluginVersion": "7.1.0", "timeFrom": null, "timeShift": null, "title": "", @@ -1135,17 +1889,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Batch Index TPS shows per-second average rate of increase in\n\n1. Total = batch_index_complete\n2. Successful = batch_index_complete - (batch_index_timeout + batch_index_error)\n3. Timeout = batch_index_timeout\n4. Error = batch_index_error", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "fill": 1, - "fillGradient": 0, + "description": "Batch Index TPS shows per-second average rate of increase in\n\n1. Total = batch_index_complete\n2. Successful = batch_index_complete - (batch_index_timeout + batch_index_error)\n3. Timeout = batch_index_timeout\n4. Error = batch_index_error", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 26 + "y": 53 }, + "hiddenSeries": false, "hideTimeOverride": false, "id": 17, "interval": "", @@ -1165,9 +1927,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1255,18 +2018,15 @@ } ], "refresh": "1m", - "schemaVersion": 19, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "text": "longevity-cluster-red", - "value": "longevity-cluster-red" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": false, @@ -1287,11 +2047,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -1312,11 +2069,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\" },ns)", "hide": 0, "includeAll": true, @@ -1337,22 +2091,21 @@ }, { "current": { - "text": "", - "value": "" + "selected": false, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" }, "hide": 0, - "label": "Tags", - "name": "tags", - "options": [ - { - "selected": true, - "text": "", - "value": "" - } - ], - "query": "", + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "textbox" + "type": "datasource" } ] }, diff --git a/config/grafana/dashboards/latency.json b/config/grafana/dashboards/latency.json index f87b21f..0deb810 100644 --- a/config/grafana/dashboards/latency.json +++ b/config/grafana/dashboards/latency.json @@ -1,4 +1,24 @@ { + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], "annotations": { "list": [ { @@ -15,12 +35,13 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 4, - "iteration": 1595275514565, + "id": null, + "iteration": 1603381556406, "links": [], "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -38,16 +59,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "$operation latency", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 }, + "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": true, @@ -64,9 +93,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -138,16 +168,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "90th percentile of $operation request durations over the last 1m", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, + "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, @@ -164,9 +202,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -224,18 +263,15 @@ } ], "refresh": "1m", - "schemaVersion": 19, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "text": "null", - "value": "null" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": false, @@ -256,13 +292,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -283,13 +314,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\" },ns)", "hide": 0, "includeAll": true, @@ -310,37 +336,15 @@ }, { "allValue": null, - "current": { - "tags": [], - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "metrics(aerospike_latencies_)", "hide": 0, "includeAll": true, "label": "Operation", "multi": true, "name": "operation", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "read", - "value": "read" - }, - { - "selected": false, - "text": "write", - "value": "write" - } - ], + "options": [], "query": "metrics(aerospike_latencies_)", "refresh": 2, "regex": "/.*aerospike_latencies_([a-z]*)_[a-z]*_count/", @@ -354,25 +358,15 @@ }, { "allValue": null, - "current": { - "tags": [], - "text": "ms", - "value": "ms" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "metrics(aerospike_latencies_)", "hide": 0, "includeAll": false, "label": "Latency Time Unit", "multi": false, "name": "latencytimeunit", - "options": [ - { - "selected": true, - "text": "ms", - "value": "ms" - } - ], + "options": [], "query": "metrics(aerospike_latencies_)", "refresh": 2, "regex": "/.*aerospike_latencies_[a-z]*_([a-z]*)_count/", @@ -383,6 +377,25 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": { + "selected": false, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" } ] }, diff --git a/config/grafana/dashboards/namespace.json b/config/grafana/dashboards/namespace.json index 0864d0a..2b50651 100644 --- a/config/grafana/dashboards/namespace.json +++ b/config/grafana/dashboards/namespace.json @@ -1,4 +1,36 @@ { + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + } + ], "annotations": { "list": [ { @@ -15,190 +47,236 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1573523847963, + "id": null, + "iteration": 1603448491597, "links": [], "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 181, + "id": 24, "panels": [], - "title": "Throughput (Read, Write)", + "repeat": "namespace", + "title": "Namespace: $namespace", "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Shows per-second average rate of increase in\n\n1. Total reads = client_read_success + client_read_error + client_read_timeout + client_read_not_found\n2. Successful reads = client_read_success\n3. Errored reads = clean_read_error\n4. Timedout reads = client_read_timeout\n5. Not found = client_read_not_found", - "fill": 1, - "fillGradient": 0, + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Migrations remaining - migrate_rx_partitions_remaining (transmit) migrate_tx_partitions_remaining (receive)", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-orange", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { - "h": 9, - "w": 12, + "h": 4, + "w": 6, "x": 0, "y": 1 }, - "hideTimeOverride": false, - "id": 177, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, + "id": 333, + "interval": null, "links": [], - "nullPointMode": "null", + "maxDataPoints": 100, "options": { - "dataLinks": [] + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "7.2.2", "targets": [ { - "expr": "rate(aerospike_namespace_client_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Total", - "refId": "B" - }, - { - "expr": "rate(aerospike_namespace_client_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Successful", - "refId": "A" - }, - { - "expr": "rate(aerospike_namespace_client_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, + "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Error", + "legendFormat": "TX", "refId": "C" }, { - "expr": "rate(aerospike_namespace_client_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Timeout", - "refId": "D" - }, - { - "expr": "rate(aerospike_namespace_client_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, + "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Not Found", - "refId": "E" + "legendFormat": "RX", + "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Client Reads (TPS)", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" + "title": "Migrations Remaining", + "type": "stat" + }, + { + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace memory usage in percentage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 1 }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "TPS", - "logBase": 1, - "max": null, - "min": "0", - "show": true + "id": 321, + "links": [], + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showUnfilled": true + }, + "pluginVersion": "7.2.2", + "targets": [ { - "format": "short", - "label": "TPS", - "logBase": 1, - "max": null, - "min": null, - "show": true + "expr": "avg(100-aerospike_namespace_memory_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "timeFrom": null, + "timeShift": null, + "title": "% Memory Used", + "type": "bargauge" }, { - "aliasColors": {}, + "aliasColors": { + "High Water Mark": "semi-dark-orange", + "Stop Writes": "dark-red", + "Used %": "blue" + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Shows per-second average rate of increase in\n\n1. Total writes = client_write_success + client_write_error + client_write_timeout\n2. Successful writes = client_write_success\n3. Errored writes = clean_write_error\n4. Timedout writes = client_write_timeout", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace memory trend (Used % vs HWM %)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 9, + "h": 6, "w": 12, "x": 12, "y": 1 }, - "id": 179, + "hiddenSeries": false, + "id": 49, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -208,43 +286,28 @@ "steppedLine": false, "targets": [ { - "expr": "rate(aerospike_namespace_client_write_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_write_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_write_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Total", - "refId": "B" - }, - { - "expr": "rate(aerospike_namespace_client_write_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(100-aerospike_namespace_memory_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Successful", + "legendFormat": "Memory Used %", "refId": "A" }, { - "expr": "rate(aerospike_namespace_client_write_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Timeout", - "refId": "C" - }, - { - "expr": "rate(aerospike_namespace_client_write_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(aerospike_namespace_high_water_memory_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Error", - "refId": "D" + "legendFormat": "Memory High Water Mark %", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Client Writes (TPS)", + "title": "% Memory Trend", "tooltip": { "shared": true, - "sort": 1, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -257,9 +320,8 @@ }, "yaxes": [ { - "decimals": 0, "format": "short", - "label": "TPS", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -267,10 +329,10 @@ }, { "format": "short", - "label": "TPS", + "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -280,105 +342,372 @@ } }, { - "collapsed": false, + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace free memory ((memory_free_pct / 100) * memory_size) in bytes\n\nNamespace total memory (memory_size) in bytes", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 + "h": 3, + "w": 6, + "x": 6, + "y": 4 }, - "id": 191, - "panels": [], - "title": "Throughput (Batch, Query, Scan, UDF)", - "type": "row" + "id": 68, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "avg(aerospike_namespace_memory_size{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_memory_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Free", + "refId": "A" + }, + { + "expr": "avg(aerospike_namespace_memory_size{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Total", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Namespace Memory Statistics", + "type": "stat" }, { - "aliasColors": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace objects (tombstones, master_objects, prole_objects, evicted_objects)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 100, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "sum(aerospike_namespace_tombstones{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Tombstones", + "refId": "C" + }, + { + "expr": "sum(aerospike_namespace_evicted_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Evicted objects", + "refId": "D" + }, + { + "expr": "sum(aerospike_namespace_prole_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Replica objects", + "refId": "B" + }, + { + "expr": "sum(aerospike_namespace_master_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Master objects", + "refId": "A" + }, + { + "expr": "sum(aerospike_namespace_expired_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Expired objects", + "refId": "E" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Objects", + "type": "stat" + }, + { + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace storage usage in percentage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 31, + "links": [], + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "avg(100-aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Device Used", + "refId": "A" + }, + { + "expr": "avg(100-aerospike_namespace_pmem_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMEM Used", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "% Storage Used", + "type": "bargauge" + }, + { + "aliasColors": { + "% Disk Used": "blue", + "% High Water Mark": "semi-dark-orange", + "% Stop Writes": "dark-red", + "Available Used %": "purple", + "Free Used %": "blue", + "High Water Mark": "semi-dark-orange", + "High Water Mark %": "semi-dark-orange", + "Stop Writes": "dark-red", + "Stop Writes %": "dark-red", + "Used %": "blue", + "Used % (Available)": "blue", + "Used % (Free)": "purple" + }, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Shows per-second average rate of increase in batch_sub_read_success, batch_sub_read_not_found, batch_sub_read_error and batch_sub_read_timeout", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace device usage trend", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 10, + "h": 6, "w": 12, - "x": 0, - "y": 11 + "x": 12, + "y": 7 }, - "hideTimeOverride": false, - "id": 183, - "interval": "", + "hiddenSeries": false, + "id": 27, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "", + "yaxis": 1 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(aerospike_namespace_batch_sub_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(100-aerospike_namespace_device_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "instant": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}: Total", - "refId": "B" + "legendFormat": "Used % (100 - device_free_pct)", + "refId": "A" }, { - "expr": "rate(aerospike_namespace_batch_sub_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(100-aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}: Not Found", - "refId": "C" + "legendFormat": "Used % (100 - device_available_pct)", + "refId": "D" }, { - "expr": "rate(aerospike_namespace_batch_sub_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(aerospike_namespace_high_water_disk_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", "instant": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}: Error", - "refId": "D" - }, - { - "expr": "rate(aerospike_namespace_batch_sub_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", - "legendFormat": "{{service}}: Timeout", - "refId": "A" + "legendFormat": "Device High Water Mark %", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Batch Sub Reads ", + "title": "% Device Trend", "tooltip": { "shared": true, - "sort": 1, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -391,9 +720,8 @@ }, "yaxes": [ { - "decimals": 0, "format": "short", - "label": "TPS", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -401,10 +729,10 @@ }, { "format": "short", - "label": "TPS", + "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -414,146 +742,531 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Shows per-second average rate of increase in\n\n1. Total (_complete + _error) scans\n2. Successful (_complete) scans\n3. Errored (_error) scans\n4. Aborted (_abort) scans\n\nIt includes all 4 types of scans - scan_basic, scan_aggr, scan_udf_bg, and scan_ops_bg and rate is computed over the last minute.", - "fill": 1, - "fillGradient": 0, + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace free disk space ((device_free_pct / 100) * device_total_bytes)\n\nNamespace device total size in bytes\n\nNamespace device available space in bytes ((device_available_pct/100) * device_total_bytes)\n\nNamespace free pmem space ((pmem_free_pct / 100) * pmem_total_bytes)\n\nNamespace pmem total size in bytes\n\nNamespace pmem available space in bytes ((pmem_available_pct/100) * pmem_total_bytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 11 + "h": 3, + "w": 6, + "x": 6, + "y": 10 }, - "hideTimeOverride": false, - "id": 185, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true + "id": 288, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "lines": true, - "linewidth": 1, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_device_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Device Free", + "refId": "A" + }, + { + "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Device Total", + "refId": "B" + }, + { + "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Device Available", + "refId": "C" + }, + { + "expr": "avg(aerospike_namespace_pmem_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_pmem_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMEM Free", + "refId": "D" + }, + { + "expr": "avg(aerospike_namespace_pmem_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMEM Total", + "refId": "E" + }, + { + "expr": "avg(aerospike_namespace_pmem_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_pmem_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMEM Available", + "refId": "F" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Namespace Storage Statistics", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace dead or unavailable partitions", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#E02F44", + "value": 1 + }, + { + "color": "#E02F44" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 358, + "interval": null, "links": [], - "nullPointMode": "null", + "maxDataPoints": 100, "options": { - "dataLinks": [] + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "7.2.2", "targets": [ { - "expr": "rate(aerospike_namespace_scan_aggr_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_aggr_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "sum(aerospike_namespace_dead_partitions{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, + "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Total", + "legendFormat": "Dead Partitions", + "refId": "A" + }, + { + "expr": "sum(aerospike_namespace_unavailable_partitions{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Unavailable Partitions", "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Partitions", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Percentage of device space used by primary index (for flash and PMEM)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 305, + "links": [], + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showUnfilled": true + }, + "pluginVersion": "7.2.2", + "targets": [ { - "expr": "rate(aerospike_namespace_scan_aggr_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(aerospike_namespace_index_flash_used_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", + "instant": true, "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Successful", + "legendFormat": "Flash", "refId": "A" }, { - "expr": "rate(aerospike_namespace_scan_udf_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_aggr_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(aerospike_namespace_index_pmem_used_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "PMEM", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "% Primary Index Device Used (Flash/PMEM)", + "type": "bargauge" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace index flash usage in bytes\n\nNamespace index PMEM usage in bytes", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 345, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "avg(aerospike_namespace_index_flash_used_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Error", - "refId": "C" + "legendFormat": "Flash", + "refId": "A" }, { - "expr": "rate(aerospike_namespace_scan_aggr_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "avg(aerospike_namespace_index_pmem_used_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "format": "time_series", - "instant": false, + "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Abort", - "refId": "D" + "legendFormat": "PMEM", + "refId": "B" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Scans (TPS)", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" + "title": "Namespace Index (Flash/PMEM) Statistics", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace stop_writes flag", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [ + { + "from": "null", + "id": 0, + "text": "N/A", + "to": "null", + "type": 2 + }, + { + "from": "0", + "id": 1, + "text": "OK", + "to": "0", + "type": 2 + }, + { + "from": "1", + "id": 2, + "text": "CRITICAL", + "to": "1000", + "type": 2 + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#E02F44", + "value": 1 + }, + { + "color": "#E02F44" + } + ] + }, + "unit": "short" + }, + "overrides": [] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 13 }, - "yaxes": [ + "id": 113, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "sum(aerospike_namespace_stop_writes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Stop Writes", + "refId": "A" + }, { - "decimals": 0, - "format": "short", - "label": "TPS", - "logBase": 1, - "max": null, - "min": "0", - "show": true + "expr": "sum(aerospike_namespace_clock_skew_stop_writes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Clock Skew Stop Writes", + "refId": "B" }, { - "format": "short", - "label": "TPS", - "logBase": 1, - "max": null, - "min": null, - "show": true + "expr": "sum(aerospike_namespace_hwm_breached{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "HWM Breached", + "refId": "C" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "timeFrom": null, + "timeShift": null, + "title": "Stop Writes / HWM Breached", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 181, + "panels": [], + "title": "Throughput (Read, Write)", + "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Shows per-second average rate of increase in\n\n1. Total (query_reqs),\n2. Successful (query_reqs - query_fail),\n3. Error (query_fail)", - "fill": 1, - "fillGradient": 0, + "description": "Shows per-second average rate of increase in\n\n1. Total reads = client_read_success + client_read_error + client_read_timeout + client_read_not_found\n2. Successful reads = client_read_success\n3. Errored reads = clean_read_error\n4. Timedout reads = client_read_timeout\n5. Not found = client_read_not_found", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 10, + "h": 9, "w": 12, "x": 0, - "y": 21 + "y": 33 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 187, + "id": 177, "interval": "", "legend": { "alignAsTable": true, @@ -571,9 +1284,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -583,7 +1297,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(aerospike_namespace_query_reqs{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", @@ -592,27 +1306,45 @@ "refId": "B" }, { - "expr": "rate(aerospike_namespace_query_reqs{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])-rate(aerospike_namespace_query_fail{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{service}}/{{ns}}: Successful", "refId": "A" }, { - "expr": "rate(aerospike_namespace_query_fail{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{service}}/{{ns}}: Error", "refId": "C" + }, + { + "expr": "rate(aerospike_namespace_client_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Timeout", + "refId": "D" + }, + { + "expr": "rate(aerospike_namespace_client_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Not Found", + "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Queries (TPS)", + "title": "Client Reads (TPS)", "tooltip": { "shared": true, "sort": 1, @@ -655,20 +1387,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Shows per-second average rate of increase in \n\n1. Total = client_udf_complete + client_udf_timeout + client_udf_error\n2. Successful = client_udf_complete\n3. Error = client_udf_error\n4. Timeout = client_udf_timeout", - "fill": 1, - "fillGradient": 0, + "description": "Shows per-second average rate of increase in\n\n1. Total writes = client_write_success + client_write_error + client_write_timeout\n2. Successful writes = client_write_success\n3. Errored writes = clean_write_error\n4. Timedout writes = client_write_timeout", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 10, + "h": 9, "w": 12, "x": 12, - "y": 21 + "y": 33 }, - "hideTimeOverride": false, - "id": 189, - "interval": "", + "hiddenSeries": false, + "id": 179, "legend": { "alignAsTable": true, "avg": true, @@ -685,9 +1423,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -697,37 +1436,32 @@ "steppedLine": false, "targets": [ { - "expr": "rate(aerospike_namespace_client_udf_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_client_udf_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_client_udf_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_write_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_write_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+rate(aerospike_namespace_client_write_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", - "instant": false, - "interval": "", "intervalFactor": 1, "legendFormat": "{{service}}/{{ns}}: Total", - "refId": "A" + "refId": "B" }, { - "expr": "rate(aerospike_namespace_client_udf_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_write_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{service}}/{{ns}}: Successful", - "refId": "B" + "refId": "A" }, { - "expr": "rate(aerospike_namespace_client_udf_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_write_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", - "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Error", + "legendFormat": "{{service}}/{{ns}}: Timeout", "refId": "C" }, { - "expr": "rate(aerospike_namespace_client_udf_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "expr": "rate(aerospike_namespace_client_write_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", - "instant": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Timeout", + "legendFormat": "{{service}}/{{ns}}: Error", "refId": "D" } ], @@ -735,7 +1469,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "UDFs (TPS)", + "title": "Client Writes (TPS)", "tooltip": { "shared": true, "sort": 1, @@ -775,15 +1509,16 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 42 }, - "id": 211, + "id": 191, "panels": [], - "title": "Objects", + "title": "Throughput (Batch, Query, Scan, UDF)", "type": "row" }, { @@ -791,26 +1526,34 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace master objects", - "fill": 1, - "fillGradient": 0, + "description": "Shows per-second average rate of increase in batch_sub_read_success, batch_sub_read_not_found, batch_sub_read_error and batch_sub_read_timeout", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 8, - "w": 8, + "h": 10, + "w": 12, "x": 0, - "y": 32 + "y": 43 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 195, + "id": 183, "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, - "max": false, - "min": false, + "max": true, + "min": true, "rightSide": true, "show": true, "total": false, @@ -821,9 +1564,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -833,111 +1577,35 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_master_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "rate(aerospike_namespace_batch_sub_read_success{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}: Total", "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Master Objects", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true }, { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace replica objects", - "decimals": 0, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 32 - }, - "hideTimeOverride": false, - "id": 199, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "expr": "rate(aerospike_namespace_batch_sub_read_not_found{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}: Not Found", + "refId": "C" + }, { - "expr": "sum(aerospike_namespace_prole_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "rate(aerospike_namespace_batch_sub_read_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}: Error", + "refId": "D" + }, + { + "expr": "rate(aerospike_namespace_batch_sub_read_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "legendFormat": "{{service}}: Timeout", "refId": "A" } ], @@ -945,7 +1613,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Prole Objects", + "title": "Batch Sub Reads ", "tooltip": { "shared": true, "sort": 1, @@ -963,19 +1631,18 @@ { "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -989,26 +1656,34 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace non-replica objects", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 32 + "description": "Shows per-second average rate of increase in\n\n1. Total (_complete + _error) scans\n2. Successful (_complete) scans\n3. Errored (_error) scans\n4. Aborted (_abort) scans\n\nIt includes all 4 types of scans - scan_basic, scan_aggr, scan_udf_bg, and scan_ops_bg and rate is computed over the last minute.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 43 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 197, + "id": 185, "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, - "max": false, - "min": false, + "max": true, + "min": true, "rightSide": true, "show": true, "total": false, @@ -1019,9 +1694,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1031,20 +1707,45 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_non_replica_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "rate(aerospike_namespace_scan_aggr_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_aggr_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}/{{ns}}: Total", "refId": "B" + }, + { + "expr": "rate(aerospike_namespace_scan_aggr_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Successful", + "refId": "A" + }, + { + "expr": "rate(aerospike_namespace_scan_udf_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_aggr_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Error", + "refId": "C" + }, + { + "expr": "rate(aerospike_namespace_scan_aggr_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_basic_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_udf_bg_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_scan_ops_bg_abort{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Abort", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Non-Replica Objects", + "title": "Scans (TPS)", "tooltip": { "shared": true, "sort": 1, @@ -1062,19 +1763,18 @@ { "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -1088,26 +1788,34 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace master tombstones", - "fill": 1, - "fillGradient": 0, + "description": "Shows per-second average rate of increase in\n\n1. Total (query_reqs),\n2. Successful (query_reqs - query_fail),\n3. Error (query_fail)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 8, - "w": 8, + "h": 10, + "w": 12, "x": 0, - "y": 40 + "y": 53 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 201, + "id": 187, "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, - "max": false, - "min": false, + "max": true, + "min": true, "rightSide": true, "show": true, "total": false, @@ -1118,9 +1826,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1130,20 +1839,36 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_master_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "rate(aerospike_namespace_query_reqs{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}/{{ns}}: Total", "refId": "B" + }, + { + "expr": "rate(aerospike_namespace_query_reqs{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])-rate(aerospike_namespace_query_fail{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Successful", + "refId": "A" + }, + { + "expr": "rate(aerospike_namespace_query_fail{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Error", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Master Tombstones", + "title": "Queries (TPS)", "tooltip": { "shared": true, "sort": 1, @@ -1161,19 +1886,18 @@ { "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -1187,26 +1911,34 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace replica tombstones", - "fill": 1, - "fillGradient": 0, + "description": "Shows per-second average rate of increase in \n\n1. Total = client_udf_complete + client_udf_timeout + client_udf_error\n2. Successful = client_udf_complete\n3. Error = client_udf_error\n4. Timeout = client_udf_timeout", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 40 + "h": 10, + "w": 12, + "x": 12, + "y": 53 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 203, + "id": 189, "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, - "max": false, - "min": false, + "max": true, + "min": true, "rightSide": true, "show": true, "total": false, @@ -1217,9 +1949,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1229,20 +1962,45 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_prole_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "rate(aerospike_namespace_client_udf_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_client_udf_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])+\nrate(aerospike_namespace_client_udf_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}/{{ns}}: Total", + "refId": "A" + }, + { + "expr": "rate(aerospike_namespace_client_udf_complete{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Successful", "refId": "B" + }, + { + "expr": "rate(aerospike_namespace_client_udf_error{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Error", + "refId": "C" + }, + { + "expr": "rate(aerospike_namespace_client_udf_timeout{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Timeout", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Prole Tombstones", + "title": "UDFs (TPS)", "tooltip": { "shared": true, "sort": 1, @@ -1260,19 +2018,18 @@ { "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": 0, "format": "short", - "label": "", + "label": "TPS", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -1281,24 +2038,46 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 211, + "panels": [], + "title": "Objects", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace non-replica tombstones", - "fill": 1, - "fillGradient": 0, + "description": "Namespace master objects", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 16, - "y": 40 + "x": 0, + "y": 64 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 205, + "id": 195, "interval": "", "legend": { "alignAsTable": true, @@ -1316,9 +2095,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1328,7 +2108,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_non_replica_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_master_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", @@ -1341,7 +2121,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Non-Replica Tombstones", + "title": "Master Objects", "tooltip": { "shared": true, "sort": 1, @@ -1385,19 +2165,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace evicted objects", - "fill": 1, - "fillGradient": 0, + "description": "Namespace replica objects", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 48 + "x": 8, + "y": 64 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 207, + "id": 199, "interval": "", "legend": { "alignAsTable": true, @@ -1415,9 +2203,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1427,21 +2216,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_evicted_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_prole_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", - "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}", - "refId": "B" + "legendFormat": "{{service}} {{ns}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Evicted Objects", + "title": "Prole Objects", "tooltip": { "shared": true, "sort": 1, @@ -1485,19 +2273,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace expired objects", - "fill": 1, - "fillGradient": 0, + "description": "Namespace non-replica objects", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 8, - "y": 48 + "x": 16, + "y": 64 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 209, + "id": 197, "interval": "", "legend": { "alignAsTable": true, @@ -1515,9 +2311,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1527,13 +2324,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_expired_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_non_replica_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", - "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}", + "legendFormat": "{{service}} {{ns}}", "refId": "B" } ], @@ -1541,7 +2337,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Expired Objects", + "title": "Non-Replica Objects", "tooltip": { "shared": true, "sort": 1, @@ -1585,18 +2381,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace migrate_tx_partitions_remaining (transmit) and migrate_rx_partitions_remaining (receive)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Namespace master tombstones", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 16, - "y": 48 + "x": 0, + "y": 72 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 193, + "id": 201, "interval": "", "legend": { "alignAsTable": true, @@ -1614,9 +2419,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1626,29 +2432,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_master_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Outgoing {{service}}/{{ns}}", + "legendFormat": "{{service}} {{ns}}", "refId": "B" - }, - { - "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Incoming {{service}}/{{ns}}", - "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Migrations Remaining", + "title": "Master Tombstones", "tooltip": { "shared": true, "sort": 1, @@ -1675,49 +2472,44 @@ { "decimals": 0, "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 259, - "panels": [], - "title": "Resources", - "type": "row" + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace Index Usage (Primary Index)", - "fill": 1, - "fillGradient": 0, + "description": "Namespace replica tombstones", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 57 + "x": 8, + "y": 72 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 257, + "id": 203, "interval": "", "legend": { "alignAsTable": true, @@ -1735,9 +2527,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1747,38 +2540,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_memory_used_index_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "shmem {{service}}/{{ns}}", - "refId": "A" - }, - { - "expr": "sum(aerospike_namespace_index_flash_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_prole_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "flash {{service}}/{{ns}}", + "legendFormat": "{{service}} {{ns}}", "refId": "B" - }, - { - "expr": "sum(aerospike_namespace_index_pmem_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "pmem {{service}}/{{ns}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Primary Index Used", + "title": "Prole Tombstones", "tooltip": { "shared": true, "sort": 1, @@ -1795,7 +2570,7 @@ "yaxes": [ { "decimals": 0, - "format": "bytes", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -1822,19 +2597,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace memory_used_sindex_bytes (Secondary Index)", - "fill": 1, - "fillGradient": 0, + "description": "Namespace non-replica tombstones", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 8, - "y": 57 + "x": 16, + "y": 72 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 255, + "id": 205, "interval": "", "legend": { "alignAsTable": true, @@ -1852,9 +2635,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1864,12 +2648,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_memory_used_sindex_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_non_replica_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}", + "legendFormat": "{{service}} {{ns}}", "refId": "B" } ], @@ -1877,7 +2661,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Secondary Index Used", + "title": "Non-Replica Tombstones", "tooltip": { "shared": true, "sort": 1, @@ -1894,7 +2678,7 @@ "yaxes": [ { "decimals": 0, - "format": "bytes", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -1921,23 +2705,31 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "description": "Namespace fail_generation (Errors due to generation mismatch), fail_key_busy (Hotkey errors), fail_record_too_big (Errors due to record being more that write block size), fail_xdr_forbidden (Forbidden errors due to XDR configs for incoming XDR traffic)", - "fill": 1, - "fillGradient": 0, + "description": "Namespace evicted objects", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 16, - "y": 57 + "x": 0, + "y": 80 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 253, + "id": 207, "interval": "", "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "max": false, "min": false, @@ -1951,9 +2743,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1963,47 +2756,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_fail_generation{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "expr": "sum(aerospike_namespace_evicted_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", + "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Fail Generation", + "legendFormat": "{{service}}/{{ns}}", "refId": "B" - }, - { - "expr": "sum(aerospike_namespace_fail_key_busy{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Key Busy", - "refId": "A" - }, - { - "expr": "sum(aerospike_namespace_fail_record_too_big{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Record Too Big", - "refId": "C" - }, - { - "expr": "sum(aerospike_namespace_fail_xdr_forbidden{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: XDR Forbidden", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Failure Rate", + "title": "Evicted Objects", "tooltip": { "shared": true, "sort": 1, @@ -2020,7 +2787,7 @@ "yaxes": [ { "decimals": 0, - "format": "percent", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -2047,18 +2814,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace memory_free_pct (available namespace memory percentage), high_water_memory_pct (memory high water mark), stop_writes_pct (stop writes threshold)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Namespace expired objects", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 65 + "x": 8, + "y": 80 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 251, + "id": 209, "interval": "", "legend": { "alignAsTable": true, @@ -2076,9 +2852,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2088,46 +2865,21 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_memory_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "sum(aerospike_namespace_expired_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", + "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Free", + "legendFormat": "{{service}}/{{ns}}", "refId": "B" - }, - { - "expr": "aerospike_namespace_high_water_memory_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: HWM", - "refId": "A" - }, - { - "expr": "aerospike_namespace_stop_writes_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service}}/{{ns}}: Stop Writes", - "refId": "C" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "yaxis": "left" } ], + "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Info %", + "title": "Expired Objects", "tooltip": { "shared": true, "sort": 1, @@ -2144,7 +2896,7 @@ "yaxes": [ { "decimals": 0, - "format": "percent", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -2171,18 +2923,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace memory_used_bytes (memory_used_data_bytes (Data) + memory_used_index_bytes (Primary Index) + memory_used_sindex_bytes (Secondary Index))", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace migrate_tx_partitions_remaining (transmit) and migrate_rx_partitions_remaining (receive)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 8, - "y": 65 + "x": 16, + "y": 80 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 249, + "id": 193, "interval": "", "legend": { "alignAsTable": true, @@ -2200,9 +2960,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2212,20 +2973,29 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Used {{service}}/{{ns}}", + "legendFormat": "Outgoing {{service}}/{{ns}}", "refId": "B" + }, + { + "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Incoming {{service}}/{{ns}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Used", + "title": "Migrations Remaining", "tooltip": { "shared": true, "sort": 1, @@ -2242,7 +3012,7 @@ "yaxes": [ { "decimals": 0, - "format": "bytes", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -2264,23 +3034,46 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 259, + "panels": [], + "title": "Resources", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace free memory = memory_size - memory_used_bytes", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Namespace Index Usage (Primary Index)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 16, - "y": 65 + "x": 0, + "y": 89 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 247, + "id": 257, "interval": "", "legend": { "alignAsTable": true, @@ -2298,9 +3091,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2310,19 +3104,38 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_memory_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} - aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "sum(aerospike_namespace_memory_used_index_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", + "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "shmem {{service}}/{{ns}}", "refId": "A" + }, + { + "expr": "sum(aerospike_namespace_index_flash_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "flash {{service}}/{{ns}}", + "refId": "B" + }, + { + "expr": "sum(aerospike_namespace_index_pmem_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pmem {{service}}/{{ns}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Free", + "title": "Primary Index Used", "tooltip": { "shared": true, "sort": 1, @@ -2366,18 +3179,27 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace device free percentage (device_free_pct)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Namespace memory_used_sindex_bytes (Secondary Index)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 73 + "x": 8, + "y": 89 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 245, + "id": 255, "interval": "", "legend": { "alignAsTable": true, @@ -2395,9 +3217,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2407,12 +3230,12 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_device_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "sum(aerospike_namespace_memory_used_sindex_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}/{{ns}}", "refId": "B" } ], @@ -2420,7 +3243,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Disk Free %", + "title": "Secondary Index Used", "tooltip": { "shared": true, "sort": 1, @@ -2437,7 +3260,7 @@ "yaxes": [ { "decimals": 0, - "format": "percent", + "format": "bytes", "label": "", "logBase": 1, "max": null, @@ -2464,22 +3287,31 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace device usage in bytes (device_used_bytes)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "decimals": 0, + "description": "Namespace fail_generation (Errors due to generation mismatch), fail_key_busy (Hotkey errors), fail_record_too_big (Errors due to record being more that write block size), fail_xdr_forbidden (Forbidden errors due to XDR configs for incoming XDR traffic)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 8, - "y": 73 + "x": 16, + "y": 89 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 243, + "id": 253, "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, "max": false, "min": false, @@ -2493,9 +3325,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2505,20 +3338,47 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "sum(aerospike_namespace_fail_generation{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Fail Generation", + "refId": "B" + }, + { + "expr": "sum(aerospike_namespace_fail_key_busy{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", - "refId": "B" + "legendFormat": "{{service}}/{{ns}}: Key Busy", + "refId": "A" + }, + { + "expr": "sum(aerospike_namespace_fail_record_too_big{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Record Too Big", + "refId": "C" + }, + { + "expr": "sum(aerospike_namespace_fail_xdr_forbidden{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) by (service, ns)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: XDR Forbidden", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Disk Used", + "title": "Failure Rate", "tooltip": { "shared": true, "sort": 1, @@ -2535,7 +3395,7 @@ "yaxes": [ { "decimals": 0, - "format": "bytes", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -2562,18 +3422,26 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace device free bytes (device_total_bytes - device_used_bytes)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace memory_free_pct (available namespace memory percentage), high_water_memory_pct (memory high water mark), stop_writes_pct (stop writes threshold)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 8, - "x": 16, - "y": 73 + "x": 0, + "y": 97 }, + "hiddenSeries": false, "hideTimeOverride": false, - "id": 241, + "id": 251, "interval": "", "legend": { "alignAsTable": true, @@ -2591,9 +3459,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -2603,18 +3472,46 @@ "steppedLine": false, "targets": [ { - "expr": "aerospike_namespace_device_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} - aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "expr": "aerospike_namespace_memory_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{service}} {{ns}}", + "legendFormat": "{{service}}/{{ns}}: Free", + "refId": "B" + }, + { + "expr": "aerospike_namespace_high_water_memory_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: HWM", "refId": "A" + }, + { + "expr": "aerospike_namespace_stop_writes_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}/{{ns}}: Stop Writes", + "refId": "C" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "yaxis": "left" } ], - "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Disk Free", + "title": "Memory Info %", "tooltip": { "shared": true, "sort": 1, @@ -2631,7 +3528,7 @@ "yaxes": [ { "decimals": 0, - "format": "bytes", + "format": "percent", "label": "", "logBase": 1, "max": null, @@ -2654,440 +3551,51 @@ } }, { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 81 - }, - "id": 24, - "panels": [], - "repeat": "namespace", - "title": "Namespace: $namespace", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#E02F44", - "#E02F44" - ], - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Namespace stop_writes flag", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 82 - }, - "id": 113, - "interval": null, - "links": [], - "mappingType": 2, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - }, - { - "from": "0", - "text": "OK", - "to": "0" - }, - { - "from": "1", - "text": "CRITICAL", - "to": "1000" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_stop_writes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1", - "timeFrom": null, - "timeShift": null, - "title": "Stop Writes", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "TRUE", - "value": "1" - }, - { - "op": "=", - "text": "FALSE", - "value": "0" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#E02F44", - "#E02F44" - ], - "datasource": "Aerospike Prometheus", - "decimals": 0, - "description": "Namespace clock_skew_stop_writes flag", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 82 - }, - "id": 114, - "interval": null, - "links": [], - "mappingType": 2, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - }, - { - "from": "0", - "text": "OK", - "to": "0" - }, - { - "from": "1", - "text": "CRITICAL", - "to": "1000" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_clock_skew_stop_writes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1", - "timeFrom": null, - "timeShift": null, - "title": "Clock Skew Stop Writes", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "TRUE", - "value": "1" - }, - { - "op": "=", - "text": "FALSE", - "value": "0" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace free memory ((memory_free_pct / 100) * memory_size)", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 6, - "y": 82 - }, - "id": 68, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_memory_size{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_memory_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Mem Free", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace total memory (memory_size)", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 9, - "y": 82 - }, - "id": 66, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_memory_size{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Mem Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": { - "High Water Mark": "semi-dark-orange", - "Stop Writes": "dark-red", - "Used %": "blue" - }, + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace memory trend (Used % vs HWM %)", - "fill": 0, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace memory_used_bytes (memory_used_data_bytes (Data) + memory_used_index_bytes (Primary Index) + memory_used_sindex_bytes (Secondary Index))", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 82 + "h": 8, + "w": 8, + "x": 8, + "y": 97 }, - "id": 49, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 249, + "interval": "", "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -3097,17 +3605,12 @@ "steppedLine": false, "targets": [ { - "expr": "avg(100-aerospike_namespace_memory_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Memory Used %", - "refId": "A" - }, - { - "expr": "avg(aerospike_namespace_high_water_memory_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Memory High Water Mark %", + "legendFormat": "Used {{service}}/{{ns}}", "refId": "B" } ], @@ -3115,10 +3618,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "% Memory Trend", + "title": "Memory Used", "tooltip": { "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -3131,16 +3634,18 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "decimals": 0, + "format": "bytes", + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { + "decimals": 0, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", @@ -3153,458 +3658,192 @@ } }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#FF9830", - "#FF9830" - ], - "datasource": "Aerospike Prometheus", - "description": "Total migrations remaining (migrate_rx_partitions_remaining (transmit) + migrate_tx_partitions_remaining (receive))", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 6, - "x": 0, - "y": 84 - }, - "id": 127, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace free memory = memory_size - memory_used_bytes", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) + sum(aerospike_namespace_migrate_tx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1", - "timeFrom": null, - "timeShift": null, - "title": "Migrations Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace free disk space ((device_free_pct / 100) * device_total_bytes)", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "overrides": [] }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 2, - "w": 3, - "x": 6, - "y": 84 - }, - "id": 288, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_device_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Disk Free", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace total device space (device_total_bytes)", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "h": 8, + "w": 8, + "x": 16, + "y": 97 }, - "gridPos": { - "h": 2, - "w": 3, - "x": 9, - "y": 84 + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 247, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true }, - "id": 289, - "interval": null, + "lines": true, + "linewidth": 1, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Disk Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "datasource": "Aerospike Prometheus", - "description": "Namespace device usage in bytes", - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 86 - }, - "id": 31, - "links": [ - { - "url": "/dashboard/db/clus" - } - ], + "nullPointMode": "null", "options": { - "fieldOptions": { - "calcs": [ - "last" - ], - "defaults": { - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "super-light-blue", - "value": null - }, - { - "color": "red", - "value": 80 - } - ], - "unit": "percent" - }, - "override": {}, - "values": false - }, - "orientation": "horizontal", - "showThresholdLabels": false, - "showThresholdMarkers": true + "alertThreshold": true }, - "pluginVersion": "6.3.2", + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "avg(100-aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_memory_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} - aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", - "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "{{service}} {{ns}}", "refId": "A" } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "% Device Used", - "type": "gauge" - }, - { - "cacheTimeout": null, - "datasource": "Aerospike Prometheus", - "description": "Percentage of device space used by primary index (for flash and PMEM)", - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 86 + "title": "Memory Free", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" }, - "id": 305, - "links": [ - { - "url": "/dashboard/db/clus" - } - ], - "options": { - "fieldOptions": { - "calcs": [ - "last" - ], - "defaults": { - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "super-light-blue", - "value": null - }, - { - "color": "red", - "value": 80 - } - ], - "unit": "percent" - }, - "override": {}, - "values": false - }, - "orientation": "horizontal", - "showThresholdLabels": false, - "showThresholdMarkers": true + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "pluginVersion": "6.3.2", - "targets": [ + "yaxes": [ { - "expr": "avg(aerospike_namespace_index_flash_used_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "Flash", - "refId": "A" + "decimals": 0, + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { - "expr": "avg(aerospike_namespace_index_pmem_used_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "PMEM", - "refId": "B" + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true } ], - "timeFrom": null, - "timeShift": null, - "title": "% Primary Index Device Used (Flash/PMEM)", - "type": "gauge" + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "aliasColors": { - "Evicted objects": "semi-dark-red", - "Master objects": "semi-dark-blue", - "Replica objects": "light-purple", - "Tombstones": "super-light-yellow" - }, + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace migration partitions remaining", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace storage free percentage (device_free_pct or pmem_free_pct)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 88 + "h": 8, + "w": 8, + "x": 0, + "y": 105 }, - "id": 141, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 245, + "interval": "", "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_device_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Transmit remaining", - "refId": "A" + "legendFormat": "device {{service}} {{ns}}", + "refId": "B" }, { - "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_pmem_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Receive remaining", - "refId": "B" + "legendFormat": "pmem {{service}} {{ns}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Migrate Partitions Remaining", + "title": "Storage Free %", "tooltip": { "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -3617,16 +3856,18 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "decimals": 0, + "format": "percent", + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { + "decimals": 0, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", @@ -3639,267 +3880,86 @@ } }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace device total size in bytes", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 91 - }, - "id": 42, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "decimals": 1, - "description": "Namespace device available space in bytes ((device_available_pct/100) * device_total_bytes)", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 91 - }, - "id": 78, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(aerospike_namespace_device_total_bytes{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})*(avg(aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})/100)", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Available", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": { - "Evicted objects": "semi-dark-red", - "Master objects": "semi-dark-blue", - "Replica objects": "light-purple", - "Tombstones": "super-light-yellow" - }, + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace objects (tombstones, master_objects, prole_objects, evicted_objects)", - "fill": 1, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace storage usage in bytes (device_used_bytes or pmem_used_bytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 93 + "h": 8, + "w": 8, + "x": 8, + "y": 105 }, - "id": 100, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 243, + "interval": "", "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_tombstones{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Tombstones", - "refId": "C" - }, - { - "expr": "sum(aerospike_namespace_evicted_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Evicted objects", - "refId": "D" - }, - { - "expr": "sum(aerospike_namespace_prole_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Replica objects", + "legendFormat": "device {{service}} {{ns}}", "refId": "B" }, { - "expr": "sum(aerospike_namespace_master_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_pmem_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Master objects", + "legendFormat": "pmem {{service}} {{ns}}", "refId": "A" - }, - { - "expr": "sum(aerospike_namespace_expired_objects{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Expired objects", - "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Objects", + "title": "Storage Used", "tooltip": { "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -3912,16 +3972,18 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "decimals": 0, + "format": "bytes", + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { + "decimals": 0, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", @@ -3934,89 +3996,73 @@ } }, { - "aliasColors": { - "% Disk Used": "blue", - "% High Water Mark": "semi-dark-orange", - "% Stop Writes": "dark-red", - "Available Used %": "purple", - "Free Used %": "blue", - "High Water Mark": "semi-dark-orange", - "High Water Mark %": "semi-dark-orange", - "Stop Writes": "dark-red", - "Stop Writes %": "dark-red", - "Used %": "blue", - "Used % (Available)": "blue", - "Used % (Free)": "purple" - }, + "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Namespace device usage trend", - "fill": 0, - "fillGradient": 0, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Namespace storage free bytes (device_total_bytes - device_used_bytes or pmem_total_bytes - pmem_used_bytes)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 94 + "h": 8, + "w": 8, + "x": 16, + "y": 105 }, - "id": 27, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 241, + "interval": "", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 2, - "links": [ - {} - ], + "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "", - "yaxis": 1 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(100-aerospike_namespace_device_free_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_device_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} - aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", - "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Used % (100 - device_free_pct)", + "legendFormat": "device {{service}} {{ns}}", "refId": "A" }, { - "expr": "avg(100-aerospike_namespace_device_available_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used % (100 - device_available_pct)", - "refId": "D" - }, - { - "expr": "avg(aerospike_namespace_high_water_disk_pct{cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "aerospike_namespace_pmem_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} - aerospike_namespace_pmem_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}", "format": "time_series", - "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "Device High Water Mark %", + "legendFormat": "pmem {{service}} {{ns}}", "refId": "B" } ], @@ -4024,10 +4070,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "% Device Trend", + "title": "Storage Free", "tooltip": { "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -4040,16 +4086,18 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "decimals": 0, + "format": "bytes", + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { + "decimals": 0, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", @@ -4063,19 +4111,15 @@ } ], "refresh": "1m", - "schemaVersion": 19, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": false, @@ -4096,13 +4140,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -4123,13 +4162,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\" },ns)", "hide": 0, "includeAll": true, @@ -4147,6 +4181,24 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": { + "selected": false, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" } ] }, diff --git a/config/grafana/dashboards/node.json b/config/grafana/dashboards/node.json index 0946852..12eca57 100644 --- a/config/grafana/dashboards/node.json +++ b/config/grafana/dashboards/node.json @@ -1,4 +1,42 @@ { + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], "annotations": { "list": [ { @@ -15,11 +53,13 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1565270958217, + "id": null, + "iteration": 1603786992057, "links": [], "panels": [ { - "collapsed": true, + "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -27,1315 +67,760 @@ "y": 0 }, "id": 4, - "panels": [ - { - "columns": [], - "datasource": "Aerospike Prometheus", - "fontSize": "100%", - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "links": [], - "options": {}, - "pageSize": 128, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true + "panels": [], + "title": "Node List", + "type": "row" + }, + { + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "color-background", + "filterable": false }, - "styles": [ - { - "alias": "Aerospike Node", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "service", - "type": "string" - }, - { - "alias": "Build", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "build", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "Time", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "__name__", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Cluster Name", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "cluster_name", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "job", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Exporter", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "instance", - "thresholds": [], - "type": "string", - "unit": "short" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "links": [], + "options": { + "showHeader": true + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Nodes", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true, + "__name__": true, + "job": true }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "hidden", - "unit": "short" - } - ], - "targets": [ - { - "expr": "aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\"}", - "format": "table", - "instant": true, - "intervalFactor": 1, - "refId": "A" + "indexByName": {}, + "renameByName": { + "Value": "", + "build": "Build", + "cluster_name": "Cluster Name", + "instance": "Exporter Instance", + "service": "Aerospike Instance" } - ], - "timeFrom": null, - "timeShift": null, - "title": "Nodes", - "transform": "table", - "type": "table" + } } ], - "title": "Node List", - "type": "row" + "type": "table" }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 10 }, "id": 6, "panels": [], "repeat": "node", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "title": "$node", "type": "row" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "Aerospike Prometheus", - "description": "Exporter status", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Exporter and Aerospike node status", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + }, + { + "id": 1, + "op": "=", + "text": "UP", + "type": 1, + "value": "1" + }, + { + "id": 2, + "op": "=", + "text": "DOWN", + "type": 1, + "value": "0" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, + "h": 4, + "w": 4, "x": 0, - "y": 2 + "y": 11 }, "id": 69, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { "expr": "min(up{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$service|$^\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "Exporter", "refId": "A" - } - ], - "thresholds": "1,1", - "timeFrom": null, - "timeShift": null, - "title": "Exporter Status", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "UP", - "value": "1" }, - { - "op": "=", - "text": "DOWN", - "value": "0" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "Aerospike Prometheus", - "description": "Aerospike server status", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 2, - "y": 2 - }, - "id": 140, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ { "expr": "aerospike_node_up{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, - "refId": "A" + "legendFormat": "Aerospike", + "refId": "B" } ], - "thresholds": "1,1", "timeFrom": null, "timeShift": null, - "title": "Server Node", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "UP", - "value": "1" - }, - { - "op": "=", - "text": "DOWN", - "value": "0" - } - ], - "valueName": "current" + "title": "Status", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Alert count", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Total records and tombstones count", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, + "h": 4, + "w": 4, "x": 4, - "y": 2 + "y": 11 }, - "id": 9, + "id": 43, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "ALERTS{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", + "expr": "sum(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "Records", "refId": "A" + }, + { + "expr": "sum(aerospike_namespace_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Tomstones", + "refId": "B" } ], - "thresholds": "1,1", "timeFrom": null, "timeShift": null, - "title": "Alerts", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Node Namespace Stats", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Client connections count", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Total namespace memory size\nTotal namespace memory used\nTotal namespace device size\nTotal namespace device used", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 6, - "y": 2 + "h": 4, + "w": 8, + "x": 8, + "y": 11 }, - "id": 14, + "id": 18, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", + "expr": "sum(aerospike_namespace_memory_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "Total Memory Size", "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Client Connections", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total objects count", - "decimals": 1, - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 8, - "y": 2 - }, - "id": 43, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "expr": "sum(aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Used", + "refId": "B" + }, { - "name": "value to text", - "value": 1 + "expr": "sum(aerospike_namespace_device_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total Device Size", + "refId": "C" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "expr": "sum(aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Device Used", + "refId": "D" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ + "expr": "sum(aerospike_namespace_pmem_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total PMEM Size", + "refId": "E" + }, { - "expr": "sum(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "sum(aerospike_namespace_pmem_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, - "refId": "A" + "legendFormat": "PMEM Used", + "refId": "F" } ], - "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "All Objects", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Node Namespace Stats", + "type": "stat" }, { + "aliasColors": {}, + "bars": false, "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total tombstones count", - "decimals": 3, - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "dashLength": 10, + "dashes": false, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "UDF transactions per second", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 3, - "w": 2, - "x": 10, - "y": 2 + "h": 8, + "w": 8, + "x": 16, + "y": 11 }, - "id": 56, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true }, - "tableColumn": "", + "percentage": false, + "pluginVersion": "7.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(aerospike_namespace_tombstones{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "avg(aerospike_latencies_udf_${latencytimeunit}_bucket{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\", le=~\"[0-9]*\"}) by (le)", "format": "time_series", - "instant": true, + "instant": false, + "interval": "", "intervalFactor": 1, + "legendFormat": "<={{le}}${latencytimeunit}", "refId": "A" + }, + { + "expr": "avg(aerospike_latencies_udf_${latencytimeunit}_count{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "legendFormat": "Ops/sec", + "refId": "B" } ], - "thresholds": "", + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "All Tombstones", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total namespace memory size", - "decimals": 1, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "title": "Latency: UDF / sec", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" }, - "gridPos": { - "h": 3, - "w": 2, - "x": 12, - "y": 2 + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "id": 18, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "yaxes": [ { - "name": "value to text", - "value": 1 + "decimals": 0, + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_memory_size{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Total Mem.", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" + "decimals": 0, + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": "0", + "show": true } ], - "valueName": "current" + "yaxis": { + "align": false, + "alignLevel": null + } }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total namespace memory used in bytes", - "decimals": 1, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Alert count", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 14, - "y": 2 + "h": 4, + "w": 4, + "x": 0, + "y": 15 }, - "id": 23, + "id": 9, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "sum(aerospike_namespace_memory_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "ALERTS{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", "format": "time_series", "instant": true, "intervalFactor": 1, "refId": "A" } ], - "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Used Mem.", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Alerts", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "Aerospike Prometheus", - "decimals": null, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Client/Heartbeat/Fabric connections count", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 16, - "y": 2 + "h": 4, + "w": 4, + "x": 4, + "y": 15 }, - "id": 24, + "id": 14, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ { - "expr": "min(aerospike_namespace_memory_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, + "legendFormat": "Client", "refId": "A" - } - ], - "thresholds": "10,5", - "timeFrom": null, - "timeShift": null, - "title": "Free Mem.", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total device size in bytes", - "decimals": 1, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 18, - "y": 2 - }, - "id": 31, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_device_total_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "aerospike_node_stats_heartbeat_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Total Disk", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Aerospike Prometheus", - "description": "Total device usage in bytes", - "decimals": 1, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 20, - "y": 2 - }, - "id": 32, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 + "legendFormat": "Heartbeat", + "refId": "B" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(aerospike_namespace_device_used_bytes{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "expr": "aerospike_node_stats_fabric_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"}", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, - "refId": "A" + "legendFormat": "Fabric", + "refId": "C" } ], - "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Used Disk", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Connections", + "type": "stat" }, { "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "Aerospike Prometheus", - "decimals": null, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Minimum of the Namespaces' Memory/Device Free Percentage", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 10 + }, + { + "color": "#299c46", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 22, - "y": 2 + "h": 4, + "w": 8, + "x": 8, + "y": 15 }, - "id": 33, + "id": 24, "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true }, - "tableColumn": "", + "pluginVersion": "7.2.2", "targets": [ + { + "expr": "min(aerospike_namespace_memory_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Free", + "refId": "A" + }, { "expr": "min(aerospike_namespace_device_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", "instant": true, + "interval": "", "intervalFactor": 1, - "refId": "A" + "legendFormat": "Device Free", + "refId": "B" + }, + { + "expr": "min(aerospike_namespace_pmem_free_pct{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMEM Free", + "refId": "C" } ], - "thresholds": "5,10", "timeFrom": null, "timeShift": null, - "title": "Free Disk", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "title": "Node Namespace Stats", + "type": "gauge" }, { "aliasColors": {}, @@ -1343,15 +828,24 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Reads per second", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, "x": 0, - "y": 5 + "y": 19 }, + "hiddenSeries": false, "id": 83, "legend": { "alignAsTable": true, @@ -1372,18 +866,14 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1453,15 +943,24 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Writes per second", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, "x": 8, - "y": 5 + "y": 19 }, + "hiddenSeries": false, "id": 98, "legend": { "alignAsTable": true, @@ -1480,18 +979,14 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1561,16 +1056,25 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "UDF transactions per second", - "fill": 1, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Proxies per second", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, "x": 16, - "y": 5 + "y": 19 }, - "id": 100, + "hiddenSeries": false, + "id": 99, "legend": { "alignAsTable": true, "avg": true, @@ -1588,25 +1092,21 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(aerospike_latencies_udf_${latencytimeunit}_bucket{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\", le=~\"[0-9]*\"}) by (le)", + "expr": "avg(aerospike_latencies_proxy_${latencytimeunit}_bucket{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\", le=~\"[0-9]*\"}) by (le)", "format": "time_series", "instant": false, "interval": "", @@ -1615,7 +1115,7 @@ "refId": "A" }, { - "expr": "avg(aerospike_latencies_udf_${latencytimeunit}_count{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", + "expr": "avg(aerospike_latencies_proxy_${latencytimeunit}_count{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", "legendFormat": "Ops/sec", "refId": "B" } @@ -1624,7 +1124,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Latency: UDF / sec", + "title": "Latency: Proxies / sec", "tooltip": { "shared": true, "sort": 1, @@ -1669,124 +1169,24 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Migrations remaining migrate_tx_partitions_remaining (transmit), migrate_rx_partitions_remaining (receive)", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": {}, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "Incoming Partitions Remaining", - "refId": "A" + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Queries per second", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] }, - { - "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "Outgoing Partitions Remaining", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Migrations (Partitions Remaining)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "overrides": [] }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "Count", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "decimals": 0, - "format": "short", - "label": "Count", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Queries per second", - "fill": 1, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, - "x": 8, - "y": 12 + "x": 0, + "y": 27 }, + "hiddenSeries": false, "id": 101, "legend": { "alignAsTable": true, @@ -1805,18 +1205,14 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1886,22 +1282,31 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Proxies per second", - "fill": 1, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Client connections, fabric connections and heartbeat connections", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, - "x": 16, - "y": 12 + "x": 8, + "y": 27 }, - "id": 99, + "hiddenSeries": false, + "id": 120, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideEmpty": false, - "hideZero": false, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, "rightSide": true, @@ -1913,43 +1318,48 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(aerospike_latencies_proxy_${latencytimeunit}_bucket{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\", le=~\"[0-9]*\"}) by (le)", + "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "<={{le}}${latencytimeunit}", + "legendFormat": "Client Connections", "refId": "A" }, { - "expr": "avg(aerospike_latencies_proxy_${latencytimeunit}_count{job=\"aerospike\",cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"})", - "legendFormat": "Ops/sec", + "expr": "aerospike_node_stats_fabric_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Fabric Connections", "refId": "B" + }, + { + "expr": "aerospike_node_stats_heartbeat_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heartbeat Connections", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Latency: Proxies / sec", + "title": "Connections", "tooltip": { "shared": true, "sort": 1, @@ -1994,82 +1404,79 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Client connections, fabric connections and heartbeat connections", - "fill": 1, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Migrations remaining migrate_tx_partitions_remaining (transmit), migrate_rx_partitions_remaining (receive)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, - "x": 0, - "y": 19 + "x": 16, + "y": 27 }, - "id": 120, + "hiddenSeries": false, + "id": 44, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, - "rightSide": true, "show": true, - "total": false, + "total": true, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "aerospike_node_stats_client_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", + "expr": "sum(aerospike_namespace_migrate_rx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", "instant": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "Client Connections", + "legendFormat": "Incoming Partitions Remaining", "refId": "A" }, { - "expr": "aerospike_node_stats_fabric_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", + "expr": "sum(aerospike_namespace_migrate_tx_partitions_remaining{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node\"})", "format": "time_series", + "instant": false, "intervalFactor": 1, - "legendFormat": "Fabric Connections", + "legendFormat": "Outgoing Partitions Remaining", "refId": "B" - }, - { - "expr": "aerospike_node_stats_heartbeat_connections{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Heartbeat Connections", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Connections", + "title": "Migrations (Partitions Remaining)", "tooltip": { "shared": true, - "sort": 1, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -2111,15 +1518,24 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Master objects, prole objects and tombstones", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { - "h": 7, + "h": 8, "w": 8, - "x": 8, - "y": 19 + "x": 0, + "y": 35 }, + "hiddenSeries": false, "id": 161, "legend": { "alignAsTable": true, @@ -2138,18 +1554,14 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "node": { - "selected": false, - "text": "172.16.224.166:3000", - "value": "172.16.224.166:3000" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -2224,18 +1636,15 @@ } ], "refresh": "1m", - "schemaVersion": 18, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "text": "null", - "value": "null" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": false, @@ -2256,11 +1665,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -2281,11 +1687,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_namespace_objects{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\" },ns)", "hide": 0, "includeAll": true, @@ -2306,24 +1709,15 @@ }, { "allValue": null, - "current": { - "text": "ms", - "value": "ms" - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "metrics(aerospike_latencies_)", "hide": 0, "includeAll": false, "label": "Latency Time Unit", "multi": false, "name": "latencytimeunit", - "options": [ - { - "selected": true, - "text": "ms", - "value": "ms" - } - ], + "options": [], "query": "metrics(aerospike_latencies_)", "refresh": 2, "regex": "/.*aerospike_latencies_[a-z]*_([a-z]*)_count/", @@ -2334,6 +1728,24 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": { + "selected": false, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" } ] }, @@ -2369,5 +1781,5 @@ "timezone": "", "title": "Node Overview", "uid": "UcZD2iHAk", - "version": 2 + "version": 1 } \ No newline at end of file diff --git a/config/grafana/dashboards/xdr.json b/config/grafana/dashboards/xdr.json index cd57adc..18cbf21 100644 --- a/config/grafana/dashboards/xdr.json +++ b/config/grafana/dashboards/xdr.json @@ -1,4 +1,30 @@ { + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + } + ], "annotations": { "list": [ { @@ -15,26 +41,220 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 7, - "iteration": 1587474352222, + "id": null, + "iteration": 1603802208531, "links": [], "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "cacheTimeout": null, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", + "description": "Total Throughput (sum across all namespaces and all DCs)\n\nAverage XDR Lag\n\nAverage Ship Latency\n\nAverage Lap us (Time taken to process records across partitions in one lap)\n\nTotal recoveries pending\n\nTotal In Progress\n\nTotal In Queue", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Avg Lag" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "dark-red", + "value": 10 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Ship Latency" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Lap us" + }, + "properties": [ + { + "id": "unit", + "value": "µs" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 34, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.2", + "targets": [ + { + "expr": "sum(aerospike_xdr_throughput{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Total Throughput", + "refId": "A" + }, + { + "expr": "avg(aerospike_xdr_lag{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Avg Lag", + "refId": "B" + }, + { + "expr": "avg(aerospike_xdr_latency_ms{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Avg Ship Latency", + "refId": "C" + }, + { + "expr": "avg(aerospike_xdr_lap_us{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Avg Lap us", + "refId": "D" + }, + { + "expr": "sum(aerospike_xdr_recoveries_pending{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Total Recoveries Pending", + "refId": "E" + }, + { + "expr": "sum(aerospike_xdr_in_progress{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Total In Progress", + "refId": "F" + }, + { + "expr": "sum(aerospike_xdr_in_queue{job=\"aerospike\", cluster_name=~\"$cluster\", service=~\"$node|$^\", dc=~\"$dc|$^\"})", + "interval": "", + "legendFormat": "Total In Queue", + "refId": "G" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "XDR Statistics Summary", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 32, + "panels": [], + "title": "Detailed Statistics", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "XDR throughput", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": 0, - "fill": 1, - "fillGradient": 0, + "description": "XDR throughput", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 6 }, + "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, @@ -51,9 +271,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -114,17 +335,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of records successfully shipped", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of records successfully shipped", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 6 }, + "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, @@ -141,9 +370,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -204,17 +434,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of ships abandoned", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of ships abandoned", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 14 }, + "hiddenSeries": false, "id": 17, "legend": { "alignAsTable": true, @@ -231,9 +469,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -294,17 +533,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of XDR local read not found", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of XDR local read not found", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 14 }, + "hiddenSeries": false, "id": 18, "legend": { "alignAsTable": true, @@ -321,9 +568,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -384,17 +632,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "How many records are skipped after XDR reading the record locally but before putting them on wire", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "How many records are skipped after XDR reading the record locally but before putting them on wire", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 22 }, + "hiddenSeries": false, "id": 19, "legend": { "alignAsTable": true, @@ -411,9 +667,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -474,17 +731,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of retries due to connection reset", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of retries due to connection reset", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 22 }, + "hiddenSeries": false, "id": 22, "legend": { "alignAsTable": true, @@ -501,9 +766,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -564,17 +830,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of retries due to temporary error from destination", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of retries due to temporary error from destination", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 30 }, + "hiddenSeries": false, "id": 21, "legend": { "alignAsTable": true, @@ -591,9 +865,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -654,17 +929,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of times a record write is skipped from processing because that record is already pending processing", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of times a record write is skipped from processing because that record is already pending processing", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 30 }, + "hiddenSeries": false, "id": 24, "legend": { "alignAsTable": true, @@ -681,9 +964,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -744,16 +1028,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Number of XDR processing record in progress", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 38 }, + "hiddenSeries": false, "id": 14, "legend": { "alignAsTable": true, @@ -769,9 +1061,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -832,16 +1125,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Number of records waiting to be processed in in-memory transaction queue", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 38 }, + "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, @@ -857,9 +1158,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -920,16 +1222,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "It signifies how much behind the destination is compared to the source. In other words, so much time worth of data is yet to be shipped from source to destination", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 46 }, + "hiddenSeries": false, "id": 16, "legend": { "alignAsTable": true, @@ -946,9 +1256,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1009,16 +1320,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Time taken to process records across partitions in one lap. A higher number indicates slowness of source in processing the records.", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 46 }, + "hiddenSeries": false, "id": 25, "legend": { "alignAsTable": true, @@ -1034,9 +1353,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1097,17 +1417,25 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", - "description": "Number of partitions that are recovered by reducing the primary index of that partition", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "decimals": null, - "fill": 1, - "fillGradient": 0, + "description": "Number of partitions that are recovered by reducing the primary index of that partition", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 54 }, + "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, @@ -1124,9 +1452,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1187,16 +1516,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Number of recoveries that are currently pending", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 54 }, + "hiddenSeries": false, "id": 23, "legend": { "alignAsTable": true, @@ -1212,9 +1549,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1275,16 +1613,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Average network latency for the successfully shipped records", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 56 + "y": 62 }, + "hiddenSeries": false, "id": 26, "legend": { "alignAsTable": true, @@ -1300,9 +1646,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1363,16 +1710,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "Average XDR compression ratio", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 56 + "y": 62 }, + "hiddenSeries": false, "id": 27, "legend": { "alignAsTable": true, @@ -1388,9 +1743,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1451,16 +1807,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Aerospike Prometheus", + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "description": "How much % of records are not compressed because they are below the compression threshold", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 5, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 64 + "y": 70 }, + "hiddenSeries": false, "id": 28, "legend": { "alignAsTable": true, @@ -1476,9 +1840,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.2.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -1536,20 +1901,15 @@ } ], "refresh": "1m", - "schemaVersion": 19, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\"},cluster_name)", "hide": 0, "includeAll": true, @@ -1570,13 +1930,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_node_stats_uptime{job=\"aerospike\", cluster_name=~\"$cluster|$^\"},service)", "hide": 0, "includeAll": true, @@ -1597,13 +1952,8 @@ }, { "allValue": null, - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Aerospike Prometheus", + "current": {}, + "datasource": "${DS_AEROSPIKE_PROMETHEUS}", "definition": "label_values(aerospike_xdr_lap_us{job=\"aerospike\", cluster_name=~\"$cluster|$^\", service=~\"$node|$^\" },dc)", "hide": 0, "includeAll": true, @@ -1624,21 +1974,22 @@ }, { "current": { - "text": "", - "value": "" + "selected": false, + "text": "Aerospike Prometheus", + "value": "Aerospike Prometheus" }, "hide": 0, - "label": "Tags", - "name": "tags", - "options": [ - { - "text": "", - "value": "" - } - ], - "query": "", + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_AEROSPIKE_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "textbox" + "type": "datasource" } ] }, diff --git a/config/prometheus/aerospike_rules.yml b/config/prometheus/aerospike_rules.yml index 4bcc6a6..6028ea0 100644 --- a/config/prometheus/aerospike_rules.yml +++ b/config/prometheus/aerospike_rules.yml @@ -5,7 +5,7 @@ groups: expr: up{job="aerospike"} == 0 for: 30s labels: - severity: "2" + severity: "1" annotations: description: '{{ $labels.instance }} has been down for more than 30 seconds.' summary: Node {{ $labels.instance }} down @@ -13,7 +13,7 @@ groups: expr: aerospike_node_up{job="aerospike"} == 0 for: 30s labels: - severity: "2" + severity: "1" annotations: description: '{{ $labels.instance }} node is down.' summary: Node {{ $labels.instance }} down @@ -25,6 +25,30 @@ groups: annotations: description: 'Used diskspace for namespace {{ $labels.ns }} in node {{ $labels.instance }} is above stop writes limit.' summary: Stop Writes for {{ $labels.instance }}/{{ $labels.ns }} + - alert: NamespaceMemoryCloseToStopWrites + expr: (aerospike_namespace_stop_writes_pct{job="aerospike"} - (100 - aerospike_namespace_memory_free_pct{job="aerospike"})) <= 10 + for: 30s + labels: + severity: "2" + annotations: + description: 'Free memory for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to stop writes limit.' + summary: Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to memory + - alert: NamespaceDiskCloseToStopWrites + expr: (aerospike_namespace_device_available_pct{job="aerospike"} - aerospike_namespace_storage_engine_min_avail_pct{job="aerospike"}) <= 10 + for: 30s + labels: + severity: "2" + annotations: + description: 'device_available_pct for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to min-avail-pct (stop writes) limit.' + summary: Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to device_available_pct + - alert: NamespacePmemCloseToStopWrites + expr: (aerospike_namespace_pmem_available_pct{job="aerospike"} - aerospike_namespace_storage_engine_min_avail_pct{job="aerospike"}) <= 10 + for: 30s + labels: + severity: "2" + annotations: + description: 'pmem_available_pct for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to min-avail-pct (stop writes) limit.' + summary: Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to pmem_available_pct - alert: ClockSkewStopWrites expr: aerospike_namespace_clock_skew_stop_writes{job="aerospike"} == 1 for: 30s @@ -34,10 +58,10 @@ groups: description: 'Clock has skewed for namespace {{ $labels.ns }} in node {{ $labels.instance }}' summary: Writes will be stopped by Aerospike - alert: ClusterSize - expr: aerospike_node_stats_cluster_size{job="aerospike"} < 3 # user has to replace his cluster size with 2 here in the example + expr: aerospike_node_stats_cluster_size{job="aerospike"} < 3 # user has to replace 3 here with their desired cluster size. for: 30s labels: - severity: "2" + severity: "1" annotations: description: 'Cluster size mismatch for namespace {{ $labels.ns }} in node {{ $labels.instance }}' summary: Some of the node(s) has gone out of the cluster @@ -53,7 +77,7 @@ groups: expr: aerospike_namespace_hwm_breached{job="aerospike"} == 1 for: 30s labels: - severity: "2" + severity: "1" annotations: description: 'High water mark has breached for namespace {{ $labels.ns }} in node {{ $labels.instance }}' summary: Eviction will start to make the space available @@ -64,45 +88,28 @@ groups: severity: "1" annotations: description: 'Some of the partition(s) is not available for namespace {{ $labels.ns }} in node {{ $labels.instance }}' - summary: Server could not found some partition(s). Check for the network issues and make sure cluster forms properly + summary: Server could not find some partition(s). Check for the network issues and make sure cluster forms properly - alert: XDRTimelag - expr: aerospike_node_stats_xdr_timelag{job="aerospike"} > 10 # user can configure the seconds. Refer XDR throttling + expr: aerospike_node_stats_xdr_timelag{job="aerospike"} > 10 # (Aerospike 4.9 and older) user can configure the seconds. Refer XDR throttling for: 30s labels: severity: "2" annotations: description: 'There seems some lag in XDR for namespace {{ $labels.ns }} in node {{ $labels.instance }}' summary: Lag can be there in XDR due to network connectivity issues or errors writing at a destination cluster - - alert: LowDeviceAvailWarning - expr: aerospike_namespace_device_available_pct{job="aerospike"} < 20 + - alert: XDRTimelag + expr: aerospike_xdr_lag{job="aerospike"} > 10 # (Aerospike 5.0 and later) user can configure the seconds. Refer XDR throttling for: 30s labels: severity: "2" annotations: - description: 'Device available percentage has dropped below 20 for namespace {{ $labels.ns }} in node {{ $labels.instance }}' - summary: Defrag is unable to keep up with the current load {{ $labels.instance }}/{{ $labels.ns }} - - alert: LowDeviceAvailCritical - expr: aerospike_namespace_device_available_pct{job="aerospike"} < 15 - for: 30s - labels: - severity: "1" - annotations: - description: 'Device available percentage has dropped below 15 for namespace {{ $labels.ns }} in node {{ $labels.instance }}' - summary: Usable disk resources are critically low which may result in a stop-writes if available percentage goes below 5 - - alert: LowPmemAvailWarning - expr: aerospike_namespace_pmem_available_pct{job="aerospike"} < 20 + description: 'There seems some lag in XDR for namespace {{ $labels.ns }} in node {{ $labels.instance }}' + summary: Lag can be there in XDR due to network connectivity issues or errors writing at a destination cluster + - alert: ActiveProxies + expr: (increase(aerospike_namespace_client_proxy_complete{job="aerospike"}[2m]) + increase(aerospike_namespace_client_proxy_timeout{job="aerospike"}[2m]) + increase(aerospike_namespace_client_proxy_error{job="aerospike"}[2m]) + increase(aerospike_namespace_batch_sub_proxy_complete{job="aerospike"}[2m]) + increase(aerospike_namespace_batch_sub_proxy_timeout{job="aerospike"}[2m]) + increase(aerospike_namespace_batch_sub_proxy_error{job="aerospike"}[2m])) > 0 for: 30s labels: severity: "2" annotations: - description: 'Pmem available percentage has dropped below 20 for namespace {{ $labels.ns }} in node {{ $labels.instance }}' - summary: Defrag is unable to keep up with the current load {{ $labels.instance }}/{{ $labels.ns }} - - alert: LowPmemAvailCritical - expr: aerospike_namespace_pmem_available_pct{job="aerospike"} < 15 - for: 30s - labels: - severity: "1" - annotations: - description: 'Pmem available percentage has dropped below 15 for namespace {{ $labels.ns }} in node {{ $labels.instance }}' - summary: Usable Pmem resource is critically low which may result in a stop-writes if available percentage goes below 5 - + description: 'Active proxies detected for {{ $labels.ns }} on node {{ $labels.instance }}' + summary: Node is doing proxy. Proxies can happen during cluster change / migrations or if there are any network issues. diff --git a/examples/swarm/aerospike.conf b/examples/swarm/aerospike.conf index 33c529f..cacd646 100644 --- a/examples/swarm/aerospike.conf +++ b/examples/swarm/aerospike.conf @@ -12,7 +12,7 @@ service { logging { console { - context any info + context any info } } diff --git a/examples/swarm/docker-stack.yaml b/examples/swarm/docker-stack.yaml index b68de71..fec46a3 100644 --- a/examples/swarm/docker-stack.yaml +++ b/examples/swarm/docker-stack.yaml @@ -62,15 +62,18 @@ services: - aeronet depends_on: - aerospike + labels: + prometheus-job: "aerospike" deploy: mode: replicated replicas: 3 endpoint_mode: dnsrr environment: - AS_HOST=aero{{.Task.Slot}}.aerospike.com + - AGENT_LOG_LEVEL=debug prometheus: - image: prom/prometheus:v2.11.1 + image: prom/prometheus:latest user: root deploy: replicas: 1 @@ -80,6 +83,7 @@ services: networks: - aeronet volumes: + - /var/run/docker.sock:/var/run/docker.sock - source: prometheus_data target: /var/lib/prometheus/ type: volume @@ -121,7 +125,7 @@ services: mode: 0444 grafana: - image: grafana/grafana:6.3.2 + image: grafana/grafana:latest user: root deploy: replicas: 1 diff --git a/examples/swarm/prometheus.yaml b/examples/swarm/prometheus.yaml index ee9ff0c..c53ed5f 100644 --- a/examples/swarm/prometheus.yaml +++ b/examples/swarm/prometheus.yaml @@ -21,4 +21,17 @@ scrape_configs: - names: - 'tasks.exporter' type: 'A' - port: 9145 \ No newline at end of file + port: 9145 + # dockerswarm_sd_configs: + # - host: unix:///var/run/docker.sock + # port: 9145 + # role: tasks + # relabel_configs: + # - source_labels: [__meta_dockerswarm_task_desired_state] + # regex: running + # action: keep + # - source_labels: [__meta_dockerswarm_task_label_prometheus_job] + # regex: .+ + # action: keep + # - source_labels: [__meta_dockerswarm_service_label_prometheus_job] + # target_label: job