diff --git a/ops/metrics/grafana/provisioning/dashboards/dashboard.json b/ops/metrics/grafana/provisioning/dashboards/dashboard.json index 52aa74cf85..e6b25d8cb9 100644 --- a/ops/metrics/grafana/provisioning/dashboards/dashboard.json +++ b/ops/metrics/grafana/provisioning/dashboards/dashboard.json @@ -31,6 +31,13 @@ "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, "mappings": [], "thresholds": { "mode": "absolute", @@ -50,11 +57,95 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 24, + "h": 10, + "w": 5, "x": 0, "y": 0 }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_bacalhau_node_info_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Node Metadata", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "node_engines", + "node_id", + "node_is_compute", + "node_is_requester", + "node_network_transport", + "node_publishers", + "node_storages" + ], + "mode": "rows" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 5, + "y": 0 + }, "id": 2, "options": { "colorMode": "value", @@ -105,16 +196,83 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 7, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_jobs_accepted_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Jobs Accepted", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, @@ -123,12 +281,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 8 + "h": 10, + "w": 2, + "x": 9, + "y": 0 }, - "id": 1, + "id": 5, "options": { "colorMode": "value", "graphMode": "area", @@ -166,22 +324,1062 @@ ], "title": "Jobs Completed", "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 11, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_docker_active_executions", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Docker Executions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 13, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_wasm_active_executions", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active WASM Executions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "editorMode": "code", + "expr": "rate(bacalhau_job_duration_milliseconds_sum[5m])\n/\nrate(bacalhau_job_duration_milliseconds_count[5m])", + "instant": false, + "legendFormat": "{{task_engine}}", + "range": true, + "refId": "A" + } + ], + "title": "Average Job Duration over 5mins", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "editorMode": "code", + "expr": "rate(bacalhau_http_server_duration_milliseconds_sum[5m])\n/\nrate(bacalhau_http_server_duration_milliseconds_count[5m])", + "instant": false, + "legendFormat": "{{http_route}}", + "range": true, + "refId": "A" + } + ], + "title": "Averagef HTTP Requests Duration over 5min", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "editorMode": "code", + "expr": "bacalhau_http_server_request_content_length_bytes_total", + "hide": false, + "instant": false, + "legendFormat": "{{http_route}}", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Request Content Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "editorMode": "code", + "expr": "bacalhau_http_server_response_content_length_bytes_total", + "hide": false, + "instant": false, + "legendFormat": "{{http_route}}", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Response Content Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_eval_broker_cancelable", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Evaluatio Broker Cancelable", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 2, + "y": 42 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_eval_broker_inflight", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Evaluatio Broker Inflight", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 4, + "y": 42 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_eval_broker_pending", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Evaluatio Broker Pending", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 6, + "y": 42 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_eval_broker_waiting", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Evaluatio Broker Waiting", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 8, + "y": 42 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_worker_ack_faults_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Worker Ack Faults", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 10, + "y": 42 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_worker_dequeue_faults_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Worker Dequeue Faults", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 12, + "y": 42 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6EBD7EB59B5FF381" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "bacalhau_worker_process_faults_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Worker Process Faults", + "type": "stat" } ], - "refresh": "", + "refresh": "10s", "schemaVersion": 39, "tags": [], "templating": { "list": [] }, "time": { - "from": "now-5m", + "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Bacalhau Metrics", "uid": "cbe6c668-d74b-4a27-be8b-431c19b2d4ca", - "version": 1, + "version": 2, "weekStart": "" } \ No newline at end of file