From 1538950c98db4cc9355ece1e61593f67138c7469 Mon Sep 17 00:00:00 2001 From: Pietro Date: Wed, 12 Nov 2025 18:08:41 +0100 Subject: [PATCH 1/2] Add node reward dashboard --- .../dashboards/node-rewards/dashboard.json | 1983 +++++++++++++++++ 1 file changed, 1983 insertions(+) create mode 100644 docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json new file mode 100644 index 0000000..9233c55 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json @@ -0,0 +1,1983 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "victoriametrics1d" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 6, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Provider Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "number" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "/^Value$/", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "last_over_time(governance_latest_reward_event_timestamp_seconds[35d]) * 1000", + "format": "table", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Last Rewards Distribution", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of providers tracked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(latest_nodes_count offset -1d)", + "range": true, + "refId": "A" + } + ], + "title": "Total Providers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total nodes across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(last_over_time(latest_nodes_count[$__range] offset -1d))", + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total adjusted rewards across all providers (XDR Permyriad)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(latest_total_adjusted_rewards_xdr_permyriad[$__range] offset -1d)) / 10000", + "range": true, + "refId": "A" + } + ], + "title": "Total Adjusted Rewards (XDR Permyriad)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Comprehensive provider metrics overview", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "provider_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Provider Details", + "url": "/d/${__dashboard.uid}?var-provider=${__value.text}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Base Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Base Total (XDR)" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Adjusted Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Adjusted Total (XDR)" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Difference" + }, + "properties": [ + { + "id": "displayName", + "value": "Adj - Base Diff" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 0 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Efficiency %" + }, + "properties": [ + { + "id": "displayName", + "value": "Rewards" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 98 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Nodes" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 105, + "options": { + "cellHeight": "sm", + "enablePagination": false, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Rewards" + } + ] + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "latest_nodes_count{provider_id=~\"$provider\"} offset -1d", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", + "format": "table", + "instant": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) - sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) / 10000", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) * 100", + "format": "table", + "instant": true, + "refId": "E" + } + ], + "title": "Provider Rewards Summary (Latest)", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "provider_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "Value #D": 4, + "Value #E": 5, + "provider_id": 0 + }, + "renameByName": { + "Value #A": "Nodes", + "Value #B": "Base Total", + "Value #C": "Adjusted Total", + "Value #D": "Difference", + "Value #E": "Efficiency %" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 200, + "panels": [], + "title": "Provider Rewards Trends", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Base vs Adjusted rewards over time for selected provider(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "XDR Permyriad", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Base.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Adjusted.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", + "legendFormat": "Base - {{provider_id}} ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", + "hide": false, + "instant": false, + "legendFormat": "Adjusted - {{provider_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Base vs Adjusted Rewards Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards efficiency percentage (Adjusted/Base * 100)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Efficiency %", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d / latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d) * 100", + "legendFormat": "{{provider_id}} - Efficiency %", + "refId": "A" + } + ], + "title": "Provider Rewards Efficiency Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Node counts over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "latest_nodes_count{provider_id=~\"$provider\"} offset -1d", + "legendFormat": "{{provider_id}} - Total Nodes", + "refId": "A" + } + ], + "title": "Node Counts Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards penalty/loss per provider (Base - Adjusted)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Penalty (XDR Permyriad)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#F2495C", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d- latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d)", + "legendFormat": "{{provider_id}} - Penalty", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Penalty (Base - Adjusted)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 300, + "panels": [], + "title": "Node-Level Metrics (Assigned)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Original failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(latest_original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Original Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Relative failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Relative Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(latest_relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Relative Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of original failure rates across all nodes", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 303, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(latest_original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Original Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of relative failure rates across all nodes.\nrelative failure rate = original failure rate - subnet failure rate", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 304, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(latest_relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Relative Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Detailed node metrics table", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Original Failure Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + { + "id": "max", + "value": 0.2 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Relative Failure Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + { + "id": "max", + "value": 0.2 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "subnet_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Subnet Details", + "url": "/d/${__dashboard.uid}?var-subnet=${__value.text}&${__url_time_range}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 305, + "options": { + "cellHeight": "sm", + "enablePagination": true, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Original Failure Rate" + } + ] + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "late{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "latest_relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", + "format": "table", + "instant": true, + "refId": "B" + } + ], + "title": "Node Performance Details", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "node_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value #A": 3, + "Value #B": 4, + "node_id": 1, + "provider_id": 0, + "subnet_id": 2 + }, + "renameByName": { + "Value #A": "Original Failure Rate", + "Value #B": "Relative Failure Rate" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 400, + "panels": [], + "title": "Subnet Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Subnet failure rates over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 401, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "subnet_failure_rate{subnet_id=~\"$subnet\"} offset -1d", + "legendFormat": "{{subnet_id}}", + "refId": "A" + } + ], + "title": "Subnet Failure Rates Over Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics1d", + "value": "victoriametrics1d" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(latest_nodes_count, provider_id)", + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": { + "query": "label_values(latest_nodes_count, provider_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "query": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(subnet_failure_rate, subnet_id)", + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "subnet", + "options": [], + "query": { + "query": "label_values(subnet_failure_rate, subnet_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "1762947155000", + "value": "1762947155000" + }, + "definition": "query_result(last_over_time(governance_latest_reward_event_timestamp_seconds[35d]))", + "hide": 2, + "includeAll": false, + "name": "last_rewards_seconds", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(last_over_time(governance_latest_reward_event_timestamp_seconds[35d]))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/\\d+\\s+(\\d+)/", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard", + "uid": "node-rewards-comprehensive", + "version": 2 + } From 2550997f40311ee71f640da6819f99e165c2a0a9 Mon Sep 17 00:00:00 2001 From: Pietro Date: Wed, 12 Nov 2025 22:15:59 +0100 Subject: [PATCH 2/2] changes --- .../dashboards/node-rewards/dashboard.json | 50 ++++++++-------- .../dashboards/observability.yaml | 11 ++++ .../datasources/observability.yaml | 8 +++ .../node-rewards-scheduler/scheduler.py | 60 +++++++++++++++---- docker/volumes/grafana/.gitignore | 5 -- docker/volumes/msd/.gitignore | 5 -- docker/volumes/victoria/.gitignore | 5 -- 7 files changed, 92 insertions(+), 52 deletions(-) delete mode 100755 docker/volumes/grafana/.gitignore delete mode 100644 docker/volumes/msd/.gitignore delete mode 100644 docker/volumes/victoria/.gitignore diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json index 9233c55..830dcb7 100644 --- a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json @@ -5,7 +5,7 @@ "builtIn": 1, "datasource": { "type": "grafana", - "uid": "victoriametrics1d" + "uid": "prometheus1d" }, "enable": true, "hide": true, @@ -171,7 +171,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "count(latest_nodes_count offset -1d)", + "expr": "count(nodes_count offset -1d)", "range": true, "refId": "A" } @@ -235,7 +235,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(last_over_time(latest_nodes_count[$__range] offset -1d))", + "expr": "sum(last_over_time(nodes_count[$__range] offset -1d))", "refId": "A" } ], @@ -299,7 +299,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(sum_over_time(latest_total_adjusted_rewards_xdr_permyriad[$__range] offset -1d)) / 10000", + "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[$__range] offset -1d)) / 10000", "range": true, "refId": "A" } @@ -539,7 +539,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "latest_nodes_count{provider_id=~\"$provider\"} offset -1d", + "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", "format": "table", "instant": true, "refId": "A" @@ -550,7 +550,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", + "expr": "sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", "format": "table", "instant": true, "refId": "B" @@ -561,7 +561,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", + "expr": "sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / 10000", "format": "table", "instant": true, "refId": "C" @@ -572,7 +572,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) - sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) / 10000", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) - sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) / 10000", "format": "table", "instant": true, "refId": "D" @@ -583,7 +583,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(sum by (provider_id) (sum_over_time(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / sum by (provider_id) (sum_over_time(latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) * 100", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) * 100", "format": "table", "instant": true, "refId": "E" @@ -766,7 +766,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", + "expr": "total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", "legendFormat": "Base - {{provider_id}} ", "range": true, "refId": "A" @@ -777,7 +777,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", + "expr": "total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d", "hide": false, "instant": false, "legendFormat": "Adjusted - {{provider_id}}", @@ -882,7 +882,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d / latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d) * 100", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d / total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d) * 100", "legendFormat": "{{provider_id}} - Efficiency %", "refId": "A" } @@ -979,7 +979,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "latest_nodes_count{provider_id=~\"$provider\"} offset -1d", + "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", "legendFormat": "{{provider_id}} - Total Nodes", "refId": "A" } @@ -1081,7 +1081,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(latest_total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d- latest_total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d)", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d- total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} offset -1d)", "legendFormat": "{{provider_id}} - Penalty", "range": true, "refId": "A" @@ -1195,7 +1195,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "label_replace(latest_original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", "legendFormat": "{{node_short}}", "range": true, "refId": "A" @@ -1300,7 +1300,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "label_replace(latest_relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", "legendFormat": "{{node_short}}", "range": true, "refId": "A" @@ -1382,7 +1382,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "label_replace(latest_original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", "format": "time_series", "legendFormat": "{{node_short}}", "range": true, @@ -1465,7 +1465,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "label_replace(latest_relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", "format": "time_series", "legendFormat": "{{node_short}}", "range": true, @@ -1670,7 +1670,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "latest_relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", + "expr": "relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", "format": "table", "instant": true, "refId": "B" @@ -1862,14 +1862,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(latest_nodes_count, provider_id)", + "definition": "label_values(nodes_count, provider_id)", "includeAll": true, "label": "Provider", "multi": true, "name": "provider", "options": [], "query": { - "query": "label_values(latest_nodes_count, provider_id)", + "query": "label_values(nodes_count, provider_id)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1887,14 +1887,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "definition": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", "includeAll": true, "label": "Node", "multi": true, "name": "node", "options": [], "query": { - "query": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "query": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1933,14 +1933,14 @@ "text": "1762947155000", "value": "1762947155000" }, - "definition": "query_result(last_over_time(governance_latest_reward_event_timestamp_seconds[35d]))", + "definition": "query_result(last_over_time(governance_reward_event_timestamp_seconds[35d]))", "hide": 2, "includeAll": false, "name": "last_rewards_seconds", "options": [], "query": { "qryType": 3, - "query": "query_result(last_over_time(governance_latest_reward_event_timestamp_seconds[35d]))", + "query": "query_result(last_over_time(governance_reward_event_timestamp_seconds[35d]))", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/docker/config/grafana/provisioning/dashboards/observability.yaml b/docker/config/grafana/provisioning/dashboards/observability.yaml index 6648df9..0a55fb6 100644 --- a/docker/config/grafana/provisioning/dashboards/observability.yaml +++ b/docker/config/grafana/provisioning/dashboards/observability.yaml @@ -11,3 +11,14 @@ providers: options: path: /etc/grafana/provisioning/dashboards/samples foldersFromFilesStructure: false + + - name: 'Node Rewards' + folder: 'Node Rewards' + folderUid: 'node-rewards' + type: file + disableDeletion: true + updateIntervalSeconds: 60 + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards/node-rewards + foldersFromFilesStructure: false diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 825b528..316f614 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -8,3 +8,11 @@ datasources: url: http://$PROM_HOST:9090 jsonData: httpMethod: POST + - name: prometheus1d + uid: prometheus1d + type: prometheus + access: proxy + url: http://$PROM_HOST:9090 + jsonData: + httpMethod: POST + timeInterval: "1d" diff --git a/docker/config/node-rewards-scheduler/scheduler.py b/docker/config/node-rewards-scheduler/scheduler.py index 3e0e30e..3c571a7 100644 --- a/docker/config/node-rewards-scheduler/scheduler.py +++ b/docker/config/node-rewards-scheduler/scheduler.py @@ -231,8 +231,33 @@ async def get_latest_governance_reward_event(self) -> Optional[float]: from ic.principal import Principal try: + # DateRangeFilter: record { start_timestamp_seconds: opt nat64; end_timestamp_seconds: opt nat64 } + date_range_filter_type = Types.Record({ + 'start_timestamp_seconds': Types.Opt(Types.Nat64), + 'end_timestamp_seconds': Types.Opt(Types.Nat64), + }) + + # ListNodeProviderRewardsRequest: record { date_filter: opt DateRangeFilter } request_type = Types.Record({ - 'date_filter': Types.Opt(Types.Nat64) + 'date_filter': Types.Opt(date_range_filter_type) + }) + + # MonthlyNodeProviderRewards (simplified - we only need timestamp field) + monthly_rewards_type = Types.Record({ + 'timestamp': Types.Nat64, + 'start_date': Types.Opt(Types.Record({'year': Types.Nat32, 'month': Types.Nat32, 'day': Types.Nat32})), + 'end_date': Types.Opt(Types.Record({'year': Types.Nat32, 'month': Types.Nat32, 'day': Types.Nat32})), + 'rewards': Types.Vec(Types.Record({})), # Simplified, not parsing individual rewards + 'xdr_conversion_rate': Types.Opt(Types.Record({})), + 'minimum_xdr_permyriad_per_icp': Types.Opt(Types.Nat64), + 'maximum_node_provider_rewards_e8s': Types.Opt(Types.Nat64), + 'registry_version': Types.Opt(Types.Nat64), + 'node_providers': Types.Vec(Types.Record({})), + }) + + # ListNodeProviderRewardsResponse: record { rewards: vec MonthlyNodeProviderRewards } + return_type = Types.Record({ + 'rewards': Types.Vec(monthly_rewards_type) }) # Build request with no date filter to get all rewards @@ -244,11 +269,12 @@ async def get_latest_governance_reward_event(self) -> Optional[float]: } }]) - # Make query + # Make query with return type response = self.agent.query_raw( self.GOVERNANCE_CANISTER_ID, 'list_node_provider_rewards', arg_bytes, + return_type ) # Extract value from response @@ -258,7 +284,7 @@ async def get_latest_governance_reward_event(self) -> Optional[float]: result = response[0].get('value', {}) rewards_list = result.get('rewards', []) - + # Rewards are in descending order (latest first) if rewards_list and len(rewards_list) > 0: first_reward = rewards_list[0] @@ -330,8 +356,8 @@ async def push_metrics_for_date(self, date_str: str) -> bool: target_date = datetime.strptime(date_str, '%Y-%m-%d') # Calculate noon timestamp in milliseconds - noon_dt = target_date.replace(hour=12, minute=0, second=0, microsecond=0) - noon_timestamp_ms = int(noon_dt.replace(tzinfo=timezone.utc).timestamp() * 1000) + target_dt = target_date.replace(hour=23, minute=59, second=59, microsecond=0) + timestamp_ms = int(target_dt.replace(tzinfo=timezone.utc).timestamp() * 1000) # Fetch data from IC canister daily_results = await self.ic_client.get_rewards_daily(date_str) @@ -351,21 +377,21 @@ async def push_metrics_for_date(self, date_str: str) -> bool: # nodes_count nodes_count = len(provider_rewards.get('daily_nodes_rewards', [])) metrics_lines.append( - f'nodes_count{{provider_id="{provider_id_str}"}} {nodes_count} {noon_timestamp_ms}' + f'nodes_count{{provider_id="{provider_id_str}"}} {nodes_count} {timestamp_ms}' ) # base_rewards base_rewards = self._unwrap_optional(provider_rewards.get('total_base_rewards_xdr_permyriad')) if base_rewards is not None: metrics_lines.append( - f'total_base_rewards_xdr_permyriad{{provider_id="{provider_id_str}"}} {base_rewards} {noon_timestamp_ms}' + f'total_base_rewards_xdr_permyriad{{provider_id="{provider_id_str}"}} {base_rewards} {timestamp_ms}' ) # adjusted_rewards adjusted_rewards = self._unwrap_optional(provider_rewards.get('total_adjusted_rewards_xdr_permyriad')) if adjusted_rewards is not None: metrics_lines.append( - f'total_adjusted_rewards_xdr_permyriad{{provider_id="{provider_id_str}"}} {adjusted_rewards} {noon_timestamp_ms}' + f'total_adjusted_rewards_xdr_permyriad{{provider_id="{provider_id_str}"}} {adjusted_rewards} {timestamp_ms}' ) # Node-level metrics @@ -388,14 +414,14 @@ async def push_metrics_for_date(self, date_str: str) -> bool: original_fr = self._unwrap_optional(node_metrics.get('original_failure_rate')) if original_fr is not None: metrics_lines.append( - f'original_failure_rate{{provider_id="{provider_id_str}",node_id="{node_id_str}",subnet_id="{subnet_id_str}"}} {original_fr} {noon_timestamp_ms}' + f'original_failure_rate{{provider_id="{provider_id_str}",node_id="{node_id_str}",subnet_id="{subnet_id_str}"}} {original_fr} {timestamp_ms}' ) # relative_failure_rate relative_fr = self._unwrap_optional(node_metrics.get('relative_failure_rate')) if relative_fr is not None: metrics_lines.append( - f'relative_failure_rate{{provider_id="{provider_id_str}",node_id="{node_id_str}",subnet_id="{subnet_id_str}"}} {relative_fr} {noon_timestamp_ms}' + f'relative_failure_rate{{provider_id="{provider_id_str}",node_id="{node_id_str}",subnet_id="{subnet_id_str}"}} {relative_fr} {timestamp_ms}' ) # Subnet-level metrics @@ -403,14 +429,22 @@ async def push_metrics_for_date(self, date_str: str) -> bool: for subnet_id, failure_rate in subnets_failure_rate.items(): subnet_id_str = str(subnet_id) metrics_lines.append( - f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {noon_timestamp_ms}' + f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {timestamp_ms}' ) # Governance timestamp gov_timestamp = await self.ic_client.get_latest_governance_reward_event() if gov_timestamp: metrics_lines.append( - f'governance_latest_reward_event_timestamp_seconds {gov_timestamp} {noon_timestamp_ms}' + f'governance_latest_reward_event_timestamp_seconds {gov_timestamp} {timestamp_ms}' + ) + + # Calculate days since last governance distribution + now_utc = datetime.now(timezone.utc) + gov_datetime: datetime = datetime.fromtimestamp(gov_timestamp, tz=timezone.utc) + days_since = int((now_utc - gov_datetime).total_seconds() / 86400) # 86400 seconds in a day + metrics_lines.append( + f'days_since_governance_distribution {days_since} {timestamp_ms}' ) # Push to VictoriaMetrics @@ -471,6 +505,8 @@ def wait_until_next_run(self): async def run_daily_scheduler(self): """Run the daily scheduler loop""" + yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).strftime('%Y-%m-%d') + logger.info(f"Running scheduled push for {yesterday}") while True: try: diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore deleted file mode 100755 index e769a26..0000000 --- a/docker/volumes/grafana/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore everything in this directory -* - -# But keep this .gitignore file -!.gitignore diff --git a/docker/volumes/msd/.gitignore b/docker/volumes/msd/.gitignore deleted file mode 100644 index e67750e..0000000 --- a/docker/volumes/msd/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore everything in this directory -* - -# But keep this .gitignore -!.gitignore diff --git a/docker/volumes/victoria/.gitignore b/docker/volumes/victoria/.gitignore deleted file mode 100644 index facdf81..0000000 --- a/docker/volumes/victoria/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore everything in this directory -* - -# But keep this .gitignore -!.gitignore