From 270ca5f67751969271dcccc18c52a839f17f09c6 Mon Sep 17 00:00:00 2001 From: Pietro Date: Thu, 13 Nov 2025 13:10:32 +0100 Subject: [PATCH 1/3] Fix ingester --- .../dashboards/node-rewards/dashboard.json | 2162 +++++++++++++++++ .../dashboards/observability.yaml | 11 + .../datasources/observability.yaml | 9 + .../node_rewards_ingester.py | 53 +- 4 files changed, 2224 insertions(+), 11 deletions(-) create mode 100644 docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json new file mode 100644 index 0000000..bdbf803 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json @@ -0,0 +1,2162 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "prometheus1d" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total nodes across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of providers tracked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Providers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 100, + "panels": [], + "title": "Current Reward Period", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "number" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "/^Value$/", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "governance_latest_reward_event_timestamp_seconds * 1000", + "format": "table", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Start Reward Period", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total base rewards across all providers (XDR Permyriad)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "hideTimeOverride": true, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_base_rewards_xdr_permyriad[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "timeShift": "1d", + "title": "Total Base Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total adjusted rewards across all providers ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "hideTimeOverride": true, + "id": 402, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "timeShift": "1d", + "title": "Total Adjusted Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "hideTimeOverride": true, + "id": 403, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[${days_since_last_distribution}d])) / sum(sum_over_time(total_base_rewards_xdr_permyriad[${days_since_last_distribution}d])) * 100", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "timeShift": "1d", + "title": "Rewards Efficiency %", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Comprehensive provider metrics overview", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "provider_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Provider Details", + "url": "/d/${__dashboard.uid}?var-provider=${__value.text}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Base Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Base Total" + }, + { + "id": "unit", + "value": "XDR" + }, + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Adjusted Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Adjusted Total" + }, + { + "id": "unit", + "value": "XDR" + }, + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Difference" + }, + "properties": [ + { + "id": "displayName", + "value": "Adj - Base Diff" + }, + { + "id": "unit", + "value": "" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 0 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Efficiency %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 98 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Nodes" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Adj - Base Diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "hideTimeOverride": true, + "id": 105, + "options": { + "cellHeight": "sm", + "enablePagination": false, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Efficiency %" + } + ] + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) - sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d]))) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d]))) * 100", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "refId": "A" + } + ], + "timeFrom": "1m", + "timeShift": "1d", + "title": "Rewards Per Provider", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "provider_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 5, + "Time 2": 6, + "Time 3": 7, + "Time 4": 8, + "Value #A": 4, + "Value #B": 1, + "Value #C": 2, + "Value #D": 3, + "provider_id": 0 + }, + "orderByMode": "manual", + "renameByName": { + "Value #A": "Nodes", + "Value #B": "Base Total", + "Value #C": "Adjusted Total", + "Value #D": "Difference", + "Value #E": "Efficiency %" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 200, + "panels": [], + "title": "Provider Rewards Trends", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Base vs Adjusted rewards over time for selected provider(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "XDR Permyriad", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Base.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Adjusted.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}", + "legendFormat": "Base - {{provider_id}} ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}", + "hide": false, + "instant": false, + "legendFormat": "Adjusted - {{provider_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Base vs Adjusted Rewards Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards efficiency percentage (Adjusted/Base * 100)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Efficiency %", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}) * 100", + "legendFormat": "{{provider_id}} - Efficiency %", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Efficiency Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Node counts over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", + "legendFormat": "{{provider_id}} - Total Nodes", + "refId": "A" + } + ], + "title": "Node Counts Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards penalty/loss per provider (Base - Adjusted)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Penalty (XDR Permyriad)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#F2495C", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} - total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} )", + "legendFormat": "{{provider_id}} - Penalty", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Penalty (Base - Adjusted)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 300, + "panels": [], + "title": "Node-Level Metrics (Assigned)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Original failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Original Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Relative failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Relative Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Relative Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of original failure rates across all nodes", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 303, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Original Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of relative failure rates across all nodes.\nrelative failure rate = original failure rate - subnet failure rate", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 304, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Relative Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Detailed node metrics table", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Original Failure Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + { + "id": "max", + "value": 0.2 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Relative Failure Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + { + "id": "max", + "value": 0.2 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "subnet_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Subnet Details", + "url": "/d/${__dashboard.uid}?var-subnet=${__value.text}&${__url_time_range}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 305, + "options": { + "cellHeight": "sm", + "enablePagination": true, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Original Failure Rate" + } + ] + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "late{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", + "format": "table", + "instant": true, + "refId": "B" + } + ], + "title": "Node Performance Details", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "node_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value #A": 3, + "Value #B": 4, + "node_id": 1, + "provider_id": 0, + "subnet_id": 2 + }, + "renameByName": { + "Value #A": "Original Failure Rate", + "Value #B": "Relative Failure Rate" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 400, + "panels": [], + "title": "Subnet Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Subnet failure rates over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 401, + "interval": "1d", + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "subnet_failure_rate{subnet_id=~\"$subnet\"}", + "legendFormat": "{{subnet_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Subnet Failure Rates Over Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus1d", + "value": "prometheus1d" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(nodes_count, provider_id)", + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": { + "query": "label_values(nodes_count, provider_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "query": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(subnet_failure_rate, subnet_id)", + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "subnet", + "options": [], + "query": { + "query": "label_values(subnet_failure_rate, subnet_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "29", + "value": "29" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus1d" + }, + "definition": "query_result(last_over_time(days_since_governance_distribution[35d]))", + "hide": 2, + "includeAll": false, + "name": "days_since_last_distribution", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(last_over_time(days_since_governance_distribution[35d]))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/\\s+(\\d+)\\s+\\d+$/", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + }, + { + "allowCustomValue": false, + "current": { + "text": "10000", + "value": "10000" + }, + "name": "scale_xdrp_to_xdr", + "options": [ + { + "selected": true, + "text": "10000", + "value": "10000" + } + ], + "query": "10000", + "type": "custom" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard", + "uid": "node-rewards-comprehensive", + "version": 1 + } \ No newline at end of file diff --git a/docker/config/grafana/provisioning/dashboards/observability.yaml b/docker/config/grafana/provisioning/dashboards/observability.yaml index 6648df9..0a55fb6 100644 --- a/docker/config/grafana/provisioning/dashboards/observability.yaml +++ b/docker/config/grafana/provisioning/dashboards/observability.yaml @@ -11,3 +11,14 @@ providers: options: path: /etc/grafana/provisioning/dashboards/samples foldersFromFilesStructure: false + + - name: 'Node Rewards' + folder: 'Node Rewards' + folderUid: 'node-rewards' + type: file + disableDeletion: true + updateIntervalSeconds: 60 + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards/node-rewards + foldersFromFilesStructure: false diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 825b528..c4fcf79 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -8,3 +8,12 @@ datasources: url: http://$PROM_HOST:9090 jsonData: httpMethod: POST + - name: prometheus1d + uid: prometheus1d + type: prometheus + access: proxy + url: http://$PROM_HOST:9090 + jsonData: + httpMethod: POST + timeInterval: "1d" + diff --git a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py index bed42f4..47e013b 100644 --- a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py +++ b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py @@ -124,7 +124,7 @@ } ) -RETURN_TYPE = Types.Variant( +GET_NODE_PROVIDERS_CALCULATION_RESPONSE_TYPE = Types.Variant( { "Ok": DAILY_RESULTS_TYPE, "Err": Types.Text, @@ -135,6 +135,27 @@ {"date_filter": Types.Opt(Types.Nat64)} ) +DATE_RANGE_FILTER_TYPE = Types.Record({ + "start_timestamp_seconds": Types.Opt(Types.Nat64), + "end_timestamp_seconds": Types.Opt(Types.Nat64), +}) + +MONTHLY_NODE_PROVIDER_REWARDS_TYPE = Types.Record({ + 'timestamp': Types.Nat64, + "start_date": Types.Opt(Types.Record({"year": Types.Nat32, "month": Types.Nat32, "day": Types.Nat32})), + "end_date": Types.Opt(Types.Record({"year": Types.Nat32, "month": Types.Nat32, "day": Types.Nat32})), + "rewards": Types.Vec(Types.Record({})), # Simplified, not parsing individual rewards + "xdr_conversion_rate": Types.Opt(Types.Record({})), + "minimum_xdr_permyriad_per_icp": Types.Opt(Types.Nat64), + "maximum_node_provider_rewards_e8s": Types.Opt(Types.Nat64), + "registry_version": Types.Opt(Types.Nat64), + "node_providers": Types.Vec(Types.Record({})), +}) + +LIST_NODE_PROVIDER_REWARDS_RESPONSE_TYPE = Types.Record({ + "rewards": Types.Vec(MONTHLY_NODE_PROVIDER_REWARDS_TYPE), +}) + class NodeRewardsClient: """Client for interacting with the node rewards canister""" @@ -164,7 +185,7 @@ def get_rewards_daily(self, date: str) -> Dict[str, Any]: self.canister_id, "get_node_providers_rewards_calculation", arg_bytes, - RETURN_TYPE, + GET_NODE_PROVIDERS_CALCULATION_RESPONSE_TYPE, ) if not response or len(response) == 0: @@ -224,6 +245,7 @@ def get_latest_governance_reward_event(self) -> Optional[float]: GOVERNANCE_CANISTER_ID, "list_node_provider_rewards", arg_bytes, + LIST_NODE_PROVIDER_REWARDS_RESPONSE_TYPE, ) if not response or len(response) == 0: @@ -261,7 +283,7 @@ def _unwrap_optional(value): @staticmethod def _make_line(metric_name: str, value: int, ts: int, **kwargs) -> str: - return f"{metric_name}{{ {' '.join([f'{key}="{value}"' for key, value in kwargs.items()])} }} {value} {ts}" + return f"{metric_name}{{{','.join([f'{key}="{value}"' for key, value in kwargs.items()])}}} {value} {ts}" def wait_for_victoria_metrics(self): """Wait for VictoriaMetrics to be ready""" @@ -289,8 +311,8 @@ def push_metrics_for_date(self, date: str): logger.info(f"Pushing node rewards data for {date}") target_date = datetime.strptime(date, "%Y-%m-%d") - noon = target_date.replace(hour=12, minute=0, second=0, microsecond=0) - noon_timestamp_ms = int(noon.replace(tzinfo=timezone.utc).timestamp() * 1000) + target_dt = target_date.replace(hour=0, minute=0, second=0, microsecond=0) + timestamp_ms = int(target_dt.replace(tzinfo=timezone.utc).timestamp() * 1000) metrics_lines = [] for client in self.nrc_clients: @@ -306,7 +328,7 @@ def add_line_helper(metric_name: str, value, **kwargs): self._make_line( metric_name, value, - noon_timestamp_ms, + timestamp_ms, canister_id=client.canister_id, **kwargs, ) @@ -401,7 +423,7 @@ def add_line_helper_with_provider( for subnet_id, failure_rate in subnets_failure_rate.items(): subnet_id_str = str(subnet_id) metrics_lines.append( - f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {noon_timestamp_ms}' + f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {timestamp_ms}' ) add_line_helper( "subnets_failure_rate", failure_rate, subnet_id=subnet_id_str @@ -413,6 +435,17 @@ def add_line_helper_with_provider( add_line_helper( "governance_latest_reward_event_timestamp_seconds", gov_timestamp ) + + # Calculate days since last governance distribution + now_utc = datetime.now(timezone.utc) + gov_datetime: datetime = datetime.fromtimestamp(gov_timestamp, tz=timezone.utc) + days_since = int((now_utc - gov_datetime).total_seconds() / 86400) # 86400 seconds in a day + metrics_lines.append( + f'days_since_governance_distribution {days_since} {timestamp_ms}' + ) + add_line_helper( + "days_since_governance_distribution", days_since + ) if not metrics_lines: raise ValueError("After evaluation there were no metrics to upload") @@ -458,13 +491,11 @@ def backfill(self, days: int = 40): logger.info("✅ Backfill complete!") def wait_until_next_run(self): - """Wait until 00:05 UTC tomorrow""" + """Wait until 00:10 UTC tomorrow""" now = datetime.now(timezone.utc) - # Calculate next 00:05 UTC - next_run = now.replace(hour=0, minute=5, second=0, microsecond=0) + next_run = now.replace(hour=0, minute=10, second=0, microsecond=0) if now >= next_run: - # Already past 00:05 today, schedule for tomorrow next_run += timedelta(days=1) wait_seconds = (next_run - now).total_seconds() From 9a3cd1c065e4e8069c28ac1dec2b5726c8511113 Mon Sep 17 00:00:00 2001 From: Pietro Date: Thu, 13 Nov 2025 17:45:12 +0100 Subject: [PATCH 2/3] Fix ingester script --- .DS_Store | Bin 0 -> 8196 bytes docker/.DS_Store | Bin 0 -> 6148 bytes .../dashboards/node-rewards/dashboard.json | 3897 ++++++++--------- .../node_rewards_ingester.py | 306 +- 4 files changed, 2047 insertions(+), 2156 deletions(-) create mode 100644 .DS_Store create mode 100644 docker/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..a59fc3fd4d70b64d9e30f14e18231e60019b886c GIT binary patch literal 8196 zcmeHMJ#W-N5S`5#xNt?TObR3`N*4(c3Mdegi5wwOA*D>ieZYaZi;qi!L}w(ZXd)W; z2l*RxDfk0)Pysbfq*QscyYbEXBXkJCjI{f#otd5IXZxISh{$Xn3^$0DiKvai*zI7d zG4}JQ*n(eK0~+Mhc(k!S$R^WDq%Noes(>n>3aA3Az~7<(-q~#JhWEbn>Z}T=0{^7~ z{QD4NFeV;zhxVrflWhTj5tgmtxXTB?*kj@`cL)#6xKyA^HNM3#E*e?7Nz|bsJ*Fq9yvx-wG92v%EN;F3-W(}8tsbT8A9|Kk!@iSj z&hYoq0pHt}Zwq`g8U*}WH)im8kL0s(T<|Hp^ZHe9P_LD6!ON~5J%>uOpz75Zhh@L4 z{q^Yem&je8U=OK4Q%u04^ND#Z(v-&3qgT|&`Dfr0UL#sssgIJJ?&MqN*QaJb$?W3D zfFH38bj9*Ju=jBJ{-L`k)d{1)#@E)F06d^KR*>ro~^?2+8B)&w@v3AIYUTD&XT@;*sSuE8jJp7MK)1 z$>IKK!9n=vPfwjctozVa#@_)6im0gqsz5~ne*R#~-2ZPifB&~ZI;jGxz`v=0iPG(K z6X#|c&X0TT7~>HJ8^`4iZ3-sajstBw4lMp*h~pTcoRfIW9byHh9|EilI;jGGRDm;C C2P7Q; literal 0 HcmV?d00001 diff --git a/docker/.DS_Store b/docker/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..f9eb51c1576a8c3c5ffd1c0b759f9102f2d01929 GIT binary patch literal 6148 zcmeHKy-ve05I&azDInA_V;+GHFo!CPOkEgJQYc6Xt^{Sy6EL&z49v_7EQ~w>FTu)p zXVd1V6%z=dJL&v6=R5oI*~-O4WQMcxfT&MI0hF=RL$gG9oV6t#Z`lDFnd6cs`D8H4 zY%y(ltKly)zP z1+CK!^01WCb^R2byw^voIfT0hUpZ9b3%kf*xkqh%lH)C2Mx`j+A2faw!g>BCto?0mdFnhp<5OM?h%MMj7~320j796l;qB literal 0 HcmV?d00001 diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json index bdbf803..edee8f5 100644 --- a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json @@ -1,2162 +1,1989 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "prometheus1d" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total nodes across all providers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 103, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(nodes_count)", - "range": true, - "refId": "A" - } - ], - "title": "Total Nodes", - "type": "stat" + "type": "grafana", + "uid": "prometheus1d" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of providers tracked", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short" + "description": "Total nodes across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 12, - "y": 0 + "unit": "short" }, - "id": 102, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "max" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(nodes_count)", - "range": true, - "refId": "A" - } - ], - "title": "Total Providers", - "type": "stat" + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 100, - "panels": [], - "title": "Current Reward Period", - "type": "row" + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 0 }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "dateTimeAsIso" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "overrides": [ - { - "matcher": { - "id": "byType", - "options": "number" - }, - "properties": [ - { - "id": "unit", - "value": "dateTimeAsIso" - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 101, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "/^Value$/", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "governance_latest_reward_event_timestamp_seconds * 1000", - "format": "table", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Start Reward Period", - "type": "stat" + "editorMode": "code", + "expr": "sum(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total base rewards across all providers (XDR Permyriad)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": 0 - } - ] - }, - "unit": "short" + "description": "Total number of providers tracked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 4 - }, - "hideTimeOverride": true, - "id": 104, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "unit": "short" }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sum_over_time(total_base_rewards_xdr_permyriad[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", - "range": true, - "refId": "A" - } - ], - "timeFrom": "1m", - "timeShift": "1d", - "title": "Total Base Rewards", - "type": "stat" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false }, - "description": "Total adjusted rewards across all providers ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 4 - }, - "hideTimeOverride": true, - "id": 402, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", - "range": true, - "refId": "A" - } - ], - "timeFrom": "1m", - "timeShift": "1d", - "title": "Total Adjusted Rewards", - "type": "stat" + "editorMode": "code", + "expr": "count(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Providers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percent" + "id": 100, + "panels": [], + "title": "Current Reward Period", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 4 - }, - "hideTimeOverride": true, - "id": 403, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "unit": "dateTimeAsIso" }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[${days_since_last_distribution}d])) / sum(sum_over_time(total_base_rewards_xdr_permyriad[${days_since_last_distribution}d])) * 100", - "range": true, - "refId": "A" - } - ], - "timeFrom": "1m", - "timeShift": "1d", - "title": "Rewards Efficiency %", - "type": "stat" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "hideTimeOverride": true, + "id": 407, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "description": "Comprehensive provider metrics overview", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "provider_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 180 - }, - { - "id": "links", - "value": [ - { - "title": "View Provider Details", - "url": "/d/${__dashboard.uid}?var-provider=${__value.text}&${__url_time_range}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Base Total" - }, - "properties": [ - { - "id": "displayName", - "value": "Base Total" - }, - { - "id": "unit", - "value": "XDR" - }, - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Adjusted Total" - }, - "properties": [ - { - "id": "displayName", - "value": "Adjusted Total" - }, - { - "id": "unit", - "value": "XDR" - }, - { - "id": "custom.width", - "value": 160 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Difference" - }, - "properties": [ - { - "id": "displayName", - "value": "Adj - Base Diff" - }, - { - "id": "unit", - "value": "" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "green", - "value": 0 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Efficiency %" - }, - { - "id": "unit", - "value": "percent" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "max", - "value": 100 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "orange", - "value": 90 - }, - { - "color": "yellow", - "value": 95 - }, - { - "color": "green", - "value": 98 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nodes" - }, - "properties": [ - { - "id": "displayName", - "value": "Total Nodes" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Adj - Base Diff" - }, - "properties": [ - { - "id": "custom.width", - "value": 207 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 7 + "editorMode": "code", + "expr": "governance_latest_reward_event_timestamp_seconds * 1000", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Last Rewards Distribution", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "(1 - Total Adjusted Rewards / Total Base Rewards) * 100", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "percent" }, - "hideTimeOverride": true, - "id": 105, - "options": { - "cellHeight": "sm", - "enablePagination": false, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Efficiency %" - } - ] + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "hideTimeOverride": true, + "id": 405, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", - "format": "table", - "instant": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / ${scale_xdrp_to_xdr}", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) - sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d]))) / ${scale_xdrp_to_xdr}", - "format": "table", - "instant": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d])) / sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[${days_since_last_distribution}d]))) * 100", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "refId": "A" - } - ], - "timeFrom": "1m", - "timeShift": "1d", - "title": "Rewards Per Provider", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "provider_id", - "mode": "outer" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "instance": true, - "job": true - }, - "includeByName": {}, - "indexByName": { - "Time 1": 5, - "Time 2": 6, - "Time 3": 7, - "Time 4": 8, - "Value #A": 4, - "Value #B": 1, - "Value #C": 2, - "Value #D": 3, - "provider_id": 0 - }, - "orderByMode": "manual", - "renameByName": { - "Value #A": "Nodes", - "Value #B": "Base Total", - "Value #C": "Adjusted Total", - "Value #D": "Difference", - "Value #E": "Efficiency %" - } - } - } - ], - "type": "table" + "editorMode": "code", + "expr": "(1 - sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[$__range])) / sum(sum_over_time(total_base_rewards_xdr_permyriad[$__range]))) * 100", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Reward Reduction", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 + "description": "Total adjusted rewards across all providers ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" }, - "id": 200, - "panels": [], - "title": "Provider Rewards Trends", - "type": "row" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "hideTimeOverride": true, + "id": 402, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "description": "Base vs Adjusted rewards over time for selected provider(s)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "XDR Permyriad", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[$__range])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Adjusted Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total base rewards ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" + ] }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Base.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Adjusted.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] + "unit": "short" }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 18 + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "hideTimeOverride": true, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "id": 201, - "options": { - "legend": { - "calcs": [ - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_base_rewards_xdr_permyriad[$__range])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Base Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Comprehensive provider metrics overview", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] } }, - "pluginVersion": "12.2.1", - "targets": [ + "overrides": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}", - "legendFormat": "Base - {{provider_id}} ", - "range": true, - "refId": "A" + "matcher": { + "id": "byName", + "options": "provider_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Provider Details", + "url": "/d/${__dashboard.uid}?var-provider=${__value.text}&${__url_time_range}" + } + ] + } + ] }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}", - "hide": false, - "instant": false, - "legendFormat": "Adjusted - {{provider_id}}", - "range": true, - "refId": "B" - } - ], - "title": "Base vs Adjusted Rewards Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rewards efficiency percentage (Adjusted/Base * 100)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Efficiency %", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "matcher": { + "id": "byName", + "options": "Base Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Base Total" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "id": "unit", + "value": "XDR" }, - "thresholdsStyle": { - "mode": "line" + { + "id": "custom.width", + "value": 140 } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percent" + ] }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 202, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last", - "sortDesc": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}) * 100", - "legendFormat": "{{provider_id}} - Efficiency %", - "range": true, - "refId": "A" - } - ], - "title": "Provider Rewards Efficiency Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Node counts over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "matcher": { + "id": "byName", + "options": "Adjusted Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Adjusted Total" }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "id": "unit", + "value": "XDR" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + { + "id": "custom.width", + "value": 160 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" + ] }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 203, - "options": { - "legend": { - "calcs": [ - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", - "legendFormat": "{{provider_id}} - Total Nodes", - "refId": "A" - } - ], - "title": "Node Counts Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rewards penalty/loss per provider (Base - Adjusted)", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": true, - "axisColorMode": "text", - "axisLabel": "Penalty (XDR Permyriad)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "matcher": { + "id": "byName", + "options": "Difference" + }, + "properties": [ + { + "id": "displayName", + "value": "Adj - Base Diff" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "id": "unit", + "value": "" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "max": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#F2495C", - "value": 0 + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 0 + } + ] } - ] - }, - "unit": "short" + } + ] }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 204, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} - total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} )", - "legendFormat": "{{provider_id}} - Penalty", - "range": true, - "refId": "A" - } - ], - "title": "Provider Rewards Penalty (Base - Adjusted)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 300, - "panels": [], - "title": "Node-Level Metrics (Assigned)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Original failure rate by node", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Failure Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Efficiency %" }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "id": "unit", + "value": "percent" }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 301, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", - "legendFormat": "{{node_short}}", - "range": true, - "refId": "A" - } - ], - "title": "Node Original Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Relative failure rate by node", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Relative Failure Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "id": "max", + "value": 100 }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "id": "min", + "value": 0 }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 0.1 + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 98 + } + ] } - ] - }, - "unit": "percentunit" + } + ] }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 302, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", - "legendFormat": "{{node_short}}", - "range": true, - "refId": "A" - } - ], - "title": "Node Relative Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Heatmap showing distribution of original failure rates across all nodes", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Nodes" }, - "scaleDistribution": { - "type": "linear" + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 100 } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 303, - "options": { - "calculate": false, - "cellGap": 2, - "cellValues": { - "unit": "short" + ] }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false - } - }, - "pluginVersion": "12.2.1", - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", - "format": "time_series", - "legendFormat": "{{node_short}}", - "range": true, - "refId": "A" - } - ], - "title": "Original Failure Rate Heatmap", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Heatmap showing distribution of relative failure rates across all nodes.\nrelative failure rate = original failure rate - subnet failure rate", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" + "matcher": { + "id": "byName", + "options": "Adj - Base Diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 304, - "options": { - "calculate": false, - "cellGap": 2, - "cellValues": { - "unit": "short" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false + ] } - }, - "pluginVersion": "12.2.1", - "targets": [ + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "hideTimeOverride": true, + "id": 105, + "options": { + "cellHeight": "sm", + "enablePagination": false, + "showHeader": true, + "sortBy": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", - "format": "time_series", - "legendFormat": "{{node_short}}", - "range": true, - "refId": "A" + "desc": false, + "displayName": "Efficiency %" } - ], - "title": "Relative Failure Rate Heatmap", - "type": "heatmap" + ] }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Detailed node metrics table", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - } + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "node_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 200 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Original Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "max", - "value": 0.2 - }, - { - "id": "min", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Relative Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "max", - "value": 0.2 - }, - { - "id": "min", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "subnet_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 180 - }, - { - "id": "links", - "value": [ - { - "title": "View Subnet Details", - "url": "/d/${__dashboard.uid}?var-subnet=${__value.text}&${__url_time_range}" - } - ] - } - ] - } - ] + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "B" }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 55 + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "C" }, - "id": 305, - "options": { - "cellHeight": "sm", - "enablePagination": true, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Original Failure Rate" - } - ] + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) - sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "D" }, - "pluginVersion": "12.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "late{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", - "format": "table", - "instant": true, - "refId": "A" + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d", - "format": "table", - "instant": true, - "refId": "B" + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) * 100", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Rewards Per Provider", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "provider_id", + "mode": "outer" } - ], - "title": "Node Performance Details", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "node_id", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "__name__": true, - "instance": true, - "job": true - }, - "indexByName": { - "Value #A": 3, - "Value #B": 4, - "node_id": 1, - "provider_id": 0, - "subnet_id": 2 - }, - "renameByName": { - "Value #A": "Original Failure Rate", - "Value #B": "Relative Failure Rate" - } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 5, + "Time 2": 6, + "Time 3": 7, + "Time 4": 8, + "Value #A": 4, + "Value #B": 2, + "Value #C": 1, + "Value #D": 3, + "provider_id": 0 + }, + "orderByMode": "manual", + "renameByName": { + "Value #A": "Nodes", + "Value #B": "Base Total", + "Value #C": "Adjusted Total", + "Value #D": "Difference", + "Value #E": "Efficiency %" } } - ], - "type": "table" + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 400, - "panels": [], - "title": "Subnet Metrics", - "type": "row" + "id": 200, + "panels": [], + "title": "Provider Rewards Trends", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Subnet failure rates over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Failure Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" + "description": "Base vs Adjusted rewards over time for selected provider(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percentunit" + ] }, - "overrides": [] + "unit": "XDR" }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 66 - }, - "id": 401, - "interval": "1d", - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.2.1", - "targets": [ + "overrides": [ { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "subnet_failure_rate{subnet_id=~\"$subnet\"}", - "legendFormat": "{{subnet_id}}", - "range": true, - "refId": "A" + "matcher": { + "id": "byRegexp", + "options": ".*Base.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Adjusted.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] } - ], - "title": "Subnet Failure Rates Over Time", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 42, - "tags": [ - "node-rewards", - "provider", - "rewards", - "performance" - ], - "templating": { - "list": [ + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "current": { - "text": "prometheus1d", - "value": "prometheus1d" - }, - "includeAll": false, - "label": "Data Source", - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} / ${scale_xdrp_to_xdr}", + "legendFormat": "Base - {{provider_id}} ", + "range": true, + "refId": "A" }, { - "allValue": ".*", - "current": { - "text": "All", - "value": [ - "$__all" + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / ${scale_xdrp_to_xdr}", + "hide": false, + "instant": false, + "legendFormat": "Adjusted - {{provider_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Base vs Adjusted Rewards Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards efficiency percentage (Adjusted/Base * 100)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Efficiency %", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } ] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(nodes_count, provider_id)", - "includeAll": true, - "label": "Provider", - "multi": true, - "name": "provider", - "options": [], - "query": { - "query": "label_values(nodes_count, provider_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}) * 100", + "legendFormat": "{{provider_id}} - Efficiency %", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Efficiency Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Node counts over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", - "includeAll": true, - "label": "Node", - "multi": true, - "name": "node", - "options": [], - "query": { - "query": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" + "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", + "legendFormat": "{{provider_id}} - Total Nodes", + "refId": "A" + } + ], + "title": "Node Counts Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards penalty/loss per provider (Base - Adjusted)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Penalty", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#F2495C", + "value": 0 + } + ] + }, + "unit": "XDR" }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(subnet_failure_rate, subnet_id)", - "includeAll": true, - "label": "Subnet", - "multi": true, - "name": "subnet", - "options": [], - "query": { - "query": "label_values(subnet_failure_rate, subnet_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} - total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} ) / ${scale_xdrp_to_xdr}", + "legendFormat": "{{provider_id}} - Penalty", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Penalty (Base - Adjusted)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 300, + "panels": [], + "title": "Node-Level Metrics (Assigned)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Original failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "allowCustomValue": false, - "current": { - "text": "29", - "value": "29" + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Original Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Relative failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Relative Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "prometheus1d" - }, - "definition": "query_result(last_over_time(days_since_governance_distribution[35d]))", - "hide": 2, - "includeAll": false, - "name": "days_since_last_distribution", - "options": [], - "query": { - "qryType": 3, - "query": "query_result(last_over_time(days_since_governance_distribution[35d]))", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/\\s+(\\d+)\\s+\\d+$/", - "type": "query" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Relative Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of original failure rates across all nodes", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 303, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "baseFilters": [], "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "filters": [], - "name": "Filters", - "type": "adhoc" + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Original Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of relative failure rates across all nodes.\nrelative failure rate = original failure rate - subnet failure rate", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 304, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ { - "allowCustomValue": false, - "current": { - "text": "10000", - "value": "10000" + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "name": "scale_xdrp_to_xdr", - "options": [ - { - "selected": true, - "text": "10000", - "value": "10000" + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Relative Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 404, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" ], - "query": "10000", - "type": "custom" + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" } - ] - }, - "time": { - "from": "now-30d", - "to": "now" + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(performance_multiplier{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Performance Multiplier", + "type": "timeseries" }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 400, + "panels": [], + "title": "Subnet Metrics", + "type": "row" }, - "timezone": "utc", - "title": "Node Provider Rewards Dashboard", - "uid": "node-rewards-comprehensive", - "version": 1 - } \ No newline at end of file + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Subnet failure rates over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 401, + "interval": "1d", + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "subnets_failure_rate{subnet_id=~\"$subnet\"}", + "legendFormat": "{{subnet_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Subnet Failure Rates Over Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus1d", + "value": "prometheus1d" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(nodes_count, provider_id)", + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": { + "query": "label_values(nodes_count, provider_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "query": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(subnet_failure_rate, subnet_id)", + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "subnet", + "options": [], + "query": { + "query": "label_values(subnet_failure_rate, subnet_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + }, + { + "allowCustomValue": false, + "current": { + "text": "10000", + "value": "10000" + }, + "name": "scale_xdrp_to_xdr", + "options": [ + { + "selected": true, + "text": "10000", + "value": "10000" + } + ], + "query": "10000", + "type": "custom" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard", + "uid": "node-rewards-comprehensive", + "version": 2 +} \ No newline at end of file diff --git a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py index 47e013b..fce60c5 100644 --- a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py +++ b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py @@ -7,6 +7,7 @@ import os import time from datetime import datetime, timedelta, timezone +from functools import wraps from typing import Any, Dict, Optional from urllib.parse import urljoin @@ -25,14 +26,72 @@ logger = logging.getLogger(__name__) +def retry_on_timeout(max_attempts=3, initial_delay=2, backoff_factor=2): + """ + Decorator to retry a function on timeout or connection errors with exponential backoff + + Args: + max_attempts: Maximum number of retry attempts + initial_delay: Initial delay in seconds before first retry + backoff_factor: Multiplier for delay between retries + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + delay = initial_delay + last_exception = None + + for attempt in range(max_attempts): + try: + return func(*args, **kwargs) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + TimeoutError, + ConnectionError, + Exception + ) as e: + last_exception = e + error_msg = str(e) + + # Check if it's a timeout-related error + is_timeout = ( + isinstance(e, (requests.exceptions.Timeout, TimeoutError)) or + "timed out" in error_msg.lower() or + "timeout" in error_msg.lower() + ) + + # Check if it's a connection error + is_connection_error = isinstance(e, (requests.exceptions.ConnectionError, ConnectionError)) + + # Only retry on timeout or connection errors + if not (is_timeout or is_connection_error): + # Re-raise if it's not a retryable error + raise + + if attempt < max_attempts - 1: + logger.warning( + f"Attempt {attempt + 1}/{max_attempts} failed with {type(e).__name__}: {error_msg}. " + f"Retrying in {delay} seconds..." + ) + time.sleep(delay) + delay *= backoff_factor + else: + logger.error( + f"All {max_attempts} attempts failed. Last error: {error_msg}" + ) + raise last_exception + + # This should never be reached, but just in case + raise last_exception + + return wrapper + return decorator + + IC_URL = "https://ic0.app" -# These have to be tuples -NODE_REWARDS_CANISTER_IDS = [ - ("uuew5-iiaaa-aaaaa-qbx4q-cai"), # Dev - # TODO: uncomment to enable prod - # ("sgymv-uiaaa-aaaaa-aaaia-cai"), # Prod -] +NODE_REWARDS_CANISTER_ID = "sgymv-uiaaa-aaaaa-aaaia-cai" GOVERNANCE_CANISTER_ID = "rrkah-fqaaa-aaaaa-aaaaq-cai" # NNS Governance canister ###################### TYPE DEFINITIONS ########################### @@ -167,6 +226,7 @@ def __init__(self, ic_url: str, canister_id: str): self.agent = Agent(self.identity, self.client) self.canister_id = canister_id + @retry_on_timeout(max_attempts=5, initial_delay=3, backoff_factor=2) def get_rewards_daily(self, date: str) -> Dict[str, Any]: """Fetch daily rewards data from node rewards canister""" parsed_date = datetime.strptime(date, "%Y-%m-%d") @@ -227,6 +287,7 @@ def get_rewards_daily(self, date: str) -> Dict[str, Any]: ) return result_dict + @retry_on_timeout(max_attempts=5, initial_delay=3, backoff_factor=2) def get_latest_governance_reward_event(self) -> Optional[float]: """Fetch latest governance reward event timestamp from governance canister""" @@ -270,9 +331,7 @@ class NodeRewardsPusher: def __init__(self, victoria_url: str): self.victoria_url = victoria_url - self.nrc_clients = [ - NodeRewardsClient(IC_URL, c) for c in NODE_REWARDS_CANISTER_IDS - ] + self.nrc_client = NodeRewardsClient(IC_URL, NODE_REWARDS_CANISTER_ID) @staticmethod def _unwrap_optional(value): @@ -302,6 +361,7 @@ def wait_for_victoria_metrics(self): logger.info(f" Waiting for VictoriaMetrics at {self.victoria_url}...") time.sleep(2) + @retry_on_timeout(max_attempts=3, initial_delay=5, backoff_factor=2) def push_metrics_for_date(self, date: str): """ Fetch node rewards data from IC canisters and push to VictoriaMetrics for a specific date @@ -315,137 +375,141 @@ def push_metrics_for_date(self, date: str): timestamp_ms = int(target_dt.replace(tzinfo=timezone.utc).timestamp() * 1000) metrics_lines = [] - for client in self.nrc_clients: - daily_results = client.get_rewards_daily(date) - - if not daily_results: - raise ValueError(f"⚠️ No data available for {date}") - - # Helper function to not repeat the labels all the - # time and to single out the place for changing labels - def add_line_helper(metric_name: str, value, **kwargs): - metrics_lines.append( - self._make_line( - metric_name, - value, - timestamp_ms, - canister_id=client.canister_id, - **kwargs, - ) + daily_results = self.nrc_client.get_rewards_daily(date) + + if not daily_results: + raise ValueError(f"⚠️ No data available for {date}") + + # Helper function to not repeat the labels all the + # time and to single out the place for changing labels + def add_line_helper(metric_name: str, value, **kwargs): + metrics_lines.append( + self._make_line( + metric_name, + value, + timestamp_ms, + canister_id=self.nrc_client.canister_id, + **kwargs, ) + ) - # Provider-level metrics - provider_results = daily_results.get("provider_results", {}) - for provider_id, provider_rewards in provider_results.items(): - provider_id_str = str(provider_id) + # Provider-level metrics + provider_results = daily_results.get("provider_results", {}) + for provider_id, provider_rewards in provider_results.items(): + provider_id_str = str(provider_id) - def add_line_helper_with_provider( - metric_name: str, value: int, **kwargs - ): - add_line_helper( - metric_name, value, provider_id=provider_id_str, **kwargs - ) + def add_line_helper_with_provider( + metric_name: str, value: int, **kwargs + ): + add_line_helper( + metric_name, value, provider_id=provider_id_str, **kwargs + ) - # nodes_count - nodes_count = len(provider_rewards.get("daily_nodes_rewards", [])) - add_line_helper_with_provider("nodes_count", nodes_count) + # nodes_count + nodes_count = len(provider_rewards.get("daily_nodes_rewards", [])) + add_line_helper_with_provider("nodes_count", nodes_count) - # base_rewards - base_rewards = self._unwrap_optional( - provider_rewards.get("total_base_rewards_xdr_permyriad") + # base_rewards + base_rewards = self._unwrap_optional( + provider_rewards.get("total_base_rewards_xdr_permyriad") + ) + if base_rewards is not None: + add_line_helper_with_provider( + "total_base_rewards_xdr_permyriad", base_rewards + ) + + adjusted_rewards = self._unwrap_optional( + provider_rewards.get("total_adjusted_rewards_xdr_permyriad") + ) + if adjusted_rewards is not None: + add_line_helper_with_provider( + "total_adjusted_rewards_xdr_permyriad", adjusted_rewards ) - if base_rewards is not None: + + # Node-level metrics + for node_result in provider_rewards.get("daily_nodes_rewards", []): + node_id = self._unwrap_optional(node_result.get("node_id")) + node_id_str = str(node_id) if node_id else "" + + # performance_multiplier + performance_multiplier = self._unwrap_optional( + node_result.get("performance_multiplier") + ) + if performance_multiplier is not None: add_line_helper_with_provider( - "total_base_rewards_xdr_permyriad", base_rewards + "performance_multiplier", + performance_multiplier, + node_id=node_id_str, ) - adjusted_rewards = self._unwrap_optional( - provider_rewards.get("total_adjusted_rewards_xdr_permyriad") + # daily_node_failure_rate is optional and contains a variant + failure_rate_data = self._unwrap_optional( + node_result.get("daily_node_failure_rate") ) - if adjusted_rewards is not None: - add_line_helper_with_provider( - "total_adjusted_rewards_xdr_permyriad", adjusted_rewards + if ( + isinstance(failure_rate_data, dict) + and "SubnetMember" in failure_rate_data + ): + # Extract node_metrics from SubnetMember variant + subnet_member = failure_rate_data["SubnetMember"] + node_metrics = self._unwrap_optional( + subnet_member.get("node_metrics") ) - # Node-level metrics - for node_result in provider_rewards.get("daily_nodes_rewards", []): - node_id = self._unwrap_optional(node_result.get("node_id")) - node_id_str = str(node_id) if node_id else "" + if not node_metrics: + continue - # daily_node_failure_rate is optional and contains a variant - failure_rate_data = self._unwrap_optional( - node_result.get("daily_node_failure_rate") + subnet_id = self._unwrap_optional( + node_metrics.get("subnet_assigned") ) - if ( - isinstance(failure_rate_data, dict) - and "SubnetMember" in failure_rate_data - ): - # Extract node_metrics from SubnetMember variant - subnet_member = failure_rate_data["SubnetMember"] - node_metrics = self._unwrap_optional( - subnet_member.get("node_metrics") - ) - - if not node_metrics: - continue + subnet_id_str = str(subnet_id) if subnet_id else "" - subnet_id = self._unwrap_optional( - node_metrics.get("subnet_assigned") + # original_failure_rate + original_fr = self._unwrap_optional( + node_metrics.get("original_failure_rate") + ) + if original_fr is not None: + add_line_helper_with_provider( + "original_failure_rate", + original_fr, + node_id=node_id_str, + subnet_id=subnet_id_str, ) - subnet_id_str = str(subnet_id) if subnet_id else "" - # original_failure_rate - original_fr = self._unwrap_optional( - node_metrics.get("original_failure_rate") - ) - if original_fr is not None: - add_line_helper_with_provider( - "original_failure_rate", - original_fr, - node_id=node_id_str, - subnet_id=subnet_id_str, - ) - - # relative_failure_rate - relative_fr = self._unwrap_optional( - node_metrics.get("relative_failure_rate") + # relative_failure_rate + relative_fr = self._unwrap_optional( + node_metrics.get("relative_failure_rate") + ) + if relative_fr is not None: + add_line_helper_with_provider( + "relative_failure_rate", + relative_fr, + node_id=node_id_str, + subnet_id=subnet_id_str, ) - if relative_fr is not None: - add_line_helper_with_provider( - "relative_failure_rate", - relative_fr, - node_id=node_id_str, - subnet_id=subnet_id_str, - ) - - # Subnet-level metrics - subnets_failure_rate = daily_results.get("subnets_failure_rate", {}) - for subnet_id, failure_rate in subnets_failure_rate.items(): - subnet_id_str = str(subnet_id) - metrics_lines.append( - f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {timestamp_ms}' - ) - add_line_helper( - "subnets_failure_rate", failure_rate, subnet_id=subnet_id_str - ) - # Governance timestamp - gov_timestamp = self.nrc_clients[0].get_latest_governance_reward_event() - if gov_timestamp: - add_line_helper( - "governance_latest_reward_event_timestamp_seconds", gov_timestamp - ) - - # Calculate days since last governance distribution - now_utc = datetime.now(timezone.utc) - gov_datetime: datetime = datetime.fromtimestamp(gov_timestamp, tz=timezone.utc) - days_since = int((now_utc - gov_datetime).total_seconds() / 86400) # 86400 seconds in a day - metrics_lines.append( - f'days_since_governance_distribution {days_since} {timestamp_ms}' - ) - add_line_helper( - "days_since_governance_distribution", days_since - ) + # Subnet-level metrics + subnets_failure_rate = daily_results.get("subnets_failure_rate", {}) + for subnet_id, failure_rate in subnets_failure_rate.items(): + subnet_id_str = str(subnet_id) + add_line_helper( + "subnets_failure_rate", failure_rate, subnet_id=subnet_id_str + ) + + # Governance timestamp + gov_timestamp = self.nrc_client.get_latest_governance_reward_event() + if gov_timestamp: + add_line_helper( + "governance_latest_reward_event_timestamp_seconds", gov_timestamp + ) + + # Calculate days since last governance distribution + now_utc = datetime.now(timezone.utc) + gov_datetime: datetime = datetime.fromtimestamp(gov_timestamp, tz=timezone.utc) + days_since = int((now_utc - gov_datetime).total_seconds() / 86400) # 86400 seconds in a day + add_line_helper( + "days_since_governance_distribution", days_since + ) if not metrics_lines: raise ValueError("After evaluation there were no metrics to upload") From 038d0177f8a27a452aa6396cd5745de4294ec92e Mon Sep 17 00:00:00 2001 From: Pietro Date: Thu, 13 Nov 2025 17:48:23 +0100 Subject: [PATCH 3/3] Add back git ignore --- docker/volumes/grafana/.gitignore | 4 ++-- docker/volumes/victoria/.gitignore | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) mode change 100755 => 100644 docker/volumes/grafana/.gitignore diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore old mode 100755 new mode 100644 index e769a26..e67750e --- a/docker/volumes/grafana/.gitignore +++ b/docker/volumes/grafana/.gitignore @@ -1,5 +1,5 @@ # Ignore everything in this directory -* +* -# But keep this .gitignore file +# But keep this .gitignore !.gitignore diff --git a/docker/volumes/victoria/.gitignore b/docker/volumes/victoria/.gitignore index facdf81..e67750e 100644 --- a/docker/volumes/victoria/.gitignore +++ b/docker/volumes/victoria/.gitignore @@ -1,5 +1,5 @@ -# Ignore everything in this directory -* +# Ignore everything in this directory +* # But keep this .gitignore !.gitignore