diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..a59fc3f Binary files /dev/null and b/.DS_Store differ diff --git a/docker/.DS_Store b/docker/.DS_Store new file mode 100644 index 0000000..f9eb51c Binary files /dev/null and b/docker/.DS_Store differ diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json new file mode 100644 index 0000000..edee8f5 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/dashboard.json @@ -0,0 +1,1989 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "prometheus1d" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total nodes across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of providers tracked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(nodes_count)", + "range": true, + "refId": "A" + } + ], + "title": "Total Providers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 100, + "panels": [], + "title": "Current Reward Period", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "hideTimeOverride": true, + "id": 407, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "governance_latest_reward_event_timestamp_seconds * 1000", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Last Rewards Distribution", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "(1 - Total Adjusted Rewards / Total Base Rewards) * 100", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "hideTimeOverride": true, + "id": 405, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(1 - sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[$__range])) / sum(sum_over_time(total_base_rewards_xdr_permyriad[$__range]))) * 100", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Reward Reduction", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total adjusted rewards across all providers ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "hideTimeOverride": true, + "id": 402, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_adjusted_rewards_xdr_permyriad[$__range])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Adjusted Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total base rewards ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "hideTimeOverride": true, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum_over_time(total_base_rewards_xdr_permyriad[$__range])) / ${scale_xdrp_to_xdr}", + "range": true, + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Total Base Rewards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Comprehensive provider metrics overview", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "provider_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "links", + "value": [ + { + "title": "View Provider Details", + "url": "/d/${__dashboard.uid}?var-provider=${__value.text}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Base Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Base Total" + }, + { + "id": "unit", + "value": "XDR" + }, + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Adjusted Total" + }, + "properties": [ + { + "id": "displayName", + "value": "Adjusted Total" + }, + { + "id": "unit", + "value": "XDR" + }, + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Difference" + }, + "properties": [ + { + "id": "displayName", + "value": "Adj - Base Diff" + }, + { + "id": "unit", + "value": "" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 0 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Efficiency %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 98 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Nodes" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Adj - Base Diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "hideTimeOverride": true, + "id": 105, + "options": { + "cellHeight": "sm", + "enablePagination": false, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Efficiency %" + } + ] + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) - sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) / ${scale_xdrp_to_xdr}", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by (provider_id) (sum_over_time(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range])) / sum by (provider_id) (sum_over_time(total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}[$__range]))) * 100", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "refId": "A" + } + ], + "timeShift": "1d", + "title": "Rewards Per Provider", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "provider_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 5, + "Time 2": 6, + "Time 3": 7, + "Time 4": 8, + "Value #A": 4, + "Value #B": 2, + "Value #C": 1, + "Value #D": 3, + "provider_id": 0 + }, + "orderByMode": "manual", + "renameByName": { + "Value #A": "Nodes", + "Value #B": "Base Total", + "Value #C": "Adjusted Total", + "Value #D": "Difference", + "Value #E": "Efficiency %" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 200, + "panels": [], + "title": "Provider Rewards Trends", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Base vs Adjusted rewards over time for selected provider(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "XDR" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Base.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Adjusted.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} / ${scale_xdrp_to_xdr}", + "legendFormat": "Base - {{provider_id}} ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / ${scale_xdrp_to_xdr}", + "hide": false, + "instant": false, + "legendFormat": "Adjusted - {{provider_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Base vs Adjusted Rewards Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards efficiency percentage (Adjusted/Base * 100)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Efficiency %", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} / total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"}) * 100", + "legendFormat": "{{provider_id}} - Efficiency %", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Efficiency Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Node counts over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "nodes_count{provider_id=~\"$provider\"} offset -1d", + "legendFormat": "{{provider_id}} - Total Nodes", + "refId": "A" + } + ], + "title": "Node Counts Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rewards penalty/loss per provider (Base - Adjusted)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Penalty", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#F2495C", + "value": 0 + } + ] + }, + "unit": "XDR" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(total_adjusted_rewards_xdr_permyriad{provider_id=~\"$provider\"} - total_base_rewards_xdr_permyriad{provider_id=~\"$provider\"} ) / ${scale_xdrp_to_xdr}", + "legendFormat": "{{provider_id}} - Penalty", + "range": true, + "refId": "A" + } + ], + "title": "Provider Rewards Penalty (Base - Adjusted)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 300, + "panels": [], + "title": "Node-Level Metrics (Assigned)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Original failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Original Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Relative failure rate by node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Relative Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Relative Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of original failure rates across all nodes", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 303, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(original_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Original Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heatmap showing distribution of relative failure rates across all nodes.\nrelative failure rate = original failure rate - subnet failure rate", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 304, + "options": { + "calculate": false, + "cellGap": 2, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(relative_failure_rate{provider_id=~\"$provider\", node_id=~\"$node\"}, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "format": "time_series", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Relative Failure Rate Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 404, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(performance_multiplier{provider_id=~\"$provider\",node_id=~\"$node\"} offset -1d, \"node_short\", \"$1\", \"node_id\", \"([^-]+)-([^-]+)-([^-]+).*\")", + "legendFormat": "{{node_short}}", + "range": true, + "refId": "A" + } + ], + "title": "Performance Multiplier", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 400, + "panels": [], + "title": "Subnet Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Subnet failure rates over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failure Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 401, + "interval": "1d", + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "subnets_failure_rate{subnet_id=~\"$subnet\"}", + "legendFormat": "{{subnet_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Subnet Failure Rates Over Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus1d", + "value": "prometheus1d" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(nodes_count, provider_id)", + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": { + "query": "label_values(nodes_count, provider_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "query": "label_values(original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(subnet_failure_rate, subnet_id)", + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "subnet", + "options": [], + "query": { + "query": "label_values(subnet_failure_rate, subnet_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + }, + { + "allowCustomValue": false, + "current": { + "text": "10000", + "value": "10000" + }, + "name": "scale_xdrp_to_xdr", + "options": [ + { + "selected": true, + "text": "10000", + "value": "10000" + } + ], + "query": "10000", + "type": "custom" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard", + "uid": "node-rewards-comprehensive", + "version": 2 +} \ No newline at end of file diff --git a/docker/config/grafana/provisioning/dashboards/observability.yaml b/docker/config/grafana/provisioning/dashboards/observability.yaml index 6648df9..0a55fb6 100644 --- a/docker/config/grafana/provisioning/dashboards/observability.yaml +++ b/docker/config/grafana/provisioning/dashboards/observability.yaml @@ -11,3 +11,14 @@ providers: options: path: /etc/grafana/provisioning/dashboards/samples foldersFromFilesStructure: false + + - name: 'Node Rewards' + folder: 'Node Rewards' + folderUid: 'node-rewards' + type: file + disableDeletion: true + updateIntervalSeconds: 60 + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards/node-rewards + foldersFromFilesStructure: false diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 825b528..c4fcf79 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -8,3 +8,12 @@ datasources: url: http://$PROM_HOST:9090 jsonData: httpMethod: POST + - name: prometheus1d + uid: prometheus1d + type: prometheus + access: proxy + url: http://$PROM_HOST:9090 + jsonData: + httpMethod: POST + timeInterval: "1d" + diff --git a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py index bed42f4..fce60c5 100644 --- a/docker/tools/node-rewards-scheduler/node_rewards_ingester.py +++ b/docker/tools/node-rewards-scheduler/node_rewards_ingester.py @@ -7,6 +7,7 @@ import os import time from datetime import datetime, timedelta, timezone +from functools import wraps from typing import Any, Dict, Optional from urllib.parse import urljoin @@ -25,14 +26,72 @@ logger = logging.getLogger(__name__) +def retry_on_timeout(max_attempts=3, initial_delay=2, backoff_factor=2): + """ + Decorator to retry a function on timeout or connection errors with exponential backoff + + Args: + max_attempts: Maximum number of retry attempts + initial_delay: Initial delay in seconds before first retry + backoff_factor: Multiplier for delay between retries + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + delay = initial_delay + last_exception = None + + for attempt in range(max_attempts): + try: + return func(*args, **kwargs) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + TimeoutError, + ConnectionError, + Exception + ) as e: + last_exception = e + error_msg = str(e) + + # Check if it's a timeout-related error + is_timeout = ( + isinstance(e, (requests.exceptions.Timeout, TimeoutError)) or + "timed out" in error_msg.lower() or + "timeout" in error_msg.lower() + ) + + # Check if it's a connection error + is_connection_error = isinstance(e, (requests.exceptions.ConnectionError, ConnectionError)) + + # Only retry on timeout or connection errors + if not (is_timeout or is_connection_error): + # Re-raise if it's not a retryable error + raise + + if attempt < max_attempts - 1: + logger.warning( + f"Attempt {attempt + 1}/{max_attempts} failed with {type(e).__name__}: {error_msg}. " + f"Retrying in {delay} seconds..." + ) + time.sleep(delay) + delay *= backoff_factor + else: + logger.error( + f"All {max_attempts} attempts failed. Last error: {error_msg}" + ) + raise last_exception + + # This should never be reached, but just in case + raise last_exception + + return wrapper + return decorator + + IC_URL = "https://ic0.app" -# These have to be tuples -NODE_REWARDS_CANISTER_IDS = [ - ("uuew5-iiaaa-aaaaa-qbx4q-cai"), # Dev - # TODO: uncomment to enable prod - # ("sgymv-uiaaa-aaaaa-aaaia-cai"), # Prod -] +NODE_REWARDS_CANISTER_ID = "sgymv-uiaaa-aaaaa-aaaia-cai" GOVERNANCE_CANISTER_ID = "rrkah-fqaaa-aaaaa-aaaaq-cai" # NNS Governance canister ###################### TYPE DEFINITIONS ########################### @@ -124,7 +183,7 @@ } ) -RETURN_TYPE = Types.Variant( +GET_NODE_PROVIDERS_CALCULATION_RESPONSE_TYPE = Types.Variant( { "Ok": DAILY_RESULTS_TYPE, "Err": Types.Text, @@ -135,6 +194,27 @@ {"date_filter": Types.Opt(Types.Nat64)} ) +DATE_RANGE_FILTER_TYPE = Types.Record({ + "start_timestamp_seconds": Types.Opt(Types.Nat64), + "end_timestamp_seconds": Types.Opt(Types.Nat64), +}) + +MONTHLY_NODE_PROVIDER_REWARDS_TYPE = Types.Record({ + 'timestamp': Types.Nat64, + "start_date": Types.Opt(Types.Record({"year": Types.Nat32, "month": Types.Nat32, "day": Types.Nat32})), + "end_date": Types.Opt(Types.Record({"year": Types.Nat32, "month": Types.Nat32, "day": Types.Nat32})), + "rewards": Types.Vec(Types.Record({})), # Simplified, not parsing individual rewards + "xdr_conversion_rate": Types.Opt(Types.Record({})), + "minimum_xdr_permyriad_per_icp": Types.Opt(Types.Nat64), + "maximum_node_provider_rewards_e8s": Types.Opt(Types.Nat64), + "registry_version": Types.Opt(Types.Nat64), + "node_providers": Types.Vec(Types.Record({})), +}) + +LIST_NODE_PROVIDER_REWARDS_RESPONSE_TYPE = Types.Record({ + "rewards": Types.Vec(MONTHLY_NODE_PROVIDER_REWARDS_TYPE), +}) + class NodeRewardsClient: """Client for interacting with the node rewards canister""" @@ -146,6 +226,7 @@ def __init__(self, ic_url: str, canister_id: str): self.agent = Agent(self.identity, self.client) self.canister_id = canister_id + @retry_on_timeout(max_attempts=5, initial_delay=3, backoff_factor=2) def get_rewards_daily(self, date: str) -> Dict[str, Any]: """Fetch daily rewards data from node rewards canister""" parsed_date = datetime.strptime(date, "%Y-%m-%d") @@ -164,7 +245,7 @@ def get_rewards_daily(self, date: str) -> Dict[str, Any]: self.canister_id, "get_node_providers_rewards_calculation", arg_bytes, - RETURN_TYPE, + GET_NODE_PROVIDERS_CALCULATION_RESPONSE_TYPE, ) if not response or len(response) == 0: @@ -206,6 +287,7 @@ def get_rewards_daily(self, date: str) -> Dict[str, Any]: ) return result_dict + @retry_on_timeout(max_attempts=5, initial_delay=3, backoff_factor=2) def get_latest_governance_reward_event(self) -> Optional[float]: """Fetch latest governance reward event timestamp from governance canister""" @@ -224,6 +306,7 @@ def get_latest_governance_reward_event(self) -> Optional[float]: GOVERNANCE_CANISTER_ID, "list_node_provider_rewards", arg_bytes, + LIST_NODE_PROVIDER_REWARDS_RESPONSE_TYPE, ) if not response or len(response) == 0: @@ -248,9 +331,7 @@ class NodeRewardsPusher: def __init__(self, victoria_url: str): self.victoria_url = victoria_url - self.nrc_clients = [ - NodeRewardsClient(IC_URL, c) for c in NODE_REWARDS_CANISTER_IDS - ] + self.nrc_client = NodeRewardsClient(IC_URL, NODE_REWARDS_CANISTER_ID) @staticmethod def _unwrap_optional(value): @@ -261,7 +342,7 @@ def _unwrap_optional(value): @staticmethod def _make_line(metric_name: str, value: int, ts: int, **kwargs) -> str: - return f"{metric_name}{{ {' '.join([f'{key}="{value}"' for key, value in kwargs.items()])} }} {value} {ts}" + return f"{metric_name}{{{','.join([f'{key}="{value}"' for key, value in kwargs.items()])}}} {value} {ts}" def wait_for_victoria_metrics(self): """Wait for VictoriaMetrics to be ready""" @@ -280,6 +361,7 @@ def wait_for_victoria_metrics(self): logger.info(f" Waiting for VictoriaMetrics at {self.victoria_url}...") time.sleep(2) + @retry_on_timeout(max_attempts=3, initial_delay=5, backoff_factor=2) def push_metrics_for_date(self, date: str): """ Fetch node rewards data from IC canisters and push to VictoriaMetrics for a specific date @@ -289,130 +371,145 @@ def push_metrics_for_date(self, date: str): logger.info(f"Pushing node rewards data for {date}") target_date = datetime.strptime(date, "%Y-%m-%d") - noon = target_date.replace(hour=12, minute=0, second=0, microsecond=0) - noon_timestamp_ms = int(noon.replace(tzinfo=timezone.utc).timestamp() * 1000) + target_dt = target_date.replace(hour=0, minute=0, second=0, microsecond=0) + timestamp_ms = int(target_dt.replace(tzinfo=timezone.utc).timestamp() * 1000) metrics_lines = [] - for client in self.nrc_clients: - daily_results = client.get_rewards_daily(date) - - if not daily_results: - raise ValueError(f"⚠️ No data available for {date}") - - # Helper function to not repeat the labels all the - # time and to single out the place for changing labels - def add_line_helper(metric_name: str, value, **kwargs): - metrics_lines.append( - self._make_line( - metric_name, - value, - noon_timestamp_ms, - canister_id=client.canister_id, - **kwargs, - ) + daily_results = self.nrc_client.get_rewards_daily(date) + + if not daily_results: + raise ValueError(f"⚠️ No data available for {date}") + + # Helper function to not repeat the labels all the + # time and to single out the place for changing labels + def add_line_helper(metric_name: str, value, **kwargs): + metrics_lines.append( + self._make_line( + metric_name, + value, + timestamp_ms, + canister_id=self.nrc_client.canister_id, + **kwargs, ) + ) - # Provider-level metrics - provider_results = daily_results.get("provider_results", {}) - for provider_id, provider_rewards in provider_results.items(): - provider_id_str = str(provider_id) + # Provider-level metrics + provider_results = daily_results.get("provider_results", {}) + for provider_id, provider_rewards in provider_results.items(): + provider_id_str = str(provider_id) - def add_line_helper_with_provider( - metric_name: str, value: int, **kwargs - ): - add_line_helper( - metric_name, value, provider_id=provider_id_str, **kwargs - ) + def add_line_helper_with_provider( + metric_name: str, value: int, **kwargs + ): + add_line_helper( + metric_name, value, provider_id=provider_id_str, **kwargs + ) - # nodes_count - nodes_count = len(provider_rewards.get("daily_nodes_rewards", [])) - add_line_helper_with_provider("nodes_count", nodes_count) + # nodes_count + nodes_count = len(provider_rewards.get("daily_nodes_rewards", [])) + add_line_helper_with_provider("nodes_count", nodes_count) - # base_rewards - base_rewards = self._unwrap_optional( - provider_rewards.get("total_base_rewards_xdr_permyriad") + # base_rewards + base_rewards = self._unwrap_optional( + provider_rewards.get("total_base_rewards_xdr_permyriad") + ) + if base_rewards is not None: + add_line_helper_with_provider( + "total_base_rewards_xdr_permyriad", base_rewards ) - if base_rewards is not None: + + adjusted_rewards = self._unwrap_optional( + provider_rewards.get("total_adjusted_rewards_xdr_permyriad") + ) + if adjusted_rewards is not None: + add_line_helper_with_provider( + "total_adjusted_rewards_xdr_permyriad", adjusted_rewards + ) + + # Node-level metrics + for node_result in provider_rewards.get("daily_nodes_rewards", []): + node_id = self._unwrap_optional(node_result.get("node_id")) + node_id_str = str(node_id) if node_id else "" + + # performance_multiplier + performance_multiplier = self._unwrap_optional( + node_result.get("performance_multiplier") + ) + if performance_multiplier is not None: add_line_helper_with_provider( - "total_base_rewards_xdr_permyriad", base_rewards + "performance_multiplier", + performance_multiplier, + node_id=node_id_str, ) - adjusted_rewards = self._unwrap_optional( - provider_rewards.get("total_adjusted_rewards_xdr_permyriad") + # daily_node_failure_rate is optional and contains a variant + failure_rate_data = self._unwrap_optional( + node_result.get("daily_node_failure_rate") ) - if adjusted_rewards is not None: - add_line_helper_with_provider( - "total_adjusted_rewards_xdr_permyriad", adjusted_rewards + if ( + isinstance(failure_rate_data, dict) + and "SubnetMember" in failure_rate_data + ): + # Extract node_metrics from SubnetMember variant + subnet_member = failure_rate_data["SubnetMember"] + node_metrics = self._unwrap_optional( + subnet_member.get("node_metrics") ) - # Node-level metrics - for node_result in provider_rewards.get("daily_nodes_rewards", []): - node_id = self._unwrap_optional(node_result.get("node_id")) - node_id_str = str(node_id) if node_id else "" + if not node_metrics: + continue - # daily_node_failure_rate is optional and contains a variant - failure_rate_data = self._unwrap_optional( - node_result.get("daily_node_failure_rate") + subnet_id = self._unwrap_optional( + node_metrics.get("subnet_assigned") ) - if ( - isinstance(failure_rate_data, dict) - and "SubnetMember" in failure_rate_data - ): - # Extract node_metrics from SubnetMember variant - subnet_member = failure_rate_data["SubnetMember"] - node_metrics = self._unwrap_optional( - subnet_member.get("node_metrics") - ) + subnet_id_str = str(subnet_id) if subnet_id else "" - if not node_metrics: - continue - - subnet_id = self._unwrap_optional( - node_metrics.get("subnet_assigned") + # original_failure_rate + original_fr = self._unwrap_optional( + node_metrics.get("original_failure_rate") + ) + if original_fr is not None: + add_line_helper_with_provider( + "original_failure_rate", + original_fr, + node_id=node_id_str, + subnet_id=subnet_id_str, ) - subnet_id_str = str(subnet_id) if subnet_id else "" - # original_failure_rate - original_fr = self._unwrap_optional( - node_metrics.get("original_failure_rate") - ) - if original_fr is not None: - add_line_helper_with_provider( - "original_failure_rate", - original_fr, - node_id=node_id_str, - subnet_id=subnet_id_str, - ) - - # relative_failure_rate - relative_fr = self._unwrap_optional( - node_metrics.get("relative_failure_rate") + # relative_failure_rate + relative_fr = self._unwrap_optional( + node_metrics.get("relative_failure_rate") + ) + if relative_fr is not None: + add_line_helper_with_provider( + "relative_failure_rate", + relative_fr, + node_id=node_id_str, + subnet_id=subnet_id_str, ) - if relative_fr is not None: - add_line_helper_with_provider( - "relative_failure_rate", - relative_fr, - node_id=node_id_str, - subnet_id=subnet_id_str, - ) - - # Subnet-level metrics - subnets_failure_rate = daily_results.get("subnets_failure_rate", {}) - for subnet_id, failure_rate in subnets_failure_rate.items(): - subnet_id_str = str(subnet_id) - metrics_lines.append( - f'subnet_failure_rate{{subnet_id="{subnet_id_str}"}} {failure_rate} {noon_timestamp_ms}' - ) - add_line_helper( - "subnets_failure_rate", failure_rate, subnet_id=subnet_id_str - ) - # Governance timestamp - gov_timestamp = self.nrc_clients[0].get_latest_governance_reward_event() - if gov_timestamp: - add_line_helper( - "governance_latest_reward_event_timestamp_seconds", gov_timestamp - ) + # Subnet-level metrics + subnets_failure_rate = daily_results.get("subnets_failure_rate", {}) + for subnet_id, failure_rate in subnets_failure_rate.items(): + subnet_id_str = str(subnet_id) + add_line_helper( + "subnets_failure_rate", failure_rate, subnet_id=subnet_id_str + ) + + # Governance timestamp + gov_timestamp = self.nrc_client.get_latest_governance_reward_event() + if gov_timestamp: + add_line_helper( + "governance_latest_reward_event_timestamp_seconds", gov_timestamp + ) + + # Calculate days since last governance distribution + now_utc = datetime.now(timezone.utc) + gov_datetime: datetime = datetime.fromtimestamp(gov_timestamp, tz=timezone.utc) + days_since = int((now_utc - gov_datetime).total_seconds() / 86400) # 86400 seconds in a day + add_line_helper( + "days_since_governance_distribution", days_since + ) if not metrics_lines: raise ValueError("After evaluation there were no metrics to upload") @@ -458,13 +555,11 @@ def backfill(self, days: int = 40): logger.info("✅ Backfill complete!") def wait_until_next_run(self): - """Wait until 00:05 UTC tomorrow""" + """Wait until 00:10 UTC tomorrow""" now = datetime.now(timezone.utc) - # Calculate next 00:05 UTC - next_run = now.replace(hour=0, minute=5, second=0, microsecond=0) + next_run = now.replace(hour=0, minute=10, second=0, microsecond=0) if now >= next_run: - # Already past 00:05 today, schedule for tomorrow next_run += timedelta(days=1) wait_seconds = (next_run - now).total_seconds() diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore old mode 100755 new mode 100644 index e769a26..e67750e --- a/docker/volumes/grafana/.gitignore +++ b/docker/volumes/grafana/.gitignore @@ -1,5 +1,5 @@ # Ignore everything in this directory -* +* -# But keep this .gitignore file +# But keep this .gitignore !.gitignore diff --git a/docker/volumes/victoria/.gitignore b/docker/volumes/victoria/.gitignore index facdf81..e67750e 100644 --- a/docker/volumes/victoria/.gitignore +++ b/docker/volumes/victoria/.gitignore @@ -1,5 +1,5 @@ -# Ignore everything in this directory -* +# Ignore everything in this directory +* # But keep this .gitignore !.gitignore