From e6a6506f9e71f4816a52e851ff86d64cc781f532 Mon Sep 17 00:00:00 2001 From: Pietro Date: Tue, 11 Nov 2025 17:15:03 +0100 Subject: [PATCH 1/4] Test influx --- .DS_Store | Bin 0 -> 8196 bytes docker/.DS_Store | Bin 0 -> 6148 bytes docker/IMPLEMENTATION_SUMMARY.md | 452 +++++++++++ docker/NODE_REWARDS_SETUP.md | 300 ++++++++ docker/QUICKSTART_NODE_REWARDS.md | 201 +++++ docker/README.md | 13 + .../alerting/notification_policy.yaml | 5 +- .../node-rewards/node-rewards-influx.json | 725 ++++++++++++++++++ .../dashboards/observability.yaml | 11 + .../datasources/observability.yaml | 13 + .../node-rewards-scheduler/entrypoint.sh | 47 ++ docker/docker-compose.yaml | 31 + docker/volumes/.DS_Store | Bin 0 -> 6148 bytes docker/volumes/influxdb/.gitignore | 3 + .../volumes/node-rewards-scheduler/.gitignore | 3 + 15 files changed, 1800 insertions(+), 4 deletions(-) create mode 100644 .DS_Store create mode 100644 docker/.DS_Store create mode 100644 docker/IMPLEMENTATION_SUMMARY.md create mode 100644 docker/NODE_REWARDS_SETUP.md create mode 100644 docker/QUICKSTART_NODE_REWARDS.md create mode 100644 docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json create mode 100755 docker/config/node-rewards-scheduler/entrypoint.sh create mode 100644 docker/volumes/.DS_Store create mode 100644 docker/volumes/influxdb/.gitignore create mode 100644 docker/volumes/node-rewards-scheduler/.gitignore diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..61f2845feb2a4e6fd6bc899f22f27aa66f08989e GIT binary patch literal 8196 zcmeHMzl#$=6n>LL%@M_vxo#0F1@Rh14q-g@uoccu5R;2WF}V#nPq5oY8!hCT^FOfh z|45;Kgq2u|jbi2ZX2#8%{c(08`UYmcW!`)9zWsJ~H*Z5kX6s|S1+BF>a`2d&%v;lLEu%HA?rnPU$a88~cb2Uf z{@y#{dzI-@8+E$2Jw^U-6dw2)Z`eKRTBW$stY z@3-H1Euu9fgC-xzWBtV!=CMX|n$nP-(Fo_Cflqvm=Z_ZvOu7oODnHRDpj}0na<*i9I9cgA zPN+ZVH`?7K9_FH@Llh7N{+j~)?G~v^37ygrWxwCb+Cz3cPNJwYNIK{*zi+g5t`A;z z^D(_&V}{4hK2dqZ6jY)Sn4E#dkj`l|V<*W(xLih0Tc3A^OD<~4W`2~%huILVfJK69 zkmXXH_@ge5tDCp&Tf@W2kCQ2f*%AKw0k2=83%&Z7VmkCW)Qg+EC1)aB9>u-;*XM6N zym?$M^P^lo>z5zQM{KbaK+R?=uNo9D3Wx%tz>fmFKLluup~cjo-a6p)5dc^~w>I4K zF9qiW7DJ1vL5#qZNd=lzWsexjq@zEuaiPW3ph+iX4D4iRpuJ&Izj`a range(start: -1d) |> limit(n: 1)' +``` + +### Logs +```bash +# Stream all logs +docker-compose logs -f + +# Specific service +docker-compose logs -f node-rewards-scheduler + +# Last 100 lines +docker-compose logs --tail=100 +``` + +## Support and Troubleshooting + +### Common Issues + +1. **No data in dashboard** + - Check backfill completed: `ls volumes/node-rewards-scheduler/backfill_done` + - Verify InfluxDB has data: See health checks above + - Check Grafana datasource is connected + +2. **Scheduler keeps restarting** + - Check logs: `docker-compose logs node-rewards-scheduler` + - Verify DRE CLI is in PATH + - Ensure InfluxDB is running and ready + +3. **Dashboard shows errors** + - Verify datasource configuration + - Check Flux query syntax + - Ensure time range includes backfilled data + +### Getting Help +- Check `QUICKSTART_NODE_REWARDS.md` for quick fixes +- See `NODE_REWARDS_SETUP.md` for detailed troubleshooting +- Review logs: `docker-compose logs -f` + +## Conclusion + +This implementation provides a complete, production-ready system for monitoring IC node provider rewards with: +- ✅ Automated daily data collection +- ✅ Historical backfill (40 days) +- ✅ Clean Grafana dashboard with discrete daily points +- ✅ Comprehensive documentation +- ✅ Easy deployment and maintenance + +The system is designed to run continuously with minimal intervention, automatically updating every day at 00:05 UTC with the latest node rewards metrics. + diff --git a/docker/NODE_REWARDS_SETUP.md b/docker/NODE_REWARDS_SETUP.md new file mode 100644 index 0000000..2913bae --- /dev/null +++ b/docker/NODE_REWARDS_SETUP.md @@ -0,0 +1,300 @@ +# Node Rewards Dashboard with InfluxDB + +This setup provides a Grafana dashboard displaying daily node rewards metrics from the IC network, stored in InfluxDB with discrete daily data points. + +## Architecture + +``` +┌─────────────────────┐ +│ DRE CLI (Rust) │ ──> Fetches daily metrics from IC Node Rewards Canister +└──────────┬──────────┘ + │ + ▼ +┌─────────────────────┐ +│ InfluxDB 2.7 │ ──> Stores daily discrete metrics +└──────────┬──────────┘ + │ + ▼ +┌─────────────────────┐ +│ Grafana 12.2.1 │ ──> Displays dashboards with daily data points +└─────────────────────┘ + +┌─────────────────────┐ +│ Node Rewards │ ──> Runs daily at 00:05 UTC + backfills 40 days on startup +│ Scheduler (cron) │ +└─────────────────────┘ +``` + +## Components + +### 1. InfluxDB +- **Image**: `influxdb:2.7` +- **Port**: `8086` +- **Organization**: `ic-org` +- **Bucket**: `node-rewards` +- **Retention**: 90 days +- **Token**: `my-super-secret-auth-token` (configured in docker-compose.yaml) + +### 2. Node Rewards Scheduler +- **Image**: `ghcr.io/dfinity/dre/dre:latest` +- **Function**: + - Backfills last 40 days of metrics on first startup + - Runs daily at 00:05 UTC to push yesterday's metrics +- **Command**: `dre node-rewards push-to-influx --date ` + +### 3. Grafana Dashboard +- **Dashboard**: `Node Provider Rewards Dashboard - InfluxDB` +- **UID**: `node-rewards-influxdb` +- **Data Source**: InfluxDB (Flux queries) +- **Features**: + - Daily discrete data points (not continuous) + - Provider overview statistics + - Base vs Adjusted rewards trends + - Efficiency metrics over time + - Provider filtering + +## Metrics Stored + +### Provider-Level Metrics +- `latest_nodes_count`: Number of nodes per provider +- `latest_total_base_rewards_xdr_permyriad`: Base rewards (before performance adjustment) +- `latest_total_adjusted_rewards_xdr_permyriad`: Adjusted rewards (after performance adjustment) + +### Node-Level Metrics +- `latest_original_failure_rate`: Original failure rate per node +- `latest_relative_failure_rate`: Relative failure rate per node + +### Subnet-Level Metrics +- `subnet_failure_rate`: Failure rate per subnet + +### Governance Metrics +- `governance_latest_reward_event_timestamp_seconds`: Latest rewards distribution timestamp + +## Data Flow + +1. **Daily Push (00:05 UTC)**: + ``` + DRE CLI → IC Node Rewards Canister → Fetch Yesterday's Data → Push to InfluxDB + ``` + +2. **Backfill (On Startup)**: + ``` + For each day in last 40 days: + DRE CLI → IC Node Rewards Canister → Fetch Day's Data → Push to InfluxDB + ``` + +3. **Grafana Visualization**: + ``` + Grafana → Flux Query → InfluxDB → Daily Discrete Data Points → Dashboard + ``` + +## Setup Instructions + +### 1. Start the Services + +```bash +cd /path/to/ic-observability-stack/docker +export UID=$(id -u) +export GID=$(id -g) +docker-compose up -d +``` + +### 2. Verify InfluxDB + +```bash +# Check InfluxDB is running +curl http://localhost:8086/health + +# Check bucket exists +curl -H "Authorization: Token my-super-secret-auth-token" \ + http://localhost:8086/api/v2/buckets?org=ic-org +``` + +### 3. Check Backfill Progress + +```bash +# View scheduler logs +docker-compose logs -f node-rewards-scheduler +``` + +The backfill will take some time as it fetches 40 days of historical data. + +### 4. Access Grafana + +1. Open browser: `http://localhost:3000` +2. Navigate to: Dashboards → Node Provider Rewards Dashboard - InfluxDB +3. Select time range: Last 30 days (or desired range) +4. Filter by provider (optional) + +## Dashboard Features + +### Overview Section +- **Latest Rewards Calculation**: Timestamp of most recent rewards distribution +- **Total Providers**: Count of all providers +- **Total Nodes**: Sum of all nodes across providers +- **Total Adjusted Rewards**: Sum of adjusted rewards (in XDR) + +### Provider Summary Table +Shows for each provider: +- Number of nodes +- Base total rewards +- Adjusted total rewards +- Difference (Adjusted - Base) +- Efficiency % (Adjusted/Base * 100) + +### Trends Charts +- **Base vs Adjusted Rewards Over Time**: Daily points showing both metrics +- **Provider Rewards Efficiency Over Time**: Daily efficiency percentage + +## Configuration + +### InfluxDB Settings +Edit in `docker-compose.yaml`: +```yaml +influxdb: + environment: + DOCKER_INFLUXDB_INIT_TOKEN: my-super-secret-auth-token # Change this! + DOCKER_INFLUXDB_INIT_BUCKET: node-rewards + DOCKER_INFLUXDB_INIT_RETENTION: 90d +``` + +### Scheduler Settings +Edit in `config/node-rewards-scheduler/entrypoint.sh`: +- Backfill days: Change `{40..1}` to different range +- Schedule time: Change `"tomorrow 00:05"` to different time + +### Grafana Datasource +Edit in `config/grafana/provisioning/datasources/observability.yaml`: +```yaml +- name: InfluxDB (Node Rewards) + uid: influxdb-node-rewards + url: http://localhost:8086 + secureJsonData: + token: my-super-secret-auth-token # Must match InfluxDB token +``` + +## Troubleshooting + +### No Data in Dashboard + +1. Check InfluxDB has data: + ```bash + docker exec -it $(docker ps -qf "name=influxdb") influx \ + --org ic-org \ + --token my-super-secret-auth-token \ + query 'from(bucket:"node-rewards") |> range(start: -30d) |> limit(n: 10)' + ``` + +2. Check scheduler logs: + ```bash + docker-compose logs node-rewards-scheduler + ``` + +3. Verify backfill completed: + ```bash + # Check if backfill marker exists + ls -la volumes/node-rewards-scheduler/backfill_done + ``` + +### Scheduler Not Running + +```bash +# Restart scheduler +docker-compose restart node-rewards-scheduler + +# Check entrypoint is executable +ls -l config/node-rewards-scheduler/entrypoint.sh +``` + +### Manual Data Push + +You can manually push data for a specific date: + +```bash +# Inside the scheduler container +docker exec -it $(docker ps -qf "name=node-rewards-scheduler") \ + dre node-rewards push-to-influx --date 2025-01-15 +``` + +### Query InfluxDB Directly + +```bash +# List measurements +curl -XPOST http://localhost:8086/api/v2/query \ + -H "Authorization: Token my-super-secret-auth-token" \ + -H "Content-Type: application/vnd.flux" \ + --data 'import "influxdata/influxdb/schema" +schema.measurements(bucket: "node-rewards")' +``` + +## Data Retention + +- InfluxDB retention policy: **90 days** +- After 90 days, old data is automatically deleted +- To change retention, modify `DOCKER_INFLUXDB_INIT_RETENTION` in docker-compose.yaml + +## Backup and Restore + +### Backup InfluxDB Data +```bash +# Backup to tar.gz +docker exec $(docker ps -qf "name=influxdb") \ + influx backup /tmp/backup -t my-super-secret-auth-token + +docker cp $(docker ps -qf "name=influxdb"):/tmp/backup ./influxdb-backup +tar -czf influxdb-backup-$(date +%Y%m%d).tar.gz influxdb-backup +``` + +### Restore InfluxDB Data +```bash +# Extract backup +tar -xzf influxdb-backup-YYYYMMDD.tar.gz + +# Copy to container +docker cp influxdb-backup $(docker ps -qf "name=influxdb"):/tmp/ + +# Restore +docker exec $(docker ps -qf "name=influxdb") \ + influx restore /tmp/influxdb-backup -t my-super-secret-auth-token +``` + +## Development + +### Testing Queries in InfluxDB UI + +1. Open http://localhost:8086 +2. Login with username/password from docker-compose +3. Go to Data Explorer +4. Write Flux queries to test data + +### Modifying the Dashboard + +1. Edit dashboard in Grafana UI +2. Export JSON: Dashboard Settings → JSON Model → Copy +3. Save to: `config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json` +4. Restart Grafana: `docker-compose restart grafana` + +## Performance Notes + +- **Data Volume**: ~1 data point per provider per day +- **Query Performance**: Flux queries are optimized for daily aggregations +- **Backfill Time**: ~5-10 minutes for 40 days (depends on number of providers) +- **Daily Push Time**: ~10-30 seconds + +## Security Notes + +⚠️ **Important**: The default token `my-super-secret-auth-token` is for development only! + +For production: +1. Generate a secure token: `openssl rand -hex 32` +2. Update in docker-compose.yaml (influxdb and scheduler) +3. Update in datasources/observability.yaml +4. Restart all services + +## References + +- [InfluxDB 2.x Documentation](https://docs.influxdata.com/influxdb/v2.7/) +- [Flux Query Language](https://docs.influxdata.com/flux/v0.x/) +- [Grafana InfluxDB Data Source](https://grafana.com/docs/grafana/latest/datasources/influxdb/) +- [DRE CLI Documentation](../../dre/rs/cli/README.md) + diff --git a/docker/QUICKSTART_NODE_REWARDS.md b/docker/QUICKSTART_NODE_REWARDS.md new file mode 100644 index 0000000..8f041ec --- /dev/null +++ b/docker/QUICKSTART_NODE_REWARDS.md @@ -0,0 +1,201 @@ +# Node Rewards Dashboard - Quick Start Guide + +## 🚀 Quick Start (5 minutes) + +### Step 1: Start All Services + +```bash +cd /Users/pietro.di.marco/ic-observability-stack/docker +export UID=$(id -u) +export GID=$(id -g) +docker-compose up -d +``` + +### Step 2: Monitor Backfill Progress + +The system will automatically backfill the last 40 days of metrics on first startup: + +```bash +# Watch the backfill progress +docker-compose logs -f node-rewards-scheduler +``` + +You should see output like: +``` +Starting backfill of last 40 days... +Backfilling data for 2024-12-02... +Backfilling data for 2024-12-03... +... +Backfill complete! +``` + +This will take approximately 5-10 minutes. + +### Step 3: Access Grafana Dashboard + +1. Open your browser to: **http://localhost:3000** +2. Navigate to: **Dashboards** → **Node Provider Rewards Dashboard - InfluxDB** +3. Set time range to: **Last 30 days** +4. Enjoy your node rewards metrics! 📊 + +## 📅 Daily Updates + +The system automatically: +- Fetches yesterday's metrics every day at **00:05 UTC** +- Stores them as discrete daily data points in InfluxDB +- Updates the Grafana dashboard automatically + +## 🎯 What You'll See + +### Provider Overview +- Total providers and nodes +- Latest rewards calculation timestamp +- Total adjusted rewards across all providers + +### Provider Summary Table +| Provider | Nodes | Base Total | Adjusted Total | Difference | Efficiency % | +|----------|-------|------------|----------------|------------|--------------| +| prov1... | 45 | 1,234,567 | 1,200,000 | -34,567 | 97.2% 🟡 | +| prov2... | 32 | 987,654 | 985,000 | -2,654 | 99.7% 🟢 | + +### Trends Charts +- **Base vs Adjusted Rewards**: Daily points showing both metrics over time +- **Efficiency Over Time**: Track provider performance day by day + +## 🔧 Quick Commands + +### Check System Status +```bash +# View all running services +docker-compose ps + +# Check InfluxDB health +curl http://localhost:8086/health + +# Check if data exists +curl -H "Authorization: Token my-super-secret-auth-token" \ + "http://localhost:8086/api/v2/buckets?org=ic-org" +``` + +### Manual Data Push +```bash +# Push data for a specific date +docker exec -it $(docker ps -qf "name=node-rewards-scheduler") \ + dre node-rewards push-to-influx --date 2025-01-15 +``` + +### View Logs +```bash +# Scheduler logs +docker-compose logs -f node-rewards-scheduler + +# InfluxDB logs +docker-compose logs -f influxdb + +# Grafana logs +docker-compose logs -f grafana +``` + +### Restart Services +```bash +# Restart everything +docker-compose restart + +# Restart specific service +docker-compose restart node-rewards-scheduler +``` + +## 🎨 Dashboard Customization + +### Filter by Provider +1. At the top of the dashboard, click the **Provider** dropdown +2. Select one or more providers to focus on +3. The dashboard updates automatically + +### Change Time Range +- Click the time range selector (top right) +- Choose from: Last 7 days, Last 30 days, Last 90 days +- Or set a custom range + +### Export Data +1. Click on any panel +2. Click the **"..."** menu +3. Select **Inspect** → **Data** → **Download CSV** + +## 📊 Understanding the Metrics + +### Base Rewards +- Rewards before any performance adjustments +- What providers would receive with perfect performance + +### Adjusted Rewards +- Actual rewards after performance multipliers +- Reduced if nodes have high failure rates + +### Efficiency % +- `(Adjusted Rewards / Base Rewards) × 100` +- **98-100%** 🟢 Excellent performance +- **95-98%** 🟡 Good performance +- **90-95%** 🟠 Warning - check node health +- **<90%** 🔴 Poor performance - investigate immediately + +## 🛠️ Troubleshooting + +### No Data Appearing? + +1. **Check backfill completed:** + ```bash + ls -la volumes/node-rewards-scheduler/backfill_done + ``` + If this file doesn't exist, backfill is still running. + +2. **Check InfluxDB has data:** + ```bash + docker exec -it $(docker ps -qf "name=influxdb") influx \ + --org ic-org \ + --token my-super-secret-auth-token \ + query 'from(bucket:"node-rewards") |> range(start: -30d) |> limit(n: 5)' + ``` + +3. **Verify Grafana datasource:** + - Go to Grafana → Configuration → Data Sources + - Check "InfluxDB (Node Rewards)" is green/connected + +### Dashboard Shows "No Data"? + +- Make sure you've selected a time range that includes the backfilled data +- Try setting the time range to "Last 30 days" +- Check that "Provider" filter is set to "All" or specific providers + +### Scheduler Container Keeps Restarting? + +```bash +# Check what's wrong +docker-compose logs node-rewards-scheduler + +# Common issues: +# - DRE CLI not in PATH → Update docker image +# - InfluxDB not ready → Wait a minute and check again +# - Auth issues → Verify INFLUXDB_TOKEN matches +``` + +## 🔐 Security Reminder + +⚠️ The default InfluxDB token `my-super-secret-auth-token` is for **development only**! + +For production, update in these files: +1. `docker-compose.yaml` (influxdb and scheduler services) +2. `config/grafana/provisioning/datasources/observability.yaml` + +## 📚 Learn More + +- Full documentation: [NODE_REWARDS_SETUP.md](./NODE_REWARDS_SETUP.md) +- InfluxDB UI: http://localhost:8086 +- Grafana: http://localhost:3000 + +## 🎉 You're All Set! + +Your node rewards monitoring system is now running and will automatically update daily. The dashboard provides real-time insights into provider performance with clean, discrete daily data points. + +**Questions?** Check the full documentation in `NODE_REWARDS_SETUP.md` + diff --git a/docker/README.md b/docker/README.md index c99b8e7..dc4810e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,6 +5,19 @@ operation of the nodes in your datacenter that are part of the Internet Computer is a collection of docker containers that, when deployed and configured, will collect metrics about the performance of selected nodes and send alerts if something is off. +## 🆕 Node Rewards Dashboard + +A new **Node Rewards Dashboard** has been added that displays daily node provider rewards metrics with discrete daily data points. This dashboard uses InfluxDB as the data source and automatically: + +- ✅ **Backfills** last 40 days of metrics on first startup +- ✅ **Updates daily** at 00:05 UTC with yesterday's data +- ✅ **Shows provider rewards**, efficiency, and performance metrics +- ✅ **Visualizes trends** over time with clean daily data points + +**Quick Start**: See [QUICKSTART_NODE_REWARDS.md](./QUICKSTART_NODE_REWARDS.md) for 5-minute setup guide. + +**Full Documentation**: See [NODE_REWARDS_SETUP.md](./NODE_REWARDS_SETUP.md) for complete details. + ## How does this work? The stack selects IC nodes and collects metrics from each node, saving them diff --git a/docker/config/grafana/provisioning/alerting/notification_policy.yaml b/docker/config/grafana/provisioning/alerting/notification_policy.yaml index 450be44..75a37fa 100644 --- a/docker/config/grafana/provisioning/alerting/notification_policy.yaml +++ b/docker/config/grafana/provisioning/alerting/notification_policy.yaml @@ -1,11 +1,8 @@ apiVersion: 1 policies: - - receiver: Obs contact points - object_matchers: - - ['severity', '=~', 'critical'] + - receiver: grafana-default-email group_by: - alertname - ic_node - job - diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json b/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json new file mode 100644 index 0000000..d438ea0 --- /dev/null +++ b/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json @@ -0,0 +1,725 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Provider Overview", + "type": "row" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Latest rewards calculation date from the node rewards canister", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"governance_latest_reward_event_timestamp_seconds\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> map(fn: (r) => ({r with _value: r._value * 1000}))", + "refId": "A" + } + ], + "title": "Latest Rewards Calculation", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Total number of providers tracked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> group()\n |> count()", + "refId": "A" + } + ], + "title": "Total Providers", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Total nodes across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> group()\n |> sum()", + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Total adjusted rewards across all providers (XDR Permyriad)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group()\n |> sum()\n |> map(fn: (r) => ({r with _value: r._value / 10000.0}))", + "refId": "A" + } + ], + "title": "Total Adjusted Rewards (XDR)", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Provider rewards summary showing latest values", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "provider_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Efficiency %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 98 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 105, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "nodes = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"nodes\"})\n\nbase = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"base_total\"})\n\nadjusted = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"adjusted_total\"})\n\n// First join nodes with base\nnodes_base = join(\n tables: {nodes: nodes, base: base},\n on: [\"_time\", \"provider_id\"]\n)\n\n// Then join the result with adjusted\njoin(\n tables: {nodes_base: nodes_base, adjusted: adjusted},\n on: [\"_time\", \"provider_id\"]\n)\n |> map(fn: (r) => ({\n _time: r._time,\n provider_id: r.provider_id,\n nodes: r.nodes,\n base_total: r.base_total,\n adjusted_total: r.adjusted_total,\n difference: r.adjusted_total - r.base_total,\n efficiency: (r.adjusted_total / r.base_total) * 100.0\n }))\n |> yield(name: \"result\")", + "refId": "A" + } + ], + "title": "Provider Rewards Summary (Latest Day)", + "transformations": [], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 200, + "panels": [], + "title": "Provider Rewards Trends", + "type": "row" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Base vs Adjusted rewards over time for selected provider(s) - Daily discrete data points", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "XDR", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Base.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Adjusted.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> map(fn: (r) => ({r with _value: r._value, provider: r.provider_id + \" - Base\"}))\n |> keep(columns: [\"_time\", \"_value\", \"provider\"])\n |> rename(columns: {provider: \"_field\"})", + "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> map(fn: (r) => ({r with _value: r._value, provider: r.provider_id + \" - Adjusted\"}))\n |> keep(columns: [\"_time\", \"_value\", \"provider\"])\n |> rename(columns: {provider: \"_field\"})", + "refId": "B" + } + ], + "title": "Base vs Adjusted Rewards Over Time (Daily Points)", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "description": "Rewards efficiency percentage (Adjusted/Base * 100) - Daily discrete data points", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Efficiency %", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "query": "base = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"base\"})\n\nadjusted = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"adjusted\"})\n\njoin(\n tables: {base: base, adjusted: adjusted},\n on: [\"_time\", \"provider_id\"]\n)\n |> map(fn: (r) => ({\n _time: r._time,\n _field: r.provider_id + \" - Efficiency %\",\n _value: (float(v: r.adjusted) / float(v: r.base)) * 100.0\n }))\n |> keep(columns: [\"_time\", \"_field\", \"_value\"])", + "refId": "A" + } + ], + "title": "Provider Rewards Efficiency Over Time (Daily Points)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1h", + "schemaVersion": 40, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance", + "influxdb" + ], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "influxdb", + "uid": "influxdb-node-rewards" + }, + "definition": "import \"influxdata/influxdb/schema\"\nschema.tagValues(\n bucket: \"node-rewards\",\n tag: \"provider_id\",\n predicate: (r) => r._measurement == \"latest_nodes_count\",\n start: -30d\n)", + "hide": 0, + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": "import \"influxdata/influxdb/schema\"\nschema.tagValues(\n bucket: \"node-rewards\",\n tag: \"provider_id\",\n predicate: (r) => r._measurement == \"latest_nodes_count\",\n start: -30d\n)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1h", + "6h", + "12h", + "1d" + ] + }, + "timezone": "browser", + "title": "Node Provider Rewards Dashboard - InfluxDB", + "uid": "node-rewards-influxdb", + "version": 1, + "weekStart": "" +} + diff --git a/docker/config/grafana/provisioning/dashboards/observability.yaml b/docker/config/grafana/provisioning/dashboards/observability.yaml index 6648df9..31b0e08 100644 --- a/docker/config/grafana/provisioning/dashboards/observability.yaml +++ b/docker/config/grafana/provisioning/dashboards/observability.yaml @@ -11,3 +11,14 @@ providers: options: path: /etc/grafana/provisioning/dashboards/samples foldersFromFilesStructure: false + + - name: 'Node Rewards' + folder: 'Node Rewards' + folderUid: 'node-rewards' + type: file + disableDeletion: false + updateIntervalSeconds: 60 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards/node-rewards + foldersFromFilesStructure: false \ No newline at end of file diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 825b528..8107b78 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -8,3 +8,16 @@ datasources: url: http://$PROM_HOST:9090 jsonData: httpMethod: POST + + - name: InfluxDB (Node Rewards) + uid: influxdb-node-rewards + type: influxdb + access: proxy + url: http://localhost:8086 + jsonData: + version: Flux + organization: ic-org + defaultBucket: node-rewards + tlsSkipVerify: true + secureJsonData: + token: my-super-secret-auth-token \ No newline at end of file diff --git a/docker/config/node-rewards-scheduler/entrypoint.sh b/docker/config/node-rewards-scheduler/entrypoint.sh new file mode 100755 index 0000000..40f7cf0 --- /dev/null +++ b/docker/config/node-rewards-scheduler/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# Backfill last 40 days on startup +backfill_data() { + echo "Starting backfill of last 40 days..." + for i in {40..1}; do + date=$(date -u -d "$i days ago" +%Y-%m-%d || date -u -v-${i}d +%Y-%m-%d) + echo "Backfilling data for $date..." + dre node-rewards push-to-influx --date "$date" || echo "Warning: Failed to backfill $date" + done + echo "Backfill complete!" +} + +# Run daily job at 00:05 UTC +run_daily_job() { + # Calculate time until next 00:05 UTC + current_time=$(date -u +%s) + next_run=$(date -u -d "tomorrow 00:05" +%s 2>/dev/null || date -u -v+1d -v00H -v05M +%s) + + if [ $next_run -le $current_time ]; then + next_run=$(date -u -d "00:05" +%s 2>/dev/null || date -u -v00H -v05M +%s) + fi + + sleep_seconds=$((next_run - current_time)) + + echo "Next run scheduled at $(date -u -d @$next_run +%Y-%m-%d\ %H:%M:%S 2>/dev/null || date -u -r $next_run +%Y-%m-%d\ %H:%M:%S) UTC (in ${sleep_seconds}s)" + sleep $sleep_seconds + + # Run the job for yesterday + yesterday=$(date -u -d "yesterday" +%Y-%m-%d 2>/dev/null || date -u -v-1d +%Y-%m-%d) + echo "Running daily job for $yesterday..." + dre node-rewards push-to-influx --date "$yesterday" +} + +# Check if backfill marker exists +BACKFILL_MARKER="/scheduler/backfill_done" +if [ ! -f "$BACKFILL_MARKER" ]; then + backfill_data + touch "$BACKFILL_MARKER" +fi + +# Run daily job in a loop +while true; do + run_daily_job +done + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 910997c..d7a0c6c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -31,6 +31,35 @@ services: reservations: memory: 12G + influxdb: + image: influxdb:2.7 + network_mode: host + environment: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: admin + DOCKER_INFLUXDB_INIT_PASSWORD: adminpassword + DOCKER_INFLUXDB_INIT_ORG: ic-org + DOCKER_INFLUXDB_INIT_BUCKET: node-rewards + DOCKER_INFLUXDB_INIT_RETENTION: 90d + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token + volumes: + - ./volumes/influxdb:/var/lib/influxdb2 + + # node-rewards-scheduler: + # image: ghcr.io/dfinity/dre/dre:latest + # network_mode: host + # environment: + # INFLUXDB_URL: http://localhost:8086 + # INFLUXDB_TOKEN: my-super-secret-auth-token + # INFLUXDB_ORG: ic-org + # INFLUXDB_BUCKET: node-rewards + # volumes: + # - ./volumes/node-rewards-scheduler:/scheduler + # - ./config/node-rewards-scheduler/entrypoint.sh:/entrypoint.sh:ro + # entrypoint: /entrypoint.sh + # depends_on: + # - influxdb + grafana: image: grafana/grafana:12.2.1 network_mode: host @@ -40,3 +69,5 @@ services: - ./config/grafana:/etc/grafana - ./volumes/grafana:/var/lib/grafana user: "${UID}:${GID}" + depends_on: + - influxdb diff --git a/docker/volumes/.DS_Store b/docker/volumes/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..55bcf86daa5c263ebe3e543bc564ee551912c9cb GIT binary patch literal 6148 zcmeHKJxat-5T3W2V#Q`lErfT4lomoV%kGEfUUT{vS-H&jqw zPUEZ>ZHC{-fV|vGx}tL`sq_BM>T1R}JluVJ0F3BUOjW08Ue-DLZjZKKpH83m*T)s# zikn@;SzGh?1xL}4D!QhEe@!1A+PPaSJ>7eFTkIX)%ROjbRWB>|z1?~)@?6iv7%&Em zf&an)dN#{wQ_xmpz!)$FS_b(0;Gv9(Vj~zn9azE$0E}P`f<4a?oZ}S}#YPYwh?7*H zq&gijoTS6KUj zi~(cdPch)4G)pIVB<-!ehvQ!Bp(julj%x(l6l_8(Ml84DEoczfJ#T=CVj~C(#Qq2b L8f-BJew2ZCv7t>G literal 0 HcmV?d00001 diff --git a/docker/volumes/influxdb/.gitignore b/docker/volumes/influxdb/.gitignore new file mode 100644 index 0000000..a5baada --- /dev/null +++ b/docker/volumes/influxdb/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/docker/volumes/node-rewards-scheduler/.gitignore b/docker/volumes/node-rewards-scheduler/.gitignore new file mode 100644 index 0000000..a5baada --- /dev/null +++ b/docker/volumes/node-rewards-scheduler/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + From 6524feaa61a2301a29e5ea1258ea9ebaa025a925 Mon Sep 17 00:00:00 2001 From: Pietro Date: Tue, 11 Nov 2025 19:50:44 +0100 Subject: [PATCH 2/4] Add victoria metrics to compose --- .DS_Store | Bin 8196 -> 8196 bytes .gitignore | 1 + docker/.DS_Store | Bin 6148 -> 6148 bytes .../node-rewards/node-rewards-dashboard.json | 2707 +++++++++++++++++ .../node-rewards/node-rewards-influx.json | 725 ----- .../datasources/observability.yaml | 17 +- .../node-rewards-scheduler/entrypoint.sh | 128 +- docker/docker-compose.yaml | 50 +- 8 files changed, 2816 insertions(+), 812 deletions(-) create mode 100644 docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-dashboard.json delete mode 100644 docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json diff --git a/.DS_Store b/.DS_Store index 61f2845feb2a4e6fd6bc899f22f27aa66f08989e..8550c36d120e6df63da289d56b66129554001f5b 100644 GIT binary patch delta 127 zcmZp1XmQw}CJ@)XhJk^Bg+Y%YogtH= 98)", + "format": "time_series", + "instant": true, + "legendFormat": "98-100%", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) >= 95 < 98)", + "format": "time_series", + "instant": true, + "legendFormat": "95-98%", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) >= 90 < 95)", + "format": "time_series", + "instant": true, + "legendFormat": "90-95%", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) < 90)", + "format": "time_series", + "instant": true, + "legendFormat": "<90%", + "refId": "D" + } + ], + "title": "Provider Efficiency Distribution", + "type": "barchart" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "node-rewards", + "provider", + "rewards", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "text": "IC Metrics (cluster local)", + "value": "PE62C54679EC3C073" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(latest_nodes_count, provider_id)", + "includeAll": true, + "label": "Provider", + "multi": true, + "name": "provider", + "options": [], + "query": { + "query": "label_values(latest_nodes_count, provider_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "query": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(subnet_failure_rate, subnet_id)", + "includeAll": true, + "label": "Subnet", + "multi": true, + "name": "subnet", + "options": [], + "query": { + "query": "label_values(subnet_failure_rate, subnet_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard - Comprehensive", + "uid": "node-rewards-comprehensive", + "version": 1 + } diff --git a/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json b/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json deleted file mode 100644 index d438ea0..0000000 --- a/docker/config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json +++ /dev/null @@ -1,725 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "panels": [], - "title": "Provider Overview", - "type": "row" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Latest rewards calculation date from the node rewards canister", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 101, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"governance_latest_reward_event_timestamp_seconds\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> map(fn: (r) => ({r with _value: r._value * 1000}))", - "refId": "A" - } - ], - "title": "Latest Rewards Calculation", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Total number of providers tracked", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 102, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> group()\n |> count()", - "refId": "A" - } - ], - "title": "Total Providers", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Total nodes across all providers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 103, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> last()\n |> group()\n |> sum()", - "refId": "A" - } - ], - "title": "Total Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Total adjusted rewards across all providers (XDR Permyriad)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 104, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group()\n |> sum()\n |> map(fn: (r) => ({r with _value: r._value / 10000.0}))", - "refId": "A" - } - ], - "title": "Total Adjusted Rewards (XDR)", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Provider rewards summary showing latest values", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "provider_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 180 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Efficiency %" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "max", - "value": 100 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 90 - }, - { - "color": "yellow", - "value": 95 - }, - { - "color": "green", - "value": 98 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 105, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "nodes = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_nodes_count\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"nodes\"})\n\nbase = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"base_total\"})\n\nadjusted = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> last()\n |> group(columns: [\"provider_id\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"adjusted_total\"})\n\n// First join nodes with base\nnodes_base = join(\n tables: {nodes: nodes, base: base},\n on: [\"_time\", \"provider_id\"]\n)\n\n// Then join the result with adjusted\njoin(\n tables: {nodes_base: nodes_base, adjusted: adjusted},\n on: [\"_time\", \"provider_id\"]\n)\n |> map(fn: (r) => ({\n _time: r._time,\n provider_id: r.provider_id,\n nodes: r.nodes,\n base_total: r.base_total,\n adjusted_total: r.adjusted_total,\n difference: r.adjusted_total - r.base_total,\n efficiency: (r.adjusted_total / r.base_total) * 100.0\n }))\n |> yield(name: \"result\")", - "refId": "A" - } - ], - "title": "Provider Rewards Summary (Latest Day)", - "transformations": [], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 200, - "panels": [], - "title": "Provider Rewards Trends", - "type": "row" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Base vs Adjusted rewards over time for selected provider(s) - Daily discrete data points", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "XDR", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 8, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Base.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Adjusted.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 201, - "options": { - "legend": { - "calcs": [ - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> map(fn: (r) => ({r with _value: r._value, provider: r.provider_id + \" - Base\"}))\n |> keep(columns: [\"_time\", \"_value\", \"provider\"])\n |> rename(columns: {provider: \"_field\"})", - "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 10000.0}))\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> map(fn: (r) => ({r with _value: r._value, provider: r.provider_id + \" - Adjusted\"}))\n |> keep(columns: [\"_time\", \"_value\", \"provider\"])\n |> rename(columns: {provider: \"_field\"})", - "refId": "B" - } - ], - "title": "Base vs Adjusted Rewards Over Time (Daily Points)", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "description": "Rewards efficiency percentage (Adjusted/Base * 100) - Daily discrete data points", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Efficiency %", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 8, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 95 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 202, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "query": "base = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_base_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"base\"})\n\nadjusted = from(bucket: \"node-rewards\")\n |> range(start: -90d)\n |> filter(fn: (r) => r._measurement == \"latest_total_adjusted_rewards_xdr_permyriad\")\n |> filter(fn: (r) => r._field == \"value\")\n |> filter(fn: (r) => r.provider_id =~ /${provider:regex}/)\n |> aggregateWindow(every: 1d, fn: last, createEmpty: false)\n |> keep(columns: [\"_time\", \"provider_id\", \"_value\"])\n |> rename(columns: {_value: \"adjusted\"})\n\njoin(\n tables: {base: base, adjusted: adjusted},\n on: [\"_time\", \"provider_id\"]\n)\n |> map(fn: (r) => ({\n _time: r._time,\n _field: r.provider_id + \" - Efficiency %\",\n _value: (float(v: r.adjusted) / float(v: r.base)) * 100.0\n }))\n |> keep(columns: [\"_time\", \"_field\", \"_value\"])", - "refId": "A" - } - ], - "title": "Provider Rewards Efficiency Over Time (Daily Points)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "1h", - "schemaVersion": 40, - "tags": [ - "node-rewards", - "provider", - "rewards", - "performance", - "influxdb" - ], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "influxdb", - "uid": "influxdb-node-rewards" - }, - "definition": "import \"influxdata/influxdb/schema\"\nschema.tagValues(\n bucket: \"node-rewards\",\n tag: \"provider_id\",\n predicate: (r) => r._measurement == \"latest_nodes_count\",\n start: -30d\n)", - "hide": 0, - "includeAll": true, - "label": "Provider", - "multi": true, - "name": "provider", - "options": [], - "query": "import \"influxdata/influxdb/schema\"\nschema.tagValues(\n bucket: \"node-rewards\",\n tag: \"provider_id\",\n predicate: (r) => r._measurement == \"latest_nodes_count\",\n start: -30d\n)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-90d", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "1h", - "6h", - "12h", - "1d" - ] - }, - "timezone": "browser", - "title": "Node Provider Rewards Dashboard - InfluxDB", - "uid": "node-rewards-influxdb", - "version": 1, - "weekStart": "" -} - diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 8107b78..2554313 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -1,23 +1,12 @@ apiVersion: 1 datasources: - - name: prometheus + - name: VictoriaMetrics uid: prometheus type: prometheus access: proxy url: http://$PROM_HOST:9090 + isDefault: true jsonData: httpMethod: POST - - - name: InfluxDB (Node Rewards) - uid: influxdb-node-rewards - type: influxdb - access: proxy - url: http://localhost:8086 - jsonData: - version: Flux - organization: ic-org - defaultBucket: node-rewards - tlsSkipVerify: true - secureJsonData: - token: my-super-secret-auth-token \ No newline at end of file + timeInterval: "1d" \ No newline at end of file diff --git a/docker/config/node-rewards-scheduler/entrypoint.sh b/docker/config/node-rewards-scheduler/entrypoint.sh index 40f7cf0..9b0f5da 100755 --- a/docker/config/node-rewards-scheduler/entrypoint.sh +++ b/docker/config/node-rewards-scheduler/entrypoint.sh @@ -1,47 +1,97 @@ #!/bin/bash set -e -# Backfill last 40 days on startup -backfill_data() { - echo "Starting backfill of last 40 days..." - for i in {40..1}; do - date=$(date -u -d "$i days ago" +%Y-%m-%d || date -u -v-${i}d +%Y-%m-%d) - echo "Backfilling data for $date..." - dre node-rewards push-to-influx --date "$date" || echo "Warning: Failed to backfill $date" - done - echo "Backfill complete!" -} - -# Run daily job at 00:05 UTC -run_daily_job() { - # Calculate time until next 00:05 UTC - current_time=$(date -u +%s) - next_run=$(date -u -d "tomorrow 00:05" +%s 2>/dev/null || date -u -v+1d -v00H -v05M +%s) +echo "=====================================" +echo "Node Rewards Scheduler for VictoriaMetrics" +echo "=====================================" + +# Install dependencies +echo "Installing dependencies..." +apt-get update -qq +apt-get install -y -qq curl cron ca-certificates > /dev/null 2>&1 + +# Set up directories +mkdir -p /scheduler +cd /scheduler + +# Download dre binary if not already present +if [ ! -f "/scheduler/.dre_installed" ]; then + echo "Downloading dre v${DRE_VERSION} binary..." + + # Detect architecture + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DRE_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + DRE_ARCH="aarch64" + else + echo "Unsupported architecture: $ARCH" + exit 1 + fi + + # Download from GitHub releases + DRE_URL="https://github.com/dfinity/dre/releases/download/v${DRE_VERSION}/dre-${DRE_ARCH}-unknown-linux-gnu" + echo "Downloading from: $DRE_URL" + + curl -L -o /usr/local/bin/dre "$DRE_URL" + chmod +x /usr/local/bin/dre + + # Verify it works + if /usr/local/bin/dre --version; then + echo "✅ dre v${DRE_VERSION} installed successfully" + touch /scheduler/.dre_installed + else + echo "❌ Failed to install dre" + exit 1 + fi +else + echo "✅ dre already installed" +fi + +# Wait for VictoriaMetrics to be ready +echo "Waiting for VictoriaMetrics to be ready..." +until curl -s "${VICTORIA_METRICS_URL}/-/ready" > /dev/null 2>&1; do + echo " Waiting for VictoriaMetrics at ${VICTORIA_METRICS_URL}..." + sleep 2 +done +echo "✅ VictoriaMetrics is ready" + +# Backfill last 40 days on initial startup +if [ ! -f /scheduler/.backfill_done ]; then + echo "" + echo "=====================================" + echo "Starting backfill of last 40 days..." + echo "=====================================" + + for i in $(seq 40 -1 1); do + DATE=$(date -u -d "$i days ago" +%Y-%m-%d 2>/dev/null || date -u -v-${i}d +%Y-%m-%d) + echo "[$(printf "%2d" $((41 - i)))/40] Backfilling data for $DATE..." - if [ $next_run -le $current_time ]; then - next_run=$(date -u -d "00:05" +%s 2>/dev/null || date -u -v00H -v05M +%s) + if /usr/local/bin/dre node-rewards push-to-victoria --date "$DATE" --victoria-url "${VICTORIA_METRICS_URL}"; then + echo " ✅ Successfully pushed data for $DATE" + else + echo " ⚠️ No data available or error for $DATE" fi - - sleep_seconds=$((next_run - current_time)) - - echo "Next run scheduled at $(date -u -d @$next_run +%Y-%m-%d\ %H:%M:%S 2>/dev/null || date -u -r $next_run +%Y-%m-%d\ %H:%M:%S) UTC (in ${sleep_seconds}s)" - sleep $sleep_seconds - - # Run the job for yesterday - yesterday=$(date -u -d "yesterday" +%Y-%m-%d 2>/dev/null || date -u -v-1d +%Y-%m-%d) - echo "Running daily job for $yesterday..." - dre node-rewards push-to-influx --date "$yesterday" -} - -# Check if backfill marker exists -BACKFILL_MARKER="/scheduler/backfill_done" -if [ ! -f "$BACKFILL_MARKER" ]; then - backfill_data - touch "$BACKFILL_MARKER" + done + + touch /scheduler/.backfill_done + echo "" + echo "✅ Backfill complete!" + echo "=====================================" fi -# Run daily job in a loop -while true; do - run_daily_job -done +# Set up cron job for daily push at 00:05 UTC (yesterday's data) +echo "" +echo "Setting up daily cron job (00:05 UTC)..." +mkdir -p /var/log +touch /var/log/node-rewards-cron.log +echo "05 00 * * * /usr/local/bin/dre node-rewards push-to-victoria --victoria-url ${VICTORIA_METRICS_URL} >> /var/log/node-rewards-cron.log 2>&1" | crontab - + +echo "✅ Cron job configured" +echo "" +echo "=====================================" +echo "Scheduler is running. Logs:" +echo "=====================================" +# Start cron in foreground +cron && tail -f /var/log/node-rewards-cron.log 2>/dev/null || cron -f diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d7a0c6c..f014358 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -12,17 +12,15 @@ services: volumes: - ./volumes/msd:/msd - prometheus: - image: quay.io/prometheus/prometheus:v2.51.1 + victoriametrics: + image: victoriametrics/victoria-metrics:v1.97.1 network_mode: host volumes: - - ./config/prometheus:/config - - ./volumes/prometheus/:/prometheus + - ./volumes/victoriametrics:/victoria-metrics-data command: - - --config.file - - /config/config.yaml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=6m + - --storageDataPath=/victoria-metrics-data + - --httpListenAddr=:9090 + - --retentionPeriod=6M user: "${UID}:${GID}" deploy: resources: @@ -31,34 +29,18 @@ services: reservations: memory: 12G - influxdb: - image: influxdb:2.7 + node-rewards-scheduler: + image: ubuntu:24.04 network_mode: host environment: - DOCKER_INFLUXDB_INIT_MODE: setup - DOCKER_INFLUXDB_INIT_USERNAME: admin - DOCKER_INFLUXDB_INIT_PASSWORD: adminpassword - DOCKER_INFLUXDB_INIT_ORG: ic-org - DOCKER_INFLUXDB_INIT_BUCKET: node-rewards - DOCKER_INFLUXDB_INIT_RETENTION: 90d - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token + VICTORIA_METRICS_URL: http://localhost:9090 + DRE_VERSION: "0.7.0" volumes: - - ./volumes/influxdb:/var/lib/influxdb2 - - # node-rewards-scheduler: - # image: ghcr.io/dfinity/dre/dre:latest - # network_mode: host - # environment: - # INFLUXDB_URL: http://localhost:8086 - # INFLUXDB_TOKEN: my-super-secret-auth-token - # INFLUXDB_ORG: ic-org - # INFLUXDB_BUCKET: node-rewards - # volumes: - # - ./volumes/node-rewards-scheduler:/scheduler - # - ./config/node-rewards-scheduler/entrypoint.sh:/entrypoint.sh:ro - # entrypoint: /entrypoint.sh - # depends_on: - # - influxdb + - ./volumes/node-rewards-scheduler:/scheduler + - ./config/node-rewards-scheduler/entrypoint.sh:/entrypoint.sh:ro + entrypoint: /entrypoint.sh + depends_on: + - victoriametrics grafana: image: grafana/grafana:12.2.1 @@ -70,4 +52,4 @@ services: - ./volumes/grafana:/var/lib/grafana user: "${UID}:${GID}" depends_on: - - influxdb + - victoriametrics From 722c13e84ce29b8e03a55f5d06d9ea42e49d6f43 Mon Sep 17 00:00:00 2001 From: Pietro Date: Tue, 11 Nov 2025 20:58:36 +0100 Subject: [PATCH 3/4] Working version --- .DS_Store | Bin 8196 -> 8196 bytes @ | 1 + docker/.DS_Store | Bin 6148 -> 6148 bytes docker/IMPLEMENTATION_SUMMARY.md | 452 ------------------ docker/NODE_REWARDS_SETUP.md | 300 ------------ docker/QUICKSTART_NODE_REWARDS.md | 201 -------- docker/README.md | 118 +++-- .../node-rewards-scheduler/entrypoint.sh | 65 ++- .../.gitignore | 0 .../config-template.yaml | 0 docker/docker-compose.tools.yaml | 2 +- docker/docker-compose.yaml | 4 +- .../prom_config_builder.py | 6 +- docker/volumes/.DS_Store | Bin 6148 -> 6148 bytes 14 files changed, 125 insertions(+), 1024 deletions(-) create mode 100644 @ delete mode 100644 docker/IMPLEMENTATION_SUMMARY.md delete mode 100644 docker/NODE_REWARDS_SETUP.md delete mode 100644 docker/QUICKSTART_NODE_REWARDS.md rename docker/config/{prometheus => victoria-metrics}/.gitignore (100%) rename docker/config/{prometheus => victoria-metrics}/config-template.yaml (100%) diff --git a/.DS_Store b/.DS_Store index 8550c36d120e6df63da289d56b66129554001f5b..61fc3ac3dc83233a098f3c836e383a984af99de7 100644 GIT binary patch delta 172 zcmZp1XmQw}CJ=XSIs*d(3xgg*IzuKyNp8N2OHxjL5>Sl8rSo3xgT&*GsPZXzSAskk63Jkj;?FP=st-&+#82YbHMx VlxKyoIECbyCG!(DYYW}v1pu*%E`9(2 diff --git a/@ b/@ new file mode 100644 index 0000000..652af04 --- /dev/null +++ b/@ @@ -0,0 +1 @@ +Please uninstall the existing Ansible and retry running this script. See installation instructions provided by Ansible: https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html diff --git a/docker/.DS_Store b/docker/.DS_Store index c3db6255cb934fd5587ea33a62d4cbda2ed80223..82e154a6a66fbc135347b3b2d6d0564078768364 100644 GIT binary patch delta 73 zcmZoMXffEJ$->OVlQ~(3MTXUYUuW6<$%QP!lM`5Y;5NebIoKO78%ypimuwzCl|5^PflRrf%6m@8zxU?S;u%{vJ-0qi-ePx X#pGkGS range(start: -1d) |> limit(n: 1)' -``` - -### Logs -```bash -# Stream all logs -docker-compose logs -f - -# Specific service -docker-compose logs -f node-rewards-scheduler - -# Last 100 lines -docker-compose logs --tail=100 -``` - -## Support and Troubleshooting - -### Common Issues - -1. **No data in dashboard** - - Check backfill completed: `ls volumes/node-rewards-scheduler/backfill_done` - - Verify InfluxDB has data: See health checks above - - Check Grafana datasource is connected - -2. **Scheduler keeps restarting** - - Check logs: `docker-compose logs node-rewards-scheduler` - - Verify DRE CLI is in PATH - - Ensure InfluxDB is running and ready - -3. **Dashboard shows errors** - - Verify datasource configuration - - Check Flux query syntax - - Ensure time range includes backfilled data - -### Getting Help -- Check `QUICKSTART_NODE_REWARDS.md` for quick fixes -- See `NODE_REWARDS_SETUP.md` for detailed troubleshooting -- Review logs: `docker-compose logs -f` - -## Conclusion - -This implementation provides a complete, production-ready system for monitoring IC node provider rewards with: -- ✅ Automated daily data collection -- ✅ Historical backfill (40 days) -- ✅ Clean Grafana dashboard with discrete daily points -- ✅ Comprehensive documentation -- ✅ Easy deployment and maintenance - -The system is designed to run continuously with minimal intervention, automatically updating every day at 00:05 UTC with the latest node rewards metrics. - diff --git a/docker/NODE_REWARDS_SETUP.md b/docker/NODE_REWARDS_SETUP.md deleted file mode 100644 index 2913bae..0000000 --- a/docker/NODE_REWARDS_SETUP.md +++ /dev/null @@ -1,300 +0,0 @@ -# Node Rewards Dashboard with InfluxDB - -This setup provides a Grafana dashboard displaying daily node rewards metrics from the IC network, stored in InfluxDB with discrete daily data points. - -## Architecture - -``` -┌─────────────────────┐ -│ DRE CLI (Rust) │ ──> Fetches daily metrics from IC Node Rewards Canister -└──────────┬──────────┘ - │ - ▼ -┌─────────────────────┐ -│ InfluxDB 2.7 │ ──> Stores daily discrete metrics -└──────────┬──────────┘ - │ - ▼ -┌─────────────────────┐ -│ Grafana 12.2.1 │ ──> Displays dashboards with daily data points -└─────────────────────┘ - -┌─────────────────────┐ -│ Node Rewards │ ──> Runs daily at 00:05 UTC + backfills 40 days on startup -│ Scheduler (cron) │ -└─────────────────────┘ -``` - -## Components - -### 1. InfluxDB -- **Image**: `influxdb:2.7` -- **Port**: `8086` -- **Organization**: `ic-org` -- **Bucket**: `node-rewards` -- **Retention**: 90 days -- **Token**: `my-super-secret-auth-token` (configured in docker-compose.yaml) - -### 2. Node Rewards Scheduler -- **Image**: `ghcr.io/dfinity/dre/dre:latest` -- **Function**: - - Backfills last 40 days of metrics on first startup - - Runs daily at 00:05 UTC to push yesterday's metrics -- **Command**: `dre node-rewards push-to-influx --date ` - -### 3. Grafana Dashboard -- **Dashboard**: `Node Provider Rewards Dashboard - InfluxDB` -- **UID**: `node-rewards-influxdb` -- **Data Source**: InfluxDB (Flux queries) -- **Features**: - - Daily discrete data points (not continuous) - - Provider overview statistics - - Base vs Adjusted rewards trends - - Efficiency metrics over time - - Provider filtering - -## Metrics Stored - -### Provider-Level Metrics -- `latest_nodes_count`: Number of nodes per provider -- `latest_total_base_rewards_xdr_permyriad`: Base rewards (before performance adjustment) -- `latest_total_adjusted_rewards_xdr_permyriad`: Adjusted rewards (after performance adjustment) - -### Node-Level Metrics -- `latest_original_failure_rate`: Original failure rate per node -- `latest_relative_failure_rate`: Relative failure rate per node - -### Subnet-Level Metrics -- `subnet_failure_rate`: Failure rate per subnet - -### Governance Metrics -- `governance_latest_reward_event_timestamp_seconds`: Latest rewards distribution timestamp - -## Data Flow - -1. **Daily Push (00:05 UTC)**: - ``` - DRE CLI → IC Node Rewards Canister → Fetch Yesterday's Data → Push to InfluxDB - ``` - -2. **Backfill (On Startup)**: - ``` - For each day in last 40 days: - DRE CLI → IC Node Rewards Canister → Fetch Day's Data → Push to InfluxDB - ``` - -3. **Grafana Visualization**: - ``` - Grafana → Flux Query → InfluxDB → Daily Discrete Data Points → Dashboard - ``` - -## Setup Instructions - -### 1. Start the Services - -```bash -cd /path/to/ic-observability-stack/docker -export UID=$(id -u) -export GID=$(id -g) -docker-compose up -d -``` - -### 2. Verify InfluxDB - -```bash -# Check InfluxDB is running -curl http://localhost:8086/health - -# Check bucket exists -curl -H "Authorization: Token my-super-secret-auth-token" \ - http://localhost:8086/api/v2/buckets?org=ic-org -``` - -### 3. Check Backfill Progress - -```bash -# View scheduler logs -docker-compose logs -f node-rewards-scheduler -``` - -The backfill will take some time as it fetches 40 days of historical data. - -### 4. Access Grafana - -1. Open browser: `http://localhost:3000` -2. Navigate to: Dashboards → Node Provider Rewards Dashboard - InfluxDB -3. Select time range: Last 30 days (or desired range) -4. Filter by provider (optional) - -## Dashboard Features - -### Overview Section -- **Latest Rewards Calculation**: Timestamp of most recent rewards distribution -- **Total Providers**: Count of all providers -- **Total Nodes**: Sum of all nodes across providers -- **Total Adjusted Rewards**: Sum of adjusted rewards (in XDR) - -### Provider Summary Table -Shows for each provider: -- Number of nodes -- Base total rewards -- Adjusted total rewards -- Difference (Adjusted - Base) -- Efficiency % (Adjusted/Base * 100) - -### Trends Charts -- **Base vs Adjusted Rewards Over Time**: Daily points showing both metrics -- **Provider Rewards Efficiency Over Time**: Daily efficiency percentage - -## Configuration - -### InfluxDB Settings -Edit in `docker-compose.yaml`: -```yaml -influxdb: - environment: - DOCKER_INFLUXDB_INIT_TOKEN: my-super-secret-auth-token # Change this! - DOCKER_INFLUXDB_INIT_BUCKET: node-rewards - DOCKER_INFLUXDB_INIT_RETENTION: 90d -``` - -### Scheduler Settings -Edit in `config/node-rewards-scheduler/entrypoint.sh`: -- Backfill days: Change `{40..1}` to different range -- Schedule time: Change `"tomorrow 00:05"` to different time - -### Grafana Datasource -Edit in `config/grafana/provisioning/datasources/observability.yaml`: -```yaml -- name: InfluxDB (Node Rewards) - uid: influxdb-node-rewards - url: http://localhost:8086 - secureJsonData: - token: my-super-secret-auth-token # Must match InfluxDB token -``` - -## Troubleshooting - -### No Data in Dashboard - -1. Check InfluxDB has data: - ```bash - docker exec -it $(docker ps -qf "name=influxdb") influx \ - --org ic-org \ - --token my-super-secret-auth-token \ - query 'from(bucket:"node-rewards") |> range(start: -30d) |> limit(n: 10)' - ``` - -2. Check scheduler logs: - ```bash - docker-compose logs node-rewards-scheduler - ``` - -3. Verify backfill completed: - ```bash - # Check if backfill marker exists - ls -la volumes/node-rewards-scheduler/backfill_done - ``` - -### Scheduler Not Running - -```bash -# Restart scheduler -docker-compose restart node-rewards-scheduler - -# Check entrypoint is executable -ls -l config/node-rewards-scheduler/entrypoint.sh -``` - -### Manual Data Push - -You can manually push data for a specific date: - -```bash -# Inside the scheduler container -docker exec -it $(docker ps -qf "name=node-rewards-scheduler") \ - dre node-rewards push-to-influx --date 2025-01-15 -``` - -### Query InfluxDB Directly - -```bash -# List measurements -curl -XPOST http://localhost:8086/api/v2/query \ - -H "Authorization: Token my-super-secret-auth-token" \ - -H "Content-Type: application/vnd.flux" \ - --data 'import "influxdata/influxdb/schema" -schema.measurements(bucket: "node-rewards")' -``` - -## Data Retention - -- InfluxDB retention policy: **90 days** -- After 90 days, old data is automatically deleted -- To change retention, modify `DOCKER_INFLUXDB_INIT_RETENTION` in docker-compose.yaml - -## Backup and Restore - -### Backup InfluxDB Data -```bash -# Backup to tar.gz -docker exec $(docker ps -qf "name=influxdb") \ - influx backup /tmp/backup -t my-super-secret-auth-token - -docker cp $(docker ps -qf "name=influxdb"):/tmp/backup ./influxdb-backup -tar -czf influxdb-backup-$(date +%Y%m%d).tar.gz influxdb-backup -``` - -### Restore InfluxDB Data -```bash -# Extract backup -tar -xzf influxdb-backup-YYYYMMDD.tar.gz - -# Copy to container -docker cp influxdb-backup $(docker ps -qf "name=influxdb"):/tmp/ - -# Restore -docker exec $(docker ps -qf "name=influxdb") \ - influx restore /tmp/influxdb-backup -t my-super-secret-auth-token -``` - -## Development - -### Testing Queries in InfluxDB UI - -1. Open http://localhost:8086 -2. Login with username/password from docker-compose -3. Go to Data Explorer -4. Write Flux queries to test data - -### Modifying the Dashboard - -1. Edit dashboard in Grafana UI -2. Export JSON: Dashboard Settings → JSON Model → Copy -3. Save to: `config/grafana/provisioning/dashboards/node-rewards/node-rewards-influx.json` -4. Restart Grafana: `docker-compose restart grafana` - -## Performance Notes - -- **Data Volume**: ~1 data point per provider per day -- **Query Performance**: Flux queries are optimized for daily aggregations -- **Backfill Time**: ~5-10 minutes for 40 days (depends on number of providers) -- **Daily Push Time**: ~10-30 seconds - -## Security Notes - -⚠️ **Important**: The default token `my-super-secret-auth-token` is for development only! - -For production: -1. Generate a secure token: `openssl rand -hex 32` -2. Update in docker-compose.yaml (influxdb and scheduler) -3. Update in datasources/observability.yaml -4. Restart all services - -## References - -- [InfluxDB 2.x Documentation](https://docs.influxdata.com/influxdb/v2.7/) -- [Flux Query Language](https://docs.influxdata.com/flux/v0.x/) -- [Grafana InfluxDB Data Source](https://grafana.com/docs/grafana/latest/datasources/influxdb/) -- [DRE CLI Documentation](../../dre/rs/cli/README.md) - diff --git a/docker/QUICKSTART_NODE_REWARDS.md b/docker/QUICKSTART_NODE_REWARDS.md deleted file mode 100644 index 8f041ec..0000000 --- a/docker/QUICKSTART_NODE_REWARDS.md +++ /dev/null @@ -1,201 +0,0 @@ -# Node Rewards Dashboard - Quick Start Guide - -## 🚀 Quick Start (5 minutes) - -### Step 1: Start All Services - -```bash -cd /Users/pietro.di.marco/ic-observability-stack/docker -export UID=$(id -u) -export GID=$(id -g) -docker-compose up -d -``` - -### Step 2: Monitor Backfill Progress - -The system will automatically backfill the last 40 days of metrics on first startup: - -```bash -# Watch the backfill progress -docker-compose logs -f node-rewards-scheduler -``` - -You should see output like: -``` -Starting backfill of last 40 days... -Backfilling data for 2024-12-02... -Backfilling data for 2024-12-03... -... -Backfill complete! -``` - -This will take approximately 5-10 minutes. - -### Step 3: Access Grafana Dashboard - -1. Open your browser to: **http://localhost:3000** -2. Navigate to: **Dashboards** → **Node Provider Rewards Dashboard - InfluxDB** -3. Set time range to: **Last 30 days** -4. Enjoy your node rewards metrics! 📊 - -## 📅 Daily Updates - -The system automatically: -- Fetches yesterday's metrics every day at **00:05 UTC** -- Stores them as discrete daily data points in InfluxDB -- Updates the Grafana dashboard automatically - -## 🎯 What You'll See - -### Provider Overview -- Total providers and nodes -- Latest rewards calculation timestamp -- Total adjusted rewards across all providers - -### Provider Summary Table -| Provider | Nodes | Base Total | Adjusted Total | Difference | Efficiency % | -|----------|-------|------------|----------------|------------|--------------| -| prov1... | 45 | 1,234,567 | 1,200,000 | -34,567 | 97.2% 🟡 | -| prov2... | 32 | 987,654 | 985,000 | -2,654 | 99.7% 🟢 | - -### Trends Charts -- **Base vs Adjusted Rewards**: Daily points showing both metrics over time -- **Efficiency Over Time**: Track provider performance day by day - -## 🔧 Quick Commands - -### Check System Status -```bash -# View all running services -docker-compose ps - -# Check InfluxDB health -curl http://localhost:8086/health - -# Check if data exists -curl -H "Authorization: Token my-super-secret-auth-token" \ - "http://localhost:8086/api/v2/buckets?org=ic-org" -``` - -### Manual Data Push -```bash -# Push data for a specific date -docker exec -it $(docker ps -qf "name=node-rewards-scheduler") \ - dre node-rewards push-to-influx --date 2025-01-15 -``` - -### View Logs -```bash -# Scheduler logs -docker-compose logs -f node-rewards-scheduler - -# InfluxDB logs -docker-compose logs -f influxdb - -# Grafana logs -docker-compose logs -f grafana -``` - -### Restart Services -```bash -# Restart everything -docker-compose restart - -# Restart specific service -docker-compose restart node-rewards-scheduler -``` - -## 🎨 Dashboard Customization - -### Filter by Provider -1. At the top of the dashboard, click the **Provider** dropdown -2. Select one or more providers to focus on -3. The dashboard updates automatically - -### Change Time Range -- Click the time range selector (top right) -- Choose from: Last 7 days, Last 30 days, Last 90 days -- Or set a custom range - -### Export Data -1. Click on any panel -2. Click the **"..."** menu -3. Select **Inspect** → **Data** → **Download CSV** - -## 📊 Understanding the Metrics - -### Base Rewards -- Rewards before any performance adjustments -- What providers would receive with perfect performance - -### Adjusted Rewards -- Actual rewards after performance multipliers -- Reduced if nodes have high failure rates - -### Efficiency % -- `(Adjusted Rewards / Base Rewards) × 100` -- **98-100%** 🟢 Excellent performance -- **95-98%** 🟡 Good performance -- **90-95%** 🟠 Warning - check node health -- **<90%** 🔴 Poor performance - investigate immediately - -## 🛠️ Troubleshooting - -### No Data Appearing? - -1. **Check backfill completed:** - ```bash - ls -la volumes/node-rewards-scheduler/backfill_done - ``` - If this file doesn't exist, backfill is still running. - -2. **Check InfluxDB has data:** - ```bash - docker exec -it $(docker ps -qf "name=influxdb") influx \ - --org ic-org \ - --token my-super-secret-auth-token \ - query 'from(bucket:"node-rewards") |> range(start: -30d) |> limit(n: 5)' - ``` - -3. **Verify Grafana datasource:** - - Go to Grafana → Configuration → Data Sources - - Check "InfluxDB (Node Rewards)" is green/connected - -### Dashboard Shows "No Data"? - -- Make sure you've selected a time range that includes the backfilled data -- Try setting the time range to "Last 30 days" -- Check that "Provider" filter is set to "All" or specific providers - -### Scheduler Container Keeps Restarting? - -```bash -# Check what's wrong -docker-compose logs node-rewards-scheduler - -# Common issues: -# - DRE CLI not in PATH → Update docker image -# - InfluxDB not ready → Wait a minute and check again -# - Auth issues → Verify INFLUXDB_TOKEN matches -``` - -## 🔐 Security Reminder - -⚠️ The default InfluxDB token `my-super-secret-auth-token` is for **development only**! - -For production, update in these files: -1. `docker-compose.yaml` (influxdb and scheduler services) -2. `config/grafana/provisioning/datasources/observability.yaml` - -## 📚 Learn More - -- Full documentation: [NODE_REWARDS_SETUP.md](./NODE_REWARDS_SETUP.md) -- InfluxDB UI: http://localhost:8086 -- Grafana: http://localhost:3000 - -## 🎉 You're All Set! - -Your node rewards monitoring system is now running and will automatically update daily. The dashboard provides real-time insights into provider performance with clean, discrete daily data points. - -**Questions?** Check the full documentation in `NODE_REWARDS_SETUP.md` - diff --git a/docker/README.md b/docker/README.md index dc4810e..f78db74 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,24 +5,15 @@ operation of the nodes in your datacenter that are part of the Internet Computer is a collection of docker containers that, when deployed and configured, will collect metrics about the performance of selected nodes and send alerts if something is off. -## 🆕 Node Rewards Dashboard - -A new **Node Rewards Dashboard** has been added that displays daily node provider rewards metrics with discrete daily data points. This dashboard uses InfluxDB as the data source and automatically: - -- ✅ **Backfills** last 40 days of metrics on first startup -- ✅ **Updates daily** at 00:05 UTC with yesterday's data -- ✅ **Shows provider rewards**, efficiency, and performance metrics -- ✅ **Visualizes trends** over time with clean daily data points - -**Quick Start**: See [QUICKSTART_NODE_REWARDS.md](./QUICKSTART_NODE_REWARDS.md) for 5-minute setup guide. - -**Full Documentation**: See [NODE_REWARDS_SETUP.md](./NODE_REWARDS_SETUP.md) for complete details. - ## How does this work? The stack selects IC nodes and collects metrics from each node, saving them -in a local Prometheus database every few seconds. This database is -queryable through a Grafana deployed side-to-side with Prometheus. +in a local VictoriaMetrics database every few seconds. This database is +queryable through a Grafana instance deployed side-by-side with VictoriaMetrics. + +Additionally, a node rewards scheduler automatically collects daily node provider +rewards data from the IC Node Rewards Canister and stores it in VictoriaMetrics, +providing historical tracking and visualization of rewards metrics. Nodes of the Internet Computer all make available to the public a series of metrics (in [Prometheus text format](https://github.com/Prometheus/docs/blob/777846211d502a287ab2b304cb515dc779de3474/content/docs/instrumenting/exposition_formats.md#text-based-format)) @@ -80,9 +71,9 @@ Example command for node provider Dfinity Stiftung for data center se1: docker compose -f ./docker-compose.tools.yaml run --rm prom-config-builder tools/prom-config-builder/prom_config_builder.py --node-provider-id bvcsg-3od6r-jnydw-eysln-aql7w-td5zn-ay5m6-sibd2-jzojt-anwag-mqe --dc-id se1 ``` -Once that executes, you should be able to see a new file at `./config/prometheus/config.yaml`. +Once that executes, you should be able to see a new file at `./config/victoria-metrics/config.yaml`. This file contains the definitions for the scraping targets. It will be slightly different -for each node provider and each data center. It is not versione controlled and you can +for each node provider and each data center. It is not version controlled and you can always recreate it with running the above command if you lose it, or delete it. ### Contact points @@ -120,17 +111,33 @@ read *Troubleshooting* ## Usage Once started, you will see the following applications: -* Prometheus - http://localhost:9090 +* VictoriaMetrics - http://localhost:9090 * Grafana - http://localhost:3000 - default creds can be see in `./config/grafana/grafana.ini` * Service discovery - http://localhost:8000 -After 5-10 minutes you should see targets discovered in prometheus on the [targets page](http://localhost:9090/targets?search=). Initially, they might apear in red and if you keep monitoring they should -slowly start getting blue, which means that the targets are successfuly scraped. +After 5-10 minutes you should see targets discovered in VictoriaMetrics. The metrics are being +collected and stored successfully when you can query them through Grafana dashboards. You should also see some data incoming in the grafana [sample dashboard for the node exporter](http://localhost:3000/d/1/node-exporter?orgId=1&from=now-3h&to=now&timezone=utc&var-datasource=prometheus&var-instance=$__all&var-diskdevice=%5Ba-z%5D%2B%7Cnvme%5B0-9%5D%2Bn%5B0-9%5D%2B%7Cmmcblk%5B0-9%5D%2B). Here, you can see various information about the performance of nodes and their health. +### Node Rewards Dashboard + +The stack includes a comprehensive Node Provider Rewards Dashboard that displays: +* **Provider Overview** - Total providers, nodes, and rewards calculations +* **Rewards Trends** - Historical tracking of base vs adjusted rewards +* **Node-Level Metrics** - Individual node failure rates and performance +* **Subnet Metrics** - Subnet-level failure rates and node distribution +* **Analysis & Insights** - Top providers, efficiency rankings, and reward penalties + +The node rewards scheduler automatically: +* Backfills the last 40 days of historical rewards data on first startup +* Pushes new rewards data daily at 00:05 UTC +* Downloads the latest `dre` binary from GitHub releases (v0.7.0) + +Access the dashboard at: http://localhost:3000/d/node-rewards + [Alerting tab](http://localhost:3000/alerting/list) will show `obs alert evaluations` which will contain a list of preconfigured alerts. Here you can see if any of the alerts are in in a problematic state. Most of the time they should be in `Normal` @@ -154,6 +161,11 @@ To access the stack remotely you can do the following: ssh -L 3000:localhost:3000 -L 9090:localhost:9090 -L 8000:localhost:8000 ``` +This will forward: +* Port 3000 - Grafana web interface +* Port 9090 - VictoriaMetrics API +* Port 8000 - Service discovery + Example command with all parameters: ```bash ssh -L 3000:localhost:3000 -L 9090:localhost:9090 -i ~/.ssh/priv_key.pem myuser@192.168.15.15 @@ -215,43 +227,56 @@ curl http://localhost:8000/prom/targets?node_provider_id=&dc_i *NOTE*: The initial sync of service discovery may take up to 15 minutes! Syncing will be clearly logged in the multiservice discovery. -### Prometheus +### VictoriaMetrics -#### No targets visible in targets view +VictoriaMetrics is the time-series database that stores all metrics from IC nodes +and node rewards data. -If you don't see anything in the [prometheus targets view](http://localhost:9090/targets?search=), -that means that prometheus failed to receive targets from the service discovery. +#### Checking VictoriaMetrics health -To check the logs run: +To verify VictoriaMetrics is running properly: +```bash +curl http://localhost:9090/-/ready +``` + +You should see: `VictoriaMetrics is Ready.` + +To check the logs: ```bash # From the same folder of this README -docker compose logs prometheus +docker compose logs victoriametrics ``` -Check if you can see your nodes by running the following command: +#### Querying metrics + +You can query metrics using the VictoriaMetrics API: ```bash -curl http://localhost:8000/prom/targets?node_provider_id=&dc_id= +# Check available metrics +curl 'http://localhost:9090/api/v1/label/__name__/values' + +# Query specific metric +curl 'http://localhost:9090/api/v1/query?query=latest_nodes_count' ``` -#### Targets visible but are being shown in read +### Node Rewards Scheduler + +The node rewards scheduler runs as a separate container and automatically collects +rewards data from the IC Node Rewards Canister. -You should now see 4 jobs: -* `host_node_exporter` -* `node_exporter` -* `orchestrator` -* `replica` +#### Checking scheduler status -If any of them are shown in read it means that some of the targets (or all of them) -are failing to be scraped. You can see that from the logs as well: +To view scheduler logs: ```bash # From the same folder of this README -docker compose logs prometheus +docker compose logs node-rewards-scheduler ``` -This means that the prometheus scraper cannot reach the nodes it is trying -to scrape. It can be because the workstation for this observability stack -isn't in the same network subnet as the nodes, or due to other network -issues. +On first startup, you should see: +* Binary download from GitHub releases +* 40-day backfill process (may take 10-20 minutes) +* Cron job setup for daily updates at 00:05 UTC + +``` ### Stack restart @@ -267,9 +292,14 @@ To make a full clean restart (or partial) you can do the following: * Stop the stack: `docker compose -f ./docker-compose.yaml down` * Clean the volumes. You don't have to clean everything, pick just ones that you wish to restart fully: - * prometheus: `rm -rf ./volumes/prometheus/` - * grafana: `rm -rf ./volumes/grafana/` - * multiservice discovery: `rm -rf ./volumes/msd/` + * VictoriaMetrics: `rm -rf ./volumes/victoriametrics/` + * Grafana: `rm -rf ./volumes/grafana/` + * Multiservice discovery: `rm -rf ./volumes/msd/` + * Node rewards scheduler: `rm -rf ./volumes/node-rewards-scheduler/` * Reset the folder structure: `git checkout -- ./volumes/` * Run the stack again: `docker compose -f ./docker-compose.yaml up -d` +**Note**: Cleaning VictoriaMetrics volumes will delete all historical metrics data, +including node rewards data. The node rewards scheduler will automatically backfill +the last 40 days when restarted. + diff --git a/docker/config/node-rewards-scheduler/entrypoint.sh b/docker/config/node-rewards-scheduler/entrypoint.sh index 9b0f5da..f307682 100755 --- a/docker/config/node-rewards-scheduler/entrypoint.sh +++ b/docker/config/node-rewards-scheduler/entrypoint.sh @@ -14,38 +14,61 @@ apt-get install -y -qq curl cron ca-certificates > /dev/null 2>&1 mkdir -p /scheduler cd /scheduler -# Download dre binary if not already present -if [ ! -f "/scheduler/.dre_installed" ]; then - echo "Downloading dre v${DRE_VERSION} binary..." + +# # Download dre binary if not already present +# if [ ! -f "/scheduler/.dre_installed" ]; then +# echo "Downloading dre v${DRE_VERSION} binary..." - # Detect architecture - ARCH=$(uname -m) - if [ "$ARCH" = "x86_64" ]; then - DRE_ARCH="x86_64" - elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - DRE_ARCH="aarch64" - else - echo "Unsupported architecture: $ARCH" - exit 1 - fi +# # Detect architecture +# ARCH=$(uname -m) +# if [ "$ARCH" = "x86_64" ]; then +# DRE_BINARY="dre-x86_64-unknown-linux" +# elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then +# # Note: v0.7.0 only has x86_64 Linux binary, no ARM64 Linux binary available +# echo "⚠️ ARM64 Linux binary not available, using x86_64 (may require emulation)" +# DRE_BINARY="dre-x86_64-unknown-linux" +# else +# echo "❌ Unsupported architecture: $ARCH" +# exit 1 +# fi - # Download from GitHub releases - DRE_URL="https://github.com/dfinity/dre/releases/download/v${DRE_VERSION}/dre-${DRE_ARCH}-unknown-linux-gnu" - echo "Downloading from: $DRE_URL" +# # Download from GitHub releases +# DRE_URL="https://github.com/dfinity/dre/releases/download/v${DRE_VERSION}/${DRE_BINARY}" +# echo "Downloading from: $DRE_URL" - curl -L -o /usr/local/bin/dre "$DRE_URL" +# curl -L -o /usr/local/bin/dre "$DRE_URL" +# chmod +x /usr/local/bin/dre + +# # Verify it works +# if /usr/local/bin/dre --version; then +# echo "✅ dre v${DRE_VERSION} installed successfully" +# touch /scheduler/.dre_installed +# else +# echo "❌ Failed to install dre" +# exit 1 +# fi +# else +# echo "✅ dre already installed" +# fi + +# Use locally built dre binary mounted from host +echo "Using locally built dre binary from /dre-binary/dre" + +if [ -f "/dre-binary/dre" ]; then + cp /dre-binary/dre /usr/local/bin/dre chmod +x /usr/local/bin/dre # Verify it works if /usr/local/bin/dre --version; then - echo "✅ dre v${DRE_VERSION} installed successfully" - touch /scheduler/.dre_installed + echo "✅ Local dre binary installed successfully" else - echo "❌ Failed to install dre" + echo "❌ Failed to run local dre binary" exit 1 fi else - echo "✅ dre already installed" + echo "❌ Local dre binary not found at /dre-binary/dre" + echo "Please build the binary with: cd /Users/pietro.di.marco/dre && cargo build --release --bin dre" + exit 1 fi # Wait for VictoriaMetrics to be ready diff --git a/docker/config/prometheus/.gitignore b/docker/config/victoria-metrics/.gitignore similarity index 100% rename from docker/config/prometheus/.gitignore rename to docker/config/victoria-metrics/.gitignore diff --git a/docker/config/prometheus/config-template.yaml b/docker/config/victoria-metrics/config-template.yaml similarity index 100% rename from docker/config/prometheus/config-template.yaml rename to docker/config/victoria-metrics/config-template.yaml diff --git a/docker/docker-compose.tools.yaml b/docker/docker-compose.tools.yaml index 1a48db6..b5742a1 100644 --- a/docker/docker-compose.tools.yaml +++ b/docker/docker-compose.tools.yaml @@ -4,6 +4,6 @@ services: context: . dockerfile: ./tools/prom-config-builder/prom-config-builder.Dockerfile volumes: - - ./config/prometheus:/config/prometheus + - ./config/victoria-metrics:/config/victoria-metrics - ./tools/prom-config-builder/:/tools/prom-config-builder diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f014358..527be9f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -20,8 +20,7 @@ services: command: - --storageDataPath=/victoria-metrics-data - --httpListenAddr=:9090 - - --retentionPeriod=6M - user: "${UID}:${GID}" + - --retentionPeriod=1y deploy: resources: limits: @@ -31,6 +30,7 @@ services: node-rewards-scheduler: image: ubuntu:24.04 + platform: linux/amd64 network_mode: host environment: VICTORIA_METRICS_URL: http://localhost:9090 diff --git a/docker/tools/prom-config-builder/prom_config_builder.py b/docker/tools/prom-config-builder/prom_config_builder.py index 8108efa..cd188fc 100644 --- a/docker/tools/prom-config-builder/prom_config_builder.py +++ b/docker/tools/prom-config-builder/prom_config_builder.py @@ -7,21 +7,21 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( "prom-config-builder", - description="Used to expand the configuration to actual prometheus config", + description="Used to expand the configuration to actual victoria-metrics config", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--template-path", dest="template_path", - default="/config/prometheus/config-template.yaml", + default="/config/victoria-metrics/config-template.yaml", help="Path to the config template that should be expanded", ) parser.add_argument( "--output-path", dest="output_path", - default="/config/prometheus/config.yaml", + default="/config/victoria-metrics/config.yaml", help="Path to where the output should be written", ) diff --git a/docker/volumes/.DS_Store b/docker/volumes/.DS_Store index 55bcf86daa5c263ebe3e543bc564ee551912c9cb..9781eee8358ac447e513fabca0daba2942bde072 100644 GIT binary patch delta 93 zcmZoMXffEZi;0Elmd%FA`?IX|}mD9galzhSc?^D~x-4Sbu~IsWnk0G(JJ2><{9 delta 93 zcmZoMXffEZi;0CncH+#*`SZ*7Q From 8d525c05401628b1859716c0fcf776e8914c147e Mon Sep 17 00:00:00 2001 From: Pietro Date: Wed, 12 Nov 2025 13:47:04 +0100 Subject: [PATCH 4/4] Add stuff --- .DS_Store | Bin 8196 -> 8196 bytes .gitignore | 2 +- docker/.DS_Store | Bin 6148 -> 6148 bytes .../node-rewards/node-rewards-dashboard.json | 4366 +++++++---------- .../datasources/observability.yaml | 11 +- .../node-rewards-scheduler/entrypoint.sh | 2 - .../victoria-metrics/config-template.yaml | 1 - docker/docker-compose.yaml | 6 + docker/volumes/.DS_Store | Bin 6148 -> 6148 bytes docker/volumes/grafana/.gitignore | 5 - 10 files changed, 1837 insertions(+), 2556 deletions(-) delete mode 100755 docker/volumes/grafana/.gitignore diff --git a/.DS_Store b/.DS_Store index 61fc3ac3dc83233a098f3c836e383a984af99de7..cc464aa3ae71464e8d0ed304d3aca12d8c817362 100644 GIT binary patch delta 130 zcmZp1XmQw}DiF`B<;=jqz`~%%kj{|FP?DSP;*yk;p9B=+Fxi*+#wF>vBdUA~UipFy u!{Frn+ybB;1_t2;43pOhNNt`cpv1_coshO_@)5xlW((d8o3({*@&W*LOCqlT delta 130 zcmZp1XmQw}DiF_eZaM=40}F#5LpnnyLrHGFi%U{YeiBfO!=>|H?SsVQj;Qh}c;yQ+ u41<&Na|?ia7#IW%7$&b1klH*?K#7q>nsa^LOJ(R8v7OB!Rr=E*Gjj4aw=AqO Svjx+HjfGbjH?wp6 delta 68 zcmZoMXffEJ$->OVlQ~(3C5`dG=E*Gjj4V?BounrpW0hlN^ILoB*kpb-(aHZ= 98)", - "format": "time_series", - "instant": true, - "legendFormat": "98-100%", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) >= 95 < 98)", - "format": "time_series", - "instant": true, - "legendFormat": "95-98%", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) >= 90 < 95)", - "format": "time_series", - "instant": true, - "legendFormat": "90-95%", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(((latest_total_adjusted_rewards_xdr_permyriad / latest_total_base_rewards_xdr_permyriad) * 100) < 90)", - "format": "time_series", - "instant": true, - "legendFormat": "<90%", - "refId": "D" - } - ], - "title": "Provider Efficiency Distribution", - "type": "barchart" + "filters": [], + "name": "Filters", + "type": "adhoc" } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 42, - "tags": [ - "node-rewards", - "provider", - "rewards", - "performance" - ], - "templating": { - "list": [ - { - "current": { - "text": "IC Metrics (cluster local)", - "value": "PE62C54679EC3C073" - }, - "includeAll": false, - "label": "Data Source", - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(latest_nodes_count, provider_id)", - "includeAll": true, - "label": "Provider", - "multi": true, - "name": "provider", - "options": [], - "query": { - "query": "label_values(latest_nodes_count, provider_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", - "includeAll": true, - "label": "Node", - "multi": true, - "name": "node", - "options": [], - "query": { - "query": "label_values(latest_original_failure_rate{provider_id=~\"$provider\"}, node_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(subnet_failure_rate, subnet_id)", - "includeAll": true, - "label": "Subnet", - "multi": true, - "name": "subnet", - "options": [], - "query": { - "query": "label_values(subnet_failure_rate, subnet_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "baseFilters": [], - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "filters": [], - "name": "Filters", - "type": "adhoc" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "utc", - "title": "Node Provider Rewards Dashboard - Comprehensive", - "uid": "node-rewards-comprehensive", - "version": 1 - } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Node Provider Rewards Dashboard - Comprehensive", + "uid": "node-rewards-comprehensive", + "version": 2 +} \ No newline at end of file diff --git a/docker/config/grafana/provisioning/datasources/observability.yaml b/docker/config/grafana/provisioning/datasources/observability.yaml index 2554313..23e4af6 100644 --- a/docker/config/grafana/provisioning/datasources/observability.yaml +++ b/docker/config/grafana/provisioning/datasources/observability.yaml @@ -1,12 +1,19 @@ apiVersion: 1 datasources: - - name: VictoriaMetrics + - name: prometheus uid: prometheus type: prometheus access: proxy url: http://$PROM_HOST:9090 + jsonData: + httpMethod: POST + - name: VictoriaMetrics1d + uid: victoriametrics1d + type: prometheus + access: proxy + url: http://$PROM_HOST:9090 isDefault: true jsonData: httpMethod: POST - timeInterval: "1d" \ No newline at end of file + timeInterval: "1d" diff --git a/docker/config/node-rewards-scheduler/entrypoint.sh b/docker/config/node-rewards-scheduler/entrypoint.sh index f307682..2ae2ba4 100755 --- a/docker/config/node-rewards-scheduler/entrypoint.sh +++ b/docker/config/node-rewards-scheduler/entrypoint.sh @@ -63,12 +63,10 @@ if [ -f "/dre-binary/dre" ]; then echo "✅ Local dre binary installed successfully" else echo "❌ Failed to run local dre binary" - exit 1 fi else echo "❌ Local dre binary not found at /dre-binary/dre" echo "Please build the binary with: cd /Users/pietro.di.marco/dre && cargo build --release --bin dre" - exit 1 fi # Wait for VictoriaMetrics to be ready diff --git a/docker/config/victoria-metrics/config-template.yaml b/docker/config/victoria-metrics/config-template.yaml index 25da4d1..9651235 100644 --- a/docker/config/victoria-metrics/config-template.yaml +++ b/docker/config/victoria-metrics/config-template.yaml @@ -28,7 +28,6 @@ anchors: http_configs: &http_configs http_sd_configs: - url: http://multiservice-discovery:8000/prom/targets - refresh_interval: 15s global: # Change this to control how frequently prometheus diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 527be9f..425b8a2 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -16,11 +16,16 @@ services: image: victoriametrics/victoria-metrics:v1.97.1 network_mode: host volumes: + - ./config/victoria-metrics:/config - ./volumes/victoriametrics:/victoria-metrics-data command: - --storageDataPath=/victoria-metrics-data - --httpListenAddr=:9090 - --retentionPeriod=1y + - --promscrape.config=/config/config.yaml + - --promscrape.configCheckInterval=30s + - --enableTCP6=true + user: "${UID}:${GID}" deploy: resources: limits: @@ -39,6 +44,7 @@ services: - ./volumes/node-rewards-scheduler:/scheduler - ./config/node-rewards-scheduler/entrypoint.sh:/entrypoint.sh:ro entrypoint: /entrypoint.sh + user: "${UID}:${GID}" depends_on: - victoriametrics diff --git a/docker/volumes/.DS_Store b/docker/volumes/.DS_Store index 9781eee8358ac447e513fabca0daba2942bde072..a8f0fa7f99ec2dd93adbf2f199bbe481acb017ef 100644 GIT binary patch delta 30 mcmZoMXffEZhl$Da@Z^0=!At@Hn-!U#u}o~>+sw}KmmdJC4how9 delta 30 mcmZoMXffEZhl$B%!{mKT!A$%cHY+keW0}~%x0#*eFFydRk_y@Y diff --git a/docker/volumes/grafana/.gitignore b/docker/volumes/grafana/.gitignore deleted file mode 100755 index e769a26..0000000 --- a/docker/volumes/grafana/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore everything in this directory -* - -# But keep this .gitignore file -!.gitignore