diff --git a/.github/workflows/backfill_aura_split.yaml b/.github/workflows/backfill_aura_split.yaml new file mode 100644 index 0000000..c1e573b --- /dev/null +++ b/.github/workflows/backfill_aura_split.yaml @@ -0,0 +1,46 @@ +name: Backfill Aura/BAL Split + +on: + schedule: + - cron: "0 8 * * 4" # Weekly Thursday 08:00 UTC + workflow_dispatch: + +jobs: + backfill: + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: biweekly-runs + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip3 install -r requirements.txt + + - name: Run backfill + run: python3 backfill_recon_aura_split.py + + - name: Create PR + uses: peter-evans/create-pull-request@v8 + with: + commit-message: "task: backfill aura/bal split data in recon" + title: "Backfill Aura/BAL Split in Recon" + body: | + Automated backfill of `auraIncentives`, `balIncentives`, and `auravebalShare` fields + in recon JSON files using VoteMarket analytics data. + + Entries with `auraIncentives == 0` and `totalIncentives > 0` were updated. + branch: gha-aura-split-backfill + branch-suffix: timestamp + delete-branch: true + labels: Aura-Split-Backfill diff --git a/backfill_recon_aura_split.py b/backfill_recon_aura_split.py new file mode 100644 index 0000000..c9f5354 --- /dev/null +++ b/backfill_recon_aura_split.py @@ -0,0 +1,175 @@ +import argparse +import datetime +import json +from decimal import Decimal +from pathlib import Path + +import pandas as pd + +from fee_allocator.votemarket_analytics import get_aura_share_per_gauge +from fee_allocator.logger import logger + +PROJECT_ROOT = Path(__file__).parent +SUMMARIES_DIR = PROJECT_ROOT / "fee_allocator" / "summaries" +INCENTIVES_DIR = PROJECT_ROOT / "fee_allocator" / "allocations" / "incentives" +BRIBES_DIR = PROJECT_ROOT / "fee_allocator" / "allocations" / "output_for_msig" + +STAKEDAO_MIGRATION_PERIOD_START = 1767225600 + + +def _ts_to_date_str(ts: int) -> str: + return datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc).strftime("%Y-%m-%d") + + +def _load_gauge_data_from_incentives_csv(csv_path: Path) -> pd.DataFrame: + df = pd.read_csv(csv_path) + if "gauge_address" in df.columns and "voting_pool_override" in df.columns: + return df + return None + + +def _load_gauge_data_from_bribe_csv(csv_path: Path) -> pd.DataFrame: + if not csv_path.exists(): + return None + df = pd.read_csv(csv_path) + if "target" not in df.columns: + return None + bribe_rows = df[df["platform"].isna()] if "platform" in df.columns else df + bribe_rows = bribe_rows[bribe_rows["amount"] > 0].copy() + if bribe_rows.empty: + return None + bribe_rows = bribe_rows.rename(columns={"target": "gauge_address", "amount": "total_incentives"}) + if "voting_pool_override" not in bribe_rows.columns: + bribe_rows["voting_pool_override"] = "" + bribe_rows["voting_pool_override"] = bribe_rows["voting_pool_override"].fillna("") + return bribe_rows + + +def _compute_aura_split(gauge_data: pd.DataFrame, gauge_aura_shares: dict) -> pd.DataFrame: + aura_list = [] + bal_list = [] + for _, row in gauge_data.iterrows(): + total = Decimal(str(row.get("total_incentives", 0))) + override = str(row.get("voting_pool_override", "")).strip() + gauge = str(row.get("gauge_address", "")).strip().lower() + + if override == "aura": + aura_share = Decimal(1) + elif override == "bal": + aura_share = Decimal(0) + elif gauge: + aura_share = gauge_aura_shares.get(gauge, Decimal(0)) + else: + aura_share = Decimal(0) + + aura_list.append(float(round(total * aura_share, 4))) + bal_list.append(float(round(total - total * aura_share, 4))) + + gauge_data = gauge_data.copy() + gauge_data["aura_incentives"] = aura_list + gauge_data["bal_incentives"] = bal_list + return gauge_data + + +def _get_total_incentives(entry: dict) -> float: + if "totalIncentives" in entry: + return entry["totalIncentives"] + aura = entry.get("auraIncentives", 0) or 0 + bal = entry.get("balIncentives", 0) or 0 + return aura + bal + + +def backfill(dry_run: bool = False): + for version in ["v2", "v3"]: + recon_path = SUMMARIES_DIR / f"{version}_recon.json" + if not recon_path.exists(): + logger.info(f"No recon file for {version}, skipping") + continue + + with open(recon_path) as f: + data = json.load(f) + + modified = False + for entry in data: + if entry["periodStart"] < STAKEDAO_MIGRATION_PERIOD_START: + continue + + total_incentives = _get_total_incentives(entry) + aura_incentives = entry.get("auraIncentives", 0) or 0 + bal_incentives = entry.get("balIncentives", 0) or 0 + + if (aura_incentives + bal_incentives) != 0: + continue + + if total_incentives == 0: + if "auraIncentives" not in entry: + entry["auraIncentives"] = 0.0 + entry["balIncentives"] = 0.0 + entry["auravebalShare"] = 0 + entry["auraIncentivesPct"] = 0.0 + entry["balIncentivesPct"] = 0.0 + modified = True + continue + + period_start = entry["periodStart"] + period_end = entry["periodEnd"] + start_str = _ts_to_date_str(period_start) + end_str = _ts_to_date_str(period_end) + + logger.info(f"[{version}] Processing period {start_str} to {end_str}") + + gauge_aura_shares = get_aura_share_per_gauge(period_start, period_end) + if not gauge_aura_shares: + logger.info(f"[{version}] No VoteMarket data for {start_str}_{end_str}, skipping") + continue + + incentives_csv = INCENTIVES_DIR / f"{version}_incentives_{start_str}_{end_str}.csv" + gauge_data = None + + if incentives_csv.exists(): + gauge_data = _load_gauge_data_from_incentives_csv(incentives_csv) + + if gauge_data is None: + end_date = _ts_to_date_str(period_end) + bribe_csv = BRIBES_DIR / f"{version}_bribes_{end_date}.csv" + gauge_data = _load_gauge_data_from_bribe_csv(bribe_csv) + + if gauge_data is None: + logger.warning(f"[{version}] No gauge data found for {start_str}_{end_str}, skipping") + continue + + gauge_data = _compute_aura_split(gauge_data, gauge_aura_shares) + + total_aura = sum(gauge_data["aura_incentives"]) + total_bal = sum(gauge_data["bal_incentives"]) + + logger.info(f"[{version}] {start_str}_{end_str}: aura={total_aura:.2f} bal={total_bal:.2f}") + + if not dry_run: + if incentives_csv.exists(): + full_df = pd.read_csv(incentives_csv) + if "gauge_address" in full_df.columns: + updated = _compute_aura_split(full_df, gauge_aura_shares) + updated.to_csv(incentives_csv, index=False) + logger.info(f"[{version}] Updated incentives CSV: {incentives_csv.name}") + + entry["auraIncentives"] = round(total_aura, 2) + entry["balIncentives"] = round(total_bal, 2) + combined = total_aura + total_bal + entry["auravebalShare"] = round(total_aura / combined, 2) if combined > 0 else 0 + total_distributed = entry.get("totalDistributed", entry.get("incentivesDistributed", 0)) + entry["auraIncentivesPct"] = round(total_aura / total_distributed, 4) if total_distributed > 0 else 0.0 + entry["balIncentivesPct"] = round(total_bal / total_distributed, 4) if total_distributed > 0 else 0.0 + modified = True + + if modified and not dry_run: + with open(recon_path, "w") as f: + json.dump(data, f, indent=2) + logger.info(f"[{version}] Wrote updated recon to {recon_path.name}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Backfill aura/bal split into recon JSON") + parser.add_argument("--dry_run", action="store_true", help="Print what would be done without writing") + args = parser.parse_args() + backfill(dry_run=args.dry_run) diff --git a/fee_allocator/accounting/core_pools.py b/fee_allocator/accounting/core_pools.py index ebbca03..bc2fd2a 100644 --- a/fee_allocator/accounting/core_pools.py +++ b/fee_allocator/accounting/core_pools.py @@ -135,7 +135,11 @@ def _get_partner_info(self): def _get_voting_pool_override(self): pool_override = self.chain.chains.pool_overrides.get(self.pool_id) - return pool_override.voting_pool_override if pool_override else None + if pool_override and pool_override.voting_pool_override: + return pool_override.voting_pool_override + if self.is_alliance_core_pool: + return "aura" + return None def _get_market_override(self): pool_override = self.chain.chains.pool_overrides.get(self.pool_id) diff --git a/fee_allocator/fee_allocator.py b/fee_allocator/fee_allocator.py index 8385c0f..12ac712 100644 --- a/fee_allocator/fee_allocator.py +++ b/fee_allocator/fee_allocator.py @@ -195,20 +195,27 @@ def generate_incentives_csv( self, output_path: Path = Path("fee_allocator/allocations/incentives") ) -> Path: logger.info("generating incentives csv") + output = [] for chain in self.run_config.all_chains: for core_pool in chain.core_pools: + total_incentives = core_pool.total_to_incentives_usd + output.append( { "pool_id": core_pool.pool_id, "chain": chain.name, "symbol": core_pool.symbol, + "gauge_address": core_pool.gauge_address or "", + "voting_pool_override": core_pool.voting_pool_override or "", "bpt_price": round(core_pool.bpt_price, 4), "earned_fees": round(core_pool.total_earned_fees_usd_twap, 4), "fees_to_vebal": round(core_pool.to_vebal_usd, 4), "fees_to_dao": round(core_pool.to_dao_usd, 4), "fees_to_beets": round(core_pool.to_beets_usd, 4), - "total_incentives": round(core_pool.total_to_incentives_usd, 4), + "total_incentives": round(total_incentives, 4), + "aura_incentives": Decimal(0), + "bal_incentives": Decimal(0), "redirected_incentives": round( core_pool.redirected_incentives_usd, 4 ), @@ -219,7 +226,7 @@ def generate_incentives_csv( ) df = pd.DataFrame(output) - + sorted_df = df.sort_values(by=["chain", "earned_fees"], ascending=False) output_path = ( PROJECT_ROOT / output_path / f"{self.run_config.protocol_version}_incentives_{self.start_date}_{self.end_date}.csv" @@ -384,8 +391,10 @@ def generate_bribe_payload( platform = StakeDAOPlatform(self.book, self.run_config) platform.process_bribes(bribe_df, builder, usdc) - usdc.transfer(payment_df["target"], dao_fee_usdc) - usdc.transfer(beets_df["target"], beets_fee_usdc) + if dao_fee_usdc > 0: + usdc.transfer(payment_df["target"], dao_fee_usdc) + if beets_fee_usdc > 0: + usdc.transfer(beets_df["target"], beets_fee_usdc) alliance_fee_usdc_spent = 0 if alliance_csv: @@ -511,6 +520,11 @@ def recon(self) -> None: "feesToVebalPct": float(round(total_vebal / total_distributed, 4)) if total_distributed > 0 else 0, "feesToPartnersPct": float(round(total_partner / total_distributed, 4)) if total_distributed > 0 else 0, "feesToBeetsPct": float(round(total_beets / total_distributed, 4)) if total_distributed > 0 else 0, + "auraIncentives": 0.0, + "balIncentives": 0.0, + "auravebalShare": 0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.0, "createdAt": int(datetime.datetime.now().timestamp()), "periodStart": self.date_range[0], "periodEnd": self.date_range[1], diff --git a/fee_allocator/summaries/v2_recon.json b/fee_allocator/summaries/v2_recon.json index ae395fe..ce1bcb4 100644 --- a/fee_allocator/summaries/v2_recon.json +++ b/fee_allocator/summaries/v2_recon.json @@ -429,7 +429,12 @@ "createdAt": 1768484870, "periodStart": 1767225600, "periodEnd": 1768435200, - "bribeThreshold": 490 + "bribeThreshold": 490, + "auraIncentives": 0.0, + "balIncentives": 0.0, + "auravebalShare": 0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.0 }, { "feesCollected": 17576.73, @@ -450,7 +455,12 @@ "createdAt": 1769701777, "periodStart": 1768435200, "periodEnd": 1769644800, - "bribeThreshold": 490 + "bribeThreshold": 490, + "auraIncentives": 0.0, + "balIncentives": 0.0, + "auravebalShare": 0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.0 }, { "feesCollected": 31650.06, @@ -471,7 +481,12 @@ "createdAt": 1770995647, "periodStart": 1769644800, "periodEnd": 1770854400, - "bribeThreshold": 390 + "bribeThreshold": 390, + "auraIncentives": 0.0, + "balIncentives": 410.32, + "auravebalShare": 0.0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.013 }, { "feesCollected": 18497.17, @@ -492,6 +507,11 @@ "createdAt": 1772132372, "periodStart": 1770854400, "periodEnd": 1772064000, - "bribeThreshold": 410 + "bribeThreshold": 410, + "auraIncentives": 0.0, + "balIncentives": 0.0, + "auravebalShare": 0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.0 } ] \ No newline at end of file diff --git a/fee_allocator/summaries/v3_recon.json b/fee_allocator/summaries/v3_recon.json index 13ba8aa..cd6ee1d 100644 --- a/fee_allocator/summaries/v3_recon.json +++ b/fee_allocator/summaries/v3_recon.json @@ -429,7 +429,12 @@ "createdAt": 1768484931, "periodStart": 1767225600, "periodEnd": 1768435200, - "bribeThreshold": 490 + "bribeThreshold": 490, + "auraIncentives": 0.0, + "balIncentives": 11025.66, + "auravebalShare": 0.0, + "auraIncentivesPct": 0.0, + "balIncentivesPct": 0.398 }, { "feesCollected": 26068.38, @@ -450,7 +455,12 @@ "createdAt": 1769701824, "periodStart": 1768435200, "periodEnd": 1769644800, - "bribeThreshold": 490 + "bribeThreshold": 490, + "auraIncentives": 9207.38, + "balIncentives": 4665.29, + "auravebalShare": 0.66, + "auraIncentivesPct": 0.3532, + "balIncentivesPct": 0.179 }, { "feesCollected": 42204.24, @@ -471,7 +481,12 @@ "createdAt": 1770995685, "periodStart": 1769644800, "periodEnd": 1770854400, - "bribeThreshold": 390 + "bribeThreshold": 390, + "auraIncentives": 9583.32, + "balIncentives": 8902.96, + "auravebalShare": 0.52, + "auraIncentivesPct": 0.2271, + "balIncentivesPct": 0.2109 }, { "feesCollected": 24064.52, @@ -492,6 +507,11 @@ "createdAt": 1772132433, "periodStart": 1770854400, "periodEnd": 1772064000, - "bribeThreshold": 410 + "bribeThreshold": 410, + "auraIncentives": 8383.21, + "balIncentives": 2601.61, + "auravebalShare": 0.76, + "auraIncentivesPct": 0.3484, + "balIncentivesPct": 0.1081 } ] \ No newline at end of file diff --git a/fee_allocator/votemarket_analytics.py b/fee_allocator/votemarket_analytics.py new file mode 100644 index 0000000..f2992bb --- /dev/null +++ b/fee_allocator/votemarket_analytics.py @@ -0,0 +1,58 @@ +from decimal import Decimal +from typing import Dict, List, Tuple +import requests + +from fee_allocator.logger import logger + +BALANCER_METADATA_URL = "https://raw.githubusercontent.com/stake-dao/votemarket-analytics/main/analytics/votemarket-analytics/balancer/rounds-metadata.json" +BALANCER_ROUND_URL = "https://raw.githubusercontent.com/stake-dao/votemarket-analytics/main/analytics/votemarket-analytics/balancer/{round_id}.json" +VLAURA_METADATA_URL = "https://raw.githubusercontent.com/stake-dao/votemarket-analytics/main/analytics/votemarket-analytics/vlaura/balancer/rounds-metadata.json" +VLAURA_ROUND_URL = "https://raw.githubusercontent.com/stake-dao/votemarket-analytics/main/analytics/votemarket-analytics/vlaura/balancer/{round_id}.json" + + +def _fetch_json(url: str) -> dict: + response = requests.get(url) + response.raise_for_status() + return response.json() + + +def _find_matching_rounds(metadata: list, period_start: int, period_end: int) -> List[int]: + return [r["id"] for r in metadata if r["endVoting"] > period_start and r["endVoting"] <= period_end] + + +def _aggregate_votes_per_gauge(round_url_template: str, round_ids: List[int]) -> Dict[str, float]: + votes = {} + for rid in round_ids: + data = _fetch_json(round_url_template.format(round_id=rid)) + for gauge in data["analytics"]: + addr = gauge["gauge"].lower() + votes[addr] = votes.get(addr, 0) + gauge["nonBlacklistedVotes"] + return votes + + +def get_aura_share_per_gauge(period_start: int, period_end: int) -> Dict[str, Decimal]: + bal_metadata = _fetch_json(BALANCER_METADATA_URL) + aura_metadata = _fetch_json(VLAURA_METADATA_URL) + + bal_round_ids = _find_matching_rounds(bal_metadata, period_start, period_end) + aura_round_ids = _find_matching_rounds(aura_metadata, period_start, period_end) + + if not bal_round_ids and not aura_round_ids: + logger.info(f"VoteMarket: no rounds found for period {period_start}-{period_end}") + return {} + + logger.info(f"VoteMarket rounds for period {period_start}-{period_end}: bal={bal_round_ids} aura={aura_round_ids}") + + bal_votes = _aggregate_votes_per_gauge(BALANCER_ROUND_URL, bal_round_ids) + aura_votes = _aggregate_votes_per_gauge(VLAURA_ROUND_URL, aura_round_ids) + + shares = {} + all_gauges = set(bal_votes) | set(aura_votes) + for gauge in all_gauges: + b = bal_votes.get(gauge, 0) + a = aura_votes.get(gauge, 0) + total = b + a + shares[gauge] = Decimal(str(a / total)) if total > 0 else Decimal(0) + + logger.info(f"VoteMarket per-gauge aura shares: { {g[:14]: float(round(s, 4)) for g, s in shares.items()} }") + return shares