feat(toolkit): exclude historical balance DBs from lite snapshot

.gitignore

@@ -57,3 +57,6 @@ Wallet
 
 /framework/propPath
 .cache
+
+#AI
+.claude

plugins/src/main/java/common/org/tron/plugins/DbLite.java

@@ -6,6 +6,7 @@
 import com.google.common.primitives.Bytes;
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
+import com.google.protobuf.ByteString;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -30,6 +31,7 @@
 import org.tron.plugins.utils.db.DBIterator;
 import org.tron.plugins.utils.db.DbTool;
 import org.tron.protos.Protocol;
+import org.tron.protos.contract.BalanceContract;
 import picocli.CommandLine;
 
 @Slf4j(topic = "lite")
@@ -57,13 +59,17 @@ public class DbLite implements Callable<Integer> {
   private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore";
   private static final String PROPERTIES_DB_NAME = "properties";
   private static final String TRANS_CACHE_DB_NAME = "trans-cache";
+  private static final String BALANCE_TRACE_DB_NAME = "balance-trace";
+  private static final String ACCOUNT_TRACE_DB_NAME = "account-trace";
 
   private static final List<String> archiveDbs = Arrays.asList(
       BLOCK_DB_NAME,
       BLOCK_INDEX_DB_NAME,
       TRANS_DB_NAME,
       TRANSACTION_RET_DB_NAME,
-      TRANSACTION_HISTORY_DB_NAME);
+      TRANSACTION_HISTORY_DB_NAME,
+      BALANCE_TRACE_DB_NAME,
+      ACCOUNT_TRACE_DB_NAME);
 
   enum Operate { split, merge }
 
@@ -522,24 +528,42 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo)
     DBInterface blockDb = DbTool.getDB(liteDir, BLOCK_DB_NAME);
     DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME);
     DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME);
-
+    DBInterface balanceTraceDb = DbTool.getDB(liteDir, BALANCE_TRACE_DB_NAME);
+    DBInterface accountTraceDb = DbTool.getDB(liteDir, ACCOUNT_TRACE_DB_NAME);
 
     ProgressBar.wrap(LongStream.rangeClosed(start, end)
         .boxed()
-        .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> {
+        .sorted((a, b) -> Long.compare(b, a)), "trimHistory")
+        .map(ByteArray::fromLong).forEach(n -> {
           try {
-            byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n));
+            byte[] blockIdHash = blockIndexDb.get(n);
             Protocol.Block block = Protocol.Block.parseFrom(blockDb.get(blockIdHash));
             // delete transactions
             for (Protocol.Transaction e : block.getTransactionsList()) {
               transDb.delete(DBUtils.getTransactionId(e).getBytes());
             }
             // delete transaction result
-            tranRetDb.delete(ByteArray.fromLong(n));
+            tranRetDb.delete(n);
             // delete block
             blockDb.delete(blockIdHash);
             // delete block index
-            blockIndexDb.delete(ByteArray.fromLong(n));
+            blockIndexDb.delete(n);
+            byte[] balanceTrace = balanceTraceDb.get(n);
+            if (balanceTrace != null) {
+              // delete account trace
+              long blockNum = ByteArray.toLong(n);
+              BalanceContract.BlockBalanceTrace.parseFrom(balanceTrace)
+                  .getTransactionBalanceTraceList().stream()
+                  .flatMap(tx -> tx.getOperationList().stream())
+                  .map(BalanceContract.TransactionBalanceTrace.Operation::getAddress)
+                  .distinct()
+                  .map(ByteString::toByteArray)
+                  .map(address -> Bytes.concat(address, Longs.toByteArray(
+                      blockNum ^ Long.MAX_VALUE)))
+                  .forEach(accountTraceDb::delete);
+              // delete balance trace
+              balanceTraceDb.delete(n);
+            }
           } catch (IOException e) {
             throw new RuntimeException(e);
           }
@@ -570,7 +594,8 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
         DBInterface bakDb = DbTool.getDB(bakDir.toString(), dbName);
         DBInterface destDb = DbTool.getDB(liteDir, dbName);
         try (DBIterator iterator = bakDb.iterator()) {
-          if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)) {
+          if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)
+              || ACCOUNT_TRACE_DB_NAME.equals(dbName)) {
             iterator.seekToFirst();
           } else {
             iterator.seek(head);

plugins/src/test/java/org/tron/plugins/DbLiteTest.java

@@ -89,7 +89,13 @@ public void clear() {
 
   public void testTools(String dbType, int checkpointVersion)
       throws InterruptedException, IOException {
-    logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion);
+    testTools(dbType, checkpointVersion, false, false);
+  }
+
+  public void testTools(String dbType, int checkpointVersion, boolean advanceSnapshot,
+      boolean historyBalanceLookup) throws InterruptedException, IOException {
+    logger.info("dbType {}, checkpointVersion {}, advanceSnapshot {}, historyBalanceLookup {}",
+        dbType, checkpointVersion, advanceSnapshot, historyBalanceLookup);
     dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis());
     init(dbType);
     final String[] argsForSnapshot =
@@ -104,6 +110,7 @@ public void testTools(String dbType, int checkpointVersion)
         new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir,
             "--dataset-path", dbPath + File.separator + "history"};
     Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion);
+    Args.getInstance().setHistoryBalanceLookup(historyBalanceLookup);
     DbLite.setRecentBlks(3);
     // start fullNode
     startApp();
@@ -117,8 +124,7 @@ public void testTools(String dbType, int checkpointVersion)
     cli.execute(argsForSnapshot);
     // start fullNode
     startApp();
-    // produce transactions
-    generateSomeTransactions(checkpointVersion == 1 ? 6 : 18);
+    generateSomeTransactions(advanceSnapshot ? (checkpointVersion == 1 ? 6 : 18) : 18);
     // stop the node
     shutdown();
     // generate history
@@ -137,11 +143,11 @@ public void testTools(String dbType, int checkpointVersion)
           String.format("rename snapshot to %s failed",
               Paths.get(dbPath, databaseDir)));
     }
-    // start and validate the snapshot
-    startApp();
-    generateSomeTransactions(checkpointVersion == 1 ? 18 : 6);
-    // stop the node
-    shutdown();
+    if (advanceSnapshot) {
+      startApp();
+      generateSomeTransactions(checkpointVersion == 1 ? 18 : 6);
+      shutdown();
+    }
     // merge history
     cli.execute(argsForMerge);
     // start and validate

plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java

@@ -8,6 +8,6 @@ public class DbLiteRocksDbV2Test extends DbLiteTest {
 
   @Test
   public void testToolsWithRocksDB() throws InterruptedException, IOException {
-    testTools("ROCKSDB", 2);
+    testTools("ROCKSDB", 2, true, false);
   }
 }

plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbTest.java

@@ -0,0 +1,13 @@
+package org.tron.plugins.rocksdb;
+
+import java.io.IOException;
+import org.junit.Test;
+import org.tron.plugins.DbLiteTest;
+
+public class DbLiteWithHistoryRocksDbTest extends DbLiteTest {
+
+  @Test
+  public void testToolsWithTrimHistory() throws InterruptedException, IOException {
+    testTools("ROCKSDB", 1, true, true);
+  }
+}

plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteWithHistoryRocksDbV2Test.java

@@ -0,0 +1,13 @@
+package org.tron.plugins.rocksdb;
+
+import java.io.IOException;
+import org.junit.Test;
+import org.tron.plugins.DbLiteTest;
+
+public class DbLiteWithHistoryRocksDbV2Test extends DbLiteTest {
+
+  @Test
+  public void testToolsWithTrimHistory() throws InterruptedException, IOException {
+    testTools("ROCKSDB", 2, false, true);
+  }
+}