Paul/add edge asset

package.json

@@ -78,6 +78,7 @@
     "@typescript-eslint/eslint-plugin": "^5.36.2",
     "@typescript-eslint/parser": "^5.36.2",
     "assert": "^2.0.0",
+    "async-mutex": "^0.5.0",
     "browserify-zlib": "^0.2.0",
     "chai": "^4.3.4",
     "eslint": "^8.19.0",

src/bin/destroyPartition.ts

@@ -3,7 +3,7 @@ import js from 'jsonfile'
 import nano from 'nano'
 
 import { pagination } from '../dbutils'
-import { datelog } from '../util'
+import { createScopedLog, datelog } from '../util'
 
 const config = js.readFileSync('./config.json')
 const nanoDb = nano(config.couchDbFullpath)
@@ -43,10 +43,11 @@ async function main(partitionName: string): Promise<void> {
     return
   }
 
+  const log = createScopedLog('destroy', partitionName)
   try {
-    await pagination(transactions, reportsTransactions)
-    datelog(`Successfully Deleted: ${transactions.length} docs`)
-    datelog(`Successfully Deleted: partition ${partitionName}`)
+    await pagination(transactions, reportsTransactions, log)
+    log(`Successfully Deleted: ${transactions.length} docs`)
+    log(`Successfully Deleted: partition ${partitionName}`)
 
     // Delete progress Cache
     const split = partitionName.split('_')

src/bin/fioPromo/fioLookup.ts

@@ -5,6 +5,7 @@ import path from 'path'
 
 import { queryChangeNow } from '../../partners/changenow'
 import { PluginParams, StandardTx } from '../../types'
+import { createScopedLog } from '../../util'
 import { defaultSettings } from './fioInfo'
 
 let addressList: string[] = []
@@ -48,12 +49,14 @@ export async function getFioTransactions(
   dateFrom: Date,
   dateTo: Date
 ): Promise<StandardTx[]> {
+  const log = createScopedLog('fio', 'changenow')
   // Get public keys from offset
   const pluginConfig: PluginParams = {
     settings: { dateFrom, dateTo, to: currencyCode },
     apiKeys: {
       changenowApiKey: config.changenowApiKey
-    }
+    },
+    log
   }
 
   const txnList = await queryChangeNow(pluginConfig)

src/bin/fixTokenIds.ts

@@ -0,0 +1,222 @@
+import js from 'jsonfile'
+import nano from 'nano'
+
+import { DbTx } from '../types'
+import { datelog } from '../util'
+import { createTokenId, tokenTypes } from '../util/asEdgeTokenId'
+
+const config = js.readFileSync('./config.json')
+const nanoDb = nano(config.couchDbFullpath)
+
+const QUERY_LIMIT = 500
+const DRY_RUN = process.argv.includes('--dry-run')
+
+const ALL_PARTITIONS = [
+  'edge_banxa',
+  'edge_bitaccess',
+  'edge_bitrefill',
+  'edge_bitsofgold',
+  'edge_bity',
+  'edge_changelly',
+  'edge_changehero',
+  'edge_changenow',
+  'edge_coinswitch',
+  'edge_exolix',
+  'edge_faast',
+  'edge_foxExchange',
+  'edge_godex',
+  'edge_kado',
+  'edge_letsexchange',
+  'edge_libertyx',
+  'edge_lifi',
+  'edge_moonpay',
+  'edge_paybis',
+  'edge_paytrie',
+  'edge_rango',
+  'edge_safello',
+  'edge_shapeshift',
+  'edge_sideshift',
+  'edge_simplex',
+  'edge_swapuz',
+  'edge_switchain',
+  'edge_thorchain',
+  'edge_totle',
+  'edge_transak',
+  'edge_wyre',
+  'edge_xanpool'
+]
+
+/**
+ * Re-derive the tokenId using createTokenId. Treats the existing tokenId
+ * value as a raw contract address and normalizes it. Returns undefined
+ * when the value is already correct or cannot be fixed.
+ */
+function normalizeTokenId(
+  pluginId: string | undefined,
+  currencyCode: string,
+  tokenId: string | null | undefined
+): string | null | undefined {
+  if (tokenId == null || pluginId == null) return undefined
+
+  const tokenType = tokenTypes[pluginId] ?? null
+  if (tokenType == null) return undefined // chain doesn't support tokens
+
+  try {
+    const normalized = createTokenId(tokenType, currencyCode, tokenId)
+    return normalized !== tokenId ? normalized : undefined
+  } catch {
+    return undefined
+  }
+}
+
+interface FixStats {
+  scanned: number
+  fixed: number
+  errors: number
+  byPartition: Record<string, number>
+  byField: { deposit: number; payout: number }
+}
+
+fixTokenIds().catch(e => {
+  datelog(e)
+  process.exit(1)
+})
+
+async function fixTokenIds(): Promise<void> {
+  const reportsTransactions = nanoDb.use('reports_transactions')
+
+  const stats: FixStats = {
+    scanned: 0,
+    fixed: 0,
+    errors: 0,
+    byPartition: {},
+    byField: { deposit: 0, payout: 0 }
+  }
+
+  const partitions =
+    process.argv[2] != null && process.argv[2] !== '--dry-run'
+      ? [process.argv[2]]
+      : ALL_PARTITIONS
+
+  if (DRY_RUN) datelog('DRY RUN — no changes will be written')
+
+  for (const partition of partitions) {
+    datelog(`Scanning partition: ${partition}`)
+    let bookmark: string | undefined
+
+    while (true) {
+      const query: any = {
+        selector: {
+          $or: [
+            { depositTokenId: { $type: 'string' } },
+            { payoutTokenId: { $type: 'string' } }
+          ]
+        },
+        bookmark,
+        limit: QUERY_LIMIT
+      }
+
+      let result: any
+      try {
+        result = await reportsTransactions.partitionedFind(partition, query)
+      } catch (e) {
+        // Partition may not exist
+        const err = e as { statusCode?: number }
+        if (err.statusCode === 404) {
+          datelog(`  Partition ${partition} not found, skipping`)
+          break
+        }
+        throw e
+      }
+
+      const { docs } = result
+      if (docs.length === 0) break
+
+      if (typeof result.bookmark === 'string' && docs.length === QUERY_LIMIT) {
+        bookmark = result.bookmark
+      } else {
+        bookmark = undefined
+      }
+
+      stats.scanned += docs.length
+
+      const docsToUpdate: any[] = []
+
+      for (const d of docs) {
+        const doc: DbTx = d
+        let changed = false
+
+        const fixedDeposit = normalizeTokenId(
+          doc.depositChainPluginId,
+          doc.depositCurrency,
+          doc.depositTokenId
+        )
+        if (fixedDeposit !== undefined) {
+          datelog(
+            `  FIX ${doc._id} depositTokenId: "${doc.depositTokenId}" -> "${fixedDeposit}"`
+          )
+          doc.depositTokenId = fixedDeposit
+          changed = true
+          stats.byField.deposit++
+        }
+
+        const fixedPayout = normalizeTokenId(
+          doc.payoutChainPluginId,
+          doc.payoutCurrency,
+          doc.payoutTokenId
+        )
+        if (fixedPayout !== undefined) {
+          datelog(
+            `  FIX ${doc._id} payoutTokenId: "${doc.payoutTokenId}" -> "${fixedPayout}"`
+          )
+          doc.payoutTokenId = fixedPayout
+          changed = true
+          stats.byField.payout++
+        }
+
+        if (changed) {
+          doc.usdValue = -1
+          docsToUpdate.push(d)
+          stats.byPartition[partition] = (stats.byPartition[partition] ?? 0) + 1
+        }
+      }
+
+      if (docsToUpdate.length > 0) {
+        stats.fixed += docsToUpdate.length
+        if (!DRY_RUN) {
+          try {
+            await reportsTransactions.bulk({ docs: docsToUpdate })
+            datelog(`  Wrote ${docsToUpdate.length} fixed docs in ${partition}`)
+          } catch (e) {
+            datelog(`  Error writing bulk update for ${partition}:`, e)
+            stats.errors++
+          }
+        } else {
+          datelog(
+            `  Would write ${docsToUpdate.length} fixed docs in ${partition}`
+          )
+        }
+      }
+
+      if (bookmark == null) break
+    }
+  }
+
+  datelog('\n=== Summary ===')
+  datelog(`Total documents scanned: ${stats.scanned}`)
+  datelog(`Total documents fixed: ${stats.fixed}`)
+  datelog(`  Deposit tokenIds fixed: ${stats.byField.deposit}`)
+  datelog(`  Payout tokenIds fixed: ${stats.byField.payout}`)
+  if (Object.keys(stats.byPartition).length > 0) {
+    datelog('Fixes by partition:')
+    for (const [partition, count] of Object.entries(stats.byPartition)) {
+      datelog(`  ${partition}: ${count}`)
+    }
+  }
+  if (stats.errors > 0) {
+    datelog(`Errors encountered: ${stats.errors}`)
+  }
+  if (DRY_RUN) {
+    datelog('DRY RUN complete — no changes were made')
+  }
+}

src/bin/resetUsdValues.ts

@@ -0,0 +1,69 @@
+import js from 'jsonfile'
+import nano from 'nano'
+
+import { DbTx } from '../types'
+import { datelog } from '../util'
+
+const config = js.readFileSync('./config.json')
+const nanoDb = nano(config.couchDbFullpath)
+
+const QUERY_LIMIT = 1000
+
+resetUsdValues().catch(e => {
+  datelog(e)
+  process.exit(1)
+})
+
+async function resetUsdValues(): Promise<void> {
+  const partitionName = process.argv[2]
+  if (partitionName == null) {
+    console.error('Usage: resetUsdValues <partitionName>')
+    process.exit(1)
+  }
+
+  const reportsTransactions = nanoDb.use('reports_transactions')
+  let bookmark: string | undefined
+  let totalUpdated = 0
+
+  while (true) {
+    const query: any = {
+      selector: {
+        usdValue: { $gte: 0 }
+      },
+      bookmark,
+      limit: QUERY_LIMIT
+    }
+
+    const result: any = await reportsTransactions.partitionedFind(
+      partitionName,
+      query
+    )
+    const { docs } = result
+    if (docs.length === 0) break
+
+    if (typeof result.bookmark === 'string' && docs.length === QUERY_LIMIT) {
+      bookmark = result.bookmark
+    } else {
+      bookmark = undefined
+    }
+
+    for (const d of docs) {
+      const doc: DbTx = d
+      doc.usdValue = -1
+    }
+
+    try {
+      await reportsTransactions.bulk({ docs })
+      totalUpdated += docs.length
+      datelog(`Updated ${docs.length} docs (${totalUpdated} total)`)
+    } catch (e) {
+      datelog('Error doing bulk update', e)
+    }
+
+    if (bookmark == null) break
+  }
+
+  datelog(
+    `Done. Set usdValue to -1 for ${totalUpdated} docs in ${partitionName}`
+  )
+}

src/bin/testpartner.ts

@@ -1,16 +1,28 @@
-import { thorchain as plugin } from '../partners/thorchain'
-import { PluginParams } from '../types.js'
+import { PluginParams } from '../types'
+import { createScopedLog } from '../util'
 
-const pluginParams: PluginParams = {
-  settings: {
-    offset: 0
-  },
-  apiKeys: {
-    thorchainAddress: ''
+async function main(): Promise<void> {
+  const partnerId = process.argv[2]
+  if (partnerId == null) {
+    console.log(
+      'Usage: node -r sucrase/register src/bin/testpartner.ts <partnerId>'
+    )
+    process.exit(1)
+  }
+
+  const pluginParams: PluginParams = {
+    settings: {},
+    apiKeys: {},
+    log: createScopedLog('edge', partnerId)
+  }
+
+  // Dynamically import the partner plugin
+  const pluginModule = await import(`../partners/${partnerId}`)
+  const plugin = pluginModule[partnerId]
+  if (plugin?.queryFunc == null) {
+    throw new Error(`Plugin ${partnerId} does not have a queryFunc`)
   }
-}
 
-async function main(): Promise<void> {
   const result = await plugin.queryFunc(pluginParams)
   console.log(JSON.stringify(result, null, 2))
 }