Skip to content

fix(indexer): calculate settlement lag from usage_reported_until #619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bajtos wants to merge 1 commit into from
Closed

fix(indexer): calculate settlement lag from usage_reported_until #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/worker-analytics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,9 @@ Each data point contains the following fields:

| Field | Type | Description |
| ------------ | ------ | ------------------------------------------------------------------------------------ |
| `doubles[0]` | number | CDN payments settled until timestamp (ms since Unix epoch) |
| `doubles[1]` | number | Settlement lag in milliseconds (Date.now() - doubles[0]) |
| `blobs[0]` | string | Data set ID with oldest unsettled CDN usage, or empty string if all usage is settled |
| `doubles[0]` | number | Waiting since: `usage_reported_until` of the data set waiting longest (ms since epoch)|
| `doubles[1]` | number | Waiting for (ms): `Date.now() - doubles[0]` |
| `blobs[0]` | string | Data set ID waiting longest, or empty string if no data set is waiting |

### Example Queries

Expand Down
25 changes: 13 additions & 12 deletions indexer/bin/indexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,39 +395,40 @@ export default {
async reportSettlementStats(env) {
/**
* @type {{
* id: string
* cdn_payments_settled_until: string
* data_set_id: string
* waiting_since: string
* } | null}
*/
const row = await env.DB.prepare(
// Find the data set waiting longest for settlement
const waitingLongest = await env.DB.prepare(
`
SELECT id, cdn_payments_settled_until
SELECT id AS data_set_id, usage_reported_until AS waiting_since
FROM data_sets
WHERE usage_reported_until > cdn_payments_settled_until
ORDER BY cdn_payments_settled_until ASC
ORDER BY usage_reported_until ASC
LIMIT 1
`,
).first()

if (!row) {
console.log('No data sets with unsettled CDN usage')
if (!waitingLongest) {
console.log('No data sets waiting for settlement')
env.SETTLEMENT_STATS.writeDataPoint({
doubles: [Date.now(), 0],
blobs: [''],
})
return
}

const timestampMs = new Date(row.cdn_payments_settled_until).getTime()
const lagMs = Date.now() - timestampMs
const waitingSinceMs = new Date(waitingLongest.waiting_since).getTime()
const waitingMs = Date.now() - waitingSinceMs

console.log(
`Oldest unsettled CDN usage: data_set=${row.id}, cdn_payments_settled_until=${row.cdn_payments_settled_until}, lag_ms=${lagMs}`,
`Reported usage waiting longest for settlement: data_set=${waitingLongest.data_set_id}, waiting_since=${waitingLongest.waiting_since}, waiting_ms=${waitingMs}`,
)

env.SETTLEMENT_STATS.writeDataPoint({
doubles: [timestampMs, lagMs],
blobs: [row.id],
doubles: [waitingSinceMs, waitingMs],
blobs: [waitingLongest.data_set_id],
})
},
}
27 changes: 21 additions & 6 deletions indexer/test/settlement-stats.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,17 @@ describe('reportSettlementStats', () => {

await workerImpl.reportSettlementStats(testEnv)

const cdnPaymentsSettledUntilMs = new Date(
'2022-11-01T12:00:00.000Z',
).getTime()
const usageReportedUntilMs = new Date('2022-11-02T00:00:00.000Z').getTime()
const nowMs = new Date('2022-11-02T12:00:00.000Z').getTime()
expect(writeDataPoint).toHaveBeenCalledWith({
doubles: [cdnPaymentsSettledUntilMs, nowMs - cdnPaymentsSettledUntilMs],
doubles: [usageReportedUntilMs, nowMs - usageReportedUntilMs],
blobs: ['ds-1'],
})
})

it('picks the data set with the oldest cdn_payments_settled_until', async () => {
it('picks the data set with the oldest usage_reported_until', async () => {
await testEnv.DB.prepare(
`INSERT INTO data_sets (id, with_cdn, usage_reported_until, cdn_payments_settled_until) VALUES ('ds-old', FALSE, '2022-11-02T00:00:00.000Z', '2022-11-01T00:00:00.000Z')`,
`INSERT INTO data_sets (id, with_cdn, usage_reported_until, cdn_payments_settled_until) VALUES ('ds-old', FALSE, '2022-11-01T00:00:00.000Z', '2022-10-31T00:00:00.000Z')`,
).run()
await testEnv.DB.prepare(
`INSERT INTO data_sets (id, with_cdn, usage_reported_until, cdn_payments_settled_until) VALUES ('ds-new', FALSE, '2022-11-02T00:00:00.000Z', '2022-11-01T12:00:00.000Z')`,
Expand All @@ -57,6 +55,23 @@ describe('reportSettlementStats', () => {
})
})

it('calculates lag from usage_reported_until, not cdn_payments_settled_until', async () => {
vi.setSystemTime(new Date('2026-02-05T15:18:57.908Z'))

await testEnv.DB.prepare(
`INSERT INTO data_sets (id, with_cdn, usage_reported_until, cdn_payments_settled_until) VALUES ('ds-11322', FALSE, '2026-02-05T00:29:30.000Z', '1970-01-01T00:00:00.000Z')`,
).run()

await workerImpl.reportSettlementStats(testEnv)

const usageReportedUntilMs = new Date('2026-02-05T00:29:30.000Z').getTime()
const nowMs = new Date('2026-02-05T15:18:57.908Z').getTime()
expect(writeDataPoint).toHaveBeenCalledWith({
doubles: [usageReportedUntilMs, nowMs - usageReportedUntilMs],
blobs: ['ds-11322'],
})
})

it('writes "no lag" data point when no unsettled usage exists', async () => {
await testEnv.DB.prepare(
`INSERT INTO data_sets (id, with_cdn, usage_reported_until, cdn_payments_settled_until) VALUES ('ds-settled', FALSE, '2022-11-01T12:00:00.000Z', '2022-11-01T12:00:00.000Z')`,
Expand Down
Loading