From 6634074ea7c143c23d133b289f228128ed536c52 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 24 Jun 2026 11:07:07 -0400 Subject: [PATCH 01/24] perf(pxe): first-miss probing for constrained tag sync Constrained delivery emits a gapless tagging-index stream, so the recipient can stop at the first missing tag instead of always probing a full UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN (20) window per secret. The initial probe is INITIAL_CONSTRAINED_PROBE_LEN tags and grows to the full window only when every probed index has a log; probe advancement is decoupled from finalized-cursor persistence so unfinalized logs at the top of the run are still fetched. Adds a unit micro-benchmark for the per-sync tag-query cost. --- yarn-project/bootstrap.sh | 1 + yarn-project/pxe/src/tagging/constants.ts | 21 +- yarn-project/pxe/src/tagging/index.ts | 2 +- .../sync_tagged_private_logs.bench.test.ts | 216 ++++++++++++++++++ .../sync_tagged_private_logs.test.ts | 84 ++++++- .../sync_tagged_private_logs.ts | 63 +++-- 6 files changed, 350 insertions(+), 37 deletions(-) create mode 100644 yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 6c84b8b7a6d3..43e78c0312d5 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -249,6 +249,7 @@ function bench_cmds { echo "$hash BENCH_OUTPUT=bench-out/sim.bench.json yarn-project/scripts/run_test.sh simulator/src/public/public_tx_simulator/apps_tests/bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/native_world_state.bench.json yarn-project/scripts/run_test.sh world-state/src/native/native_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/kv_store.bench.json yarn-project/scripts/run_test.sh kv-store/src/bench/map_bench.test.ts" + echo "$hash BENCH_OUTPUT=bench-out/tag_sync.bench.json yarn-project/scripts/run_test.sh pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_pool_v2.bench.json yarn-project/scripts/run_test.sh p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_validator.bench.json yarn-project/scripts/run_test.sh p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts" echo "$hash:ISOLATE=1:CPUS=16:MEM=32g:TIMEOUT=1800 BENCH_OUTPUT=bench-out/p2p_client_batch_tx_requester.bench.json yarn-project/scripts/run_test.sh p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts" diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts index 3c40d9734e6d..bd002f297dc4 100644 --- a/yarn-project/pxe/src/tagging/constants.ts +++ b/yarn-project/pxe/src/tagging/constants.ts @@ -1,10 +1,15 @@ -// This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app -// tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function. -// This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than -// MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note -// and the note is nullified in the same tx. +// The maximum number of tagging indexes a sender may use ahead of the highest finalized index for a given directional +// app tagging secret. A sender that tries to use an index beyond it throws in the `PXE::proveTx` function. It therefore +// has to be at least as large as the largest number of indexes a single tx is expected to consume for one directional +// app secret. That number can in theory exceed MAX_PRIVATE_LOGS_PER_TX (currently 64), because squashing (a note +// created and nullified in the same tx) consumes an index without emitting a persisted log, but in practice it stays +// well below that. // -// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than -// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use -// cases. +// A large window significantly slowed down the `e2e_l1_with_wall_time` test, which syncs more than 1000 secrets, so we +// set it to a relatively low value of 20, which is sufficient for current use cases. export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; + +// Initial number of tags probed per constrained secret before falling back to the full window. Constrained delivery is +// gapless, so a single missing tag proves the stream has ended. At steady state (no new logs) this turns a full +// WINDOW_LEN probe into one tag. +export const INITIAL_CONSTRAINED_PROBE_LEN = 1; diff --git a/yarn-project/pxe/src/tagging/index.ts b/yarn-project/pxe/src/tagging/index.ts index e4f7765c35a2..462d0bdaec4b 100644 --- a/yarn-project/pxe/src/tagging/index.ts +++ b/yarn-project/pxe/src/tagging/index.ts @@ -12,7 +12,7 @@ export { syncTaggedPrivateLogs } from './recipient_sync/sync_tagged_private_logs.js'; export { syncSenderTaggingIndexes } from './sender_sync/sync_sender_tagging_indexes.js'; export { persistSenderTaggingIndexRangesForTx } from './persist_sender_tagging_index_ranges.js'; -export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from './constants.js'; +export { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, INITIAL_CONSTRAINED_PROBE_LEN } from './constants.js'; export { getAllPrivateLogsByTags, getAllPublicLogsByTagsFromContract } from './get_all_logs_by_tags.js'; // Re-export tagging-related types from stdlib diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts new file mode 100644 index 000000000000..a077b05d0788 --- /dev/null +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -0,0 +1,216 @@ +import { MAX_TX_LIFETIME } from '@aztec/constants'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { createLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; +import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { + type AppTaggingSecret, + AppTaggingSecretKind, + type PrivateLogsQuery, + SiloedTag, + type TagQuery, + randomLogResult, +} from '@aztec/stdlib/logs'; +import { randomAppTaggingSecret } from '@aztec/stdlib/testing'; +import { BlockHeader } from '@aztec/stdlib/tx'; + +import { mkdir, writeFile } from 'fs/promises'; +import { type MockProxy, mock } from 'jest-mock-extended'; +import path from 'path'; + +import { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; +import { + INITIAL_CONSTRAINED_PROBE_LEN, + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, + syncTaggedPrivateLogs, +} from '../index.js'; + +/** + * Benchmark for constrained recipient tag-sync. + * + * Measures the per-sync tag-query cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are + * gapless, so the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and stops at the first missing + * tag instead of always fetching the full `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window. The headline metric is + * the total number of tags queried across the node (`getPrivateLogsByTags`) calls; at steady state (no new logs) it + * drops from a full window per secret to a single tag. + * + * The "first-miss optimum" column is the theoretical floor a first-miss scan targets ("new logs + one miss" per + * secret); the reduction column shows how close each scenario gets to it. An unconstrained row is included as a + * control: it cannot first-miss (windowed scan), so its optimum equals its cost. + */ + +const logger = createLogger('pxe:tagging:bench'); + +const FINALIZED_BLOCK_NUMBER = BlockNumber(10); +const ANCHOR_BLOCK_NUMBER = BlockNumber(100); +const CURRENT_TIMESTAMP = BigInt(Math.floor(Date.now() / 1000)); +const ANCHOR_BLOCK_HEADER = BlockHeader.random({ blockNumber: ANCHOR_BLOCK_NUMBER, timestamp: CURRENT_TIMESTAMP }); +const AGED_TIMESTAMP = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; +const JOB_ID = 'bench-job'; + +/** One benchmark measurement in the GitHub-action benchmark JSON shape. */ +type BenchResult = { name: string; value: number; unit: string }; + +type Scenario = { + label: string; + kind: AppTaggingSecretKind; + /** Number of secrets the recipient holds for this directional app. */ + secretCount: number; + /** Highest finalized index already persisted before the sync (recipient has `priorCursor + 1` prior messages). */ + priorCursor: number; + /** New contiguous finalized logs available per secret since the last sync (0 = steady state). */ + newLogs: number; +}; + +const SCENARIOS: Scenario[] = [ + // Steady state (no new logs) across recipient secret counts — the dominant case and where first-miss wins ~20x. + ...[1, 10, 100, 1000].map(secretCount => ({ + label: `constrained/steady-state/secrets=${secretCount}`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount, + priorCursor: 0, + newLogs: 0, + })), + // Single new message per secret since last sync — the case that most distinguishes INITIAL_CONSTRAINED_PROBE_LEN + // values (a probe of P resolves K < P new logs in a single round). + { + label: `constrained/catch-up-1/secrets=100`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount: 100, + priorCursor: 0, + newLogs: 1, + }, + // Light catch-up: a couple of new messages per secret since last sync. + { + label: `constrained/catch-up-3/secrets=100`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount: 100, + priorCursor: 0, + newLogs: 3, + }, + // Window-edge catch-up: a full window of new messages forces a second round under the fixed-window scan. + { + label: `constrained/catch-up-${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN}/secrets=100`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount: 100, + priorCursor: 0, + newLogs: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, + }, + // Control: unconstrained steady state is unaffected by the optimization (windowed scan cannot first-miss). + { + label: `unconstrained/steady-state/secrets=100`, + kind: AppTaggingSecretKind.UNCONSTRAINED, + secretCount: 100, + priorCursor: 0, + newLogs: 0, + }, +]; + +describe('syncTaggedPrivateLogs constrained-sync bench', () => { + const aztecNode: MockProxy = mock(); + + function computeSiloedTagForIndex(secret: AppTaggingSecret, index: number) { + return SiloedTag.compute({ extendedSecret: secret, index }); + } + + function extractTags(query: PrivateLogsQuery): SiloedTag[] { + return query.tags.map((entry: TagQuery) => (entry instanceof SiloedTag ? entry : entry.tag)); + } + + function makeFinalizedLog() { + return { + ...randomLogResult(/* includeEffects */ true), + blockNumber: FINALIZED_BLOCK_NUMBER, + blockTimestamp: AGED_TIMESTAMP, + }; + } + + async function runScenario(scenario: Scenario) { + const { kind, secretCount, priorCursor, newLogs } = scenario; + + aztecNode.getPrivateLogsByTags.mockReset(); + const taggingStore = new RecipientTaggingStore(await openTmpStore('bench')); + const secrets = await Promise.all(Array.from({ length: secretCount }, () => randomAppTaggingSecret(kind))); + + // Seed the persisted cursor(s) to simulate a recipient that already synced prior finalized messages. + for (const secret of secrets) { + await taggingStore.updateHighestFinalizedIndex(secret, priorCursor, JOB_ID); + if (kind === AppTaggingSecretKind.UNCONSTRAINED) { + await taggingStore.updateHighestAgedIndex(secret, priorCursor, JOB_ID); + } + } + + // Tags that should resolve to a finalized log: the contiguous run (priorCursor, priorCursor + newLogs]. + const hitTags = new Set(); + for (const secret of secrets) { + for (let k = 1; k <= newLogs; k++) { + hitTags.add((await computeSiloedTagForIndex(secret, priorCursor + k)).toString()); + } + } + + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => + Promise.resolve(extractTags(query).map(tag => (hitTags.has(tag.toString()) ? [makeFinalizedLog()] : []))), + ); + + const timer = new Timer(); + const logs = await syncTaggedPrivateLogs( + secrets, + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + FINALIZED_BLOCK_NUMBER, + JOB_ID, + ); + const syncMs = timer.ms(); + + const calls = aztecNode.getPrivateLogsByTags.mock.calls; + const tagQueries = calls.reduce((sum, [query]) => sum + extractTags(query).length, 0); + const nodeCalls = calls.length; + + // First-miss floor: each secret pays `newLogs` hits + 1 miss. Unconstrained cannot first-miss, so its floor is + // its current cost. + const firstMissOptimum = kind === AppTaggingSecretKind.CONSTRAINED ? secretCount * (newLogs + 1) : tagQueries; + + return { ...scenario, logsFound: logs.length, tagQueries, nodeCalls, syncMs, firstMissOptimum }; + } + + it('measures tag-query cost per sync', async () => { + const rows = []; + for (const scenario of SCENARIOS) { + const row = await runScenario(scenario); + rows.push(row); + logger.info( + `${row.label.padEnd(42)} tag-queries=${String(row.tagQueries).padStart(6)} ` + + `first-miss-optimum=${String(row.firstMissOptimum).padStart(6)} ` + + `reduction=${(row.tagQueries / row.firstMissOptimum).toFixed(1)}x ` + + `node-calls=${String(row.nodeCalls).padStart(4)} logs=${String(row.logsFound).padStart(5)} ` + + `time=${row.syncMs.toFixed(1)}ms`, + ); + + // Pin behavior as an executable assertion, not just a printout. + if (scenario.newLogs === 0) { + if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { + // Constrained steady state: the first missing tag ends the scan, so we query only the initial probe. + expect(row.tagQueries).toBe(scenario.secretCount * INITIAL_CONSTRAINED_PROBE_LEN); + } else { + // Unconstrained steady state: still the full window (control for the optimization). + expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + } + expect(row.logsFound).toBe(0); + } + expect(row.logsFound).toBe(scenario.secretCount * scenario.newLogs); + } + + const results: BenchResult[] = rows.flatMap(row => [ + { name: `TagSync/${row.label}/tag-queries`, value: row.tagQueries, unit: 'tag-queries' }, + { name: `TagSync/${row.label}/node-calls`, value: row.nodeCalls, unit: 'calls' }, + { name: `TagSync/${row.label}/sync-time`, value: Number(row.syncMs.toFixed(2)), unit: 'ms' }, + ]); + + if (process.env.BENCH_OUTPUT) { + await mkdir(path.dirname(process.env.BENCH_OUTPUT), { recursive: true }); + await writeFile(process.env.BENCH_OUTPUT, JSON.stringify(results, null, 2)); + } + }, 600_000); +}); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 6beb31dbf95e..141037c79fe1 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -17,7 +17,11 @@ import { BlockHeader } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; import { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; -import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, syncTaggedPrivateLogs } from '../index.js'; +import { + INITIAL_CONSTRAINED_PROBE_LEN, + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, + syncTaggedPrivateLogs, +} from '../index.js'; const FAR_FUTURE_BLOCK_NUMBER = BlockNumber(100); const CURRENT_TIMESTAMP = BigInt(Math.floor(Date.now() / 1000)); @@ -264,7 +268,7 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('continues when all indexes in the batch have logs', async () => { + it('fully drains a contiguous run that spans multiple windows', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -285,8 +289,11 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); + // Every log in the run is returned and the cursor lands on the last index, even though the run is longer than a + // single window, so the scan has to advance across more than one round to drain it. expect(logs).toHaveLength(totalLogs); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(2); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeGreaterThan(1); }); it('persists cursor only up to finalized block', async () => { @@ -322,15 +329,50 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); + // The unfinalized logs (4, 5) are returned to the caller, but the durable cursor only advances to the finalized + // prefix (3): probe advancement is decoupled from cursor persistence. expect(logs).toHaveLength(6); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(3); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); }); - it('respects pre-existing finalized index', async () => { + it('advances the probe past an unfinalized-only first probe', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const existingFinalizedIndex = 5; - await taggingStore.updateHighestFinalizedIndex(secret, existingFinalizedIndex, JOB_ID); + const finalizedBlockNumber = BlockNumber(5); + const recentTimestamp = CURRENT_TIMESTAMP - 5000n; + + // Indexes 0 and 1 are hits in an unfinalized block (8 > finalized 5); index 2 is the gap. With the small initial + // probe, index 0 is probed alone and is unfinalized, so the scan must still advance to discover index 1 — gating + // advancement on finalization (as an earlier design did) would drop it. + const unfinalizedTags = await Promise.all([0, 1].map(i => computeSiloedTagForIndex(secret, i))); + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); + return Promise.resolve( + tags.map((t: SiloedTag) => + unfinalizedTags.find(tag => tag.equals(t)) ? [makeLog(8, recentTimestamp, t.value)] : [], + ), + ); + }); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + expect(logs).toHaveLength(2); + // Nothing finalized, so the durable cursor must not advance. + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); + }); + }); + + describe('constrained vs unconstrained probing', () => { + it('constrained steady-state probes only INITIAL_CONSTRAINED_PROBE_LEN tags', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedIndex = 8; + await taggingStore.updateHighestFinalizedIndex(secret, finalizedIndex, JOB_ID); aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => Promise.resolve(query.tags.map(() => [])), @@ -338,16 +380,38 @@ describe('syncTaggedPrivateLogs', () => { await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + // Gapless stream: the first missing tag ends the scan, so steady state probes just the initial window. const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + const expectedTags = await Promise.all( + Array.from({ length: INITIAL_CONSTRAINED_PROBE_LEN }, (_, i) => + computeSiloedTagForIndex(secret, finalizedIndex + 1 + i), + ), + ); + expect(calledTags).toEqual(expectedTags); + }); - const expectedStart = existingFinalizedIndex + 1; - const expectedEnd = existingFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; + it('unconstrained steady-state probes the full window', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + const cursor = 8; + await taggingStore.updateHighestAgedIndex(secret, cursor, JOB_ID); + await taggingStore.updateHighestFinalizedIndex(secret, cursor, JOB_ID); + + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => + Promise.resolve(query.tags.map(() => [])), + ); + + await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + + // Unconstrained streams can have gaps, so the full window is always probed — the same cursor that costs one tag + // for a constrained secret costs a full window here. + const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + const expectedStart = cursor + 1; + const expectedEnd = cursor + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; const expectedTags = await Promise.all( Array.from({ length: expectedEnd - expectedStart + 1 }, (_, i) => computeSiloedTagForIndex(secret, expectedStart + i), ), ); - expect(calledTags).toEqual(expectedTags); }); }); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index c2aeeededcf4..c9aed9cdbb6c 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -7,7 +7,7 @@ import { AppTaggingSecretKind, SiloedTag } from '@aztec/stdlib/logs'; import type { BlockHeader } from '@aztec/stdlib/tx'; import type { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; -import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../constants.js'; +import { INITIAL_CONSTRAINED_PROBE_LEN, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../constants.js'; import { getAllPrivateLogsByTags } from '../get_all_logs_by_tags.js'; import { findHighestIndexes } from './utils/find_highest_indexes.js'; @@ -62,9 +62,13 @@ import { findHighestIndexes } from './utils/find_highest_indexes.js'; * - The lower bound is `highestFinalizedIndex + 1`. The `highestAgedIndex` mechanism is unnecessary because * the nullifier chain prevents out-of-order index usage: no device can fill in a lower index after a higher * one is already on-chain. - * - The scan stops at the first gap in the index sequence. If all indexes in the batch have logs, the window - * advances. If a gap is found, the sequence has ended and no further logs exist. - * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. + * - Because the sequence is gapless, the scan stops at the first missing index: that gap proves the sequence has + * ended and no further logs exist. We exploit this by probing only a small initial window + * (`INITIAL_CONSTRAINED_PROBE_LEN`) and growing to the full window only when every probed index has a log, so a + * steady-state sync with no new logs costs a single tag instead of the whole window. + * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. Advancing the probe is + * decoupled from persisting the finalized cursor, so unfinalized logs at the top of the run are still fetched + * while only the finalized prefix is persisted. * * # Batching across secrets * @@ -147,9 +151,16 @@ function getIndexRangesForSecrets( : await taggingStore.getHighestAgedIndex(secret, jobId); const start = highestIndexBeforeStart === undefined ? 0 : highestIndexBeforeStart + 1; - const end = (currentHighestFinalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; + const boundEnd = (currentHighestFinalizedIndex ?? 0) + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; - return { secret, start, end }; + // Constrained streams are gapless, so probe a small initial window and stop at the first missing tag instead of + // always fetching the full window. Unconstrained secrets can have gaps, so they still scan the whole window. + const end = + secret.kind === AppTaggingSecretKind.CONSTRAINED + ? Math.min(boundEnd, start + INITIAL_CONSTRAINED_PROBE_LEN) + : boundEnd; + + return { secret, start, end, boundEnd }; }), ); } @@ -223,21 +234,32 @@ async function processConstrainedResults( firstMissingIndex++; } - // Finalize any logs before the gap. The nullifier for a constrained-delivery log is emitted in the same - // transaction, guaranteeing the log cannot disappear once included, so we persist the highest finalized - // index even when a gap stops the scan. This lets the next sync round skip already-finalized indexes. + // Persist the highest finalized index found. The nullifier for a constrained-delivery log is emitted in the same + // transaction, guaranteeing the log cannot disappear once included, so we persist even when a gap stops the scan. + // This lets the next sync round skip already-finalized indexes. const { highestFinalizedIndex } = findHighestIndexes(logsWithIndexes, currentTimestamp, finalizedBlockNumber); - if (highestFinalizedIndex !== undefined) { await taggingStore.updateHighestFinalizedIndex(pending.secret, highestFinalizedIndex, jobId); + } - if (firstMissingIndex >= pending.end) { - return { - secret: pending.secret, - start: pending.end, - end: highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, - }; - } + // Advancing the probe is decoupled from persisting the cursor: keep scanning as long as the probe was entirely hits + // (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. Gating this on + // finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. The bound + // re-anchors to any finalized index found this round. Once the small initial probe is fully consumed, this jumps to + // the full window, so later rounds behave exactly as before. + const probeFullyConsumed = firstMissingIndex >= pending.end; + const boundEnd = + highestFinalizedIndex !== undefined + ? Math.max(pending.boundEnd, highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1) + : pending.boundEnd; + + if (probeFullyConsumed && pending.end < boundEnd) { + return { + secret: pending.secret, + start: pending.end, + end: boundEnd, + boundEnd, + }; } return undefined; @@ -282,10 +304,12 @@ async function processUnconstrainedResults( // For the next iteration we want to look only at indexes for which we have not yet fetched logs while // ensuring that we do not look further than WINDOW_LEN ahead of the highest finalized index. + const end = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; return { secret: pending.secret, start: pending.end, - end: highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, + end, + boundEnd: end, }; } @@ -293,6 +317,9 @@ type PendingSecret = { secret: AppTaggingSecret; start: number; end: number; + // Highest index this secret may probe this sync (highest finalized index + WINDOW_LEN + 1, the protocol bound on how + // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets starts as a small probe. + boundEnd: number; }; type LogWithIndex = { From b24784547f9df970a7a4af90a5b139a2840763dd Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 24 Jun 2026 14:15:39 -0400 Subject: [PATCH 02/24] sync one tag at a time --- yarn-project/pxe/src/tagging/constants.ts | 7 ++-- .../sync_tagged_private_logs.bench.test.ts | 18 ++++---- .../sync_tagged_private_logs.test.ts | 42 +++++++++++++++++-- .../sync_tagged_private_logs.ts | 15 ++++--- 4 files changed, 60 insertions(+), 22 deletions(-) diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts index bd002f297dc4..756c61121349 100644 --- a/yarn-project/pxe/src/tagging/constants.ts +++ b/yarn-project/pxe/src/tagging/constants.ts @@ -9,7 +9,8 @@ // set it to a relatively low value of 20, which is sufficient for current use cases. export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; -// Initial number of tags probed per constrained secret before falling back to the full window. Constrained delivery is -// gapless, so a single missing tag proves the stream has ended. At steady state (no new logs) this turns a full -// WINDOW_LEN probe into one tag. +// Number of tags probed per constrained secret per round: both the initial probe size and the step the probe grows by +// each round until the first missing tag. Constrained delivery is gapless, so a single missing tag proves the stream +// has ended. At steady state (no new logs) this turns a full WINDOW_LEN probe into one tag, and a secret with K new +// logs costs exactly K + 1 tags instead of the whole window. export const INITIAL_CONSTRAINED_PROBE_LEN = 1; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index a077b05d0788..217cc82de3c0 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -189,15 +189,15 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { ); // Pin behavior as an executable assertion, not just a printout. - if (scenario.newLogs === 0) { - if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { - // Constrained steady state: the first missing tag ends the scan, so we query only the initial probe. - expect(row.tagQueries).toBe(scenario.secretCount * INITIAL_CONSTRAINED_PROBE_LEN); - } else { - // Unconstrained steady state: still the full window (control for the optimization). - expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); - } - expect(row.logsFound).toBe(0); + if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { + // Fixed-step probing is tag-optimal: K hits plus 1 terminating miss, rounded up to whole probe steps. At + // INITIAL_CONSTRAINED_PROBE_LEN = 1 this equals the first-miss floor (reduction = 1.0x), and at steady state + // (K = 0) it collapses to a single initial probe per secret. + const stepsToFirstMiss = Math.ceil((scenario.newLogs + 1) / INITIAL_CONSTRAINED_PROBE_LEN); + expect(row.tagQueries).toBe(scenario.secretCount * stepsToFirstMiss * INITIAL_CONSTRAINED_PROBE_LEN); + } else if (scenario.newLogs === 0) { + // Unconstrained steady state: still the full window (control for the optimization). + expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); } expect(row.logsFound).toBe(scenario.secretCount * scenario.newLogs); } diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 141037c79fe1..2e0ea06a1da8 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -268,7 +268,7 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('fully drains a contiguous run that spans multiple windows', async () => { + it('fully drains a long contiguous run one probe at a time', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -289,11 +289,12 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // Every log in the run is returned and the cursor lands on the last index, even though the run is longer than a - // single window, so the scan has to advance across more than one round to drain it. + // The whole run is returned and the cursor lands on the last index, even though the run is longer than a single + // window. With INITIAL_CONSTRAINED_PROBE_LEN = 1 the scan advances one index per round, so it takes one round + // per log plus a final round for the terminating miss. expect(logs).toHaveLength(totalLogs); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeGreaterThan(1); + expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(totalLogs + 1); }); it('persists cursor only up to finalized block', async () => { @@ -366,6 +367,39 @@ describe('syncTaggedPrivateLogs', () => { // Nothing finalized, so the durable cursor must not advance. expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); }); + + // Pins the committed P=1 behavior: a fully-consumed probe advances by one INITIAL_CONSTRAINED_PROBE_LEN step, + // not the whole window. + it('catch-up probes one step at a time, not the full window', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Recipient already synced index 0; exactly one new finalized log sits at index 1. + await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); + const newLogTag = await computeSiloedTagForIndex(secret, 1); + mockNodeWithLogs([newLogTag], Number(finalizedBlockNumber), agedTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + expect(logs).toHaveLength(1); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(1); + + // Round 1 probes index 1 (the hit); round 2 probes only index 2 (the terminating miss) — a single + // INITIAL_CONSTRAINED_PROBE_LEN step, not the full window. The old full-window fallback queried indexes + // 2..(1 + WINDOW_LEN) in round 2. + const calls = aztecNode.getPrivateLogsByTags.mock.calls; + expect(calls).toHaveLength(2); + expect(extractTags(calls[0][0])).toEqual([await computeSiloedTagForIndex(secret, 1)]); + expect(extractTags(calls[1][0])).toEqual([await computeSiloedTagForIndex(secret, 2)]); + }); }); describe('constrained vs unconstrained probing', () => { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index c9aed9cdbb6c..eec5ce3a7813 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -64,8 +64,9 @@ import { findHighestIndexes } from './utils/find_highest_indexes.js'; * one is already on-chain. * - Because the sequence is gapless, the scan stops at the first missing index: that gap proves the sequence has * ended and no further logs exist. We exploit this by probing only a small initial window - * (`INITIAL_CONSTRAINED_PROBE_LEN`) and growing to the full window only when every probed index has a log, so a - * steady-state sync with no new logs costs a single tag instead of the whole window. + * (`INITIAL_CONSTRAINED_PROBE_LEN`) and growing by another such step each round while every probed index has a + * log, stopping at the first miss. A steady-state sync with no new logs costs a single tag instead of the whole + * window, and a sync with K new logs costs exactly K + 1 tags (K hits plus the terminating miss). * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. Advancing the probe is * decoupled from persisting the finalized cursor, so unfinalized logs at the top of the run are still fetched * while only the finalized prefix is persisted. @@ -245,8 +246,9 @@ async function processConstrainedResults( // Advancing the probe is decoupled from persisting the cursor: keep scanning as long as the probe was entirely hits // (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. Gating this on // finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. The bound - // re-anchors to any finalized index found this round. Once the small initial probe is fully consumed, this jumps to - // the full window, so later rounds behave exactly as before. + // re-anchors to any finalized index found this round. Because the stream is gapless, we only need to fetch up to the + // first missing tag, so the probe grows by another INITIAL_CONSTRAINED_PROBE_LEN step each round (capped at the + // bound) rather than jumping to the full window: a secret with K new logs costs exactly K + 1 tags. const probeFullyConsumed = firstMissingIndex >= pending.end; const boundEnd = highestFinalizedIndex !== undefined @@ -257,7 +259,7 @@ async function processConstrainedResults( return { secret: pending.secret, start: pending.end, - end: boundEnd, + end: Math.min(boundEnd, pending.end + INITIAL_CONSTRAINED_PROBE_LEN), boundEnd, }; } @@ -318,7 +320,8 @@ type PendingSecret = { start: number; end: number; // Highest index this secret may probe this sync (highest finalized index + WINDOW_LEN + 1, the protocol bound on how - // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets starts as a small probe. + // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets advances by + // INITIAL_CONSTRAINED_PROBE_LEN per round up to this bound. boundEnd: number; }; From 2ad7237ee5d01900353c978981950d1f6a5f0a96 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 24 Jun 2026 16:04:42 -0400 Subject: [PATCH 03/24] more benches --- .../sync_tagged_private_logs.bench.test.ts | 94 +++++++++++++++---- 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 217cc82de3c0..499ee36c6634 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -1,8 +1,10 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { createLogger } from '@aztec/foundation/log'; +import { sleep } from '@aztec/foundation/sleep'; import { Timer } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; +import { MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { type AppTaggingSecret, @@ -19,6 +21,7 @@ import { mkdir, writeFile } from 'fs/promises'; import { type MockProxy, mock } from 'jest-mock-extended'; import path from 'path'; +import { BenchmarkedNodeFactory } from '../../contract_function_simulator/benchmarked_node.js'; import { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; import { INITIAL_CONSTRAINED_PROBE_LEN, @@ -29,15 +32,22 @@ import { /** * Benchmark for constrained recipient tag-sync. * - * Measures the per-sync tag-query cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are - * gapless, so the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and stops at the first missing - * tag instead of always fetching the full `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window. The headline metric is - * the total number of tags queried across the node (`getPrivateLogsByTags`) calls; at steady state (no new logs) it - * drops from a full window per secret to a single tag. + * Measures the per-sync cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are gapless, so + * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and grows one such step at a time, stopping + * at the first missing tag instead of fetching the full `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window. * - * The "first-miss optimum" column is the theoretical floor a first-miss scan targets ("new logs + one miss" per - * secret); the reduction column shows how close each scenario gets to it. An unconstrained row is included as a - * control: it cannot first-miss (windowed scan), so its optimum equals its cost. + * Metrics, per scenario: + * - `tag-queries`: total tags queried — the throughput win. At steady state it drops from a full window per secret to + * a single tag; a secret with K new logs costs exactly K + 1 (the "first-miss optimum" floor, reduction 1.0x). + * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` chunks within a round count as + * one), via `BenchmarkedNodeFactory`. This is the latency cost: fixed-step trades fewer tags for more round-trips, + * so it grows ~linearly with K during catch-up while tag-queries stay at the floor. + * - `node-calls`: total `getPrivateLogsByTags` invocations. Equals `rpc-round-trips` at <=100 secrets, but above that + * a round fans out into `ceil(tags / MAX_RPC_LEN)` parallel chunks, so node-calls overstates the real latency. + * - `rpc-blocking-time`: wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` per + * call. Demonstrates the parallelism: a 1000-secret round is many calls but ~one round-trip of blocking time. + * + * An unconstrained row is included as a control: it cannot first-miss (windowed scan), so its optimum equals its cost. */ const logger = createLogger('pxe:tagging:bench'); @@ -49,6 +59,10 @@ const ANCHOR_BLOCK_HEADER = BlockHeader.random({ blockNumber: ANCHOR_BLOCK_NUMBE const AGED_TIMESTAMP = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; const JOB_ID = 'bench-job'; +// Models per-call node RPC latency so round-trip blocking time is meaningful against an otherwise-instant mock node. +// The round-trip *count* is independent of this value; only `rpc-blocking-time` scales with it. +const MODELED_NODE_RPC_LATENCY_MS = 5; + /** One benchmark measurement in the GitHub-action benchmark JSON shape. */ type BenchResult = { name: string; value: number; unit: string }; @@ -97,6 +111,24 @@ const SCENARIOS: Scenario[] = [ priorCursor: 0, newLogs: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, }, + // Deep catch-up (the negative case): round-trips grow linearly with K (one per probe step) while tag-queries stay at + // the K + 1 floor. node-calls == round-trips here since each round's <=100 tags fit one RPC chunk. + ...[50, 100].map(newLogs => ({ + label: `constrained/catch-up-${newLogs}/secrets=100`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount: 100, + priorCursor: 0, + newLogs, + })), + // Chunked regime: at 1000 secrets a round fans out into ceil(1000 / MAX_RPC_LEN) parallel chunks, so node-calls is + // ~10x the round-trips — the case where raw call count overstates latency. + { + label: `constrained/catch-up-5/secrets=1000`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount: 1000, + priorCursor: 0, + newLogs: 5, + }, // Control: unconstrained steady state is unaffected by the optimization (windowed scan cannot first-miss). { label: `unconstrained/steady-state/secrets=100`, @@ -149,14 +181,19 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { } } - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(extractTags(query).map(tag => (hitTags.has(tag.toString()) ? [makeFinalizedLog()] : []))), - ); + aztecNode.getPrivateLogsByTags.mockImplementation(async (query: PrivateLogsQuery) => { + await sleep(MODELED_NODE_RPC_LATENCY_MS); + return extractTags(query).map(tag => (hitTags.has(tag.toString()) ? [makeFinalizedLog()] : [])); + }); + + // Wrap the node so we capture round-trips and blocking time the same way the client_flows app benches do. The + // Proxy delegates to the underlying mock, so `mock.calls` still records every query for tag counting. + const benchmarkedNode = BenchmarkedNodeFactory.create(aztecNode); const timer = new Timer(); const logs = await syncTaggedPrivateLogs( secrets, - aztecNode, + benchmarkedNode, taggingStore, ANCHOR_BLOCK_HEADER, FINALIZED_BLOCK_NUMBER, @@ -168,11 +205,26 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { const tagQueries = calls.reduce((sum, [query]) => sum + extractTags(query).length, 0); const nodeCalls = calls.length; + // Round-trips and blocking time from the same instrumentation the app benches use. `syncTaggedPrivateLogs` only + // ever calls `getPrivateLogsByTags`, so every round-trip is that method. + const { roundTrips } = benchmarkedNode.getStats(); + const rpcRoundTrips = roundTrips.roundTrips; + const rpcBlockingTimeMs = roundTrips.totalBlockingTime; + // First-miss floor: each secret pays `newLogs` hits + 1 miss. Unconstrained cannot first-miss, so its floor is // its current cost. const firstMissOptimum = kind === AppTaggingSecretKind.CONSTRAINED ? secretCount * (newLogs + 1) : tagQueries; - return { ...scenario, logsFound: logs.length, tagQueries, nodeCalls, syncMs, firstMissOptimum }; + return { + ...scenario, + logsFound: logs.length, + tagQueries, + nodeCalls, + rpcRoundTrips, + rpcBlockingTimeMs, + syncMs, + firstMissOptimum, + }; } it('measures tag-query cost per sync', async () => { @@ -184,27 +236,37 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { `${row.label.padEnd(42)} tag-queries=${String(row.tagQueries).padStart(6)} ` + `first-miss-optimum=${String(row.firstMissOptimum).padStart(6)} ` + `reduction=${(row.tagQueries / row.firstMissOptimum).toFixed(1)}x ` + - `node-calls=${String(row.nodeCalls).padStart(4)} logs=${String(row.logsFound).padStart(5)} ` + + `round-trips=${String(row.rpcRoundTrips).padStart(4)} node-calls=${String(row.nodeCalls).padStart(4)} ` + + `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)} ` + `time=${row.syncMs.toFixed(1)}ms`, ); - // Pin behavior as an executable assertion, not just a printout. + // Pin behavior as an executable assertion, not just a printout. Timings are reported only (they vary run to run). if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { // Fixed-step probing is tag-optimal: K hits plus 1 terminating miss, rounded up to whole probe steps. At // INITIAL_CONSTRAINED_PROBE_LEN = 1 this equals the first-miss floor (reduction = 1.0x), and at steady state // (K = 0) it collapses to a single initial probe per secret. const stepsToFirstMiss = Math.ceil((scenario.newLogs + 1) / INITIAL_CONSTRAINED_PROBE_LEN); expect(row.tagQueries).toBe(scenario.secretCount * stepsToFirstMiss * INITIAL_CONSTRAINED_PROBE_LEN); + // One sequential round-trip per probe step — the negative-case cost, independent of chunking. + expect(row.rpcRoundTrips).toBe(stepsToFirstMiss); + // node-calls = round-trips x chunks per round. Each round queries `secretCount * P` tags split into + // MAX_RPC_LEN chunks, so node-calls equals round-trips only while a round fits one chunk, and fans out above. + const chunksPerRound = Math.ceil((scenario.secretCount * INITIAL_CONSTRAINED_PROBE_LEN) / MAX_RPC_LEN); + expect(row.nodeCalls).toBe(stepsToFirstMiss * chunksPerRound); } else if (scenario.newLogs === 0) { - // Unconstrained steady state: still the full window (control for the optimization). + // Unconstrained steady state: still the full window (control for the optimization), drained in one round. expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + expect(row.rpcRoundTrips).toBe(1); } expect(row.logsFound).toBe(scenario.secretCount * scenario.newLogs); } const results: BenchResult[] = rows.flatMap(row => [ { name: `TagSync/${row.label}/tag-queries`, value: row.tagQueries, unit: 'tag-queries' }, + { name: `TagSync/${row.label}/rpc-round-trips`, value: row.rpcRoundTrips, unit: 'round_trips' }, { name: `TagSync/${row.label}/node-calls`, value: row.nodeCalls, unit: 'calls' }, + { name: `TagSync/${row.label}/rpc-blocking-time`, value: Number(row.rpcBlockingTimeMs.toFixed(2)), unit: 'ms' }, { name: `TagSync/${row.label}/sync-time`, value: Number(row.syncMs.toFixed(2)), unit: 'ms' }, ]); From 91cabe677cf2cd45746f6a8787f2b50535f4b805 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 24 Jun 2026 22:48:58 -0400 Subject: [PATCH 04/24] simplify bench metrics --- yarn-project/pxe/src/tagging/constants.ts | 4 +- .../sync_tagged_private_logs.bench.test.ts | 90 +++++++------------ 2 files changed, 35 insertions(+), 59 deletions(-) diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts index 756c61121349..22844c73b585 100644 --- a/yarn-project/pxe/src/tagging/constants.ts +++ b/yarn-project/pxe/src/tagging/constants.ts @@ -12,5 +12,7 @@ export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; // Number of tags probed per constrained secret per round: both the initial probe size and the step the probe grows by // each round until the first missing tag. Constrained delivery is gapless, so a single missing tag proves the stream // has ended. At steady state (no new logs) this turns a full WINDOW_LEN probe into one tag, and a secret with K new -// logs costs exactly K + 1 tags instead of the whole window. +// logs costs exactly K + 1 tags instead of the whole window. Larger values trade more steady-state tags (P per secret) +// for fewer sequential round-trips during catch-up (a probe of P resolves K < P new logs in a single round), so 1 is +// the right default while idle secrets dominate. export const INITIAL_CONSTRAINED_PROBE_LEN = 1; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 499ee36c6634..b40bed78bc35 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -4,7 +4,6 @@ import { createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { Timer } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import { MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { type AppTaggingSecret, @@ -37,17 +36,21 @@ import { * at the first missing tag instead of fetching the full `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window. * * Metrics, per scenario: - * - `tag-queries`: total tags queried — the throughput win. At steady state it drops from a full window per secret to - * a single tag; a secret with K new logs costs exactly K + 1 (the "first-miss optimum" floor, reduction 1.0x). - * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` chunks within a round count as - * one), via `BenchmarkedNodeFactory`. This is the latency cost: fixed-step trades fewer tags for more round-trips, - * so it grows ~linearly with K during catch-up while tag-queries stay at the floor. - * - `node-calls`: total `getPrivateLogsByTags` invocations. Equals `rpc-round-trips` at <=100 secrets, but above that - * a round fans out into `ceil(tags / MAX_RPC_LEN)` parallel chunks, so node-calls overstates the real latency. - * - `rpc-blocking-time`: wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` per - * call. Demonstrates the parallelism: a 1000-secret round is many calls but ~one round-trip of blocking time. + * - `tag-queries`: total tags queried, the throughput win. At steady state it drops from a full window per secret to a + * single tag; a secret with K new logs costs exactly K + 1 (the "first-miss optimum" floor, reduction 1.0x). + * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as one), + * via `BenchmarkedNodeFactory`. The latency axis: fixed-step trades fewer tags for more round-trips, so it grows + * ~linearly with K during catch-up while tag-queries stay at the floor. Depends only on K and P, not on secret count. + * A round's tags are chunked at MAX_RPC_LEN (=100) into parallel calls internally, but those overlap, so a wide round + * is still one round-trip; that is why round-trips, not raw call count, is the latency axis. + * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` + * per call plus a little per-round overhead. Parallel calls within a round overlap, so it tracks round-trips (a + * 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported only (varies run to run). * - * An unconstrained row is included as a control: it cannot first-miss (windowed scan), so its optimum equals its cost. + * Scenario labels: `steady-state` is no new logs (K = 0); `catch-up-K` is K new contiguous logs per secret since the + * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K and P + * (not N), the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while + * round-trips do not. The `unconstrained` row is the control: it cannot first-miss (windowed scan), so its cost is fixed. */ const logger = createLogger('pxe:tagging:bench'); @@ -78,7 +81,8 @@ type Scenario = { }; const SCENARIOS: Scenario[] = [ - // Steady state (no new logs) across recipient secret counts — the dominant case and where first-miss wins ~20x. + // steady-state (K = 0): no new logs since the last sync, across recipient secret counts. The dominant case and where + // first-miss wins ~20x. ...[1, 10, 100, 1000].map(secretCount => ({ label: `constrained/steady-state/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, @@ -86,49 +90,26 @@ const SCENARIOS: Scenario[] = [ priorCursor: 0, newLogs: 0, })), - // Single new message per secret since last sync — the case that most distinguishes INITIAL_CONSTRAINED_PROBE_LEN - // values (a probe of P resolves K < P new logs in a single round). - { - label: `constrained/catch-up-1/secrets=100`, - kind: AppTaggingSecretKind.CONSTRAINED, - secretCount: 100, - priorCursor: 0, - newLogs: 1, - }, - // Light catch-up: a couple of new messages per secret since last sync. - { - label: `constrained/catch-up-3/secrets=100`, - kind: AppTaggingSecretKind.CONSTRAINED, - secretCount: 100, - priorCursor: 0, - newLogs: 3, - }, - // Window-edge catch-up: a full window of new messages forces a second round under the fixed-window scan. - { - label: `constrained/catch-up-${UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN}/secrets=100`, - kind: AppTaggingSecretKind.CONSTRAINED, - secretCount: 100, - priorCursor: 0, - newLogs: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, - }, - // Deep catch-up (the negative case): round-trips grow linearly with K (one per probe step) while tag-queries stay at - // the K + 1 floor. node-calls == round-trips here since each round's <=100 tags fit one RPC chunk. - ...[50, 100].map(newLogs => ({ + // Light catch-up (K new logs per secret) at 100 and 1000 secrets. Round-trips depend only on K and P, not on N, so + // the two secret counts share a round-trip count and differ only in tag-queries (10x) and blocking time. + ...[100, 1000].flatMap(secretCount => + [1, 3].map(newLogs => ({ + label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount, + priorCursor: 0, + newLogs, + })), + ), + // Deep catch-up at 100 secrets (the negative case): round-trips grow one per probe step while tag-queries stay at the + // K + 1 floor. From a full window (catch-up-20) up, the WINDOW_LEN cap forces multiple rounds at any P. + ...[UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, 50, 100].map(newLogs => ({ label: `constrained/catch-up-${newLogs}/secrets=100`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount: 100, priorCursor: 0, newLogs, })), - // Chunked regime: at 1000 secrets a round fans out into ceil(1000 / MAX_RPC_LEN) parallel chunks, so node-calls is - // ~10x the round-trips — the case where raw call count overstates latency. - { - label: `constrained/catch-up-5/secrets=1000`, - kind: AppTaggingSecretKind.CONSTRAINED, - secretCount: 1000, - priorCursor: 0, - newLogs: 5, - }, // Control: unconstrained steady state is unaffected by the optimization (windowed scan cannot first-miss). { label: `unconstrained/steady-state/secrets=100`, @@ -203,7 +184,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { const calls = aztecNode.getPrivateLogsByTags.mock.calls; const tagQueries = calls.reduce((sum, [query]) => sum + extractTags(query).length, 0); - const nodeCalls = calls.length; // Round-trips and blocking time from the same instrumentation the app benches use. `syncTaggedPrivateLogs` only // ever calls `getPrivateLogsByTags`, so every round-trip is that method. @@ -219,7 +199,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { ...scenario, logsFound: logs.length, tagQueries, - nodeCalls, rpcRoundTrips, rpcBlockingTimeMs, syncMs, @@ -236,7 +215,7 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { `${row.label.padEnd(42)} tag-queries=${String(row.tagQueries).padStart(6)} ` + `first-miss-optimum=${String(row.firstMissOptimum).padStart(6)} ` + `reduction=${(row.tagQueries / row.firstMissOptimum).toFixed(1)}x ` + - `round-trips=${String(row.rpcRoundTrips).padStart(4)} node-calls=${String(row.nodeCalls).padStart(4)} ` + + `round-trips=${String(row.rpcRoundTrips).padStart(4)} ` + `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)} ` + `time=${row.syncMs.toFixed(1)}ms`, ); @@ -248,12 +227,8 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { // (K = 0) it collapses to a single initial probe per secret. const stepsToFirstMiss = Math.ceil((scenario.newLogs + 1) / INITIAL_CONSTRAINED_PROBE_LEN); expect(row.tagQueries).toBe(scenario.secretCount * stepsToFirstMiss * INITIAL_CONSTRAINED_PROBE_LEN); - // One sequential round-trip per probe step — the negative-case cost, independent of chunking. + // One sequential round-trip per probe step (the negative-case cost), independent of chunking and secret count. expect(row.rpcRoundTrips).toBe(stepsToFirstMiss); - // node-calls = round-trips x chunks per round. Each round queries `secretCount * P` tags split into - // MAX_RPC_LEN chunks, so node-calls equals round-trips only while a round fits one chunk, and fans out above. - const chunksPerRound = Math.ceil((scenario.secretCount * INITIAL_CONSTRAINED_PROBE_LEN) / MAX_RPC_LEN); - expect(row.nodeCalls).toBe(stepsToFirstMiss * chunksPerRound); } else if (scenario.newLogs === 0) { // Unconstrained steady state: still the full window (control for the optimization), drained in one round. expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); @@ -265,7 +240,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { const results: BenchResult[] = rows.flatMap(row => [ { name: `TagSync/${row.label}/tag-queries`, value: row.tagQueries, unit: 'tag-queries' }, { name: `TagSync/${row.label}/rpc-round-trips`, value: row.rpcRoundTrips, unit: 'round_trips' }, - { name: `TagSync/${row.label}/node-calls`, value: row.nodeCalls, unit: 'calls' }, { name: `TagSync/${row.label}/rpc-blocking-time`, value: Number(row.rpcBlockingTimeMs.toFixed(2)), unit: 'ms' }, { name: `TagSync/${row.label}/sync-time`, value: Number(row.syncMs.toFixed(2)), unit: 'ms' }, ]); From f738f6c8ac0b9035360f0ae606ef661cca38ad18 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 24 Jun 2026 23:21:35 -0400 Subject: [PATCH 05/24] refactor(pxe): drop sync-time from constrained tag-sync bench The mocked node makes sync-time an unfaithful end-to-end latency; real full-sync timing already lives in the client_flows /sync e2e bench (ProvingTimings.sync). Keeps the metrics the micro-bench is for: tag-queries, round-trips, and blocking-time. --- .../recipient_sync/sync_tagged_private_logs.bench.test.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index b40bed78bc35..cbd501759d13 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -2,7 +2,6 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; -import { Timer } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { @@ -171,7 +170,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { // Proxy delegates to the underlying mock, so `mock.calls` still records every query for tag counting. const benchmarkedNode = BenchmarkedNodeFactory.create(aztecNode); - const timer = new Timer(); const logs = await syncTaggedPrivateLogs( secrets, benchmarkedNode, @@ -180,7 +178,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { FINALIZED_BLOCK_NUMBER, JOB_ID, ); - const syncMs = timer.ms(); const calls = aztecNode.getPrivateLogsByTags.mock.calls; const tagQueries = calls.reduce((sum, [query]) => sum + extractTags(query).length, 0); @@ -201,7 +198,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { tagQueries, rpcRoundTrips, rpcBlockingTimeMs, - syncMs, firstMissOptimum, }; } @@ -216,8 +212,7 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { `first-miss-optimum=${String(row.firstMissOptimum).padStart(6)} ` + `reduction=${(row.tagQueries / row.firstMissOptimum).toFixed(1)}x ` + `round-trips=${String(row.rpcRoundTrips).padStart(4)} ` + - `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)} ` + - `time=${row.syncMs.toFixed(1)}ms`, + `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)}`, ); // Pin behavior as an executable assertion, not just a printout. Timings are reported only (they vary run to run). @@ -241,7 +236,6 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { { name: `TagSync/${row.label}/tag-queries`, value: row.tagQueries, unit: 'tag-queries' }, { name: `TagSync/${row.label}/rpc-round-trips`, value: row.rpcRoundTrips, unit: 'round_trips' }, { name: `TagSync/${row.label}/rpc-blocking-time`, value: Number(row.rpcBlockingTimeMs.toFixed(2)), unit: 'ms' }, - { name: `TagSync/${row.label}/sync-time`, value: Number(row.syncMs.toFixed(2)), unit: 'ms' }, ]); if (process.env.BENCH_OUTPUT) { From 0926ae6b6fc3a3cba1c8970f167fa4492ced5c87 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 25 Jun 2026 15:42:14 -0400 Subject: [PATCH 06/24] test(pxe): add mixed-load scenario to constrained tag-sync bench Add a realistic mixed-population row (999 idle + 1 deep straggler at K=100) and run deep catch-up at 1000 secrets as well as 100, so every value in the PR-description tables is produced by the committed bench. Generalize scenario seeding and assertions to a per-secret new-log distribution and compute firstMissOptimum from it. --- .../sync_tagged_private_logs.bench.test.ts | 87 ++++++++++++++----- 1 file changed, 64 insertions(+), 23 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index cbd501759d13..26fe624d2cf6 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -49,7 +49,10 @@ import { * Scenario labels: `steady-state` is no new logs (K = 0); `catch-up-K` is K new contiguous logs per secret since the * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K and P * (not N), the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while - * round-trips do not. The `unconstrained` row is the control: it cannot first-miss (windowed scan), so its cost is fixed. + * round-trips do not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it + * isolates that tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count + * (round-trips therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed + * scan), so its cost is fixed. */ const logger = createLogger('pxe:tagging:bench'); @@ -77,8 +80,25 @@ type Scenario = { priorCursor: number; /** New contiguous finalized logs available per secret since the last sync (0 = steady state). */ newLogs: number; + /** + * Optional heterogeneous load: the first `count` secrets get `newLogs` new logs each, the remaining + * `secretCount - count` are idle (K = 0). When set, the scenario-level `newLogs` is ignored. Models a realistic + * sync where most senders are quiet and a few are deep in catch-up. + */ + deepCohort?: { count: number; newLogs: number }; }; +/** + * The per-secret new-log distribution for a scenario: `deepCohort.count` secrets at `deepCohort.newLogs`, the rest idle, + * or a uniform `newLogs` for every secret when no cohort is set. Single source for both log seeding and the expected + * tag-query / round-trip assertions. + */ +function newLogsPerSecret(scenario: Scenario): number[] { + return Array.from({ length: scenario.secretCount }, (_, i) => + scenario.deepCohort && i < scenario.deepCohort.count ? scenario.deepCohort.newLogs : scenario.newLogs, + ); +} + const SCENARIOS: Scenario[] = [ // steady-state (K = 0): no new logs since the last sync, across recipient secret counts. The dominant case and where // first-miss wins ~20x. @@ -100,15 +120,30 @@ const SCENARIOS: Scenario[] = [ newLogs, })), ), - // Deep catch-up at 100 secrets (the negative case): round-trips grow one per probe step while tag-queries stay at the - // K + 1 floor. From a full window (catch-up-20) up, the WINDOW_LEN cap forces multiple rounds at any P. - ...[UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, 50, 100].map(newLogs => ({ - label: `constrained/catch-up-${newLogs}/secrets=100`, + // Deep catch-up at 100 and 1000 secrets (the negative case): round-trips grow one per probe step while tag-queries stay + // at the K + 1 floor. From a full window (catch-up-20) up, the WINDOW_LEN cap forces multiple rounds at any P. As with + // light catch-up, round-trips depend only on K and P (not N), so the two secret counts share a round-trip count and + // differ only in tag-queries (10x) and blocking time. + ...[100, 1000].flatMap(secretCount => + [UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, 50, 100].map(newLogs => ({ + label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, + kind: AppTaggingSecretKind.CONSTRAINED, + secretCount, + priorCursor: 0, + newLogs, + })), + ), + // Mixed (the realistic active sync): 999 idle senders + 1 deep straggler (K = 100). Tag-queries stay dominated by the + // idle majority, but the single straggler alone sets the round-trip count (tags are batched across all secrets per + // round, so the round count equals the deepest secret's), so round-trips match catch-up-100. + { + label: `constrained/mixed/secrets=1000`, kind: AppTaggingSecretKind.CONSTRAINED, - secretCount: 100, + secretCount: 1000, priorCursor: 0, - newLogs, - })), + newLogs: 0, + deepCohort: { count: 1, newLogs: 100 }, + }, // Control: unconstrained steady state is unaffected by the optimization (windowed scan cannot first-miss). { label: `unconstrained/steady-state/secrets=100`, @@ -139,7 +174,8 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { } async function runScenario(scenario: Scenario) { - const { kind, secretCount, priorCursor, newLogs } = scenario; + const { kind, secretCount, priorCursor } = scenario; + const perSecretNewLogs = newLogsPerSecret(scenario); aztecNode.getPrivateLogsByTags.mockReset(); const taggingStore = new RecipientTaggingStore(await openTmpStore('bench')); @@ -153,11 +189,11 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { } } - // Tags that should resolve to a finalized log: the contiguous run (priorCursor, priorCursor + newLogs]. + // Tags that should resolve to a finalized log: per secret, the contiguous run (priorCursor, priorCursor + K]. const hitTags = new Set(); - for (const secret of secrets) { - for (let k = 1; k <= newLogs; k++) { - hitTags.add((await computeSiloedTagForIndex(secret, priorCursor + k)).toString()); + for (let s = 0; s < secrets.length; s++) { + for (let k = 1; k <= perSecretNewLogs[s]; k++) { + hitTags.add((await computeSiloedTagForIndex(secrets[s], priorCursor + k)).toString()); } } @@ -188,9 +224,10 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { const rpcRoundTrips = roundTrips.roundTrips; const rpcBlockingTimeMs = roundTrips.totalBlockingTime; - // First-miss floor: each secret pays `newLogs` hits + 1 miss. Unconstrained cannot first-miss, so its floor is + // First-miss floor: each secret pays its K hits + 1 miss. Unconstrained cannot first-miss, so its floor is // its current cost. - const firstMissOptimum = kind === AppTaggingSecretKind.CONSTRAINED ? secretCount * (newLogs + 1) : tagQueries; + const firstMissOptimum = + kind === AppTaggingSecretKind.CONSTRAINED ? perSecretNewLogs.reduce((sum, k) => sum + k + 1, 0) : tagQueries; return { ...scenario, @@ -216,20 +253,24 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { ); // Pin behavior as an executable assertion, not just a printout. Timings are reported only (they vary run to run). + const perSecretNewLogs = newLogsPerSecret(scenario); if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { - // Fixed-step probing is tag-optimal: K hits plus 1 terminating miss, rounded up to whole probe steps. At - // INITIAL_CONSTRAINED_PROBE_LEN = 1 this equals the first-miss floor (reduction = 1.0x), and at steady state - // (K = 0) it collapses to a single initial probe per secret. - const stepsToFirstMiss = Math.ceil((scenario.newLogs + 1) / INITIAL_CONSTRAINED_PROBE_LEN); - expect(row.tagQueries).toBe(scenario.secretCount * stepsToFirstMiss * INITIAL_CONSTRAINED_PROBE_LEN); - // One sequential round-trip per probe step (the negative-case cost), independent of chunking and secret count. - expect(row.rpcRoundTrips).toBe(stepsToFirstMiss); + // Fixed-step probing is tag-optimal: each secret pays its K hits plus 1 terminating miss, rounded up to whole + // probe steps. At INITIAL_CONSTRAINED_PROBE_LEN = 1 this equals the first-miss floor (reduction = 1.0x), and at + // steady state (K = 0) it collapses to a single initial probe per secret. + const stepsPerSecret = perSecretNewLogs.map(k => Math.ceil((k + 1) / INITIAL_CONSTRAINED_PROBE_LEN)); + expect(row.tagQueries).toBe( + stepsPerSecret.reduce((sum, steps) => sum + steps * INITIAL_CONSTRAINED_PROBE_LEN, 0), + ); + // Tags are batched across all secrets per round, so the round-trip count is the deepest secret's: one sequential + // round-trip per probe step, independent of chunking and secret count. + expect(row.rpcRoundTrips).toBe(Math.max(...stepsPerSecret)); } else if (scenario.newLogs === 0) { // Unconstrained steady state: still the full window (control for the optimization), drained in one round. expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); expect(row.rpcRoundTrips).toBe(1); } - expect(row.logsFound).toBe(scenario.secretCount * scenario.newLogs); + expect(row.logsFound).toBe(perSecretNewLogs.reduce((sum, k) => sum + k, 0)); } const results: BenchResult[] = rows.flatMap(row => [ From 66a5ff0700b141563d87dd333d595a57a600328f Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 25 Jun 2026 17:07:02 -0400 Subject: [PATCH 07/24] doubling --- yarn-project/pxe/src/tagging/constants.ts | 11 ++- .../sync_tagged_private_logs.bench.test.ts | 83 +++++++++++++------ .../sync_tagged_private_logs.test.ts | 72 ++++++++++++---- .../sync_tagged_private_logs.ts | 29 +++++-- 4 files changed, 139 insertions(+), 56 deletions(-) diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts index 22844c73b585..caf54a02dc20 100644 --- a/yarn-project/pxe/src/tagging/constants.ts +++ b/yarn-project/pxe/src/tagging/constants.ts @@ -9,10 +9,9 @@ // set it to a relatively low value of 20, which is sufficient for current use cases. export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; -// Number of tags probed per constrained secret per round: both the initial probe size and the step the probe grows by -// each round until the first missing tag. Constrained delivery is gapless, so a single missing tag proves the stream -// has ended. At steady state (no new logs) this turns a full WINDOW_LEN probe into one tag, and a secret with K new -// logs costs exactly K + 1 tags instead of the whole window. Larger values trade more steady-state tags (P per secret) -// for fewer sequential round-trips during catch-up (a probe of P resolves K < P new logs in a single round), so 1 is -// the right default while idle secrets dominate. +// The number of tags probed per constrained secret in the first round. The probe then doubles each round (capped at +// UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) while every probed index is a hit, stopping at the first missing tag. +// Constrained delivery is gapless, so a single missing tag proves the stream has ended: at steady state this turns a +// full WINDOW_LEN probe into a single tag, and a secret K logs behind catches up in ~log2(K) round-trips (probing +// 1, 2, 4, ... tags) rather than the full window or one round per log. export const INITIAL_CONSTRAINED_PROBE_LEN = 1; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 26fe624d2cf6..13ccfa282f53 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -31,28 +31,31 @@ import { * Benchmark for constrained recipient tag-sync. * * Measures the per-sync cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are gapless, so - * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and grows one such step at a time, stopping - * at the first missing tag instead of fetching the full `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window. + * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubles it each round (capped at the + * `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window), stopping at the first missing tag instead of fetching the + * full window. This validates the shipped doubling policy; the fixed-step sweep that motivated choosing it lives in the + * PR description (reproducible from this branch's pre-doubling history). * * Metrics, per scenario: * - `tag-queries`: total tags queried, the throughput win. At steady state it drops from a full window per secret to a - * single tag; a secret with K new logs costs exactly K + 1 (the "first-miss optimum" floor, reduction 1.0x). + * single tag; a secret K logs behind costs a little above the K + 1 first-miss floor (the doubling probe overshoots + * slightly to cut round-trips). The floor is reported as the reduction baseline. * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as one), - * via `BenchmarkedNodeFactory`. The latency axis: fixed-step trades fewer tags for more round-trips, so it grows - * ~linearly with K during catch-up while tag-queries stay at the floor. Depends only on K and P, not on secret count. - * A round's tags are chunked at MAX_RPC_LEN (=100) into parallel calls internally, but those overlap, so a wide round - * is still one round-trip; that is why round-trips, not raw call count, is the latency axis. + * via `BenchmarkedNodeFactory`. The latency axis: doubling grows the probe geometrically, so round-trips rise only + * ~log2(K) during catch-up while tag-queries stay near the floor. Depends only on K, not on secret count. A round's + * tags are chunked at MAX_RPC_LEN (=100) into parallel calls internally, but those overlap, so a wide round is still + * one round-trip; that is why round-trips, not raw call count, is the latency axis. * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` * per call plus a little per-round overhead. Parallel calls within a round overlap, so it tracks round-trips (a * 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported only (varies run to run). * * Scenario labels: `steady-state` is no new logs (K = 0); `catch-up-K` is K new contiguous logs per secret since the - * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K and P - * (not N), the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while - * round-trips do not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it - * isolates that tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count - * (round-trips therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed - * scan), so its cost is fixed. + * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K (not N), + * the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while round-trips do + * not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it isolates that + * tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count (round-trips + * therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed scan), so its + * cost is fixed. */ const logger = createLogger('pxe:tagging:bench'); @@ -99,6 +102,42 @@ function newLogsPerSecret(scenario: Scenario): number[] { ); } +/** + * Mirrors `processConstrainedResults` for a single secret with `k` new contiguous logs (`priorCursor = 0`): the probe + * starts at `INITIAL_CONSTRAINED_PROBE_LEN`, doubles each round (capped at WINDOW_LEN), and the queried range is bounded + * by the WINDOW_LEN-ahead-of-finalized frontier. Returns the total tags queried and the sequential round-trip count. + * Single source for the expected tag-query / round-trip assertions. + */ +function probeSchedule(k: number): { tags: number; rounds: number } { + let start = 1; // priorCursor (0) + 1 + let highestFinalized = 0; // priorCursor + let probeLen = INITIAL_CONSTRAINED_PROBE_LEN; + let tags = 0; + let rounds = 0; + for (;;) { + const boundEnd = highestFinalized + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; + const end = Math.min(boundEnd, start + probeLen); + if (end <= start) { + break; + } + tags += end - start; + rounds++; + // Hits occupy indexes 1..k; the round stops at the first index in [start, end) without a log. + const lastHit = Math.min(end - 1, k); + const fullyConsumed = lastHit === end - 1; + if (lastHit >= start) { + highestFinalized = lastHit; + } + // Stop on the first miss, or once the queried range has reached the (re-anchored) bound. + if (!fullyConsumed || end >= highestFinalized + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1) { + break; + } + start = end; + probeLen = Math.min(probeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + } + return { tags, rounds }; +} + const SCENARIOS: Scenario[] = [ // steady-state (K = 0): no new logs since the last sync, across recipient secret counts. The dominant case and where // first-miss wins ~20x. @@ -252,19 +291,15 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)}`, ); - // Pin behavior as an executable assertion, not just a printout. Timings are reported only (they vary run to run). + // Pin behavior as an executable assertion, not just a printout. Timings are reported only (vary run to run). const perSecretNewLogs = newLogsPerSecret(scenario); if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { - // Fixed-step probing is tag-optimal: each secret pays its K hits plus 1 terminating miss, rounded up to whole - // probe steps. At INITIAL_CONSTRAINED_PROBE_LEN = 1 this equals the first-miss floor (reduction = 1.0x), and at - // steady state (K = 0) it collapses to a single initial probe per secret. - const stepsPerSecret = perSecretNewLogs.map(k => Math.ceil((k + 1) / INITIAL_CONSTRAINED_PROBE_LEN)); - expect(row.tagQueries).toBe( - stepsPerSecret.reduce((sum, steps) => sum + steps * INITIAL_CONSTRAINED_PROBE_LEN, 0), - ); - // Tags are batched across all secrets per round, so the round-trip count is the deepest secret's: one sequential - // round-trip per probe step, independent of chunking and secret count. - expect(row.rpcRoundTrips).toBe(Math.max(...stepsPerSecret)); + // `probeSchedule` mirrors the real doubling scan: a secret K logs behind resolves in ~log2(K) round-trips at a + // little above the K + 1 tag floor. Tags are batched across all secrets per round, so the round-trip count is + // the deepest secret's. + const sched = perSecretNewLogs.map(k => probeSchedule(k)); + expect(row.tagQueries).toBe(sched.reduce((sum, s) => sum + s.tags, 0)); + expect(row.rpcRoundTrips).toBe(Math.max(...sched.map(s => s.rounds))); } else if (scenario.newLogs === 0) { // Unconstrained steady state: still the full window (control for the optimization), drained in one round. expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 2e0ea06a1da8..c1a3d56a2998 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -268,7 +268,7 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('fully drains a long contiguous run one probe at a time', async () => { + it('fully drains a long contiguous run beyond the window', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -290,11 +290,12 @@ describe('syncTaggedPrivateLogs', () => { ); // The whole run is returned and the cursor lands on the last index, even though the run is longer than a single - // window. With INITIAL_CONSTRAINED_PROBE_LEN = 1 the scan advances one index per round, so it takes one round - // per log plus a final round for the terminating miss. + // window: boundEnd re-anchors to each finalized index found, so the doubling probe keeps advancing past one + // window until the terminating miss. expect(logs).toHaveLength(totalLogs); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(totalLogs + 1); + // Doubling drains the run in far fewer rounds than one-per-log. + expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeLessThan(totalLogs); }); it('persists cursor only up to finalized block', async () => { @@ -368,17 +369,18 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); }); - // Pins the committed P=1 behavior: a fully-consumed probe advances by one INITIAL_CONSTRAINED_PROBE_LEN step, - // not the whole window. - it('catch-up probes one step at a time, not the full window', async () => { + // Pins the probe schedule: the probe doubles each round (1, 2, 4, ...) until the first miss, so K-deep + // catch-up resolves in ~log2(K) round-trips instead of K + 1. With 3 new logs the windows are [1], [2,3], + // [4,5,6,7] - round 3 (probe length 4) contains the terminating miss at index 4. + it('doubling catch-up grows the probe geometrically and stops at the first miss', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // Recipient already synced index 0; exactly one new finalized log sits at index 1. + // Recipient already synced index 0; three new finalized logs sit at indexes 1..3. await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); - const newLogTag = await computeSiloedTagForIndex(secret, 1); - mockNodeWithLogs([newLogTag], Number(finalizedBlockNumber), agedTimestamp); + const hitTags = await Promise.all([1, 2, 3].map(i => computeSiloedTagForIndex(secret, i))); + mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); const logs = await syncTaggedPrivateLogs( [secret], @@ -389,16 +391,50 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - expect(logs).toHaveLength(1); - expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(1); + expect(logs).toHaveLength(3); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(3); - // Round 1 probes index 1 (the hit); round 2 probes only index 2 (the terminating miss) — a single - // INITIAL_CONSTRAINED_PROBE_LEN step, not the full window. The old full-window fallback queried indexes - // 2..(1 + WINDOW_LEN) in round 2. + // Probe windows double each round: [1], then [2,3], then [4,5,6,7] where index 4 is the terminating miss. const calls = aztecNode.getPrivateLogsByTags.mock.calls; - expect(calls).toHaveLength(2); - expect(extractTags(calls[0][0])).toEqual([await computeSiloedTagForIndex(secret, 1)]); - expect(extractTags(calls[1][0])).toEqual([await computeSiloedTagForIndex(secret, 2)]); + expect(calls).toHaveLength(3); + expect(extractTags(calls[0][0])).toEqual(await Promise.all([1].map(i => computeSiloedTagForIndex(secret, i)))); + expect(extractTags(calls[1][0])).toEqual(await Promise.all([2, 3].map(i => computeSiloedTagForIndex(secret, i)))); + expect(extractTags(calls[2][0])).toEqual( + await Promise.all([4, 5, 6, 7].map(i => computeSiloedTagForIndex(secret, i))), + ); + }); + + // Pins the effective per-round query cap for the doubling probe: it ramps 1, 2, 4, 8, 16 and then saturates at + // WINDOW_LEN (=20) per round rather than growing without bound, because a probe can never reach more than WINDOW_LEN + // past the finalized frontier (the `boundEnd` cap, which the stored-intent `Math.min` mirrors). With 100 new logs + // the per-round probe sizes are [1, 2, 4, 8, 16, 20, 20, 20, 20]. + it('doubling caps the probe at WINDOW_LEN on deep catch-up', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Recipient already synced index 0; a deep run of 100 new finalized logs sits at indexes 1..100. + const newLogs = 100; + await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); + const hitTags = await Promise.all( + Array.from({ length: newLogs }, (_, i) => computeSiloedTagForIndex(secret, i + 1)), + ); + mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + expect(logs).toHaveLength(newLogs); + // Geometric ramp, then saturation at WINDOW_LEN: no round ever queries more than 20 tags. + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes).toEqual([1, 2, 4, 8, 16, 20, 20, 20, 20]); + expect(Math.max(...callSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); }); }); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index eec5ce3a7813..7f77203d788c 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -64,9 +64,9 @@ import { findHighestIndexes } from './utils/find_highest_indexes.js'; * one is already on-chain. * - Because the sequence is gapless, the scan stops at the first missing index: that gap proves the sequence has * ended and no further logs exist. We exploit this by probing only a small initial window - * (`INITIAL_CONSTRAINED_PROBE_LEN`) and growing by another such step each round while every probed index has a + * (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubling it each round (capped at the window) while every probed index has a * log, stopping at the first miss. A steady-state sync with no new logs costs a single tag instead of the whole - * window, and a sync with K new logs costs exactly K + 1 tags (K hits plus the terminating miss). + * window, and a secret K logs behind catches up in ~log2(K) round-trips instead of one round per log. * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. Advancing the probe is * decoupled from persisting the finalized cursor, so unfinalized logs at the top of the run are still fetched * while only the finalized prefix is persisted. @@ -161,7 +161,10 @@ function getIndexRangesForSecrets( ? Math.min(boundEnd, start + INITIAL_CONSTRAINED_PROBE_LEN) : boundEnd; - return { secret, start, end, boundEnd }; + // Only constrained secrets grow their probe round to round; unconstrained always scans the full window. + const probeLen = secret.kind === AppTaggingSecretKind.CONSTRAINED ? INITIAL_CONSTRAINED_PROBE_LEN : undefined; + + return { secret, start, end, boundEnd, probeLen }; }), ); } @@ -247,20 +250,27 @@ async function processConstrainedResults( // (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. Gating this on // finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. The bound // re-anchors to any finalized index found this round. Because the stream is gapless, we only need to fetch up to the - // first missing tag, so the probe grows by another INITIAL_CONSTRAINED_PROBE_LEN step each round (capped at the - // bound) rather than jumping to the full window: a secret with K new logs costs exactly K + 1 tags. + // first missing tag, so the probe advances by the growth policy below (capped at the bound) rather than jumping to + // the full window. const probeFullyConsumed = firstMissingIndex >= pending.end; const boundEnd = highestFinalizedIndex !== undefined ? Math.max(pending.boundEnd, highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1) : pending.boundEnd; + // Double the probe each round so a secret K logs behind resolves in ~log2(K) round-trips instead of one per log. The + // queried range is already bounded by boundEnd (a probe can never reach more than WINDOW_LEN past the finalized + // frontier); the cap on the stored length just keeps it from growing unbounded once the probe saturates the window. + const currentProbeLen = pending.probeLen ?? INITIAL_CONSTRAINED_PROBE_LEN; + const nextProbeLen = Math.min(currentProbeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + if (probeFullyConsumed && pending.end < boundEnd) { return { secret: pending.secret, start: pending.end, - end: Math.min(boundEnd, pending.end + INITIAL_CONSTRAINED_PROBE_LEN), + end: Math.min(boundEnd, pending.end + nextProbeLen), boundEnd, + probeLen: nextProbeLen, }; } @@ -320,9 +330,12 @@ type PendingSecret = { start: number; end: number; // Highest index this secret may probe this sync (highest finalized index + WINDOW_LEN + 1, the protocol bound on how - // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets advances by - // INITIAL_CONSTRAINED_PROBE_LEN per round up to this bound. + // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets advances by doubling the + // probe each round up to this bound. boundEnd: number; + // Intended constrained probe length for the round that produced `end`; the next round doubles it (capped at + // WINDOW_LEN). Only set/read for constrained secrets - unconstrained secrets always scan the full window. + probeLen?: number; }; type LogWithIndex = { From fbae65861358366f625785b56a2dc744be4c8b04 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 25 Jun 2026 21:34:49 -0400 Subject: [PATCH 08/24] refactor(pxe): dedupe constrained tag-sync comments The constrained tag-sync source restated "gapless stream stops at the first missing tag" ~5x and the doubling-probe mechanics ~4x. Keep one canonical home for each (the module JSDoc for the why, the nextProbeLen comment for the mechanics) and reduce the rest to one-line local facts; also correct boundEnd's comment to note it is an exclusive bound and probeLen's to note the queried span can be smaller when boundEnd caps it. Drop the tag-sync micro-bench from bench_cmds: it existed to find the probe-growth numbers locally, which is done, so we no longer track it on the benchmark dashboard. It stays runnable as a local tool, and the doubling behavior remains covered in CI by the unit-test file. --- yarn-project/bootstrap.sh | 1 - .../sync_tagged_private_logs.bench.test.ts | 2 +- .../sync_tagged_private_logs.ts | 22 ++++++++----------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 43e78c0312d5..6c84b8b7a6d3 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -249,7 +249,6 @@ function bench_cmds { echo "$hash BENCH_OUTPUT=bench-out/sim.bench.json yarn-project/scripts/run_test.sh simulator/src/public/public_tx_simulator/apps_tests/bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/native_world_state.bench.json yarn-project/scripts/run_test.sh world-state/src/native/native_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/kv_store.bench.json yarn-project/scripts/run_test.sh kv-store/src/bench/map_bench.test.ts" - echo "$hash BENCH_OUTPUT=bench-out/tag_sync.bench.json yarn-project/scripts/run_test.sh pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_pool_v2.bench.json yarn-project/scripts/run_test.sh p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_validator.bench.json yarn-project/scripts/run_test.sh p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts" echo "$hash:ISOLATE=1:CPUS=16:MEM=32g:TIMEOUT=1800 BENCH_OUTPUT=bench-out/p2p_client_batch_tx_requester.bench.json yarn-project/scripts/run_test.sh p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts" diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 13ccfa282f53..d160944f5553 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -71,7 +71,7 @@ const JOB_ID = 'bench-job'; // The round-trip *count* is independent of this value; only `rpc-blocking-time` scales with it. const MODELED_NODE_RPC_LATENCY_MS = 5; -/** One benchmark measurement in the GitHub-action benchmark JSON shape. */ +/** One benchmark measurement in the benchmark JSON shape. */ type BenchResult = { name: string; value: number; unit: string }; type Scenario = { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index 7f77203d788c..00f088ee8a33 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -161,7 +161,7 @@ function getIndexRangesForSecrets( ? Math.min(boundEnd, start + INITIAL_CONSTRAINED_PROBE_LEN) : boundEnd; - // Only constrained secrets grow their probe round to round; unconstrained always scans the full window. + // Constrained-only: continued probes grow each round up to the window cap (see processConstrainedResults). const probeLen = secret.kind === AppTaggingSecretKind.CONSTRAINED ? INITIAL_CONSTRAINED_PROBE_LEN : undefined; return { secret, start, end, boundEnd, probeLen }; @@ -218,8 +218,8 @@ async function fetchLogsForSecrets( } /** - * Processes fetched logs for a constrained secret. Constrained delivery guarantees contiguous index sequences, - * so we scan from the start of the queried range and stop at the first missing index. Only `highestFinalizedIndex` + * Processes fetched logs for a constrained secret: persists the finalized cursor and advances the probe. See the + * "# Constrained secrets" section above for why the scan stops at the first missing index. Only `highestFinalizedIndex` * is tracked (no aged index). */ async function processConstrainedResults( @@ -230,8 +230,7 @@ async function processConstrainedResults( finalizedBlockNumber: BlockNumber, jobId: string, ): Promise { - // Find where the contiguous run of indexes ends. Constrained delivery guarantees there are no logs after the - // first gap, so all logs in the batch are within this prefix. + // Find where the contiguous run of indexes ends; all logs in the batch fall within this prefix. const indexesWithLogs = new Set(logsWithIndexes.map(l => l.taggingIndex)); let firstMissingIndex = pending.start; while (firstMissingIndex < pending.end && indexesWithLogs.has(firstMissingIndex)) { @@ -249,9 +248,7 @@ async function processConstrainedResults( // Advancing the probe is decoupled from persisting the cursor: keep scanning as long as the probe was entirely hits // (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. Gating this on // finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. The bound - // re-anchors to any finalized index found this round. Because the stream is gapless, we only need to fetch up to the - // first missing tag, so the probe advances by the growth policy below (capped at the bound) rather than jumping to - // the full window. + // re-anchors to any finalized index found this round. const probeFullyConsumed = firstMissingIndex >= pending.end; const boundEnd = highestFinalizedIndex !== undefined @@ -329,12 +326,11 @@ type PendingSecret = { secret: AppTaggingSecret; start: number; end: number; - // Highest index this secret may probe this sync (highest finalized index + WINDOW_LEN + 1, the protocol bound on how - // far ahead of finalized a log can sit). Distinct from `end`, which for constrained secrets advances by doubling the - // probe each round up to this bound. + // Exclusive upper bound on the indexes this secret may probe this sync (highest finalized index + WINDOW_LEN + 1, the + // protocol bound on how far ahead of finalized a log can sit). Constrained `end` grows toward it each round. boundEnd: number; - // Intended constrained probe length for the round that produced `end`; the next round doubles it (capped at - // WINDOW_LEN). Only set/read for constrained secrets - unconstrained secrets always scan the full window. + // Intended constrained probe length for the round that produced `end` (constrained-only). The queried span can be + // smaller when boundEnd caps it. probeLen?: number; }; From 9eb3c8618163d1a37b71870c26e40d46fd01a3b8 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 25 Jun 2026 21:59:27 -0400 Subject: [PATCH 09/24] comment --- .../src/tagging/recipient_sync/sync_tagged_private_logs.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index c1a3d56a2998..55cdf5493e6f 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -344,7 +344,7 @@ describe('syncTaggedPrivateLogs', () => { // Indexes 0 and 1 are hits in an unfinalized block (8 > finalized 5); index 2 is the gap. With the small initial // probe, index 0 is probed alone and is unfinalized, so the scan must still advance to discover index 1 — gating - // advancement on finalization (as an earlier design did) would drop it. + // advancement on finalization would drop it. const unfinalizedTags = await Promise.all([0, 1].map(i => computeSiloedTagForIndex(secret, i))); aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { const tags = extractTags(query); From eb6d3b5e505981cce7f3542133eecd63bca74916 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 25 Jun 2026 23:00:40 -0400 Subject: [PATCH 10/24] cleanup tests --- .../sync_tagged_private_logs.test.ts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 55cdf5493e6f..749c2a9913e4 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -102,6 +102,13 @@ describe('syncTaggedPrivateLogs', () => { await syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); + // The single call must carry tags from every secret, not just one - otherwise "batches into a single call" would + // pass for a per-secret call that merely returned early. + const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + const firstTagPerSecret = await Promise.all(secrets.map(s => computeSiloedTagForIndex(s, 0))); + for (const tag of firstTagPerSecret) { + expect(calledTags.some(t => t.equals(tag))).toBe(true); + } }); it('syncs logs and updates store independently per secret', async () => { @@ -251,7 +258,10 @@ describe('syncTaggedPrivateLogs', () => { const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - const logTags = await Promise.all([0, 1, 2].map(i => computeSiloedTagForIndex(secret, i))); + // Logs sit at 0,1,2 with the gap at 3. The sentinel at index 10 lies beyond the round that discovers the gap + // (the doubling probe reaches [3..6] when it first misses): if the scan failed to stop at the gap and kept + // doubling it would eventually fetch index 10, so the sentinel's absence from the results proves the scan halted. + const logTags = await Promise.all([0, 1, 2, 10].map(i => computeSiloedTagForIndex(secret, i))); mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); const logs = await syncTaggedPrivateLogs( @@ -268,16 +278,17 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('fully drains a long contiguous run beyond the window', async () => { + it('drains a contiguous run longer than one window in a single sync', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + // Deliberately a couple of indexes past one window. With a frozen bound the scan would stop at WINDOW_LEN and + // miss the tail, so draining the whole run is what proves boundEnd re-anchors to each finalized index found. const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; const logTags = await Promise.all( Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), ); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); const logs = await syncTaggedPrivateLogs( @@ -289,13 +300,14 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // The whole run is returned and the cursor lands on the last index, even though the run is longer than a single - // window: boundEnd re-anchors to each finalized index found, so the doubling probe keeps advancing past one - // window until the terminating miss. + // The whole run is drained and the durable cursor lands on the last index even though the run outgrows one + // window: re-anchoring carries the probe past the cold-start bound to the terminating miss. expect(logs).toHaveLength(totalLogs); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - // Doubling drains the run in far fewer rounds than one-per-log. - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeLessThan(totalLogs); + // The probe doubles each round: [1, 2, 4, 8, 16]. Round 5 queries indexes [15, 30], reaching well past the + // cold-start bound (index 20) - only possible because boundEnd re-anchored to each finalized index found. + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes).toEqual([1, 2, 4, 8, 16]); }); it('persists cursor only up to finalized block', async () => { @@ -519,6 +531,16 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); + // Both kinds share the first RPC round: its tag list must carry tags from each secret rather than one call + // per secret. + const firstCallTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + const constrainedProbeTag = await computeSiloedTagForIndex(constrainedSecret, 0); + const unconstrainedProbeTag = await computeSiloedTagForIndex(unconstrainedSecret, 0); + expect(firstCallTags.some(t => t.equals(constrainedProbeTag))).toBe(true); + expect(firstCallTags.some(t => t.equals(unconstrainedProbeTag))).toBe(true); + + // Different stop conditions: the constrained scan halts at its first gap (index 2 missing, so cursor = 1) while + // the unconstrained scan tracks its highest aged/finalized hit (index 5). expect(logs).toHaveLength(4); expect(await taggingStore.getHighestFinalizedIndex(constrainedSecret, JOB_ID)).toBe(1); expect(await taggingStore.getHighestAgedIndex(constrainedSecret, JOB_ID)).toBeUndefined(); From c11b1b7da7a6a9993663a9cb1bb417bbfd400671 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 12:33:11 -0400 Subject: [PATCH 11/24] test(pxe): pin constrained tag-sync behaviors in dedicated tests Revert the in-place strengthening of the existing unconstrained and mixed tests and re-express the constrained-specific coverage as three new tests in the constrained-secrets block: multi-secret batching, halting at the first all-miss batch (sentinel proof), and doubling-probe re-anchoring past the cold-start bound. --- .../sync_tagged_private_logs.test.ts | 109 +++++++++++++----- 1 file changed, 79 insertions(+), 30 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 749c2a9913e4..3f7cd50823f1 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -102,13 +102,6 @@ describe('syncTaggedPrivateLogs', () => { await syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); - // The single call must carry tags from every secret, not just one - otherwise "batches into a single call" would - // pass for a per-secret call that merely returned early. - const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - const firstTagPerSecret = await Promise.all(secrets.map(s => computeSiloedTagForIndex(s, 0))); - for (const tag of firstTagPerSecret) { - expect(calledTags.some(t => t.equals(tag))).toBe(true); - } }); it('syncs logs and updates store independently per secret', async () => { @@ -258,10 +251,7 @@ describe('syncTaggedPrivateLogs', () => { const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // Logs sit at 0,1,2 with the gap at 3. The sentinel at index 10 lies beyond the round that discovers the gap - // (the doubling probe reaches [3..6] when it first misses): if the scan failed to stop at the gap and kept - // doubling it would eventually fetch index 10, so the sentinel's absence from the results proves the scan halted. - const logTags = await Promise.all([0, 1, 2, 10].map(i => computeSiloedTagForIndex(secret, i))); + const logTags = await Promise.all([0, 1, 2].map(i => computeSiloedTagForIndex(secret, i))); mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); const logs = await syncTaggedPrivateLogs( @@ -278,17 +268,16 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('drains a contiguous run longer than one window in a single sync', async () => { + it('fully drains a long contiguous run beyond the window', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // Deliberately a couple of indexes past one window. With a frozen bound the scan would stop at WINDOW_LEN and - // miss the tail, so draining the whole run is what proves boundEnd re-anchors to each finalized index found. const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; const logTags = await Promise.all( Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), ); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); const logs = await syncTaggedPrivateLogs( @@ -300,14 +289,13 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // The whole run is drained and the durable cursor lands on the last index even though the run outgrows one - // window: re-anchoring carries the probe past the cold-start bound to the terminating miss. + // The whole run is returned and the cursor lands on the last index, even though the run is longer than a single + // window: boundEnd re-anchors to each finalized index found, so the doubling probe keeps advancing past one + // window until the terminating miss. expect(logs).toHaveLength(totalLogs); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - // The probe doubles each round: [1, 2, 4, 8, 16]. Round 5 queries indexes [15, 30], reaching well past the - // cold-start bound (index 20) - only possible because boundEnd re-anchored to each finalized index found. - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual([1, 2, 4, 8, 16]); + // Doubling drains the run in far fewer rounds than one-per-log. + expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeLessThan(totalLogs); }); it('persists cursor only up to finalized block', async () => { @@ -448,6 +436,77 @@ describe('syncTaggedPrivateLogs', () => { expect(callSizes).toEqual([1, 2, 4, 8, 16, 20, 20, 20, 20]); expect(Math.max(...callSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); }); + + it('batches tags from multiple constrained secrets into a single RPC call', async () => { + const secrets = await makeSecrets(3, AppTaggingSecretKind.CONSTRAINED); + + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => + Promise.resolve(query.tags.map(() => [])), + ); + + await syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + + // One batched call carrying exactly each secret's initial probe tag (index 0): constrained secrets share the same + // RPC round as everything else rather than getting a call apiece. + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); + const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + const expectedTags = await Promise.all(secrets.map(s => computeSiloedTagForIndex(s, 0))); + expect(calledTags).toEqual(expectedTags); + }); + + it('halts at the first all-miss batch and never probes past the gap', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Contiguous run 0,1,2 ends at the gap (index 3); a sentinel log sits far past it at index 10. The doubling probe + // reaches the all-miss batch [3..6] and stops there, starting no further round, so index 10 is never queried. The + // sentinel is a falsifying witness: a scan that failed to stop would eventually fetch it. + const logTags = await Promise.all([0, 1, 2, 10].map(i => computeSiloedTagForIndex(secret, i))); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + // Only the contiguous prefix is returned, and the sentinel's index was never queried. + expect(logs).toHaveLength(3); + const sentinelTag = await computeSiloedTagForIndex(secret, 10); + const queriedTags = aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([q]) => extractTags(q)); + expect(queriedTags.some(t => t.equals(sentinelTag))).toBe(false); + }); + + it('doubling probe re-anchors past the cold-start bound in a single sync', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // A cold-start run two indexes past one window. The cold-start bound caps the probe at index 20, so reaching the + // tail is only possible because boundEnd re-anchors forward to each finalized index found. + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; + const logTags = await Promise.all( + Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), + ); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + + await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + + // The probe doubles each round: [1, 2, 4, 8, 16]. The final round queries indexes [15..30], reaching past the + // cold-start bound (20) - only possible because boundEnd re-anchored forward to each finalized index found. + const calls = aztecNode.getPrivateLogsByTags.mock.calls; + const callSizes = calls.map(([q]) => extractTags(q).length); + expect(callSizes).toEqual([1, 2, 4, 8, 16]); + const lastCallTags = extractTags(calls[calls.length - 1][0]); + const expectedLastRange = await Promise.all( + Array.from({ length: 16 }, (_, i) => computeSiloedTagForIndex(secret, 15 + i)), + ); + expect(lastCallTags).toEqual(expectedLastRange); + }); }); describe('constrained vs unconstrained probing', () => { @@ -531,16 +590,6 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // Both kinds share the first RPC round: its tag list must carry tags from each secret rather than one call - // per secret. - const firstCallTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - const constrainedProbeTag = await computeSiloedTagForIndex(constrainedSecret, 0); - const unconstrainedProbeTag = await computeSiloedTagForIndex(unconstrainedSecret, 0); - expect(firstCallTags.some(t => t.equals(constrainedProbeTag))).toBe(true); - expect(firstCallTags.some(t => t.equals(unconstrainedProbeTag))).toBe(true); - - // Different stop conditions: the constrained scan halts at its first gap (index 2 missing, so cursor = 1) while - // the unconstrained scan tracks its highest aged/finalized hit (index 5). expect(logs).toHaveLength(4); expect(await taggingStore.getHighestFinalizedIndex(constrainedSecret, JOB_ID)).toBe(1); expect(await taggingStore.getHighestAgedIndex(constrainedSecret, JOB_ID)).toBeUndefined(); From 6e95bdb67538304468c52cf7d5275d3ee9fae518 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 14:31:02 -0400 Subject: [PATCH 12/24] unit test on probe reset between syncs --- .../sync_tagged_private_logs.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 3f7cd50823f1..4c614a63f826 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -507,6 +507,38 @@ describe('syncTaggedPrivateLogs', () => { ); expect(lastCallTags).toEqual(expectedLastRange); }); + + it('resets the probe on the sync after a catch-up', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // A cold-start run long enough that the first sync grows the in-memory probe through [1, 2, 4, 8, 16]. The probe + // length lives only on the in-memory PendingSecret (never persisted), so the next sync must start fresh at the + // initial probe rather than inheriting the grown length. + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; + const logTags = await Promise.all( + Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), + ); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + + // First sync catches up the whole run; the cursor lands on the last index. + await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + + // Drop the catch-up's recorded calls (mockClear keeps the implementation; mockReset would not) so the next + // assertions see only the second sync. + aztecNode.getPrivateLogsByTags.mockClear(); + + // Second sync, no new logs: it must probe a single tag at cursor + 1, not the grown probe from the catch-up. + await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + + const calls = aztecNode.getPrivateLogsByTags.mock.calls; + expect(calls).toHaveLength(1); + const probedTags = extractTags(calls[0][0]); + expect(probedTags).toHaveLength(INITIAL_CONSTRAINED_PROBE_LEN); + expect(probedTags).toEqual([await computeSiloedTagForIndex(secret, totalLogs)]); + }); }); describe('constrained vs unconstrained probing', () => { From f8634274daa593dd72d8d9ae494dfaa54bfad1c1 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 15:04:46 -0400 Subject: [PATCH 13/24] test(pxe): pin unconstrained full-window drain of a long contiguous run The constrained-delivery optimization rewrote the old "continues when all indexes in the batch have logs" test (which scanned the full window every round) into the constrained doubling-probe test, leaving the unconstrained full-window-scan drain of a run longer than one window without direct coverage. Add an unconstrained test pinning that a contiguous run filling and exceeding the cold-start window drains fully: all logs returned, both cursors advanced to the last index, and per-round query sizes of [WINDOW_LEN + 1, WINDOW_LEN] showing full-window re-anchoring rather than a doubling probe. --- .../sync_tagged_private_logs.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 4c614a63f826..747a5868cbd4 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -181,6 +181,43 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(newWindowIndex); }); + it('unconstrained drains a contiguous run that fills and exceeds the window', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Unconstrained streams can have gaps, so each round scans the full window and re-anchors the next window to the + // highest finalized index found that round, rather than first-miss stopping or doubling a probe (both + // constrained-only). A contiguous run that fills the cold-start window and extends past it drains across rounds. + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; + const logTags = await Promise.all(Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i))); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + // Every log is returned and both cursors land on the last index. The aged cursor advances since the logs are old + // enough, unlike a constrained secret, which never tracks an aged index. + expect(logs).toHaveLength(totalLogs); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + + // The first round spans the full cold-start window (WINDOW_LEN + 1). Because every index hit, the next round + // re-anchors to another full WINDOW_LEN window ahead of the new finalized index: no small initial probe and no + // doubling, in contrast to the constrained scan. + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes.slice(0, 2)).toEqual([ + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, + ]); + }); + it('respects pre-existing store indexes', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); const finalizedBlockNumber = BlockNumber(10); From 9c6da20fc5b8cfa962eacc1a9beb358ac9269950 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 16:22:25 -0400 Subject: [PATCH 14/24] Apply suggestion from @vezenovm --- .../src/tagging/recipient_sync/sync_tagged_private_logs.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 747a5868cbd4..f8145c01d5b5 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -205,6 +205,7 @@ describe('syncTaggedPrivateLogs', () => { // Every log is returned and both cursors land on the last index. The aged cursor advances since the logs are old // enough, unlike a constrained secret, which never tracks an aged index. expect(logs).toHaveLength(totalLogs); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(2); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(totalLogs - 1); From b8c9ed7cfb91102fb179ecc9e69b5ce4a157a883 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 16:23:24 -0400 Subject: [PATCH 15/24] Apply suggestion from @vezenovm --- .../src/tagging/recipient_sync/sync_tagged_private_logs.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index f8145c01d5b5..747a5868cbd4 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -205,7 +205,6 @@ describe('syncTaggedPrivateLogs', () => { // Every log is returned and both cursors land on the last index. The aged cursor advances since the logs are old // enough, unlike a constrained secret, which never tracks an aged index. expect(logs).toHaveLength(totalLogs); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(2); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(totalLogs - 1); From 62cd3349b0ea94ba4b36d4d853a4cbfd6c8d6646 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Fri, 26 Jun 2026 18:32:21 -0400 Subject: [PATCH 16/24] test(pxe): pin constrained vs unconstrained tag-sync scan behavior in CI The constrained-sync micro-benchmark is skipped in CI (test_cmds skips *.bench.test.ts and it is not in bench_cmds), so bring its coverage into sync_tagged_private_logs.test.ts, which runs in CI: - log-then-linear round-count law (K=1,3,20,50,100 -> 2,3,5,6,9 rounds) - batched straggler: a single deep straggler among idle secrets sets the round count while idle secrets each cost one probe and drop - cold-start constrained-vs-unconstrained scan-shape contrast - tighten the deep-cap test to pin the linear-past-cap regime Qualify the overstated ~log2(K) catch-up claim in constants.ts, sync_tagged_private_logs.ts, and the bench: catch-up is logarithmic only while the probe doubles, then linear at ~K/WINDOW_LEN once it saturates the cap. No behavior change. --- yarn-project/pxe/src/tagging/constants.ts | 11 +- .../sync_tagged_private_logs.bench.test.ts | 13 +- .../sync_tagged_private_logs.test.ts | 137 +++++++++++++++++- .../sync_tagged_private_logs.ts | 10 +- 4 files changed, 151 insertions(+), 20 deletions(-) diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts index caf54a02dc20..1006c6353637 100644 --- a/yarn-project/pxe/src/tagging/constants.ts +++ b/yarn-project/pxe/src/tagging/constants.ts @@ -9,9 +9,10 @@ // set it to a relatively low value of 20, which is sufficient for current use cases. export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; -// The number of tags probed per constrained secret in the first round. The probe then doubles each round (capped at -// UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) while every probed index is a hit, stopping at the first missing tag. -// Constrained delivery is gapless, so a single missing tag proves the stream has ended: at steady state this turns a -// full WINDOW_LEN probe into a single tag, and a secret K logs behind catches up in ~log2(K) round-trips (probing -// 1, 2, 4, ... tags) rather than the full window or one round per log. +// The number of tags probed per constrained secret in the first round. The probe then doubles each round (1, 2, 4, ..., +// capped at UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) while every probed index is a hit, stopping at the first missing tag. +// Constrained delivery is gapless, so a single missing tag proves the stream has ended: at steady state this turns a full +// WINDOW_LEN probe into a single tag. A secret K logs behind catches up in ~log2(K) round-trips while the probe is still +// doubling (1, 2, 4, 8, 16), but once it saturates the cap and advances WINDOW_LEN tags per round, deeper catch-up is +// linear at ~K/WINDOW_LEN rounds. Either way it beats both the full window every round and one round per log. export const INITIAL_CONSTRAINED_PROBE_LEN = 1; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index d160944f5553..1fe0c528b9f6 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -42,9 +42,10 @@ import { * slightly to cut round-trips). The floor is reported as the reduction baseline. * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as one), * via `BenchmarkedNodeFactory`. The latency axis: doubling grows the probe geometrically, so round-trips rise only - * ~log2(K) during catch-up while tag-queries stay near the floor. Depends only on K, not on secret count. A round's - * tags are chunked at MAX_RPC_LEN (=100) into parallel calls internally, but those overlap, so a wide round is still - * one round-trip; that is why round-trips, not raw call count, is the latency axis. + * ~log2(K) while the probe is still doubling, then linearly at ~K/WINDOW_LEN once it saturates the cap, while + * tag-queries stay near the floor. Depends only on K, not on secret count. A round's tags are chunked at MAX_RPC_LEN + * (=100) into parallel calls internally, but those overlap, so a wide round is still one round-trip; that is why + * round-trips, not raw call count, is the latency axis. * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` * per call plus a little per-round overhead. Parallel calls within a round overlap, so it tracks round-trips (a * 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported only (varies run to run). @@ -294,9 +295,9 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { // Pin behavior as an executable assertion, not just a printout. Timings are reported only (vary run to run). const perSecretNewLogs = newLogsPerSecret(scenario); if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { - // `probeSchedule` mirrors the real doubling scan: a secret K logs behind resolves in ~log2(K) round-trips at a - // little above the K + 1 tag floor. Tags are batched across all secrets per round, so the round-trip count is - // the deepest secret's. + // `probeSchedule` mirrors the real doubling scan: a secret K logs behind resolves in ~log2(K) round-trips while + // the probe is still doubling and ~K/WINDOW_LEN once it saturates the cap, at a little above the K + 1 tag floor. + // Tags are batched across all secrets per round, so the round-trip count is the deepest secret's. const sched = perSecretNewLogs.map(k => probeSchedule(k)); expect(row.tagQueries).toBe(sched.reduce((sum, s) => sum + s.tags, 0)); expect(row.rpcRoundTrips).toBe(Math.max(...sched.map(s => s.rounds))); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 747a5868cbd4..dbb98076bdfb 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -441,11 +441,13 @@ describe('syncTaggedPrivateLogs', () => { ); }); - // Pins the effective per-round query cap for the doubling probe: it ramps 1, 2, 4, 8, 16 and then saturates at - // WINDOW_LEN (=20) per round rather than growing without bound, because a probe can never reach more than WINDOW_LEN - // past the finalized frontier (the `boundEnd` cap, which the stored-intent `Math.min` mirrors). With 100 new logs - // the per-round probe sizes are [1, 2, 4, 8, 16, 20, 20, 20, 20]. - it('doubling caps the probe at WINDOW_LEN on deep catch-up', async () => { + // Pins the effective per-round query cap AND that deep catch-up is linear past the cap, not logarithmic: the probe + // ramps 1, 2, 4, 8, 16 (5 doubling rounds covering indexes 1..31), then saturates at WINDOW_LEN (=20) per round + // rather than growing without bound, because a probe can never reach more than WINDOW_LEN past the finalized frontier + // (the `boundEnd` cap, which the stored-intent `Math.min` mirrors). With 100 new logs the per-round probe sizes are + // [1, 2, 4, 8, 16, 20, 20, 20, 20]: 9 rounds (5 doubling + 4 capped windows), the linear regime the ~log2(K) claim is + // qualified against, where ~log2(100) would be roughly 7. + it('doubling caps the probe at WINDOW_LEN and grows linearly past the cap on deep catch-up', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -576,6 +578,92 @@ describe('syncTaggedPrivateLogs', () => { expect(probedTags).toHaveLength(INITIAL_CONSTRAINED_PROBE_LEN); expect(probedTags).toEqual([await computeSiloedTagForIndex(secret, totalLogs)]); }); + + // Pins the round-trip growth law the ~log2(K) claim is qualified against: catch-up is logarithmic only while the + // probe is still doubling (1, 2, 4, 8, 16 cover indexes 1..31 in 5 rounds), then linear at roughly one extra round + // per WINDOW_LEN window once the probe saturates the cap. Every round here queries fewer than MAX_RPC_LEN tags and + // at most one log per tag, so one sync round is exactly one RPC call. + it.each([ + { newLogs: 1, expectedRounds: 2 }, // [1], then [2,3] finds the miss + { newLogs: 3, expectedRounds: 3 }, // [1], [2,3], [4..7] (miss at 4) + { newLogs: 20, expectedRounds: 5 }, // last doubling round [16..31] contains the miss + { newLogs: 50, expectedRounds: 6 }, // 5 doubling rounds reach 31, then 1 capped window + { newLogs: 100, expectedRounds: 9 }, // 5 doubling + 4 capped windows; ~log2(100) would be roughly 7 + ])( + 'catch-up round-trips grow logarithmically then linearly: K=$newLogs resolves in $expectedRounds rounds', + async ({ newLogs, expectedRounds }) => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Recipient already synced index 0; K new contiguous finalized logs sit at indexes 1..K. + await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); + const hitTags = await Promise.all( + Array.from({ length: newLogs }, (_, i) => computeSiloedTagForIndex(secret, i + 1)), + ); + mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + expect(logs).toHaveLength(newLogs); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(newLogs); + // One sync round is exactly one RPC call here, so the call count is the round-trip count. + expect(aztecNode.getPrivateLogsByTags.mock.calls).toHaveLength(expectedRounds); + }, + ); + + // Pins the batched-round semantics under heterogeneous load: idle secrets each cost exactly one probe and drop out + // after the first round, while a single deep straggler alone drives the round count. Round-trips track the deepest + // secret (here probeSchedule(3) = 3 rounds, the same as a lone K=3 secret), NOT the idle count and NOT one round per + // log; tag-queries are the idle probes plus the straggler's doubling schedule. This is the unit-level analogue of the + // bench `mixed` scenario, which the bench (skipped in CI) cannot pin. + it('a single deep straggler among idle secrets sets the round count, not the idle count', async () => { + const idleSecrets = await makeSecrets(4, AppTaggingSecretKind.CONSTRAINED); + const [straggler] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + // Idle secrets are caught up at index 5, so their next probe (index 6) misses and drops them after round 1. The + // straggler is at index 0 with 3 new contiguous logs at indexes 1..3. + const idleCursor = 5; + for (const secret of idleSecrets) { + await taggingStore.updateHighestFinalizedIndex(secret, idleCursor, JOB_ID); + } + await taggingStore.updateHighestFinalizedIndex(straggler, 0, JOB_ID); + const stragglerHits = await Promise.all([1, 2, 3].map(i => computeSiloedTagForIndex(straggler, i))); + mockNodeWithLogs(stragglerHits, Number(finalizedBlockNumber), agedTimestamp); + + const logs = await syncTaggedPrivateLogs( + [...idleSecrets, straggler], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + // Only the straggler's logs are found; its cursor advances while the idle cursors are untouched. + expect(logs).toHaveLength(3); + expect(await taggingStore.getHighestFinalizedIndex(straggler, JOB_ID)).toBe(3); + for (const secret of idleSecrets) { + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(idleCursor); + } + + // 3 rounds = the straggler's doubling schedule; the 4 idle secrets add no rounds. Each round stays under + // MAX_RPC_LEN, so call count is round count. Round 1: 4 idle probes + straggler[1]; round 2: straggler[2,3]; + // round 3: straggler[4..7] (terminating miss at 4). + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes).toEqual([5, 2, 4]); + // Total tag-queries: 4 idle probes + the straggler's (1 + 2 + 4) doubling probe = 11. + expect(callSizes.reduce((a, b) => a + b, 0)).toBe(11); + }); }); describe('constrained vs unconstrained probing', () => { @@ -624,6 +712,45 @@ describe('syncTaggedPrivateLogs', () => { ); expect(calledTags).toEqual(expectedTags); }); + + it('cold start: constrained probes a single tag where unconstrained probes the full window', async () => { + const [constrained] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const [unconstrained] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => + Promise.resolve(query.tags.map(() => [])), + ); + + // Fresh secret, nothing stored. The constrained scan opens at the initial probe (a single tag at index 0); the + // unconstrained scan must cover the whole [0, WINDOW_LEN] window because gaps are possible. Same starting point, + // a single tag vs WINDOW_LEN + 1 tags: the headline of the optimization, at cold start as well as steady state. + await syncTaggedPrivateLogs([constrained], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + const constrainedTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + expect(constrainedTags).toEqual( + await Promise.all( + Array.from({ length: INITIAL_CONSTRAINED_PROBE_LEN }, (_, i) => computeSiloedTagForIndex(constrained, i)), + ), + ); + + aztecNode.getPrivateLogsByTags.mockClear(); + + await syncTaggedPrivateLogs( + [unconstrained], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + BlockNumber(10), + JOB_ID, + ); + const unconstrainedTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + expect(unconstrainedTags).toEqual( + await Promise.all( + Array.from({ length: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1 }, (_, i) => + computeSiloedTagForIndex(unconstrained, i), + ), + ), + ); + }); }); describe('mixed constrained and unconstrained secrets', () => { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index 00f088ee8a33..dbf2af99f008 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -66,7 +66,8 @@ import { findHighestIndexes } from './utils/find_highest_indexes.js'; * ended and no further logs exist. We exploit this by probing only a small initial window * (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubling it each round (capped at the window) while every probed index has a * log, stopping at the first miss. A steady-state sync with no new logs costs a single tag instead of the whole - * window, and a secret K logs behind catches up in ~log2(K) round-trips instead of one round per log. + * window. A secret K logs behind catches up in ~log2(K) round-trips while the probe is still doubling, then linearly + * at ~K/WINDOW_LEN rounds once the probe saturates the cap, still far below one round per log. * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. Advancing the probe is * decoupled from persisting the finalized cursor, so unfinalized logs at the top of the run are still fetched * while only the finalized prefix is persisted. @@ -255,9 +256,10 @@ async function processConstrainedResults( ? Math.max(pending.boundEnd, highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1) : pending.boundEnd; - // Double the probe each round so a secret K logs behind resolves in ~log2(K) round-trips instead of one per log. The - // queried range is already bounded by boundEnd (a probe can never reach more than WINDOW_LEN past the finalized - // frontier); the cap on the stored length just keeps it from growing unbounded once the probe saturates the window. + // Double the probe each round so catch-up costs ~log2(K) round-trips while the probe is still doubling, then + // ~K/WINDOW_LEN once it saturates the cap, far below one round per log either way. The queried range is already + // bounded by boundEnd (a probe can never reach more than WINDOW_LEN past the finalized frontier); the cap on the + // stored length just keeps it from growing unbounded once the probe saturates the window. const currentProbeLen = pending.probeLen ?? INITIAL_CONSTRAINED_PROBE_LEN; const nextProbeLen = Math.min(currentProbeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); From fb0c9f5eeb5f63fe94c8da887c57a4caf07da7a6 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Tue, 7 Jul 2026 15:25:24 -0400 Subject: [PATCH 17/24] test: update tagged sync expectations for window size --- .../sync_tagged_private_logs.bench.test.ts | 10 +-- .../sync_tagged_private_logs.test.ts | 84 ++++++++++++------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 1fe0c528b9f6..ed156cbf3631 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -32,7 +32,7 @@ import { * * Measures the per-sync cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are gapless, so * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubles it each round (capped at the - * `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` (=20) window), stopping at the first missing tag instead of fetching the + * `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` window), stopping at the first missing tag instead of fetching the * full window. This validates the shipped doubling policy; the fixed-step sweep that motivated choosing it lives in the * PR description (reproducible from this branch's pre-doubling history). * @@ -141,7 +141,7 @@ function probeSchedule(k: number): { tags: number; rounds: number } { const SCENARIOS: Scenario[] = [ // steady-state (K = 0): no new logs since the last sync, across recipient secret counts. The dominant case and where - // first-miss wins ~20x. + // first-miss wins by roughly the window size. ...[1, 10, 100, 1000].map(secretCount => ({ label: `constrained/steady-state/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, @@ -161,9 +161,9 @@ const SCENARIOS: Scenario[] = [ })), ), // Deep catch-up at 100 and 1000 secrets (the negative case): round-trips grow one per probe step while tag-queries stay - // at the K + 1 floor. From a full window (catch-up-20) up, the WINDOW_LEN cap forces multiple rounds at any P. As with - // light catch-up, round-trips depend only on K and P (not N), so the two secret counts share a round-trip count and - // differ only in tag-queries (10x) and blocking time. + // at the K + 1 floor. From a full window up, the WINDOW_LEN cap forces multiple rounds at any P. As with light + // catch-up, round-trips depend only on K and P (not N), so the two secret counts share a round-trip count and differ + // only in tag-queries (10x) and blocking time. ...[100, 1000].flatMap(secretCount => [UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, 50, 100].map(newLogs => ({ label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index ca19e6caee8d..c7f970e65725 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -60,6 +60,34 @@ describe('syncTaggedPrivateLogs', () => { }); } + function expectedConstrainedProbeRanges(logCount: number, firstLogIndex = 1): { start: number; end: number }[] { + const ranges: { start: number; end: number }[] = []; + const lastLogIndex = firstLogIndex + logCount - 1; + let start = firstLogIndex; + let highestFinalizedIndex = firstLogIndex - 1; + let boundEnd = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; + let probeLen = INITIAL_CONSTRAINED_PROBE_LEN; + + for (;;) { + const end = Math.min(boundEnd, start + probeLen); + ranges.push({ start, end }); + + const highestHitInRange = Math.min(end - 1, lastLogIndex); + const probeFullyConsumed = highestHitInRange === end - 1; + if (highestHitInRange >= start) { + highestFinalizedIndex = highestHitInRange; + boundEnd = Math.max(boundEnd, highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1); + } + + if (!probeFullyConsumed || end >= boundEnd) { + return ranges; + } + + start = end; + probeLen = Math.min(probeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + } + } + beforeEach(async () => { aztecNode.getPrivateLogsByTags.mockReset(); taggingStore = new RecipientTaggingStore(await openTmpStore('test')); @@ -462,19 +490,15 @@ describe('syncTaggedPrivateLogs', () => { ); }); - // Pins the effective per-round query cap AND that deep catch-up is linear past the cap, not logarithmic: the probe - // ramps 1, 2, 4, 8, 16 (5 doubling rounds covering indexes 1..31), then saturates at WINDOW_LEN (=20) per round - // rather than growing without bound, because a probe can never reach more than WINDOW_LEN past the finalized frontier - // (the `boundEnd` cap, which the stored-intent `Math.min` mirrors). With 100 new logs the per-round probe sizes are - // [1, 2, 4, 8, 16, 20, 20, 20, 20]: 9 rounds (5 doubling + 4 capped windows), the linear regime the ~log2(K) claim is - // qualified against, where ~log2(100) would be roughly 7. + // Pins the effective per-round query cap and that deep catch-up is linear past the cap, not logarithmic: the probe + // doubles until it saturates at WINDOW_LEN per round rather than growing without bound. it('doubling caps the probe at WINDOW_LEN and grows linearly past the cap on deep catch-up', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // Recipient already synced index 0; a deep run of 100 new finalized logs sits at indexes 1..100. - const newLogs = 100; + // Recipient already synced index 0; a deep run of finalized logs sits past multiple capped probe windows. + const newLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3; await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); const hitTags = await Promise.all( Array.from({ length: newLogs }, (_, i) => computeSiloedTagForIndex(secret, i + 1)), @@ -491,10 +515,10 @@ describe('syncTaggedPrivateLogs', () => { ); expect(logs).toHaveLength(newLogs); - // Geometric ramp, then saturation at WINDOW_LEN: no round ever queries more than 20 tags. const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual([1, 2, 4, 8, 16, 20, 20, 20, 20]); + expect(callSizes).toEqual(expectedConstrainedProbeRanges(newLogs).map(({ start, end }) => end - start)); expect(Math.max(...callSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + expect(callSizes.filter(size => size === UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN)).toHaveLength(2); }); it('batches tags from multiple constrained secrets into a single RPC call', async () => { @@ -546,8 +570,8 @@ describe('syncTaggedPrivateLogs', () => { const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // A cold-start run two indexes past one window. The cold-start bound caps the probe at index 20, so reaching the - // tail is only possible because boundEnd re-anchors forward to each finalized index found. + // A cold-start run two indexes past one window. Reaching the tail is only possible because boundEnd re-anchors + // forward to each finalized index found. const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; const logTags = await Promise.all( Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), @@ -556,16 +580,20 @@ describe('syncTaggedPrivateLogs', () => { await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); - // The probe doubles each round: [1, 2, 4, 8, 16]. The final round queries indexes [15..30], reaching past the - // cold-start bound (20) - only possible because boundEnd re-anchored forward to each finalized index found. + // The final round reaches past the initial cold-start bound only because boundEnd re-anchored forward. const calls = aztecNode.getPrivateLogsByTags.mock.calls; const callSizes = calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual([1, 2, 4, 8, 16]); + const expectedRanges = expectedConstrainedProbeRanges(totalLogs, 0); + expect(callSizes).toEqual(expectedRanges.map(({ start, end }) => end - start)); const lastCallTags = extractTags(calls[calls.length - 1][0]); + const lastRange = expectedRanges[expectedRanges.length - 1]; const expectedLastRange = await Promise.all( - Array.from({ length: 16 }, (_, i) => computeSiloedTagForIndex(secret, 15 + i)), + Array.from({ length: lastRange.end - lastRange.start }, (_, i) => + computeSiloedTagForIndex(secret, lastRange.start + i), + ), ); expect(lastCallTags).toEqual(expectedLastRange); + expect(lastRange.end - 1).toBeGreaterThan(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); }); it('resets the probe on the sync after a catch-up', async () => { @@ -573,9 +601,8 @@ describe('syncTaggedPrivateLogs', () => { const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // A cold-start run long enough that the first sync grows the in-memory probe through [1, 2, 4, 8, 16]. The probe - // length lives only on the in-memory PendingSecret (never persisted), so the next sync must start fresh at the - // initial probe rather than inheriting the grown length. + // A cold-start run long enough that the first sync grows the in-memory probe. The probe length lives only on the + // in-memory PendingSecret, so the next sync must start fresh at the initial probe rather than inheriting it. const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; const logTags = await Promise.all( Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), @@ -601,21 +628,16 @@ describe('syncTaggedPrivateLogs', () => { }); // Pins the round-trip growth law the ~log2(K) claim is qualified against: catch-up is logarithmic only while the - // probe is still doubling (1, 2, 4, 8, 16 cover indexes 1..31 in 5 rounds), then linear at roughly one extra round - // per WINDOW_LEN window once the probe saturates the cap. Every round here queries fewer than MAX_RPC_LEN tags and - // at most one log per tag, so one sync round is exactly one RPC call. - it.each([ - { newLogs: 1, expectedRounds: 2 }, // [1], then [2,3] finds the miss - { newLogs: 3, expectedRounds: 3 }, // [1], [2,3], [4..7] (miss at 4) - { newLogs: 20, expectedRounds: 5 }, // last doubling round [16..31] contains the miss - { newLogs: 50, expectedRounds: 6 }, // 5 doubling rounds reach 31, then 1 capped window - { newLogs: 100, expectedRounds: 9 }, // 5 doubling + 4 capped windows; ~log2(100) would be roughly 7 - ])( - 'catch-up round-trips grow logarithmically then linearly: K=$newLogs resolves in $expectedRounds rounds', - async ({ newLogs, expectedRounds }) => { + // probe is still doubling, then linear at roughly one extra round per WINDOW_LEN once the probe saturates the cap. + // Every round here queries fewer than MAX_RPC_LEN tags and at most one log per tag, so one sync round is exactly one + // RPC call. + it.each([1, 3, 20, 50, 100, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3])( + 'catch-up round-trips grow logarithmically then linearly: K=%i', + async newLogs => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const expectedRounds = expectedConstrainedProbeRanges(newLogs).length; // Recipient already synced index 0; K new contiguous finalized logs sit at indexes 1..K. await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); From e872988140044f81d79bac0849a7a4c810be2448 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Tue, 7 Jul 2026 16:37:45 -0400 Subject: [PATCH 18/24] opt in bench --- .../sync_tagged_private_logs.bench.test.ts | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index ed156cbf3631..3fcca4028431 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -33,30 +33,37 @@ import { * Measures the per-sync cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are gapless, so * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubles it each round (capped at the * `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` window), stopping at the first missing tag instead of fetching the - * full window. This validates the shipped doubling policy; the fixed-step sweep that motivated choosing it lives in the - * PR description (reproducible from this branch's pre-doubling history). + * full window. This validates the shipped doubling policy against the fixed-step alternatives considered when choosing + * it. + * + * Manual run: + * ```bash + * RUN_TAG_SYNC_BENCH=1 JEST_MAX_WORKERS=1 BENCH_OUTPUT=/tmp/tag-sync-bench-current.json \ + * yarn workspace @aztec/pxe test src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts + * ``` * * Metrics, per scenario: * - `tag-queries`: total tags queried, the throughput win. At steady state it drops from a full window per secret to a * single tag; a secret K logs behind costs a little above the K + 1 first-miss floor (the doubling probe overshoots * slightly to cut round-trips). The floor is reported as the reduction baseline. - * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as one), - * via `BenchmarkedNodeFactory`. The latency axis: doubling grows the probe geometrically, so round-trips rise only - * ~log2(K) while the probe is still doubling, then linearly at ~K/WINDOW_LEN once it saturates the cap, while + * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as + * one), via `BenchmarkedNodeFactory`. The latency axis: doubling grows the probe geometrically, so round-trips rise + * only ~log2(K) while the probe is still doubling, then linearly at ~K/WINDOW_LEN once it saturates the cap, while * tag-queries stay near the floor. Depends only on K, not on secret count. A round's tags are chunked at MAX_RPC_LEN * (=100) into parallel calls internally, but those overlap, so a wide round is still one round-trip; that is why * round-trips, not raw call count, is the latency axis. - * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled `MODELED_NODE_RPC_LATENCY_MS` - * per call plus a little per-round overhead. Parallel calls within a round overlap, so it tracks round-trips (a - * 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported only (varies run to run). + * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled + * `MODELED_NODE_RPC_LATENCY_MS` per call plus a little per-round overhead. Parallel calls within a round overlap, so + * it tracks round-trips (a 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported + * only (varies run to run). * * Scenario labels: `steady-state` is no new logs (K = 0); `catch-up-K` is K new contiguous logs per secret since the - * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K (not N), - * the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while round-trips do - * not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it isolates that - * tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count (round-trips - * therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed scan), so its - * cost is fixed. + * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K (not + * N), the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while round-trips + * do not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it isolates + * that tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count + * (round-trips therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed + * scan), so its cost is fixed. */ const logger = createLogger('pxe:tagging:bench'); @@ -72,6 +79,8 @@ const JOB_ID = 'bench-job'; // The round-trip *count* is independent of this value; only `rpc-blocking-time` scales with it. const MODELED_NODE_RPC_LATENCY_MS = 5; +const describeBench = process.env.RUN_TAG_SYNC_BENCH ? describe : describe.skip; + /** One benchmark measurement in the benchmark JSON shape. */ type BenchResult = { name: string; value: number; unit: string }; @@ -93,9 +102,9 @@ type Scenario = { }; /** - * The per-secret new-log distribution for a scenario: `deepCohort.count` secrets at `deepCohort.newLogs`, the rest idle, - * or a uniform `newLogs` for every secret when no cohort is set. Single source for both log seeding and the expected - * tag-query / round-trip assertions. + * The per-secret new-log distribution for a scenario: `deepCohort.count` secrets at `deepCohort.newLogs`, the rest + * idle, or a uniform `newLogs` for every secret when no cohort is set. Single source for both log seeding and the + * expected tag-query / round-trip assertions. */ function newLogsPerSecret(scenario: Scenario): number[] { return Array.from({ length: scenario.secretCount }, (_, i) => @@ -105,9 +114,9 @@ function newLogsPerSecret(scenario: Scenario): number[] { /** * Mirrors `processConstrainedResults` for a single secret with `k` new contiguous logs (`priorCursor = 0`): the probe - * starts at `INITIAL_CONSTRAINED_PROBE_LEN`, doubles each round (capped at WINDOW_LEN), and the queried range is bounded - * by the WINDOW_LEN-ahead-of-finalized frontier. Returns the total tags queried and the sequential round-trip count. - * Single source for the expected tag-query / round-trip assertions. + * starts at `INITIAL_CONSTRAINED_PROBE_LEN`, doubles each round (capped at WINDOW_LEN), and the queried range is + * bounded by the WINDOW_LEN-ahead-of-finalized frontier. Returns the total tags queried and the sequential round-trip + * count. Single source for the expected tag-query / round-trip assertions. */ function probeSchedule(k: number): { tags: number; rounds: number } { let start = 1; // priorCursor (0) + 1 @@ -160,8 +169,8 @@ const SCENARIOS: Scenario[] = [ newLogs, })), ), - // Deep catch-up at 100 and 1000 secrets (the negative case): round-trips grow one per probe step while tag-queries stay - // at the K + 1 floor. From a full window up, the WINDOW_LEN cap forces multiple rounds at any P. As with light + // Deep catch-up at 100 and 1000 secrets (the negative case): round-trips grow one per probe step while tag-queries + // stay at the K + 1 floor. From a full window up, the WINDOW_LEN cap forces multiple rounds at any P. As with light // catch-up, round-trips depend only on K and P (not N), so the two secret counts share a round-trip count and differ // only in tag-queries (10x) and blocking time. ...[100, 1000].flatMap(secretCount => @@ -194,7 +203,7 @@ const SCENARIOS: Scenario[] = [ }, ]; -describe('syncTaggedPrivateLogs constrained-sync bench', () => { +describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { const aztecNode: MockProxy = mock(); function computeSiloedTagForIndex(secret: AppTaggingSecret, index: number) { @@ -296,7 +305,8 @@ describe('syncTaggedPrivateLogs constrained-sync bench', () => { const perSecretNewLogs = newLogsPerSecret(scenario); if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { // `probeSchedule` mirrors the real doubling scan: a secret K logs behind resolves in ~log2(K) round-trips while - // the probe is still doubling and ~K/WINDOW_LEN once it saturates the cap, at a little above the K + 1 tag floor. + // the probe is still doubling and ~K/WINDOW_LEN once it saturates the cap, at a little above the K + 1 tag + // floor. // Tags are batched across all secrets per round, so the round-trip count is the deepest secret's. const sched = perSecretNewLogs.map(k => probeSchedule(k)); expect(row.tagQueries).toBe(sched.reduce((sum, s) => sum + s.tags, 0)); From 4a31283028caaaac8845f71caffe68219be9ba6b Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 11:49:09 -0400 Subject: [PATCH 19/24] simplfiy bench --- .../sync_tagged_private_logs.bench.test.ts | 93 +++++-------------- 1 file changed, 21 insertions(+), 72 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 3fcca4028431..48b252d1f776 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -21,11 +21,7 @@ import path from 'path'; import { BenchmarkedNodeFactory } from '../../contract_function_simulator/benchmarked_node.js'; import { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; -import { - INITIAL_CONSTRAINED_PROBE_LEN, - UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, - syncTaggedPrivateLogs, -} from '../index.js'; +import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, syncTaggedPrivateLogs } from '../index.js'; /** * Benchmark for constrained recipient tag-sync. @@ -33,8 +29,8 @@ import { * Measures the per-sync cost of `syncTaggedPrivateLogs` for constrained secrets. Constrained streams are gapless, so * the scan probes a small initial window (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubles it each round (capped at the * `UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN` window), stopping at the first missing tag instead of fetching the - * full window. This validates the shipped doubling policy against the fixed-step alternatives considered when choosing - * it. + * full window. This only reports costs; the scan behavior itself (probe schedules, round counts) is pinned by the + * unit tests in `sync_tagged_private_logs.test.ts`, which run in CI while this bench is opt-in. * * Manual run: * ```bash @@ -43,27 +39,22 @@ import { * ``` * * Metrics, per scenario: - * - `tag-queries`: total tags queried, the throughput win. At steady state it drops from a full window per secret to a - * single tag; a secret K logs behind costs a little above the K + 1 first-miss floor (the doubling probe overshoots - * slightly to cut round-trips). The floor is reported as the reduction baseline. - * - `rpc-round-trips`: sequential blocking waits on the node (parallel `Promise.all` calls within a round count as - * one), via `BenchmarkedNodeFactory`. The latency axis: doubling grows the probe geometrically, so round-trips rise - * only ~log2(K) while the probe is still doubling, then linearly at ~K/WINDOW_LEN once it saturates the cap, while - * tag-queries stay near the floor. Depends only on K, not on secret count. A round's tags are chunked at MAX_RPC_LEN - * (=100) into parallel calls internally, but those overlap, so a wide round is still one round-trip; that is why + * - `tag-queries`: total tags queried, the throughput win. + * - `rpc-round-trips`: sequential blocking waits on the node, via `BenchmarkedNodeFactory`. + * The latency axis: doubling grows the probe geometrically, so round-trips rise only ~log2(K) while the + * probe is still doubling, then linearly at ~K/WINDOW_LEN once it saturates the cap. + * Depends only on K, not on secret count. A round's tags are chunked at MAX_RPC_LEN (=100) into parallel calls + * internally, but those overlap, so a wide round is still one round-trip; that is why * round-trips, not raw call count, is the latency axis. * - `rpc-blocking-time`: measured wall-clock the caller blocks on the node, under a modeled * `MODELED_NODE_RPC_LATENCY_MS` per call plus a little per-round overhead. Parallel calls within a round overlap, so - * it tracks round-trips (a 1000-secret round is many parallel chunks but ~one round-trip of blocking time). Reported - * only (varies run to run). + * it tracks round-trips (a 1000-secret round is many parallel chunks but ~one round-trip of blocking time). * * Scenario labels: `steady-state` is no new logs (K = 0); `catch-up-K` is K new contiguous logs per secret since the * last sync; `secrets=N` is N secrets synced together in one batched pass. Because round-trips depend only on K (not * N), the light catch-up scenarios run at both 100 and 1000 secrets to show tag-queries scale with N while round-trips * do not. The `mixed` row is the realistic active sync (999 idle secrets + 1 deep straggler at K = 100): it isolates - * that tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count - * (round-trips therefore match catch-up-100). The `unconstrained` row is the control: it cannot first-miss (windowed - * scan), so its cost is fixed. + * that tag-queries stay dominated by the idle majority while a single straggler alone sets the round-trip count. */ const logger = createLogger('pxe:tagging:bench'); @@ -104,7 +95,7 @@ type Scenario = { /** * The per-secret new-log distribution for a scenario: `deepCohort.count` secrets at `deepCohort.newLogs`, the rest * idle, or a uniform `newLogs` for every secret when no cohort is set. Single source for both log seeding and the - * expected tag-query / round-trip assertions. + * seeding sanity check. */ function newLogsPerSecret(scenario: Scenario): number[] { return Array.from({ length: scenario.secretCount }, (_, i) => @@ -112,42 +103,6 @@ function newLogsPerSecret(scenario: Scenario): number[] { ); } -/** - * Mirrors `processConstrainedResults` for a single secret with `k` new contiguous logs (`priorCursor = 0`): the probe - * starts at `INITIAL_CONSTRAINED_PROBE_LEN`, doubles each round (capped at WINDOW_LEN), and the queried range is - * bounded by the WINDOW_LEN-ahead-of-finalized frontier. Returns the total tags queried and the sequential round-trip - * count. Single source for the expected tag-query / round-trip assertions. - */ -function probeSchedule(k: number): { tags: number; rounds: number } { - let start = 1; // priorCursor (0) + 1 - let highestFinalized = 0; // priorCursor - let probeLen = INITIAL_CONSTRAINED_PROBE_LEN; - let tags = 0; - let rounds = 0; - for (;;) { - const boundEnd = highestFinalized + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; - const end = Math.min(boundEnd, start + probeLen); - if (end <= start) { - break; - } - tags += end - start; - rounds++; - // Hits occupy indexes 1..k; the round stops at the first index in [start, end) without a log. - const lastHit = Math.min(end - 1, k); - const fullyConsumed = lastHit === end - 1; - if (lastHit >= start) { - highestFinalized = lastHit; - } - // Stop on the first miss, or once the queried range has reached the (re-anchored) bound. - if (!fullyConsumed || end >= highestFinalized + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1) { - break; - } - start = end; - probeLen = Math.min(probeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); - } - return { tags, rounds }; -} - const SCENARIOS: Scenario[] = [ // steady-state (K = 0): no new logs since the last sync, across recipient secret counts. The dominant case and where // first-miss wins by roughly the window size. @@ -288,7 +243,7 @@ describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { }; } - it('measures tag-query cost per sync', async () => { + it('reports per-sync tag-queries, round-trips, and blocking time', async () => { const rows = []; for (const scenario of SCENARIOS) { const row = await runScenario(scenario); @@ -301,22 +256,16 @@ describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { `blocking=${row.rpcBlockingTimeMs.toFixed(0).padStart(4)}ms logs=${String(row.logsFound).padStart(5)}`, ); - // Pin behavior as an executable assertion, not just a printout. Timings are reported only (vary run to run). - const perSecretNewLogs = newLogsPerSecret(scenario); - if (scenario.kind === AppTaggingSecretKind.CONSTRAINED) { - // `probeSchedule` mirrors the real doubling scan: a secret K logs behind resolves in ~log2(K) round-trips while - // the probe is still doubling and ~K/WINDOW_LEN once it saturates the cap, at a little above the K + 1 tag - // floor. - // Tags are batched across all secrets per round, so the round-trip count is the deepest secret's. - const sched = perSecretNewLogs.map(k => probeSchedule(k)); - expect(row.tagQueries).toBe(sched.reduce((sum, s) => sum + s.tags, 0)); - expect(row.rpcRoundTrips).toBe(Math.max(...sched.map(s => s.rounds))); - } else if (scenario.newLogs === 0) { - // Unconstrained steady state: still the full window (control for the optimization), drained in one round. - expect(row.tagQueries).toBe(scenario.secretCount * UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + // Seeding sanity check only: every seeded log must be found, or the reported numbers measure a broken harness. + // Scan behavior (probe schedules, round counts) is pinned by the unit tests, which run in CI while this does not. + expect(row.logsFound).toBe(newLogsPerSecret(scenario).reduce((sum, k) => sum + k, 0)); + + // Steady state is one round trip by construction: every secret first-misses in round one, independent of secret + // count, so a wide round's chunked parallel RPC calls must count as a single blocking wait. Guards the round-trip + // accounting at widths the unit tests don't reach (they count RPC calls, not round trips). + if (scenario.newLogs === 0 && !scenario.deepCohort) { expect(row.rpcRoundTrips).toBe(1); } - expect(row.logsFound).toBe(perSecretNewLogs.reduce((sum, k) => sum + k, 0)); } const results: BenchResult[] = rows.flatMap(row => [ From 781ab5051a61db5662957e48ef1438295360912e Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 12:21:22 -0400 Subject: [PATCH 20/24] test(pxe): consolidate constrained tag sync tests and simplify bench Merge the constrained drain test into the boundEnd re-anchor test, move the unconstrained drain next to its constrained counterpart, and drop the steady-state full-window test subsumed by the store-index test. Strengthen the unfinalized-probe test with a second-sync restart assertion and make the probe reset test saturate the WINDOW_LEN cap first. Remove the bench's probe-schedule mirror (behavior is pinned by the CI unit tests) and switch remaining "cursor" wording to "finalized index". --- .../sync_tagged_private_logs.bench.test.ts | 27 +-- .../sync_tagged_private_logs.test.ts | 201 ++++++++---------- .../sync_tagged_private_logs.ts | 15 +- 3 files changed, 115 insertions(+), 128 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 48b252d1f776..2d6380deadb3 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -80,8 +80,8 @@ type Scenario = { kind: AppTaggingSecretKind; /** Number of secrets the recipient holds for this directional app. */ secretCount: number; - /** Highest finalized index already persisted before the sync (recipient has `priorCursor + 1` prior messages). */ - priorCursor: number; + /** Highest finalized index already persisted before the sync (recipient has `priorFinalizedIndex + 1` prior messages). */ + priorFinalizedIndex: number; /** New contiguous finalized logs available per secret since the last sync (0 = steady state). */ newLogs: number; /** @@ -110,7 +110,7 @@ const SCENARIOS: Scenario[] = [ label: `constrained/steady-state/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorCursor: 0, + priorFinalizedIndex: 0, newLogs: 0, })), // Light catch-up (K new logs per secret) at 100 and 1000 secrets. Round-trips depend only on K and P, not on N, so @@ -120,7 +120,7 @@ const SCENARIOS: Scenario[] = [ label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorCursor: 0, + priorFinalizedIndex: 0, newLogs, })), ), @@ -133,7 +133,7 @@ const SCENARIOS: Scenario[] = [ label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorCursor: 0, + priorFinalizedIndex: 0, newLogs, })), ), @@ -144,7 +144,7 @@ const SCENARIOS: Scenario[] = [ label: `constrained/mixed/secrets=1000`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount: 1000, - priorCursor: 0, + priorFinalizedIndex: 0, newLogs: 0, deepCohort: { count: 1, newLogs: 100 }, }, @@ -153,7 +153,7 @@ const SCENARIOS: Scenario[] = [ label: `unconstrained/steady-state/secrets=100`, kind: AppTaggingSecretKind.UNCONSTRAINED, secretCount: 100, - priorCursor: 0, + priorFinalizedIndex: 0, newLogs: 0, }, ]; @@ -178,26 +178,27 @@ describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { } async function runScenario(scenario: Scenario) { - const { kind, secretCount, priorCursor } = scenario; + const { kind, secretCount, priorFinalizedIndex } = scenario; const perSecretNewLogs = newLogsPerSecret(scenario); aztecNode.getPrivateLogsByTags.mockReset(); const taggingStore = new RecipientTaggingStore(await openTmpStore('bench')); const secrets = await Promise.all(Array.from({ length: secretCount }, () => randomAppTaggingSecret(kind))); - // Seed the persisted cursor(s) to simulate a recipient that already synced prior finalized messages. + // Seed the persisted finalized indexes to simulate a recipient that already synced prior finalized messages. for (const secret of secrets) { - await taggingStore.updateHighestFinalizedIndex(secret, priorCursor, JOB_ID); + await taggingStore.updateHighestFinalizedIndex(secret, priorFinalizedIndex, JOB_ID); if (kind === AppTaggingSecretKind.UNCONSTRAINED) { - await taggingStore.updateHighestAgedIndex(secret, priorCursor, JOB_ID); + await taggingStore.updateHighestAgedIndex(secret, priorFinalizedIndex, JOB_ID); } } - // Tags that should resolve to a finalized log: per secret, the contiguous run (priorCursor, priorCursor + K]. + // Tags that should resolve to a finalized log: per secret, the contiguous run + // (priorFinalizedIndex, priorFinalizedIndex + K]. const hitTags = new Set(); for (let s = 0; s < secrets.length; s++) { for (let k = 1; k <= perSecretNewLogs[s]; k++) { - hitTags.add((await computeSiloedTagForIndex(secrets[s], priorCursor + k)).toString()); + hitTags.add((await computeSiloedTagForIndex(secrets[s], priorFinalizedIndex + k)).toString()); } } diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index c7f970e65725..e0c2228a0911 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -230,43 +230,6 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(newWindowIndex); }); - it('unconstrained drains a contiguous run that fills and exceeds the window', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - // Unconstrained streams can have gaps, so each round scans the full window and re-anchors the next window to the - // highest finalized index found that round, rather than first-miss stopping or doubling a probe (both - // constrained-only). A contiguous run that fills the cold-start window and extends past it drains across rounds. - const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; - const logTags = await Promise.all(Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i))); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); - - // Every log is returned and both cursors land on the last index. The aged cursor advances since the logs are old - // enough, unlike a constrained secret, which never tracks an aged index. - expect(logs).toHaveLength(totalLogs); - expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - - // The first round spans the full cold-start window (WINDOW_LEN + 1). Because every index hit, the next round - // re-anchors to another full WINDOW_LEN window ahead of the new finalized index: no small initial probe and no - // doubling, in contrast to the constrained scan. - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes.slice(0, 2)).toEqual([ - UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, - UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, - ]); - }); - it('respects pre-existing store indexes', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); const finalizedBlockNumber = BlockNumber(10); @@ -354,37 +317,7 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); - it('fully drains a long contiguous run beyond the window', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; - const logTags = await Promise.all( - Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), - ); - - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); - - // The whole run is returned and the cursor lands on the last index, even though the run is longer than a single - // window: boundEnd re-anchors to each finalized index found, so the doubling probe keeps advancing past one - // window until the terminating miss. - expect(logs).toHaveLength(totalLogs); - expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - // Doubling drains the run in far fewer rounds than one-per-log. - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBeLessThan(totalLogs); - }); - - it('persists cursor only up to finalized block', async () => { + it('advances the finalized index only through the finalized prefix', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(5); const oldTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -417,8 +350,8 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // The unfinalized logs (4, 5) are returned to the caller, but the durable cursor only advances to the finalized - // prefix (3): probe advancement is decoupled from cursor persistence. + // The unfinalized logs (4, 5) are returned to the caller, but the finalized index only advances to the finalized + // prefix (3): probe advancement is decoupled from the finalized index. expect(logs).toHaveLength(6); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(3); }); @@ -451,8 +384,28 @@ describe('syncTaggedPrivateLogs', () => { ); expect(logs).toHaveLength(2); - // Nothing finalized, so the durable cursor must not advance. + // Nothing finalized, so the finalized index must not advance. expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); + + // Round 1 probes [0] and advances on the unfinalized hit; round 2 probes [1, 2] and stops at the gap (2). + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes).toEqual([1, 2]); + + // The store only ever advances through finalized logs, so an unfinalized log can never be skipped: since nothing + // was persisted, a second sync restarts at index 0 and re-fetches both unfinalized logs. + aztecNode.getPrivateLogsByTags.mockClear(); + const secondSyncLogs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + expect(secondSyncLogs).toHaveLength(2); + expect(extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0])).toEqual([ + await computeSiloedTagForIndex(secret, 0), + ]); }); // Pins the probe schedule: the probe doubles each round (1, 2, 4, ...) until the first miss, so K-deep @@ -565,7 +518,7 @@ describe('syncTaggedPrivateLogs', () => { expect(queriedTags.some(t => t.equals(sentinelTag))).toBe(false); }); - it('doubling probe re-anchors past the cold-start bound in a single sync', async () => { + it('fully drains a cold-start run beyond the window by re-anchoring boundEnd', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; @@ -578,7 +531,19 @@ describe('syncTaggedPrivateLogs', () => { ); mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + // The whole run is returned and the finalized index lands on the last index, even though the run is longer than + // a single window. + expect(logs).toHaveLength(totalLogs); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); // The final round reaches past the initial cold-start bound only because boundEnd re-anchored forward. const calls = aztecNode.getPrivateLogsByTags.mock.calls; @@ -596,28 +561,34 @@ describe('syncTaggedPrivateLogs', () => { expect(lastRange.end - 1).toBeGreaterThan(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); }); - it('resets the probe on the sync after a catch-up', async () => { + it('resets the probe to the initial length on the sync after a cap-saturating catch-up', async () => { const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(10); const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - // A cold-start run long enough that the first sync grows the in-memory probe. The probe length lives only on the - // in-memory PendingSecret, so the next sync must start fresh at the initial probe rather than inheriting it. - const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; + // A cold-start run deep enough that the first sync doubles the in-memory probe all the way to the WINDOW_LEN + // cap. The probe length lives only on the in-memory PendingSecret, so the next sync must start fresh at the + // initial probe rather than inheriting the saturated one, which would forfeit the steady-state optimization + // after every catch-up. + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3; const logTags = await Promise.all( Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), ); mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - // First sync catches up the whole run; the cursor lands on the last index. + // First sync catches up the whole run; the finalized index lands on the last index and the probe saturated the + // cap along the way. await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + const catchUpCallSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(Math.max(...catchUpCallSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); // Drop the catch-up's recorded calls (mockClear keeps the implementation; mockReset would not) so the next // assertions see only the second sync. aztecNode.getPrivateLogsByTags.mockClear(); - // Second sync, no new logs: it must probe a single tag at cursor + 1, not the grown probe from the catch-up. + // Second sync, no new logs: it must probe a single tag at the finalized index + 1, not the saturated probe from + // the catch-up. await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); const calls = aztecNode.getPrivateLogsByTags.mock.calls; @@ -675,9 +646,9 @@ describe('syncTaggedPrivateLogs', () => { // Idle secrets are caught up at index 5, so their next probe (index 6) misses and drops them after round 1. The // straggler is at index 0 with 3 new contiguous logs at indexes 1..3. - const idleCursor = 5; + const idleFinalizedIndex = 5; for (const secret of idleSecrets) { - await taggingStore.updateHighestFinalizedIndex(secret, idleCursor, JOB_ID); + await taggingStore.updateHighestFinalizedIndex(secret, idleFinalizedIndex, JOB_ID); } await taggingStore.updateHighestFinalizedIndex(straggler, 0, JOB_ID); const stragglerHits = await Promise.all([1, 2, 3].map(i => computeSiloedTagForIndex(straggler, i))); @@ -692,11 +663,11 @@ describe('syncTaggedPrivateLogs', () => { JOB_ID, ); - // Only the straggler's logs are found; its cursor advances while the idle cursors are untouched. + // Only the straggler's logs are found; its finalized index advances while the idle secrets' stay untouched. expect(logs).toHaveLength(3); expect(await taggingStore.getHighestFinalizedIndex(straggler, JOB_ID)).toBe(3); for (const secret of idleSecrets) { - expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(idleCursor); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(idleFinalizedIndex); } // 3 rounds = the straggler's doubling schedule; the 4 idle secrets add no rounds. Each round stays under @@ -731,31 +702,6 @@ describe('syncTaggedPrivateLogs', () => { expect(calledTags).toEqual(expectedTags); }); - it('unconstrained steady-state probes the full window', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const cursor = 8; - await taggingStore.updateHighestAgedIndex(secret, cursor, JOB_ID); - await taggingStore.updateHighestFinalizedIndex(secret, cursor, JOB_ID); - - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(query.tags.map(() => [])), - ); - - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); - - // Unconstrained streams can have gaps, so the full window is always probed — the same cursor that costs one tag - // for a constrained secret costs a full window here. - const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - const expectedStart = cursor + 1; - const expectedEnd = cursor + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; - const expectedTags = await Promise.all( - Array.from({ length: expectedEnd - expectedStart + 1 }, (_, i) => - computeSiloedTagForIndex(secret, expectedStart + i), - ), - ); - expect(calledTags).toEqual(expectedTags); - }); - it('cold start: constrained probes a single tag where unconstrained probes the full window', async () => { const [constrained] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); const [unconstrained] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); @@ -794,6 +740,45 @@ describe('syncTaggedPrivateLogs', () => { ), ); }); + + // The unconstrained counterpart of the constrained drain test ('fully drains a cold-start run beyond the window by + // re-anchoring boundEnd'): the same run drains with full-window rounds because gaps are possible, so there is no + // first-miss stop and no doubling probe. + it('unconstrained drains the same cold-start run with full-window rounds, not a doubling probe', async () => { + const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + const finalizedBlockNumber = BlockNumber(10); + const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; + const logTags = await Promise.all( + Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), + ); + mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + + const logs = await syncTaggedPrivateLogs( + [secret], + aztecNode, + taggingStore, + ANCHOR_BLOCK_HEADER, + finalizedBlockNumber, + JOB_ID, + ); + + // Every log is returned and both stored indexes land on the last index. The aged index advances since the logs + // are old enough, unlike a constrained secret, which never tracks an aged index. + expect(logs).toHaveLength(totalLogs); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(totalLogs - 1); + + // The first round spans the full cold-start window (WINDOW_LEN + 1). Because every index hit, the next round + // re-anchors to another full WINDOW_LEN window ahead of the new finalized index: no small initial probe and no + // doubling, in contrast to the constrained scan. + const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); + expect(callSizes.slice(0, 2)).toEqual([ + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, + ]); + }); }); describe('mixed constrained and unconstrained secrets', () => { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index dbf2af99f008..eed87a9745d9 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -67,9 +67,10 @@ import { findHighestIndexes } from './utils/find_highest_indexes.js'; * (`INITIAL_CONSTRAINED_PROBE_LEN`) and doubling it each round (capped at the window) while every probed index has a * log, stopping at the first miss. A steady-state sync with no new logs costs a single tag instead of the whole * window. A secret K logs behind catches up in ~log2(K) round-trips while the probe is still doubling, then linearly - * at ~K/WINDOW_LEN rounds once the probe saturates the cap, still far below one round per log. + * at ~K/WINDOW_LEN rounds once the probe saturates the cap, still far below one round per log. The probe length is + * in-memory state scoped to one sync call, so every sync restarts at the initial probe. * - The upper bound is the same as unconstrained: `highestFinalizedIndex + WINDOW_LEN`. Advancing the probe is - * decoupled from persisting the finalized cursor, so unfinalized logs at the top of the run are still fetched + * decoupled from persisting the finalized index, so unfinalized logs at the top of the run are still fetched * while only the finalized prefix is persisted. * * # Batching across secrets @@ -219,7 +220,7 @@ async function fetchLogsForSecrets( } /** - * Processes fetched logs for a constrained secret: persists the finalized cursor and advances the probe. See the + * Processes fetched logs for a constrained secret: persists the finalized index and advances the probe. See the * "# Constrained secrets" section above for why the scan stops at the first missing index. Only `highestFinalizedIndex` * is tracked (no aged index). */ @@ -246,10 +247,10 @@ async function processConstrainedResults( await taggingStore.updateHighestFinalizedIndex(pending.secret, highestFinalizedIndex, jobId); } - // Advancing the probe is decoupled from persisting the cursor: keep scanning as long as the probe was entirely hits - // (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. Gating this on - // finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. The bound - // re-anchors to any finalized index found this round. + // Advancing the probe is decoupled from persisting the finalized index: keep scanning as long as the probe was + // entirely hits (no gap) and there is room before the protocol bound, even if none of those hits is finalized yet. + // Gating this on finalization would drop logs in unfinalized blocks, which sit at the top of the contiguous run. + // The bound re-anchors to any finalized index found this round. const probeFullyConsumed = firstMissingIndex >= pending.end; const boundEnd = highestFinalizedIndex !== undefined From 3a93f518ad5efeb19dfe25c57968b43936bcf93a Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 12:34:14 -0400 Subject: [PATCH 21/24] test(pxe): drop constant priorFinalizedIndex scenario field from bench --- .../sync_tagged_private_logs.bench.test.ts | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts index 2d6380deadb3..69b43797c1b6 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.bench.test.ts @@ -66,6 +66,9 @@ const ANCHOR_BLOCK_HEADER = BlockHeader.random({ blockNumber: ANCHOR_BLOCK_NUMBE const AGED_TIMESTAMP = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; const JOB_ID = 'bench-job'; +// Every scenario starts warm: index 0 is already persisted, so the scan resumes at index 1 rather than cold-starting. +const PRIOR_FINALIZED_INDEX = 0; + // Models per-call node RPC latency so round-trip blocking time is meaningful against an otherwise-instant mock node. // The round-trip *count* is independent of this value; only `rpc-blocking-time` scales with it. const MODELED_NODE_RPC_LATENCY_MS = 5; @@ -80,8 +83,6 @@ type Scenario = { kind: AppTaggingSecretKind; /** Number of secrets the recipient holds for this directional app. */ secretCount: number; - /** Highest finalized index already persisted before the sync (recipient has `priorFinalizedIndex + 1` prior messages). */ - priorFinalizedIndex: number; /** New contiguous finalized logs available per secret since the last sync (0 = steady state). */ newLogs: number; /** @@ -110,7 +111,6 @@ const SCENARIOS: Scenario[] = [ label: `constrained/steady-state/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorFinalizedIndex: 0, newLogs: 0, })), // Light catch-up (K new logs per secret) at 100 and 1000 secrets. Round-trips depend only on K and P, not on N, so @@ -120,7 +120,6 @@ const SCENARIOS: Scenario[] = [ label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorFinalizedIndex: 0, newLogs, })), ), @@ -133,7 +132,6 @@ const SCENARIOS: Scenario[] = [ label: `constrained/catch-up-${newLogs}/secrets=${secretCount}`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount, - priorFinalizedIndex: 0, newLogs, })), ), @@ -144,7 +142,6 @@ const SCENARIOS: Scenario[] = [ label: `constrained/mixed/secrets=1000`, kind: AppTaggingSecretKind.CONSTRAINED, secretCount: 1000, - priorFinalizedIndex: 0, newLogs: 0, deepCohort: { count: 1, newLogs: 100 }, }, @@ -153,7 +150,6 @@ const SCENARIOS: Scenario[] = [ label: `unconstrained/steady-state/secrets=100`, kind: AppTaggingSecretKind.UNCONSTRAINED, secretCount: 100, - priorFinalizedIndex: 0, newLogs: 0, }, ]; @@ -178,7 +174,7 @@ describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { } async function runScenario(scenario: Scenario) { - const { kind, secretCount, priorFinalizedIndex } = scenario; + const { kind, secretCount } = scenario; const perSecretNewLogs = newLogsPerSecret(scenario); aztecNode.getPrivateLogsByTags.mockReset(); @@ -187,18 +183,18 @@ describeBench('syncTaggedPrivateLogs constrained-sync bench', () => { // Seed the persisted finalized indexes to simulate a recipient that already synced prior finalized messages. for (const secret of secrets) { - await taggingStore.updateHighestFinalizedIndex(secret, priorFinalizedIndex, JOB_ID); + await taggingStore.updateHighestFinalizedIndex(secret, PRIOR_FINALIZED_INDEX, JOB_ID); if (kind === AppTaggingSecretKind.UNCONSTRAINED) { - await taggingStore.updateHighestAgedIndex(secret, priorFinalizedIndex, JOB_ID); + await taggingStore.updateHighestAgedIndex(secret, PRIOR_FINALIZED_INDEX, JOB_ID); } } // Tags that should resolve to a finalized log: per secret, the contiguous run - // (priorFinalizedIndex, priorFinalizedIndex + K]. + // (PRIOR_FINALIZED_INDEX, PRIOR_FINALIZED_INDEX + K]. const hitTags = new Set(); for (let s = 0; s < secrets.length; s++) { for (let k = 1; k <= perSecretNewLogs[s]; k++) { - hitTags.add((await computeSiloedTagForIndex(secrets[s], priorFinalizedIndex + k)).toString()); + hitTags.add((await computeSiloedTagForIndex(secrets[s], PRIOR_FINALIZED_INDEX + k)).toString()); } } From 06f7d2d2f79f9f1ae24139e5334f5ad45dffdaad Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 14:45:52 -0400 Subject: [PATCH 22/24] test(pxe): dedupe tagged sync test boilerplate and strengthen assertions --- .../sync_tagged_private_logs.test.ts | 713 ++++++------------ 1 file changed, 237 insertions(+), 476 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index e0c2228a0911..3598f3bc932a 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -1,6 +1,6 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; import { BlockNumber } from '@aztec/foundation/branded-types'; -import type { Fr } from '@aztec/foundation/curves/bn254'; +import { times, timesParallel } from '@aztec/foundation/collection'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; @@ -28,6 +28,11 @@ const FAR_FUTURE_BLOCK_NUMBER = BlockNumber(100); const CURRENT_TIMESTAMP = BigInt(Math.floor(Date.now() / 1000)); const ANCHOR_BLOCK_HEADER = BlockHeader.random({ blockNumber: FAR_FUTURE_BLOCK_NUMBER, timestamp: CURRENT_TIMESTAMP }); const JOB_ID = 'test-job'; +const FINALIZED_BLOCK_NUMBER = BlockNumber(10); +// Old enough that the log is past MAX_TX_LIFETIME and may advance the aged index. +const AGED_TIMESTAMP = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; +// Recent enough that the log could still belong to a pending tx and must not advance the aged index. +const RECENT_TIMESTAMP = CURRENT_TIMESTAMP - 5000n; describe('syncTaggedPrivateLogs', () => { const aztecNode: MockProxy = mock(); @@ -37,7 +42,19 @@ describe('syncTaggedPrivateLogs', () => { return SiloedTag.compute({ extendedSecret: secret, index }); } - function makeLog(blockNumber: number, blockTimestamp: bigint, _tag: Fr) { + function computeSiloedTags(secret: AppTaggingSecret, indexes: number[]): Promise { + return Promise.all(indexes.map(i => computeSiloedTagForIndex(secret, i))); + } + + /** Computes the tags of `count` contiguous indexes starting at `firstIndex`. */ + function computeSiloedTagRange(secret: AppTaggingSecret, count: number, firstIndex = 0): Promise { + return computeSiloedTags( + secret, + times(count, i => firstIndex + i), + ); + } + + function makeLog(blockNumber: number, blockTimestamp: bigint) { return { ...randomLogResult(/* includeEffects */ true), blockNumber: BlockNumber(blockNumber), blockTimestamp }; } @@ -48,18 +65,53 @@ describe('syncTaggedPrivateLogs', () => { return query.tags.map((entry: TagQuery) => (entry instanceof SiloedTag ? entry : entry.tag)); } - function mockNodeWithLogs(logTags: SiloedTag[], blockNumber: number, blockTimestamp: bigint) { + /** Mocks the node to return one log per matching tag per group, at the group's block number and timestamp. */ + function mockNodeWithLogGroups(groups: { tags: SiloedTag[]; blockNumber?: number; blockTimestamp?: bigint }[]) { aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { const tags = extractTags(query); return Promise.resolve( - tags.map((t: SiloedTag) => { - const match = logTags.find(tag => tag.equals(t)); - return match ? [makeLog(blockNumber, blockTimestamp, match.value)] : []; - }), + tags.map((t: SiloedTag) => + groups.flatMap(group => + group.tags + .filter(tag => tag.equals(t)) + .map(() => + makeLog(group.blockNumber ?? Number(FINALIZED_BLOCK_NUMBER), group.blockTimestamp ?? AGED_TIMESTAMP), + ), + ), + ), ); }); } + /** Mocks the node to return one aged, finalized log per matching tag; repeat a tag to return multiple logs for it. */ + function mockNodeWithLogs(logTags: SiloedTag[], blockNumber?: number, blockTimestamp?: bigint) { + mockNodeWithLogGroups([{ tags: logTags, blockNumber, blockTimestamp }]); + } + + /** Runs syncTaggedPrivateLogs against the mocked node and the store under test. */ + function sync( + secrets: AppTaggingSecret[], + finalizedBlockNumber = FINALIZED_BLOCK_NUMBER, + header = ANCHOR_BLOCK_HEADER, + ) { + return syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, header, finalizedBlockNumber, JOB_ID); + } + + /** The tags queried by the `callIndex`-th RPC call. */ + function calledTags(callIndex = 0): SiloedTag[] { + return extractTags(aztecNode.getPrivateLogsByTags.mock.calls[callIndex][0]); + } + + /** The number of tags queried by each RPC call, in call order. */ + function callSizes(): number[] { + return aztecNode.getPrivateLogsByTags.mock.calls.map(([query]) => extractTags(query).length); + } + + /** Every tag queried across all RPC calls, in call order. */ + function allCalledTags(): SiloedTag[] { + return aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([query]) => extractTags(query)); + } + function expectedConstrainedProbeRanges(logCount: number, firstLogIndex = 1): { start: number; end: number }[] { const ranges: { start: number; end: number }[] = []; const lastLogIndex = firstLogIndex + logCount - 1; @@ -94,7 +146,7 @@ describe('syncTaggedPrivateLogs', () => { }); it('returns empty array when given no secrets', async () => { - const logs = await syncTaggedPrivateLogs([], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + const logs = await sync([]); expect(logs).toHaveLength(0); expect(aztecNode.getPrivateLogsByTags).not.toHaveBeenCalled(); @@ -102,49 +154,31 @@ describe('syncTaggedPrivateLogs', () => { it('returns empty array when no logs found for any secret', async () => { const secrets = await makeSecrets(3, AppTaggingSecretKind.UNCONSTRAINED); - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve(tags.map(() => [])); - }); + mockNodeWithLogs([]); - const logs = await syncTaggedPrivateLogs( - secrets, - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - BlockNumber(10), - JOB_ID, - ); + const logs = await sync(secrets); expect(logs).toHaveLength(0); }); - it('batches tags from multiple secrets across as few RPC calls as MAX_RPC_LEN allows', async () => { + it('batches tags from multiple secrets across as few RPC calls as the RPC limit allows', async () => { // Pick enough secrets that the total tag count spans several MAX_RPC_LEN chunks. A per-secret // implementation would need one RPC per secret; batched behavior needs ceil(totalTags / MAX_RPC_LEN). const numSecrets = 10; const secrets = await makeSecrets(numSecrets, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve(tags.map(() => [])); - }); + mockNodeWithLogs([]); - await syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + await sync(secrets); - const expectedTags = await Promise.all( - secrets.flatMap(secret => - Array.from({ length: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1 }, (_, i) => - computeSiloedTagForIndex(secret, i), - ), - ), - ); - const calledTags = aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([query]) => extractTags(query)); + const expectedTags = ( + await Promise.all( + secrets.map(secret => computeSiloedTagRange(secret, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1)), + ) + ).flat(); const asStrings = (tags: SiloedTag[]) => tags.map(t => t.toString()).sort(); // Every expected (secret, index) tag was queried exactly once, in some order across the batched RPC calls. - expect(asStrings(calledTags)).toEqual(asStrings(expectedTags)); + expect(asStrings(allCalledTags())).toEqual(asStrings(expectedTags)); // Batching invariant: the sync issues ceil(totalTags / MAX_RPC_LEN) calls, which is strictly fewer than one // RPC per secret. This is what a per-secret implementation would degenerate to. @@ -155,24 +189,15 @@ describe('syncTaggedPrivateLogs', () => { it('syncs logs and updates store independently per secret', async () => { const secrets = await makeSecrets(3, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; const log1Index = 3; const log2Index = 7; - const log1Tag = await computeSiloedTagForIndex(secrets[0], log1Index); - const log2Tag = await computeSiloedTagForIndex(secrets[1], log2Index); - - mockNodeWithLogs([log1Tag, log2Tag], Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - secrets, - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs([ + await computeSiloedTagForIndex(secrets[0], log1Index), + await computeSiloedTagForIndex(secrets[1], log2Index), + ]); + + const logs = await sync(secrets); expect(logs).toHaveLength(2); expect(await taggingStore.getHighestAgedIndex(secrets[0], JOB_ID)).toBe(log1Index); @@ -185,45 +210,34 @@ describe('syncTaggedPrivateLogs', () => { }); it('does not advance aged index for recent logs', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - 5000n; // not aged + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); const logIndex = 5; - const logTag = await computeSiloedTagForIndex(secret, logIndex); - - mockNodeWithLogs([logTag], Number(finalizedBlockNumber), logBlockTimestamp); + mockNodeWithLogs( + [await computeSiloedTagForIndex(secret, logIndex)], + Number(FINALIZED_BLOCK_NUMBER), + RECENT_TIMESTAMP, + ); - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + const logs = await sync([secret]); + // The recent log is still returned to the caller: recency only gates the aged index, not delivery. + expect(logs).toHaveLength(1); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(logIndex); expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBeUndefined(); }); it('updates store correctly when multiple iterations are needed', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const agedBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); // A log at the last index of the initial window [0, WINDOW_LEN] moves the finalized index to WINDOW_LEN, - // which shifts the next window forward and triggers a second iteration. + // which shifts the next window forward and triggers a second iteration. A second log sits in the advanced + // window, only reachable in the second iteration. const lastIndexInInitialWindow = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; - const log1Tag = await computeSiloedTagForIndex(secret, lastIndexInInitialWindow); - - // A second log sits in the advanced window, only reachable in the second iteration. const newWindowIndex = lastIndexInInitialWindow + 3; - const log2Tag = await computeSiloedTagForIndex(secret, newWindowIndex); + mockNodeWithLogs(await computeSiloedTags(secret, [lastIndexInInitialWindow, newWindowIndex])); - mockNodeWithLogs([log1Tag, log2Tag], Number(finalizedBlockNumber), agedBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const logs = await sync([secret]); expect(logs).toHaveLength(2); expect(await taggingStore.getHighestAgedIndex(secret, JOB_ID)).toBe(newWindowIndex); @@ -231,86 +245,40 @@ describe('syncTaggedPrivateLogs', () => { }); it('respects pre-existing store indexes', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); const existingAgedIndex = 5; const existingFinalizedIndex = 8; await taggingStore.updateHighestAgedIndex(secret, existingAgedIndex, JOB_ID); await taggingStore.updateHighestFinalizedIndex(secret, existingFinalizedIndex, JOB_ID); + mockNodeWithLogs([]); - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(query.tags.map(() => [])), - ); - - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); - - const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); + await sync([secret]); // The query window must start at existingAgedIndex+1 and end at existingFinalizedIndex+WINDOW_LEN (inclusive). const expectedStart = existingAgedIndex + 1; const expectedEnd = existingFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN; - const expectedTags = await Promise.all( - Array.from({ length: expectedEnd - expectedStart + 1 }, (_, i) => - computeSiloedTagForIndex(secret, expectedStart + i), - ), - ); - - expect(calledTags).toEqual(expectedTags); + expect(calledTags()).toEqual(await computeSiloedTagRange(secret, expectedEnd - expectedStart + 1, expectedStart)); }); it('handles multiple logs at the same tag index', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); - const logIndex = 3; - const logTag = await computeSiloedTagForIndex(secret, logIndex); + // The tag appears twice, so the node returns two logs for it. + const logTag = await computeSiloedTagForIndex(secret, 3); + mockNodeWithLogs([logTag, logTag]); - // Two logs returned for the same tag - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve( - tags.map((t: SiloedTag) => - t.equals(logTag) - ? [ - makeLog(Number(finalizedBlockNumber), logBlockTimestamp, logTag.value), - makeLog(Number(finalizedBlockNumber), logBlockTimestamp, logTag.value), - ] - : [], - ), - ); - }); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const logs = await sync([secret]); expect(logs).toHaveLength(2); }); describe('constrained secrets', () => { it('stops at first gap and does not track aged index', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - const logTags = await Promise.all([0, 1, 2].map(i => computeSiloedTagForIndex(secret, i))); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); + mockNodeWithLogs(await computeSiloedTags(secret, [0, 1, 2])); + + const logs = await sync([secret]); expect(logs).toHaveLength(3); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(2); @@ -318,37 +286,16 @@ describe('syncTaggedPrivateLogs', () => { }); it('advances the finalized index only through the finalized prefix', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(5); - const oldTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - const recentTimestamp = CURRENT_TIMESTAMP - 5000n; - - const finalizedTags = await Promise.all([0, 1, 2, 3].map(i => computeSiloedTagForIndex(secret, i))); - const unfinalizedTags = await Promise.all([4, 5].map(i => computeSiloedTagForIndex(secret, i))); - - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve( - tags.map((t: SiloedTag) => { - if (finalizedTags.find(tag => tag.equals(t))) { - return [makeLog(Number(finalizedBlockNumber), oldTimestamp, t.value)]; - } - if (unfinalizedTags.find(tag => tag.equals(t))) { - return [makeLog(8, recentTimestamp, t.value)]; - } - return []; - }), - ); - }); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + + // Indexes 0..3 are finalized; indexes 4 and 5 are hits in a block past the finalized one. + mockNodeWithLogGroups([ + { tags: await computeSiloedTags(secret, [0, 1, 2, 3]), blockNumber: Number(finalizedBlockNumber) }, + { tags: await computeSiloedTags(secret, [4, 5]), blockNumber: 8, blockTimestamp: RECENT_TIMESTAMP }, + ]); + + const logs = await sync([secret], finalizedBlockNumber); // The unfinalized logs (4, 5) are returned to the caller, but the finalized index only advances to the finalized // prefix (3): probe advancement is decoupled from the finalized index. @@ -357,231 +304,119 @@ describe('syncTaggedPrivateLogs', () => { }); it('advances the probe past an unfinalized-only first probe', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); const finalizedBlockNumber = BlockNumber(5); - const recentTimestamp = CURRENT_TIMESTAMP - 5000n; // Indexes 0 and 1 are hits in an unfinalized block (8 > finalized 5); index 2 is the gap. With the small initial // probe, index 0 is probed alone and is unfinalized, so the scan must still advance to discover index 1 — gating // advancement on finalization would drop it. - const unfinalizedTags = await Promise.all([0, 1].map(i => computeSiloedTagForIndex(secret, i))); - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve( - tags.map((t: SiloedTag) => - unfinalizedTags.find(tag => tag.equals(t)) ? [makeLog(8, recentTimestamp, t.value)] : [], - ), - ); - }); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTags(secret, [0, 1]), 8, RECENT_TIMESTAMP); + + const logs = await sync([secret], finalizedBlockNumber); expect(logs).toHaveLength(2); // Nothing finalized, so the finalized index must not advance. expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); // Round 1 probes [0] and advances on the unfinalized hit; round 2 probes [1, 2] and stops at the gap (2). - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual([1, 2]); + expect(callSizes()).toEqual([1, 2]); // The store only ever advances through finalized logs, so an unfinalized log can never be skipped: since nothing // was persisted, a second sync restarts at index 0 and re-fetches both unfinalized logs. aztecNode.getPrivateLogsByTags.mockClear(); - const secondSyncLogs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const secondSyncLogs = await sync([secret], finalizedBlockNumber); expect(secondSyncLogs).toHaveLength(2); - expect(extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0])).toEqual([ - await computeSiloedTagForIndex(secret, 0), - ]); + expect(calledTags()).toEqual(await computeSiloedTags(secret, [0])); + // The repeat sync saw the same unfinalized-only hits, so the finalized index must still not advance. + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBeUndefined(); }); // Pins the probe schedule: the probe doubles each round (1, 2, 4, ...) until the first miss, so K-deep // catch-up resolves in ~log2(K) round-trips instead of K + 1. With 3 new logs the windows are [1], [2,3], // [4,5,6,7] - round 3 (probe length 4) contains the terminating miss at index 4. it('doubling catch-up grows the probe geometrically and stops at the first miss', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); // Recipient already synced index 0; three new finalized logs sit at indexes 1..3. await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); - const hitTags = await Promise.all([1, 2, 3].map(i => computeSiloedTagForIndex(secret, i))); - mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTagRange(secret, 3, 1)); + + const logs = await sync([secret]); expect(logs).toHaveLength(3); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(3); // Probe windows double each round: [1], then [2,3], then [4,5,6,7] where index 4 is the terminating miss. - const calls = aztecNode.getPrivateLogsByTags.mock.calls; - expect(calls).toHaveLength(3); - expect(extractTags(calls[0][0])).toEqual(await Promise.all([1].map(i => computeSiloedTagForIndex(secret, i)))); - expect(extractTags(calls[1][0])).toEqual(await Promise.all([2, 3].map(i => computeSiloedTagForIndex(secret, i)))); - expect(extractTags(calls[2][0])).toEqual( - await Promise.all([4, 5, 6, 7].map(i => computeSiloedTagForIndex(secret, i))), - ); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(3); + expect(calledTags(0)).toEqual(await computeSiloedTags(secret, [1])); + expect(calledTags(1)).toEqual(await computeSiloedTags(secret, [2, 3])); + expect(calledTags(2)).toEqual(await computeSiloedTags(secret, [4, 5, 6, 7])); }); // Pins the effective per-round query cap and that deep catch-up is linear past the cap, not logarithmic: the probe // doubles until it saturates at WINDOW_LEN per round rather than growing without bound. - it('doubling caps the probe at WINDOW_LEN and grows linearly past the cap on deep catch-up', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + it('caps the doubling probe at the window length so deep catch-up advances linearly', async () => { + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); // Recipient already synced index 0; a deep run of finalized logs sits past multiple capped probe windows. const newLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3; await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); - const hitTags = await Promise.all( - Array.from({ length: newLogs }, (_, i) => computeSiloedTagForIndex(secret, i + 1)), - ); - mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTagRange(secret, newLogs, 1)); + + const logs = await sync([secret]); expect(logs).toHaveLength(newLogs); - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual(expectedConstrainedProbeRanges(newLogs).map(({ start, end }) => end - start)); - expect(Math.max(...callSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); - expect(callSizes.filter(size => size === UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN)).toHaveLength(2); + // The capped catch-up still drains the run fully. + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(newLogs); + const sizes = callSizes(); + expect(sizes).toEqual(expectedConstrainedProbeRanges(newLogs).map(({ start, end }) => end - start)); + expect(Math.max(...sizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + expect(sizes.filter(size => size === UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN)).toHaveLength(2); }); it('batches tags from multiple constrained secrets into a single RPC call', async () => { const secrets = await makeSecrets(3, AppTaggingSecretKind.CONSTRAINED); + mockNodeWithLogs([]); - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(query.tags.map(() => [])), - ); - - await syncTaggedPrivateLogs(secrets, aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); + await sync(secrets); // One batched call carrying exactly each secret's initial probe tag (index 0): constrained secrets share the same // RPC round as everything else rather than getting a call apiece. expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); - const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - const expectedTags = await Promise.all(secrets.map(s => computeSiloedTagForIndex(s, 0))); - expect(calledTags).toEqual(expectedTags); + expect(calledTags()).toEqual(await Promise.all(secrets.map(s => computeSiloedTagForIndex(s, 0)))); }); it('halts at the first all-miss batch and never probes past the gap', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); // Contiguous run 0,1,2 ends at the gap (index 3); a sentinel log sits far past it at index 10. The doubling probe // reaches the all-miss batch [3..6] and stops there, starting no further round, so index 10 is never queried. The // sentinel is a falsifying witness: a scan that failed to stop would eventually fetch it. - const logTags = await Promise.all([0, 1, 2, 10].map(i => computeSiloedTagForIndex(secret, i))); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTags(secret, [0, 1, 2, 10])); + + const logs = await sync([secret]); // Only the contiguous prefix is returned, and the sentinel's index was never queried. expect(logs).toHaveLength(3); const sentinelTag = await computeSiloedTagForIndex(secret, 10); - const queriedTags = aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([q]) => extractTags(q)); - expect(queriedTags.some(t => t.equals(sentinelTag))).toBe(false); - }); - - it('fully drains a cold-start run beyond the window by re-anchoring boundEnd', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - // A cold-start run two indexes past one window. Reaching the tail is only possible because boundEnd re-anchors - // forward to each finalized index found. - const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; - const logTags = await Promise.all( - Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), - ); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); - - // The whole run is returned and the finalized index lands on the last index, even though the run is longer than - // a single window. - expect(logs).toHaveLength(totalLogs); - expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - - // The final round reaches past the initial cold-start bound only because boundEnd re-anchored forward. - const calls = aztecNode.getPrivateLogsByTags.mock.calls; - const callSizes = calls.map(([q]) => extractTags(q).length); - const expectedRanges = expectedConstrainedProbeRanges(totalLogs, 0); - expect(callSizes).toEqual(expectedRanges.map(({ start, end }) => end - start)); - const lastCallTags = extractTags(calls[calls.length - 1][0]); - const lastRange = expectedRanges[expectedRanges.length - 1]; - const expectedLastRange = await Promise.all( - Array.from({ length: lastRange.end - lastRange.start }, (_, i) => - computeSiloedTagForIndex(secret, lastRange.start + i), - ), - ); - expect(lastCallTags).toEqual(expectedLastRange); - expect(lastRange.end - 1).toBeGreaterThan(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + expect(allCalledTags().some(t => t.equals(sentinelTag))).toBe(false); }); it('resets the probe to the initial length on the sync after a cap-saturating catch-up', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); // A cold-start run deep enough that the first sync doubles the in-memory probe all the way to the WINDOW_LEN // cap. The probe length lives only on the in-memory PendingSecret, so the next sync must start fresh at the // initial probe rather than inheriting the saturated one, which would forfeit the steady-state optimization // after every catch-up. const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3; - const logTags = await Promise.all( - Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), - ); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); + mockNodeWithLogs(await computeSiloedTagRange(secret, totalLogs)); // First sync catches up the whole run; the finalized index lands on the last index and the probe saturated the // cap along the way. - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + await sync([secret]); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - const catchUpCallSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(Math.max(...catchUpCallSizes)).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + expect(Math.max(...callSizes())).toBe(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); // Drop the catch-up's recorded calls (mockClear keeps the implementation; mockReset would not) so the next // assertions see only the second sync. @@ -589,46 +424,49 @@ describe('syncTaggedPrivateLogs', () => { // Second sync, no new logs: it must probe a single tag at the finalized index + 1, not the saturated probe from // the catch-up. - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); + await sync([secret]); - const calls = aztecNode.getPrivateLogsByTags.mock.calls; - expect(calls).toHaveLength(1); - const probedTags = extractTags(calls[0][0]); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); + const probedTags = calledTags(); expect(probedTags).toHaveLength(INITIAL_CONSTRAINED_PROBE_LEN); expect(probedTags).toEqual([await computeSiloedTagForIndex(secret, totalLogs)]); }); + it('steady state probes only the initial probe length in a single round', async () => { + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); + const finalizedIndex = 8; + await taggingStore.updateHighestFinalizedIndex(secret, finalizedIndex, JOB_ID); + mockNodeWithLogs([]); + + await sync([secret]); + + // Gapless stream: the first missing tag ends the scan, so steady state costs a single round of a single probe. + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); + expect(calledTags()).toEqual( + await computeSiloedTagRange(secret, INITIAL_CONSTRAINED_PROBE_LEN, finalizedIndex + 1), + ); + }); + // Pins the round-trip growth law the ~log2(K) claim is qualified against: catch-up is logarithmic only while the // probe is still doubling, then linear at roughly one extra round per WINDOW_LEN once the probe saturates the cap. - // Every round here queries fewer than MAX_RPC_LEN tags and at most one log per tag, so one sync round is exactly one - // RPC call. it.each([1, 3, 20, 50, 100, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN * 3])( 'catch-up round-trips grow logarithmically then linearly: K=%i', async newLogs => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + // A round queries at most WINDOW_LEN tags (the probe cap) with at most one log per tag, so as long as the cap + // fits in a single RPC, one sync round is exactly one RPC call and the call count is the round-trip count. + expect(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN).toBeLessThanOrEqual(MAX_RPC_LEN); + + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); const expectedRounds = expectedConstrainedProbeRanges(newLogs).length; // Recipient already synced index 0; K new contiguous finalized logs sit at indexes 1..K. await taggingStore.updateHighestFinalizedIndex(secret, 0, JOB_ID); - const hitTags = await Promise.all( - Array.from({ length: newLogs }, (_, i) => computeSiloedTagForIndex(secret, i + 1)), - ); - mockNodeWithLogs(hitTags, Number(finalizedBlockNumber), agedTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTagRange(secret, newLogs, 1)); + + const logs = await sync([secret]); expect(logs).toHaveLength(newLogs); expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(newLogs); - // One sync round is exactly one RPC call here, so the call count is the round-trip count. expect(aztecNode.getPrivateLogsByTags.mock.calls).toHaveLength(expectedRounds); }, ); @@ -640,9 +478,7 @@ describe('syncTaggedPrivateLogs', () => { // bench `mixed` scenario, which the bench (skipped in CI) cannot pin. it('a single deep straggler among idle secrets sets the round count, not the idle count', async () => { const idleSecrets = await makeSecrets(4, AppTaggingSecretKind.CONSTRAINED); - const [straggler] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const agedTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; + const straggler = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); // Idle secrets are caught up at index 5, so their next probe (index 6) misses and drops them after round 1. The // straggler is at index 0 with 3 new contiguous logs at indexes 1..3. @@ -651,17 +487,9 @@ describe('syncTaggedPrivateLogs', () => { await taggingStore.updateHighestFinalizedIndex(secret, idleFinalizedIndex, JOB_ID); } await taggingStore.updateHighestFinalizedIndex(straggler, 0, JOB_ID); - const stragglerHits = await Promise.all([1, 2, 3].map(i => computeSiloedTagForIndex(straggler, i))); - mockNodeWithLogs(stragglerHits, Number(finalizedBlockNumber), agedTimestamp); - - const logs = await syncTaggedPrivateLogs( - [...idleSecrets, straggler], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + mockNodeWithLogs(await computeSiloedTags(straggler, [1, 2, 3])); + + const logs = await sync([...idleSecrets, straggler]); // Only the straggler's logs are found; its finalized index advances while the idle secrets' stay untouched. expect(logs).toHaveLength(3); @@ -673,96 +501,48 @@ describe('syncTaggedPrivateLogs', () => { // 3 rounds = the straggler's doubling schedule; the 4 idle secrets add no rounds. Each round stays under // MAX_RPC_LEN, so call count is round count. Round 1: 4 idle probes + straggler[1]; round 2: straggler[2,3]; // round 3: straggler[4..7] (terminating miss at 4). - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes).toEqual([5, 2, 4]); + const sizes = callSizes(); + expect(sizes).toEqual([5, 2, 4]); // Total tag-queries: 4 idle probes + the straggler's (1 + 2 + 4) doubling probe = 11. - expect(callSizes.reduce((a, b) => a + b, 0)).toBe(11); + expect(sizes.reduce((a, b) => a + b, 0)).toBe(11); }); }); - describe('constrained vs unconstrained probing', () => { - it('constrained steady-state probes only INITIAL_CONSTRAINED_PROBE_LEN tags', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const finalizedIndex = 8; - await taggingStore.updateHighestFinalizedIndex(secret, finalizedIndex, JOB_ID); + describe('fully drains a cold-start run longer than the window', () => { + const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(query.tags.map(() => [])), - ); + it('constrained', async () => { + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); - await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); - - // Gapless stream: the first missing tag ends the scan, so steady state probes just the initial window. - const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - const expectedTags = await Promise.all( - Array.from({ length: INITIAL_CONSTRAINED_PROBE_LEN }, (_, i) => - computeSiloedTagForIndex(secret, finalizedIndex + 1 + i), - ), - ); - expect(calledTags).toEqual(expectedTags); - }); + // A cold-start run two indexes past one window. Reaching the tail is only possible because the scan bound + // re-anchors forward to each finalized index found. + mockNodeWithLogs(await computeSiloedTagRange(secret, totalLogs)); - it('cold start: constrained probes a single tag where unconstrained probes the full window', async () => { - const [constrained] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const [unconstrained] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + const logs = await sync([secret]); - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => - Promise.resolve(query.tags.map(() => [])), - ); + // The whole run is returned and the finalized index lands on the last index, even though the run is longer than + // a single window. + expect(logs).toHaveLength(totalLogs); + expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(totalLogs - 1); - // Fresh secret, nothing stored. The constrained scan opens at the initial probe (a single tag at index 0); the - // unconstrained scan must cover the whole [0, WINDOW_LEN] window because gaps are possible. Same starting point, - // a single tag vs WINDOW_LEN + 1 tags: the headline of the optimization, at cold start as well as steady state. - await syncTaggedPrivateLogs([constrained], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, BlockNumber(10), JOB_ID); - const constrainedTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - expect(constrainedTags).toEqual( - await Promise.all( - Array.from({ length: INITIAL_CONSTRAINED_PROBE_LEN }, (_, i) => computeSiloedTagForIndex(constrained, i)), - ), + // The final round reaches past the initial cold-start bound only because the bound re-anchored forward. + const expectedRanges = expectedConstrainedProbeRanges(totalLogs, 0); + expect(callSizes()).toEqual(expectedRanges.map(({ start, end }) => end - start)); + const lastRange = expectedRanges[expectedRanges.length - 1]; + expect(calledTags(expectedRanges.length - 1)).toEqual( + await computeSiloedTagRange(secret, lastRange.end - lastRange.start, lastRange.start), ); + expect(lastRange.end - 1).toBeGreaterThan(UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); + }); - aztecNode.getPrivateLogsByTags.mockClear(); + it('unconstrained', async () => { + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); - await syncTaggedPrivateLogs( - [unconstrained], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - BlockNumber(10), - JOB_ID, - ); - const unconstrainedTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); - expect(unconstrainedTags).toEqual( - await Promise.all( - Array.from({ length: UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1 }, (_, i) => - computeSiloedTagForIndex(unconstrained, i), - ), - ), - ); - }); + // The same run as the constrained case, but gaps are possible so there is no first-miss stop and no doubling + // probe: every round covers a full window. + mockNodeWithLogs(await computeSiloedTagRange(secret, totalLogs)); - // The unconstrained counterpart of the constrained drain test ('fully drains a cold-start run beyond the window by - // re-anchoring boundEnd'): the same run drains with full-window rounds because gaps are possible, so there is no - // first-miss stop and no doubling probe. - it('unconstrained drains the same cold-start run with full-window rounds, not a doubling probe', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - const totalLogs = UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 2; - const logTags = await Promise.all( - Array.from({ length: totalLogs }, (_, i) => computeSiloedTagForIndex(secret, i)), - ); - mockNodeWithLogs(logTags, Number(finalizedBlockNumber), logBlockTimestamp); - - const logs = await syncTaggedPrivateLogs( - [secret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const logs = await sync([secret]); // Every log is returned and both stored indexes land on the last index. The aged index advances since the logs // are old enough, unlike a constrained secret, which never tracks an aged index. @@ -773,8 +553,7 @@ describe('syncTaggedPrivateLogs', () => { // The first round spans the full cold-start window (WINDOW_LEN + 1). Because every index hit, the next round // re-anchors to another full WINDOW_LEN window ahead of the new finalized index: no small initial probe and no // doubling, in contrast to the constrained scan. - const callSizes = aztecNode.getPrivateLogsByTags.mock.calls.map(([q]) => extractTags(q).length); - expect(callSizes.slice(0, 2)).toEqual([ + expect(callSizes().slice(0, 2)).toEqual([ UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN, ]); @@ -783,53 +562,35 @@ describe('syncTaggedPrivateLogs', () => { describe('mixed constrained and unconstrained secrets', () => { it('batches both kinds into a single RPC call with different stop conditions', async () => { - const [constrainedSecret] = await makeSecrets(1, AppTaggingSecretKind.CONSTRAINED); - const [unconstrainedSecret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - const constrainedLogTags = await Promise.all([0, 1].map(i => computeSiloedTagForIndex(constrainedSecret, i))); - const unconstrainedLogTags = await Promise.all([0, 5].map(i => computeSiloedTagForIndex(unconstrainedSecret, i))); - - aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { - const tags = extractTags(query); - return Promise.resolve( - tags.map((t: SiloedTag) => { - const cMatch = constrainedLogTags.find(tag => tag.equals(t)); - const uMatch = unconstrainedLogTags.find(tag => tag.equals(t)); - if (cMatch || uMatch) { - return [makeLog(Number(finalizedBlockNumber), logBlockTimestamp, t.value)]; - } - return []; - }), - ); - }); - - const logs = await syncTaggedPrivateLogs( - [constrainedSecret, unconstrainedSecret], - aztecNode, - taggingStore, - ANCHOR_BLOCK_HEADER, - finalizedBlockNumber, - JOB_ID, - ); + const constrainedSecret = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); + const unconstrainedSecret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); + + mockNodeWithLogs([ + ...(await computeSiloedTags(constrainedSecret, [0, 1])), + ...(await computeSiloedTags(unconstrainedSecret, [0, 5])), + ]); + + const logs = await sync([constrainedSecret, unconstrainedSecret]); expect(logs).toHaveLength(4); expect(await taggingStore.getHighestFinalizedIndex(constrainedSecret, JOB_ID)).toBe(1); expect(await taggingStore.getHighestAgedIndex(constrainedSecret, JOB_ID)).toBeUndefined(); expect(await taggingStore.getHighestFinalizedIndex(unconstrainedSecret, JOB_ID)).toBe(5); + + // Both kinds' probes ride the same first RPC call rather than getting a call per kind. + const firstCallTags = calledTags(); + const constrainedProbeTag = await computeSiloedTagForIndex(constrainedSecret, 0); + const unconstrainedProbeTag = await computeSiloedTagForIndex(unconstrainedSecret, 0); + expect(firstCallTags.some(t => t.equals(constrainedProbeTag))).toBe(true); + expect(firstCallTags.some(t => t.equals(unconstrainedProbeTag))).toBe(true); }); }); - it('caps the node query at anchorBlockNumber + 1 (toBlock exclusive)', async () => { - const [secret] = await makeSecrets(1, AppTaggingSecretKind.UNCONSTRAINED); + it('caps the node query to blocks at or before the anchor block', async () => { + const secret = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); const anchorBlock = BlockNumber(10); const header = BlockHeader.random({ blockNumber: anchorBlock, timestamp: CURRENT_TIMESTAMP }); - const finalizedBlockNumber = BlockNumber(10); - const logBlockTimestamp = CURRENT_TIMESTAMP - BigInt(MAX_TX_LIFETIME) - 1000n; - - const logIndex = 3; - const logTag = await computeSiloedTagForIndex(secret, logIndex); + const logTag = await computeSiloedTagForIndex(secret, 3); // The mock simulates the node honoring `toBlock` (exclusive). Recipient sync now relies on the node // for this filter rather than dropping post-anchor logs client-side. @@ -837,9 +598,9 @@ describe('syncTaggedPrivateLogs', () => { const tags = extractTags(query); const toBlockExclusive = Number(query.toBlock ?? Infinity); const allCandidates = [ - makeLog(Number(anchorBlock) - 1, logBlockTimestamp, logTag.value), - makeLog(Number(anchorBlock), logBlockTimestamp, logTag.value), - makeLog(Number(anchorBlock) + 1, logBlockTimestamp, logTag.value), + makeLog(Number(anchorBlock) - 1, AGED_TIMESTAMP), + makeLog(Number(anchorBlock), AGED_TIMESTAMP), + makeLog(Number(anchorBlock) + 1, AGED_TIMESTAMP), ]; return Promise.resolve( tags.map((t: SiloedTag) => @@ -848,7 +609,7 @@ describe('syncTaggedPrivateLogs', () => { ); }); - const logs = await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, header, finalizedBlockNumber, JOB_ID); + const logs = await sync([secret], FINALIZED_BLOCK_NUMBER, header); // Only logs at or before the anchor block should be included — node-side filter drops the post-anchor log. expect(logs).toHaveLength(2); @@ -858,5 +619,5 @@ describe('syncTaggedPrivateLogs', () => { }); function makeSecrets(count: number, kind: AppTaggingSecretKind): Promise { - return Promise.all(Array.from({ length: count }, () => randomAppTaggingSecret(kind))); + return timesParallel(count, () => randomAppTaggingSecret(kind)); } From a8bef534373b125044fd029113fcaf37227262fb Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 16:43:40 -0400 Subject: [PATCH 23/24] test(pxe): document constrained probe model and slim straggler test --- .../sync_tagged_private_logs.test.ts | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 3598f3bc932a..d07f73be0f5d 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -112,11 +112,19 @@ describe('syncTaggedPrivateLogs', () => { return aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([query]) => extractTags(query)); } + /** + * Models the constrained probe schedule for draining `logCount` contiguous finalized logs starting at + * `firstLogIndex`: the half-open [start, end) tag-index range each sync round is expected to query. + * + * E.g. 3 logs at indexes 1..3 give [1, 2), [2, 4), [4, 8): the probe doubles after each fully-hit round and the + * miss at index 4 ends the scan. + */ function expectedConstrainedProbeRanges(logCount: number, firstLogIndex = 1): { start: number; end: number }[] { const ranges: { start: number; end: number }[] = []; const lastLogIndex = firstLogIndex + logCount - 1; let start = firstLogIndex; let highestFinalizedIndex = firstLogIndex - 1; + // Exclusive end of the probing window, re-anchored past the highest finalized index as hits land. let boundEnd = highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1; let probeLen = INITIAL_CONSTRAINED_PROBE_LEN; @@ -131,10 +139,12 @@ describe('syncTaggedPrivateLogs', () => { boundEnd = Math.max(boundEnd, highestFinalizedIndex + UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + 1); } + // A miss inside the probe means the gapless stream ended; hitting the window bound also stops the scan. if (!probeFullyConsumed || end >= boundEnd) { return ranges; } + // The probe doubles after every fully-hit round, capped at the window length. start = end; probeLen = Math.min(probeLen * 2, UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN); } @@ -471,12 +481,9 @@ describe('syncTaggedPrivateLogs', () => { }, ); - // Pins the batched-round semantics under heterogeneous load: idle secrets each cost exactly one probe and drop out - // after the first round, while a single deep straggler alone drives the round count. Round-trips track the deepest - // secret (here probeSchedule(3) = 3 rounds, the same as a lone K=3 secret), NOT the idle count and NOT one round per - // log; tag-queries are the idle probes plus the straggler's doubling schedule. This is the unit-level analogue of the - // bench `mixed` scenario, which the bench (skipped in CI) cannot pin. - it('a single deep straggler among idle secrets sets the round count, not the idle count', async () => { + // Secrets whose probe misses drop out of the sync and are not re-probed in the rounds a straggler keeps driving. + // This is the unit-level analogue of the bench `mixed` scenario, which the bench (skipped in CI) cannot pin. + it('drops caught-up secrets from later rounds while a straggler keeps syncing', async () => { const idleSecrets = await makeSecrets(4, AppTaggingSecretKind.CONSTRAINED); const straggler = await randomAppTaggingSecret(AppTaggingSecretKind.CONSTRAINED); @@ -490,21 +497,17 @@ describe('syncTaggedPrivateLogs', () => { mockNodeWithLogs(await computeSiloedTags(straggler, [1, 2, 3])); const logs = await sync([...idleSecrets, straggler]); - - // Only the straggler's logs are found; its finalized index advances while the idle secrets' stay untouched. expect(logs).toHaveLength(3); - expect(await taggingStore.getHighestFinalizedIndex(straggler, JOB_ID)).toBe(3); + + // Round 1: 4 idle probes + straggler[1]. Rounds 2 and 3 are straggler-only: [2,3], then [4..7] (terminating + // miss at 4). + expect(callSizes()).toEqual([5, 2, 4]); + + // Dropping out also means no writes: the caught-up secrets' finalized indexes are untouched by the + // straggler-driven rounds. for (const secret of idleSecrets) { expect(await taggingStore.getHighestFinalizedIndex(secret, JOB_ID)).toBe(idleFinalizedIndex); } - - // 3 rounds = the straggler's doubling schedule; the 4 idle secrets add no rounds. Each round stays under - // MAX_RPC_LEN, so call count is round count. Round 1: 4 idle probes + straggler[1]; round 2: straggler[2,3]; - // round 3: straggler[4..7] (terminating miss at 4). - const sizes = callSizes(); - expect(sizes).toEqual([5, 2, 4]); - // Total tag-queries: 4 idle probes + the straggler's (1 + 2 + 4) doubling probe = 11. - expect(sizes.reduce((a, b) => a + b, 0)).toBe(11); }); }); @@ -577,7 +580,7 @@ describe('syncTaggedPrivateLogs', () => { expect(await taggingStore.getHighestAgedIndex(constrainedSecret, JOB_ID)).toBeUndefined(); expect(await taggingStore.getHighestFinalizedIndex(unconstrainedSecret, JOB_ID)).toBe(5); - // Both kinds' probes ride the same first RPC call rather than getting a call per kind. + // Both kinds share one batched query rather than one query per kind. const firstCallTags = calledTags(); const constrainedProbeTag = await computeSiloedTagForIndex(constrainedSecret, 0); const unconstrainedProbeTag = await computeSiloedTagForIndex(unconstrainedSecret, 0); From 1d76c97dea652e106d43127b581ee4f67a9b67f6 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 9 Jul 2026 16:55:41 -0400 Subject: [PATCH 24/24] test(pxe): restore straggler finalized index assertion --- .../src/tagging/recipient_sync/sync_tagged_private_logs.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index d07f73be0f5d..e5ae328df61c 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -502,6 +502,7 @@ describe('syncTaggedPrivateLogs', () => { // Round 1: 4 idle probes + straggler[1]. Rounds 2 and 3 are straggler-only: [2,3], then [4..7] (terminating // miss at 4). expect(callSizes()).toEqual([5, 2, 4]); + expect(await taggingStore.getHighestFinalizedIndex(straggler, JOB_ID)).toBe(3); // Dropping out also means no writes: the caught-up secrets' finalized indexes are untouched by the // straggler-driven rounds.