diff --git a/docs/docs-developers/docs/resources/migration_notes.md b/docs/docs-developers/docs/resources/migration_notes.md index 821acf763150..0ff37ad3c221 100644 --- a/docs/docs-developers/docs/resources/migration_notes.md +++ b/docs/docs-developers/docs/resources/migration_notes.md @@ -9,6 +9,12 @@ Aztec is in active development. Each version may introduce breaking changes that ## TBD +### [PXE] Local PXE database is reset on upgrade + +The persisted tagging stores now key every entry by the self-describing `::` form of `AppTaggingSecret`; unconstrained secrets previously used a two-part `:` key. This bumps the PXE data schema version, and there is no forward migration for the old keys: on first open the PXE clears any database whose stored schema version differs from the current one. The wipe resets the entire PXE store, not just the tagging data, because all of it shares one backing database. + +**Impact**: On upgrade your local PXE state is reset. You must re-register accounts and re-sync from genesis. Wallets should surface a "your local state was reset, please re-register accounts and re-sync" path. + ### [Aztec.nr] `TestEnvironmentOptions::with_tagging_secret_strategy` replaced `TestEnvironmentOptions::with_tagging_secret_strategy` is now `with_default_tag_secret_strategy_all_modes` for tests diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/RecipientTaggingStore.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/RecipientTaggingStore.json index 86234edfe1fc..8b9271d749e3 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/RecipientTaggingStore.json +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/RecipientTaggingStore.json @@ -1,17 +1,25 @@ { "highest_aged_index": [ { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", + "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017", + "value": "num:13" + }, + { + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", "value": "num:13" } ], "highest_finalized_index": [ { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", + "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017", + "value": "num:11" + }, + { + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", "value": "num:11" }, { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007", + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007", "value": "num:17" } ] diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/SenderTaggingStore.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/SenderTaggingStore.json index 487674b3cde4..dcf8654b3ecf 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/SenderTaggingStore.json +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/SenderTaggingStore.json @@ -1,21 +1,29 @@ { "pending_indexes": [ { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", + "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017", + "value": "[{\"lowestIndex\":4,\"highestIndex\":7,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000025\"}]" + }, + { + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", "value": "[{\"lowestIndex\":4,\"highestIndex\":7,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000013\"},{\"lowestIndex\":8,\"highestIndex\":11,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000017\"}]" }, { - "key": "utf8:0x000000000000000000000000000000000000000000000000000000000000000b:0x000000000000000000000000000000000000000000000000000000000000000d", + "key": "utf8:unconstrained:0x000000000000000000000000000000000000000000000000000000000000000b:0x000000000000000000000000000000000000000000000000000000000000000d", "value": "[{\"lowestIndex\":1,\"highestIndex\":9,\"txHash\":\"0x000000000000000000000000000000000000000000000000000000000000001d\"}]" } ], "last_finalized_indexes": [ { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", + "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017", + "value": "num:3" + }, + { + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003", "value": "num:3" }, { - "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007", + "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007", "value": "num:5" } ] diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json index 5afea8e276cf..fe8a7f6856ea 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json @@ -1,5 +1,5 @@ { - "schemaVersion": 12, + "schemaVersion": 13, "stores": [ { "name": "capsules", diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/pxe_db_compatibility.test.ts b/yarn-project/pxe/src/storage/backwards_compatibility_tests/pxe_db_compatibility.test.ts index 0cfe79e80992..e40e0209e4a1 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/pxe_db_compatibility.test.ts +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/pxe_db_compatibility.test.ts @@ -1,5 +1,7 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { KeyStore } from '@aztec/key-store'; +import type { AztecAsyncKVStore } from '@aztec/kv-store'; import { createStore, openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/stdlib/block'; @@ -23,6 +25,9 @@ expect.extend({ toMatchFile }); const __dirname = dirname(fileURLToPath(import.meta.url)); // The last schema in which the key store still persisted the message-signing and fallback secret keys. const PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION = 10; +// The last schema in which the tagging stores keyed unconstrained entries by the legacy two-part AppTaggingSecret +// format (`:`), before every key moved to the self-describing `::` form. +const PRE_KIND_PREFIXED_TAGGING_KEY_PXE_SCHEMA_VERSION = 12; /** * Asserts that `value` matches the per-store snapshot file `__snapshots__/.json`. Each store gets its own file @@ -140,6 +145,42 @@ async function collectOpenedStores() { } } +/** + * Seeds rows into a `pxe_data` store opened at `oldSchemaVersion`, then reopens it at the current + * `PXE_DATA_SCHEMA_VERSION`. The version mismatch makes DatabaseVersionManager wipe the database on open, so + * `assertWiped` runs against a cleared store and can prove the legacy rows are gone. + */ +async function expectStoreWipedOnUpgradeFrom( + oldSchemaVersion: number, + seedLegacyRows: (oldStore: AztecAsyncKVStore) => Promise, + assertWiped: (currentStore: AztecAsyncKVStore) => Promise, +) { + const dataDirectory = await mkdtemp(join(tmpdir(), 'pxe-schema-wipe-')); + const config = { + dataDirectory, + dataStoreMapSizeKb: 1024, + rollupAddress: EthAddress.ZERO, + }; + + try { + const oldStore = await createStore('pxe_data', oldSchemaVersion, config); + try { + await seedLegacyRows(oldStore); + } finally { + await oldStore.close(); + } + + const currentStore = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, config); + try { + await assertWiped(currentStore); + } finally { + await currentStore.close(); + } + } finally { + await rm(dataDirectory, { recursive: true, force: true, maxRetries: 3 }); + } +} + /** * Backwards-compatibility test suite for PXE storage. The intent is to detect any change to the bytes PXE writes to * disk that would render existing on-device data unreadable after a version bump. Each schema test (in @@ -159,42 +200,35 @@ async function collectOpenedStores() { describe('PXE storage compatibility test suite', () => { it('wipes key-store rows written before the message-signing and fallback secret keys were removed', async () => { const account = AztecAddress.fromStringUnsafe('0x0b3683ee9df3ed6ed7027145bd6093f783b0bb4d8354501d906db7bb8cb58ea3'); - const dataDirectory = await mkdtemp(join(tmpdir(), 'pxe-schema-reset-')); - const config = { - dataDirectory, - dataStoreMapSizeKb: 1024, - rollupAddress: EthAddress.ZERO, - }; - - try { - const oldStore = await createStore( - 'pxe_data', - PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION, - config, - ); - try { - await oldStore + await expectStoreWipedOnUpgradeFrom( + PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION, + oldStore => + oldStore .openMap('key_store') .set( `${account.toString()}-ivsk_m`, Buffer.from('1fb01c42d1aaa2662041b899c77cb19e08192193acc5a94405f1b43c974eba7a', 'hex'), - ); - } finally { - await oldStore.close(); - } - - const currentStore = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, config); - try { + ), + async currentStore => { const keyStore = new KeyStore(currentStore); - // Opening a below-current-version DB triggers DatabaseVersionManager to wipe it, so the account written under - // the old schema is gone and the new code never reads its now-incompatible rows. await expect(keyStore.hasAccount(account)).resolves.toBe(false); - } finally { - await currentStore.close(); - } - } finally { - await rm(dataDirectory, { recursive: true, force: true, maxRetries: 3 }); - } + }, + ); + }); + + it('wipes tagging-store rows written under the legacy two-part AppTaggingSecret key format', async () => { + // The current toString() only emits the three-part `::` form, so build the legacy two-part + // `:` key by hand. + const legacyKey = `${new Fr(2n).toString()}:${AztecAddress.fromBigIntUnsafe(3n).toString()}`; + await expectStoreWipedOnUpgradeFrom( + PRE_KIND_PREFIXED_TAGGING_KEY_PXE_SCHEMA_VERSION, + oldStore => oldStore.openMap('highest_aged_index').set(legacyKey, 13), + async currentStore => { + // Assert on the raw map, not a high-level getter: the new three-part toString() never reconstructs the + // legacy two-part key, so a getter would report it absent (false-pass) whether or not the wipe happened. + await expect(currentStore.openMap('highest_aged_index').sizeAsync()).resolves.toBe(0); + }, + ); }); it('opens the expected set of stores', async () => { diff --git a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts index 196f97b4cd67..a8561507d41d 100644 --- a/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts +++ b/yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts @@ -18,6 +18,7 @@ import { GasFees } from '@aztec/stdlib/gas'; import { PublicKey, PublicKeys, deriveKeys } from '@aztec/stdlib/keys'; import { AppTaggingSecret, + AppTaggingSecretKind, ContractClassLog, ContractClassLogFields, PrivateLog, @@ -530,10 +531,18 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [ const jobId = 'fixture-job'; const secretA = new AppTaggingSecret(new Fr(2n), AztecAddress.fromBigIntUnsafe(3n)); const secretB = new AppTaggingSecret(new Fr(5n), AztecAddress.fromBigIntUnsafe(7n)); + // A constrained secret keys under the `constrained:` prefix, so the snapshot pins both kinds side by side. + const secretConstrained = new AppTaggingSecret( + new Fr(19n), + AztecAddress.fromBigIntUnsafe(23n), + AppTaggingSecretKind.CONSTRAINED, + ); await recipientTaggingStore.updateHighestFinalizedIndex(secretA, 11, jobId); await recipientTaggingStore.updateHighestAgedIndex(secretA, 13, jobId); await recipientTaggingStore.updateHighestFinalizedIndex(secretB, 17, jobId); + await recipientTaggingStore.updateHighestFinalizedIndex(secretConstrained, 11, jobId); + await recipientTaggingStore.updateHighestAgedIndex(secretConstrained, 13, jobId); await kvStore.transactionAsync(() => recipientTaggingStore.commit(jobId)); }, snapshotStore: async kvStore => ({ @@ -584,10 +593,17 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [ const secretA = new AppTaggingSecret(new Fr(2n), AztecAddress.fromBigIntUnsafe(3n)); const secretB = new AppTaggingSecret(new Fr(5n), AztecAddress.fromBigIntUnsafe(7n)); const secretC = new AppTaggingSecret(new Fr(11n), AztecAddress.fromBigIntUnsafe(13n)); + const secretConstrained = new AppTaggingSecret( + new Fr(19n), + AztecAddress.fromBigIntUnsafe(23n), + AppTaggingSecretKind.CONSTRAINED, + ); const txHashA = TxHash.fromBigInt(17n); const txHashB = TxHash.fromBigInt(19n); const txHashC = TxHash.fromBigInt(23n); const txHashD = TxHash.fromBigInt(29n); + const txHashE = TxHash.fromBigInt(31n); + const txHashF = TxHash.fromBigInt(37n); // secretA receives three pending ranges (one per tx); secretB receives one. After finalizing txHashA below, // secretA's array shrinks to two elements (the txHashB and txHashC ranges, both with highestIndex > 3) which @@ -626,7 +642,20 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [ jobId, ); - await senderTaggingStore.finalizePendingIndexes([txHashA], jobId); + // secretConstrained gets a finalized range (txHashE) plus a surviving higher pending range (txHashF), so the + // `constrained:` key lands in both pending_indexes and last_finalized_indexes. + await senderTaggingStore.storePendingIndexes( + [{ extendedSecret: secretConstrained, lowestIndex: 1, highestIndex: 3 }], + txHashE, + jobId, + ); + await senderTaggingStore.storePendingIndexes( + [{ extendedSecret: secretConstrained, lowestIndex: 4, highestIndex: 7 }], + txHashF, + jobId, + ); + + await senderTaggingStore.finalizePendingIndexes([txHashA, txHashE], jobId); await kvStore.transactionAsync(() => senderTaggingStore.commit(jobId)); }, diff --git a/yarn-project/pxe/src/storage/metadata.ts b/yarn-project/pxe/src/storage/metadata.ts index ca02f2aa5901..71cc183a6299 100644 --- a/yarn-project/pxe/src/storage/metadata.ts +++ b/yarn-project/pxe/src/storage/metadata.ts @@ -1 +1 @@ -export const PXE_DATA_SCHEMA_VERSION = 12; +export const PXE_DATA_SCHEMA_VERSION = 13; diff --git a/yarn-project/stdlib/src/logs/app_tagging_secret.test.ts b/yarn-project/stdlib/src/logs/app_tagging_secret.test.ts index e20729703003..04b6a28a7f48 100644 --- a/yarn-project/stdlib/src/logs/app_tagging_secret.test.ts +++ b/yarn-project/stdlib/src/logs/app_tagging_secret.test.ts @@ -128,15 +128,12 @@ describe('AppTaggingSecret', () => { expect(parsed.toString()).toBe(original.toString()); }); - // TODO(F-680): Remove once unconstrained `toString()` always emits the kind-prefixed format. - it('parses kind-prefixed unconstrained secrets', async () => { + it('rejects a string that is not three-part', async () => { const original = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED); - const parsed = AppTaggingSecret.fromString( - `${AppTaggingSecretKind.UNCONSTRAINED}:${original.secret.toString()}:${original.app.toString()}`, - ); - expect(parsed.kind).toBe(AppTaggingSecretKind.UNCONSTRAINED); - expect(parsed.toString()).toBe(original.toString()); + expect(() => AppTaggingSecret.fromString(`${original.secret.toString()}:${original.app.toString()}`)).toThrow( + /Invalid AppTaggingSecret string/, + ); }); it('rejects unknown kind prefixes', async () => { diff --git a/yarn-project/stdlib/src/logs/app_tagging_secret.ts b/yarn-project/stdlib/src/logs/app_tagging_secret.ts index ad263cee3b06..779618a2461b 100644 --- a/yarn-project/stdlib/src/logs/app_tagging_secret.ts +++ b/yarn-project/stdlib/src/logs/app_tagging_secret.ts @@ -89,29 +89,20 @@ export class AppTaggingSecret { } toString(): string { - // TODO(F-680): Migrate stored tagging keys and remove the legacy unconstrained format. - if (this.kind === AppTaggingSecretKind.UNCONSTRAINED) { - return `${this.secret.toString()}:${this.app.toString()}`; - } return `${this.kind}:${this.secret.toString()}:${this.app.toString()}`; } static fromString(str: string): AppTaggingSecret { const parts = str.split(':'); - if (parts.length === 2) { - // TODO(F-680): Remove legacy two-part parsing after stored tagging keys are migrated. - const [secretStr, appStr] = parts; - return new AppTaggingSecret(Fr.fromString(secretStr), AztecAddress.fromStringUnsafe(appStr)); - } - if (parts.length === 3) { - const [kindStr, secretStr, appStr] = parts; - return new AppTaggingSecret( - Fr.fromString(secretStr), - AztecAddress.fromStringUnsafe(appStr), - appTaggingSecretKindFromString(kindStr), - ); + if (parts.length !== 3) { + throw new Error(`Invalid AppTaggingSecret string: ${str}`); } - throw new Error(`Invalid AppTaggingSecret string: ${str}`); + const [kindStr, secretStr, appStr] = parts; + return new AppTaggingSecret( + Fr.fromString(secretStr), + AztecAddress.fromStringUnsafe(appStr), + appTaggingSecretKindFromString(kindStr), + ); } }