feat(pxe)!: Add AppTaggingSecret kinds to keys in tagging stores

docs/docs-developers/docs/resources/migration_notes.md

@@ -9,6 +9,12 @@ Aztec is in active development. Each version may introduce breaking changes that
 
 ## TBD
 
+### [PXE] Local PXE database is reset on upgrade
+
+The persisted tagging stores now key every entry by the self-describing `<kind>:<secret>:<app>` form of `AppTaggingSecret`; unconstrained secrets previously used a two-part `<secret>:<app>` key. This bumps the PXE data schema version, and there is no forward migration for the old keys: on first open the PXE clears any database whose stored schema version differs from the current one. The wipe resets the entire PXE store, not just the tagging data, because all of it shares one backing database.
+
+**Impact**: On upgrade your local PXE state is reset. You must re-register accounts and re-sync from genesis. Wallets should surface a "your local state was reset, please re-register accounts and re-sync" path.
+
 ### [Aztec.nr] `TestEnvironmentOptions::with_tagging_secret_strategy` replaced
 
 `TestEnvironmentOptions::with_tagging_secret_strategy` is now `with_default_tag_secret_strategy_all_modes` for tests

yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/RecipientTaggingStore.json

@@ -1,17 +1,25 @@
 {
   "highest_aged_index": [
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
+      "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017",
+      "value": "num:13"
+    },
+    {
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
       "value": "num:13"
     }
   ],
   "highest_finalized_index": [
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
+      "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017",
+      "value": "num:11"
+    },
+    {
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
       "value": "num:11"
     },
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007",
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007",
       "value": "num:17"
     }
   ]

yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/SenderTaggingStore.json

@@ -1,21 +1,29 @@
 {
   "pending_indexes": [
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
+      "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017",
+      "value": "[{\"lowestIndex\":4,\"highestIndex\":7,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000025\"}]"
+    },
+    {
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
       "value": "[{\"lowestIndex\":4,\"highestIndex\":7,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000013\"},{\"lowestIndex\":8,\"highestIndex\":11,\"txHash\":\"0x0000000000000000000000000000000000000000000000000000000000000017\"}]"
     },
     {
-      "key": "utf8:0x000000000000000000000000000000000000000000000000000000000000000b:0x000000000000000000000000000000000000000000000000000000000000000d",
+      "key": "utf8:unconstrained:0x000000000000000000000000000000000000000000000000000000000000000b:0x000000000000000000000000000000000000000000000000000000000000000d",
       "value": "[{\"lowestIndex\":1,\"highestIndex\":9,\"txHash\":\"0x000000000000000000000000000000000000000000000000000000000000001d\"}]"
     }
   ],
   "last_finalized_indexes": [
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
+      "key": "utf8:constrained:0x0000000000000000000000000000000000000000000000000000000000000013:0x0000000000000000000000000000000000000000000000000000000000000017",
+      "value": "num:3"
+    },
+    {
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000002:0x0000000000000000000000000000000000000000000000000000000000000003",
       "value": "num:3"
     },
     {
-      "key": "utf8:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007",
+      "key": "utf8:unconstrained:0x0000000000000000000000000000000000000000000000000000000000000005:0x0000000000000000000000000000000000000000000000000000000000000007",
       "value": "num:5"
     }
   ]

yarn-project/pxe/src/storage/backwards_compatibility_tests/__snapshots__/opened_stores.json

@@ -1,5 +1,5 @@
 {
-  "schemaVersion": 12,
+  "schemaVersion": 13,
   "stores": [
     {
       "name": "capsules",

yarn-project/pxe/src/storage/backwards_compatibility_tests/pxe_db_compatibility.test.ts

@@ -1,5 +1,7 @@
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { KeyStore } from '@aztec/key-store';
+import type { AztecAsyncKVStore } from '@aztec/kv-store';
 import { createStore, openTmpStore } from '@aztec/kv-store/lmdb-v2';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/stdlib/block';
@@ -23,6 +25,9 @@ expect.extend({ toMatchFile });
 const __dirname = dirname(fileURLToPath(import.meta.url));
 // The last schema in which the key store still persisted the message-signing and fallback secret keys.
 const PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION = 10;
+// The last schema in which the tagging stores keyed unconstrained entries by the legacy two-part AppTaggingSecret
+// format (`<secret>:<app>`), before every key moved to the self-describing `<kind>:<secret>:<app>` form.
+const PRE_KIND_PREFIXED_TAGGING_KEY_PXE_SCHEMA_VERSION = 12;
 
 /**
  * Asserts that `value` matches the per-store snapshot file `__snapshots__/<name>.json`. Each store gets its own file
@@ -140,6 +145,42 @@ async function collectOpenedStores() {
   }
 }
 
+/**
+ * Seeds rows into a `pxe_data` store opened at `oldSchemaVersion`, then reopens it at the current
+ * `PXE_DATA_SCHEMA_VERSION`. The version mismatch makes DatabaseVersionManager wipe the database on open, so
+ * `assertWiped` runs against a cleared store and can prove the legacy rows are gone.
+ */
+async function expectStoreWipedOnUpgradeFrom(
+  oldSchemaVersion: number,
+  seedLegacyRows: (oldStore: AztecAsyncKVStore) => Promise<void>,
+  assertWiped: (currentStore: AztecAsyncKVStore) => Promise<void>,
+) {
+  const dataDirectory = await mkdtemp(join(tmpdir(), 'pxe-schema-wipe-'));
+  const config = {
+    dataDirectory,
+    dataStoreMapSizeKb: 1024,
+    rollupAddress: EthAddress.ZERO,
+  };
+
+  try {
+    const oldStore = await createStore('pxe_data', oldSchemaVersion, config);
+    try {
+      await seedLegacyRows(oldStore);
+    } finally {
+      await oldStore.close();
+    }
+
+    const currentStore = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, config);
+    try {
+      await assertWiped(currentStore);
+    } finally {
+      await currentStore.close();
+    }
+  } finally {
+    await rm(dataDirectory, { recursive: true, force: true, maxRetries: 3 });
+  }
+}
+
 /**
  * Backwards-compatibility test suite for PXE storage. The intent is to detect any change to the bytes PXE writes to
  * disk that would render existing on-device data unreadable after a version bump. Each schema test (in
@@ -159,42 +200,35 @@ async function collectOpenedStores() {
 describe('PXE storage compatibility test suite', () => {
   it('wipes key-store rows written before the message-signing and fallback secret keys were removed', async () => {
     const account = AztecAddress.fromStringUnsafe('0x0b3683ee9df3ed6ed7027145bd6093f783b0bb4d8354501d906db7bb8cb58ea3');
-    const dataDirectory = await mkdtemp(join(tmpdir(), 'pxe-schema-reset-'));
-    const config = {
-      dataDirectory,
-      dataStoreMapSizeKb: 1024,
-      rollupAddress: EthAddress.ZERO,
-    };
-
-    try {
-      const oldStore = await createStore(
-        'pxe_data',
-        PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION,
-        config,
-      );
-      try {
-        await oldStore
+    await expectStoreWipedOnUpgradeFrom(
+      PRE_MESSAGE_AND_FALLBACK_SECRET_KEY_REMOVAL_PXE_SCHEMA_VERSION,
+      oldStore =>
+        oldStore
           .openMap<string, Buffer>('key_store')
           .set(
             `${account.toString()}-ivsk_m`,
             Buffer.from('1fb01c42d1aaa2662041b899c77cb19e08192193acc5a94405f1b43c974eba7a', 'hex'),
-          );
-      } finally {
-        await oldStore.close();
-      }
-
-      const currentStore = await createStore('pxe_data', PXE_DATA_SCHEMA_VERSION, config);
-      try {
+          ),
+      async currentStore => {
         const keyStore = new KeyStore(currentStore);
-        // Opening a below-current-version DB triggers DatabaseVersionManager to wipe it, so the account written under
-        // the old schema is gone and the new code never reads its now-incompatible rows.
         await expect(keyStore.hasAccount(account)).resolves.toBe(false);
-      } finally {
-        await currentStore.close();
-      }
-    } finally {
-      await rm(dataDirectory, { recursive: true, force: true, maxRetries: 3 });
-    }
+      },
+    );
+  });
+
+  it('wipes tagging-store rows written under the legacy two-part AppTaggingSecret key format', async () => {
+    // The current toString() only emits the three-part `<kind>:<secret>:<app>` form, so build the legacy two-part
+    // `<secret>:<app>` key by hand.
+    const legacyKey = `${new Fr(2n).toString()}:${AztecAddress.fromBigIntUnsafe(3n).toString()}`;
+    await expectStoreWipedOnUpgradeFrom(
+      PRE_KIND_PREFIXED_TAGGING_KEY_PXE_SCHEMA_VERSION,
+      oldStore => oldStore.openMap<string, number>('highest_aged_index').set(legacyKey, 13),
+      async currentStore => {
+        // Assert on the raw map, not a high-level getter: the new three-part toString() never reconstructs the
+        // legacy two-part key, so a getter would report it absent (false-pass) whether or not the wipe happened.
+        await expect(currentStore.openMap<string, number>('highest_aged_index').sizeAsync()).resolves.toBe(0);
+      },
+    );
   });
 
   it('opens the expected set of stores', async () => {

yarn-project/pxe/src/storage/backwards_compatibility_tests/schema_tests.ts

@@ -18,6 +18,7 @@ import { GasFees } from '@aztec/stdlib/gas';
 import { PublicKey, PublicKeys, deriveKeys } from '@aztec/stdlib/keys';
 import {
   AppTaggingSecret,
+  AppTaggingSecretKind,
   ContractClassLog,
   ContractClassLogFields,
   PrivateLog,
@@ -530,10 +531,18 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [
       const jobId = 'fixture-job';
       const secretA = new AppTaggingSecret(new Fr(2n), AztecAddress.fromBigIntUnsafe(3n));
       const secretB = new AppTaggingSecret(new Fr(5n), AztecAddress.fromBigIntUnsafe(7n));
+      // A constrained secret keys under the `constrained:` prefix, so the snapshot pins both kinds side by side.
+      const secretConstrained = new AppTaggingSecret(
+        new Fr(19n),
+        AztecAddress.fromBigIntUnsafe(23n),
+        AppTaggingSecretKind.CONSTRAINED,
+      );
 
       await recipientTaggingStore.updateHighestFinalizedIndex(secretA, 11, jobId);
       await recipientTaggingStore.updateHighestAgedIndex(secretA, 13, jobId);
       await recipientTaggingStore.updateHighestFinalizedIndex(secretB, 17, jobId);
+      await recipientTaggingStore.updateHighestFinalizedIndex(secretConstrained, 11, jobId);
+      await recipientTaggingStore.updateHighestAgedIndex(secretConstrained, 13, jobId);
       await kvStore.transactionAsync(() => recipientTaggingStore.commit(jobId));
     },
     snapshotStore: async kvStore => ({
@@ -584,10 +593,17 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [
       const secretA = new AppTaggingSecret(new Fr(2n), AztecAddress.fromBigIntUnsafe(3n));
       const secretB = new AppTaggingSecret(new Fr(5n), AztecAddress.fromBigIntUnsafe(7n));
       const secretC = new AppTaggingSecret(new Fr(11n), AztecAddress.fromBigIntUnsafe(13n));
+      const secretConstrained = new AppTaggingSecret(
+        new Fr(19n),
+        AztecAddress.fromBigIntUnsafe(23n),
+        AppTaggingSecretKind.CONSTRAINED,
+      );
       const txHashA = TxHash.fromBigInt(17n);
       const txHashB = TxHash.fromBigInt(19n);
       const txHashC = TxHash.fromBigInt(23n);
       const txHashD = TxHash.fromBigInt(29n);
+      const txHashE = TxHash.fromBigInt(31n);
+      const txHashF = TxHash.fromBigInt(37n);
 
       // secretA receives three pending ranges (one per tx); secretB receives one. After finalizing txHashA below,
       // secretA's array shrinks to two elements (the txHashB and txHashC ranges, both with highestIndex > 3) which
@@ -626,7 +642,20 @@ export const SCHEMA_TESTS: readonly SchemaTest[] = [
         jobId,
       );
 
-      await senderTaggingStore.finalizePendingIndexes([txHashA], jobId);
+      // secretConstrained gets a finalized range (txHashE) plus a surviving higher pending range (txHashF), so the
+      // `constrained:` key lands in both pending_indexes and last_finalized_indexes.
+      await senderTaggingStore.storePendingIndexes(
+        [{ extendedSecret: secretConstrained, lowestIndex: 1, highestIndex: 3 }],
+        txHashE,
+        jobId,
+      );
+      await senderTaggingStore.storePendingIndexes(
+        [{ extendedSecret: secretConstrained, lowestIndex: 4, highestIndex: 7 }],
+        txHashF,
+        jobId,
+      );
+
+      await senderTaggingStore.finalizePendingIndexes([txHashA, txHashE], jobId);
 
       await kvStore.transactionAsync(() => senderTaggingStore.commit(jobId));
     },

yarn-project/pxe/src/storage/metadata.ts

@@ -1 +1 @@
-export const PXE_DATA_SCHEMA_VERSION = 12;
+export const PXE_DATA_SCHEMA_VERSION = 13;

yarn-project/stdlib/src/logs/app_tagging_secret.test.ts

@@ -128,15 +128,12 @@ describe('AppTaggingSecret', () => {
       expect(parsed.toString()).toBe(original.toString());
     });
 
-    // TODO(F-680): Remove once unconstrained `toString()` always emits the kind-prefixed format.
-    it('parses kind-prefixed unconstrained secrets', async () => {
+    it('rejects a string that is not three-part', async () => {
       const original = await randomAppTaggingSecret(AppTaggingSecretKind.UNCONSTRAINED);
-      const parsed = AppTaggingSecret.fromString(
-        `${AppTaggingSecretKind.UNCONSTRAINED}:${original.secret.toString()}:${original.app.toString()}`,
-      );
 
-      expect(parsed.kind).toBe(AppTaggingSecretKind.UNCONSTRAINED);
-      expect(parsed.toString()).toBe(original.toString());
+      expect(() => AppTaggingSecret.fromString(`${original.secret.toString()}:${original.app.toString()}`)).toThrow(
+        /Invalid AppTaggingSecret string/,
+      );
     });
 
     it('rejects unknown kind prefixes', async () => {

yarn-project/stdlib/src/logs/app_tagging_secret.ts

@@ -89,29 +89,20 @@ export class AppTaggingSecret {
   }
 
   toString(): string {
-    // TODO(F-680): Migrate stored tagging keys and remove the legacy unconstrained format.
-    if (this.kind === AppTaggingSecretKind.UNCONSTRAINED) {
-      return `${this.secret.toString()}:${this.app.toString()}`;
-    }
     return `${this.kind}:${this.secret.toString()}:${this.app.toString()}`;
   }
 
   static fromString(str: string): AppTaggingSecret {
     const parts = str.split(':');
-    if (parts.length === 2) {
-      // TODO(F-680): Remove legacy two-part parsing after stored tagging keys are migrated.
-      const [secretStr, appStr] = parts;
-      return new AppTaggingSecret(Fr.fromString(secretStr), AztecAddress.fromStringUnsafe(appStr));
-    }
-    if (parts.length === 3) {
-      const [kindStr, secretStr, appStr] = parts;
-      return new AppTaggingSecret(
-        Fr.fromString(secretStr),
-        AztecAddress.fromStringUnsafe(appStr),
-        appTaggingSecretKindFromString(kindStr),
-      );
+    if (parts.length !== 3) {
+      throw new Error(`Invalid AppTaggingSecret string: ${str}`);
     }
-    throw new Error(`Invalid AppTaggingSecret string: ${str}`);
+    const [kindStr, secretStr, appStr] = parts;
+    return new AppTaggingSecret(
+      Fr.fromString(secretStr),
+      AztecAddress.fromStringUnsafe(appStr),
+      appTaggingSecretKindFromString(kindStr),
+    );
   }
 }