From 207d71b6066009d8d2d942c34635c01bd25f927b Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 8 Jul 2026 14:58:08 +0900 Subject: [PATCH 01/10] [ci] Simplify now that minimumInitialization packages are present Signed-off-by: Tim Emiola --- ...wardCouponV2TimeBasedIntegrationTest.scala | 78 +++---------------- 1 file changed, 9 insertions(+), 69 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala index a52052a1a9..daf9739c55 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala @@ -1,7 +1,6 @@ package org.lfdecentralizedtrust.splice.integration.tests import com.digitalasset.canton.HasExecutionContext -import com.digitalasset.canton.config.CantonRequireTypes.InstanceName import com.digitalasset.canton.config.NonNegativeFiniteDuration import com.digitalasset.canton.logging.SuppressionRule import com.digitalasset.canton.metrics.MetricValue @@ -10,7 +9,6 @@ import com.digitalasset.canton.topology.transaction.VettedPackage import com.digitalasset.canton.topology.{ForceFlag, ForceFlags, ParticipantId, PartyId} import com.digitalasset.daml.lf.data.Ref.{PackageId, PackageVersion} import java.time.Duration -import org.lfdecentralizedtrust.splice.codegen.java.splice.amulet.FeaturedAppRight import org.lfdecentralizedtrust.splice.codegen.java.splice.api.rewardassignmentv1.{ RewardBeneficiary, RewardCoupon, @@ -66,27 +64,15 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest DarResources.amulet_0_1_19.metadata.version, ) - private val minV2AmuletPackageId = - DarResources.amulet.getPackageIdWithVersion(minV2AmuletVersion.toString).value - - private val latestAmuletDar: DarResource = DarResources.amulet.latest - private val v2CapableAmuletPackageIds: Seq[String] = DarResources.amulet.all .filter(_.metadata.version >= minV2AmuletVersion) .map(_.packageId) .distinct - private val amuletVersionsAboveOldestV2: Seq[String] = - DarResources.amulet.all - .filter(_.metadata.version > minV2AmuletVersion) - .map(_.packageId) - .distinct - - // Only the latest is unvetted, as this would still cause - // ProcessRewardsTrigger to create hidden coupons - private val darsUnvettedOnAliceAtStart: Seq[DarResource] = { - val latestAmuletIds = Set(latestAmuletDar.packageId) + // DARs that need to be uploaded and vetted when re-vetting alice for v2 + private val v2CapableDarsUnvettedOnAlice: Seq[DarResource] = { + val v2CapableAmuletIds = v2CapableAmuletPackageIds.toSet Seq( DarResources.amulet, DarResources.amuletNameService, @@ -104,22 +90,6 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest override def environmentDefinition: SpliceEnvironmentDefinition = EnvironmentDefinition .simpleTopology1SvWithSimTime(this.getClass.getSimpleName) - .withNoVettedPackages(implicit env => Seq(aliceValidatorBackend.participantClient)) - .addConfigTransforms((_, config) => { - val aliceValidator = InstanceName.tryCreate("aliceValidator") - config.copy( - validatorApps = config.validatorApps + - (aliceValidator -> config - .validatorApps(aliceValidator) - .copy( - additionalPackagesToUnvet = darsUnvettedOnAliceAtStart - .groupBy(_.metadata.name) - .map { case (name, resources) => - name -> resources.map(_.metadata.version).toSet - } - )) - ) - }) .addConfigTransform((_, config) => updateAutomationConfig(ConfigurableApp.Validator)( _.withPausedTrigger[AcceptedTransferOfferTrigger] @@ -144,7 +114,6 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest "Unhide and expire of RewardCouponV2" in { implicit env => val aliceParticipantId = aliceValidatorBackend.appState.participantAdminConnection.getParticipantId().futureValue - assertAliceVettedBelowLatest(aliceParticipantId) val (aliceParty, bobParty) = onboardAliceAndBobWithFeaturedRights() @@ -178,6 +147,11 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest assertOldestOpenRound(round.toLong) } // FA right now effective from round 4 + + // Unvet v2-capable amulet packages from alice after onboarding so + // ProcessRewardsTrigger sees her as having wrong vetting state. + unvetV2AmuletOnAlice(aliceParticipantId) + doTransfer() advanceRoundsToNextRoundOpening @@ -480,22 +454,6 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest .futureValue .flatMap(_.mapping.packages.map(_.packageId)) - private def assertAliceVettedBelowLatest( - aliceParticipantId: ParticipantId - )(implicit env: SpliceTestConsoleEnvironment): Unit = - clue("Alice's validator vets the second-latest amulet version but not the latest") { - eventually() { - val vetted = vettedPackagesOnSv1View(aliceParticipantId) - vetted should contain( - DarResources.amulet.others - .filter(_.metadata.version < latestAmuletDar.metadata.version) - .maxBy(_.metadata.version) - .packageId - ) - vetted should not contain latestAmuletDar.packageId - } - } - private def onboardAliceAndBobWithFeaturedRights()(implicit env: SpliceTestConsoleEnvironment ): (PartyId, PartyId) = { @@ -504,25 +462,7 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest bobWalletClient.tap(100) grantFeaturedAppRight(bobWalletClient) - - // Alice can't self-grant while unvetted, so use dso directly - actAndCheck( - "DSO directly creates Alice's FeaturedAppRight", - sv1Backend.participantClientWithAdminToken.ledger_api_extensions.commands - .submitWithResult( - userId = sv1Backend.config.ledgerApiUser, - actAs = Seq(dsoParty), - readAs = Seq.empty, - update = new FeaturedAppRight( - dsoParty.toProtoPrimitive, - aliceParty.toProtoPrimitive, - java.util.Optional.empty(), - ).create, - ), - )( - "Alice's featured app right is visible in scan", - _ => sv1ScanBackend.lookupFeaturedAppRight(aliceParty) should not be empty, - ) + grantFeaturedAppRight(aliceWalletClient) (aliceParty, bobParty) } From 26570561a2dd20752100febaf229e1a673959ad6 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 9 Jul 2026 19:30:59 +0900 Subject: [PATCH 02/10] [ci] Allow a warning in UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest Signed-off-by: Tim Emiola --- ...dExpireRewardCouponV2TimeBasedIntegrationTest.scala | 10 +--------- .../ignore-patterns/canton_network_test_log.ignore.txt | 9 ++++++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala index daf9739c55..ca705431b8 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala @@ -27,10 +27,7 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ IntegrationTestWithIsolatedEnvironment, SpliceTestConsoleEnvironment, } -import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.{ - ExpireRewardCouponV2Trigger, - UnhideRewardCouponV2Trigger, -} +import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.ExpireRewardCouponV2Trigger import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, TimeTestUtil, @@ -95,11 +92,6 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest _.withPausedTrigger[AcceptedTransferOfferTrigger] )(config) ) - .addConfigTransform((_, config) => - updateAutomationConfig(ConfigurableApp.Sv)( - _.withPausedTrigger[UnhideRewardCouponV2Trigger] - )(config) - ) .addConfigTransform((_, config) => ConfigTransforms.updateAllSvAppConfigs_(svConfig => svConfig.copy( diff --git a/project/ignore-patterns/canton_network_test_log.ignore.txt b/project/ignore-patterns/canton_network_test_log.ignore.txt index ab044cad67..fd66e29b6a 100644 --- a/project/ignore-patterns/canton_network_test_log.ignore.txt +++ b/project/ignore-patterns/canton_network_test_log.ignore.txt @@ -173,6 +173,9 @@ api/sv/v0/onboard/validator \(POST\) resulted in a timeout # In BaseStorePerformanceTest, we run migrations in an unforked sbt JVM, so Flyway # finds the sbt test jar (apps-app_*-tests.jar). But, Flyway cannot open it, # so it skips it with this WARN. It contains no migrations, so this is harmless and unrelated to DB migrations. -# The same warning is emitted by both the FlywayExecutor and the ClassPathScanner loggers. -# a single pattern matching the message (the unloadable sbt tests jar) covers both. -Skipping unloadable jar file: file:.*-tests\.jar +Skipping unloadable jar file:.*FlywayExecutor + +# Expected in UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest: the test dynamically unvets all +# v2-capable amulet packages from alice, leaving no common vetted amulet version between alice and SV1 +# (minimumInitialization is 0.1.19, and all versions >= 0.1.19 are v2-capable). +No vetted splice-amulet version for.*UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest From c1dbf5ea67080b82c48650188704876903a96357 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 9 Jul 2026 02:44:36 +0000 Subject: [PATCH 03/10] Initial attempt at adding the new integration test Signed-off-by: Tim Emiola --- ...ZeroRoundBootstrapBftIntegrationTest.scala | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala new file mode 100644 index 0000000000..88dcb68c82 --- /dev/null +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala @@ -0,0 +1,222 @@ +package org.lfdecentralizedtrust.splice.integration.tests + +import com.digitalasset.canton.HasExecutionContext +import com.digitalasset.canton.config.NonNegativeFiniteDuration +import org.lfdecentralizedtrust.splice.codegen.java.splice.cometbft.{ + CometBftConfig, + CometBftNodeConfig, + GovernanceKeyConfig, + SequencingKeyConfig, +} +import org.lfdecentralizedtrust.splice.codegen.java.splice.dso.decentralizedsynchronizer.{ + ScanConfig, + SynchronizerNodeConfig, +} +import org.lfdecentralizedtrust.splice.store.AppStoreWithIngestion.SpliceLedgerConnectionPriority +import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules_AddSv +import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.actionrequiringconfirmation.ARC_DsoRules +import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.dsorules_actionrequiringconfirmation.SRARC_AddSv +import org.lfdecentralizedtrust.splice.codegen.java.splice.types.Round +import org.lfdecentralizedtrust.splice.config.ConfigTransforms +import org.lfdecentralizedtrust.splice.http.v0.definitions +import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition +import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ + IntegrationTestWithIsolatedEnvironment, + SpliceTestConsoleEnvironment, +} +import org.lfdecentralizedtrust.splice.sv.config.InitialRewardConfig +import org.scalatest.concurrent.PatienceConfiguration +import org.lfdecentralizedtrust.splice.util.{TimeTestUtil, WalletTestUtil} + +import java.util.Optional +import scala.jdk.CollectionConverters.* + +/** Tests that non-firstSV scans can provide reward totals for the + * initial round when bootstrapping TBAR at a non-zero round with + * BFT f >= 1. + * + * Adds a dummy 5th SV to increase the BFT quorum from 1 to 2. + * Currently fails because only SV1's scan has the initial round's + * data — will pass after implementing randomSingleCall for the + * initial round's BFT reads. + * + * Related: issue #6110 + */ +class NonZeroRoundBootstrapBftIntegrationTest + extends IntegrationTestWithIsolatedEnvironment + with HasExecutionContext + with WalletTestUtil + with TimeTestUtil { + + private val initialRound = 4815L + + override def environmentDefinition: SpliceEnvironmentDefinition = + EnvironmentDefinition + .simpleTopology4Svs(this.getClass.getSimpleName) + .withoutAutomaticRewardsCollectionAndAmuletMerging + .addConfigTransform((_, config) => + ConfigTransforms.withRewardConfig( + InitialRewardConfig( + mintingVersion = "RewardVersion_TrafficBasedAppRewards", + dryRunVersion = None, + appRewardCouponThreshold = BigDecimal("0"), + ) + )(config) + ) + .addConfigTransforms((_, config) => + ConfigTransforms.updateAllSvAppFoundDsoConfigs_( + _.copy(initialRound = initialRound) + )(config) + ) + .addConfigTransform((_, config) => ConfigTransforms.withNoVoteCooldown(config)) + // Short tick duration so rounds advance quickly in real time, + // allowing CalculateRewardsV2 contracts to be created. + .addConfigTransform((_, config) => + ConfigTransforms.updateInitialTickDuration(NonNegativeFiniteDuration.ofMillis(500))(config) + ) + + "non-firstSV scans can provide reward totals for the initial round" in { implicit env => + // Add a dummy 5th SV with a fake scan URL. This increases BFT f + // from 0 to 1, requiring 2 agreeing Ok responses. Since only + // SV1's scan has the initial round data, BFT reads from SV2/3/4 + // can't reach quorum. + addDummySvWithFakeScanUrl() + + import definitions.GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk + + import scala.concurrent.duration.DurationInt + + // SV1's scan has the data (firstSV seeding). Wait up to 60s because + // in real time the reward pipeline (verdict ingestion → activity + // completeness → reward computation) needs multiple trigger cycles. + eventually(timeUntilSuccess = 60.seconds) { + sv1ScanBackend + .getRewardAccountingActivityTotals(initialRound) shouldBe a[RewardAccountingActivityTotalsOk] + } + + // SV2's scan should also have the data once the BFT fix lets it + // read from SV1. Currently fails — will pass after implementing + // randomSingleCall for the initial round's BFT reads. + eventually(timeUntilSuccess = 60.seconds) { + sv2ScanBackend + .getRewardAccountingActivityTotals(initialRound) shouldBe a[RewardAccountingActivityTotalsOk] + } + } + + private def addDummySvWithFakeScanUrl()(implicit + env: SpliceTestConsoleEnvironment + ): com.digitalasset.canton.topology.PartyId = { + val dsoParty = sv1Backend.getDsoInfo().svParty + + // Random suffix avoids collisions with stale parties from + // previous runs (databases persist between local test runs). + val dummySvParty = sv1Backend.participantClientWithAdminToken + .ledger_api.parties + .allocate(s"dummy-sv5-${scala.util.Random.nextInt().toHexString}") + .party + + val addSvAction = new ARC_DsoRules( + new SRARC_AddSv( + new DsoRules_AddSv( + dummySvParty.toProtoPrimitive, + "Dummy-SV5", + 1000L, + "dummy-participant-id", + new Round(initialRound), + ) + ) + ) + + val (_, voteRequest) = actAndCheck( + "sv1 creates vote request to add dummy SV5", + eventuallySucceeds() { + sv1Backend.createVoteRequest( + dsoParty.toProtoPrimitive, + addSvAction, + "url", + "Add dummy SV5 for BFT threshold test", + sv1Backend.getDsoInfo().dsoRules.payload.config.voteRequestTimeout, + None, + ) + }, + )( + "vote request exists", + _ => sv1Backend.listVoteRequests().loneElement, + ) + + actAndCheck( + "sv2 and sv3 vote yes (3 votes total → executes)", { + Seq(sv2Backend, sv3Backend).foreach { sv => + eventuallySucceeds() { + sv.castVote(voteRequest.contractId, true, "url", "description") + } + } + }, + )( + "dummy SV5 is in DsoRules", + _ => { + val svs = sv1Backend.getDsoInfo().dsoRules.payload.svs + svs.asScala should contain key dummySvParty.toProtoPrimitive + }, + ) + + clue("Set fake scan URL on dummy SV") { + setDummySvScanUrl(dummySvParty) + } + + dummySvParty + } + + private def setDummySvScanUrl( + dummySvParty: com.digitalasset.canton.topology.PartyId + )(implicit env: SpliceTestConsoleEnvironment): Unit = { + val synchronizerId = decentralizedSynchronizerId.toProtoPrimitive + + val nodeConfig = new SynchronizerNodeConfig( + new CometBftConfig( + Map.empty[String, CometBftNodeConfig].asJava, + Seq.empty[GovernanceKeyConfig].asJava, + Seq.empty[SequencingKeyConfig].asJava, + ), + Optional.empty(), // sequencer + Optional.empty(), // mediator + // Unreachable URL — BFT marks it as a failed peer, increasing + // totalNumber (and thus f) without needing a running scan. + Optional.of(new ScanConfig("http://localhost:1")), + Optional.empty(), // legacySequencerConfig + Optional.empty(), // sequencerIdentity + Optional.empty(), // physicalSynchronizers + ) + + // SV1's ledger API user needs actAs rights for the dummy party to + // submit SetSynchronizerNodeConfig (controller = sv party) via + // SV1's SpliceLedgerConnection. + sv1Backend.participantClientWithAdminToken.ledger_api.users.rights + .grant(sv1Backend.config.ledgerApiUser, actAs = Set(dummySvParty)) + + // Use SV1's SpliceLedgerConnection (not raw submitJava) because it + // has built-in retry for CONTRACT_NOT_FOUND caused by concurrent + // DsoRules churn from SV automation. + // See ValidatorSequencerConnectionIntegrationTest for the same pattern. + + val dsoStore = sv1Backend.appState.dsoStore + val connection = + sv1Backend.appState.svAutomation.connection(SpliceLedgerConnectionPriority.Low) + import scala.concurrent.duration.DurationInt + (for { + rulesAndState <- dsoStore.getDsoRulesWithSvNodeState(dummySvParty) + cmd = rulesAndState.dsoRules.exercise( + _.exerciseDsoRules_SetSynchronizerNodeConfig( + dummySvParty.toProtoPrimitive, + synchronizerId, + nodeConfig, + rulesAndState.svNodeState.contractId, + ) + ) + _ <- connection + .submit(Seq(dummySvParty), Seq(dsoParty), cmd) + .noDedup + .yieldResult() + } yield ()).futureValue(timeout = PatienceConfiguration.Timeout(60.seconds)) + } +} From 346058901292b0c3c8ee1a62f36f129232f5194e Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 9 Jul 2026 14:10:53 +0900 Subject: [PATCH 04/10] Add BFT integration test for non-zero round bootstrap reward totals Signed-off-by: Tim Emiola --- ...ZeroRoundBootstrapBftIntegrationTest.scala | 124 +++++++++--------- test-full-class-names.log | 1 + 2 files changed, 65 insertions(+), 60 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala index 88dcb68c82..b9abb1bab7 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala @@ -1,7 +1,7 @@ package org.lfdecentralizedtrust.splice.integration.tests import com.digitalasset.canton.HasExecutionContext -import com.digitalasset.canton.config.NonNegativeFiniteDuration +import com.digitalasset.canton.concurrent.Threading import org.lfdecentralizedtrust.splice.codegen.java.splice.cometbft.{ CometBftConfig, CometBftNodeConfig, @@ -12,7 +12,6 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.dso.decentralizedsync ScanConfig, SynchronizerNodeConfig, } -import org.lfdecentralizedtrust.splice.store.AppStoreWithIngestion.SpliceLedgerConnectionPriority import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules_AddSv import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.actionrequiringconfirmation.ARC_DsoRules import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.dsorules_actionrequiringconfirmation.SRARC_AddSv @@ -24,8 +23,8 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ IntegrationTestWithIsolatedEnvironment, SpliceTestConsoleEnvironment, } +import org.lfdecentralizedtrust.splice.automation.Trigger import org.lfdecentralizedtrust.splice.sv.config.InitialRewardConfig -import org.scalatest.concurrent.PatienceConfiguration import org.lfdecentralizedtrust.splice.util.{TimeTestUtil, WalletTestUtil} import java.util.Optional @@ -52,7 +51,7 @@ class NonZeroRoundBootstrapBftIntegrationTest override def environmentDefinition: SpliceEnvironmentDefinition = EnvironmentDefinition - .simpleTopology4Svs(this.getClass.getSimpleName) + .simpleTopology4SvsWithSimTime(this.getClass.getSimpleName) .withoutAutomaticRewardsCollectionAndAmuletMerging .addConfigTransform((_, config) => ConfigTransforms.withRewardConfig( @@ -69,49 +68,51 @@ class NonZeroRoundBootstrapBftIntegrationTest )(config) ) .addConfigTransform((_, config) => ConfigTransforms.withNoVoteCooldown(config)) - // Short tick duration so rounds advance quickly in real time, - // allowing CalculateRewardsV2 contracts to be created. - .addConfigTransform((_, config) => - ConfigTransforms.updateInitialTickDuration(NonNegativeFiniteDuration.ofMillis(500))(config) - ) "non-firstSV scans can provide reward totals for the initial round" in { implicit env => - // Add a dummy 5th SV with a fake scan URL. This increases BFT f - // from 0 to 1, requiring 2 agreeing Ok responses. Since only - // SV1's scan has the initial round data, BFT reads from SV2/3/4 - // can't reach quorum. - addDummySvWithFakeScanUrl() - import definitions.GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk - import scala.concurrent.duration.DurationInt + // Advance rounds so the reward pipeline runs for the initial round: + // AdvanceOpenMiningRoundTrigger archives oldest round → creates + // CalculateRewardsV2 → RewardComputationTrigger computes totals. + advanceTimeForRewardAutomationToRunForCurrentRound + actAndCheck( + "Advance to next round opening", + advanceRoundsToNextRoundOpening, + )( + "SV1's scan has reward totals for the initial round", + _ => + sv1ScanBackend + .getRewardAccountingActivityTotals(initialRound) shouldBe a[ + RewardAccountingActivityTotalsOk + ], + ) - // SV1's scan has the data (firstSV seeding). Wait up to 60s because - // in real time the reward pipeline (verdict ingestion → activity - // completeness → reward computation) needs multiple trigger cycles. - eventually(timeUntilSuccess = 60.seconds) { - sv1ScanBackend - .getRewardAccountingActivityTotals(initialRound) shouldBe a[RewardAccountingActivityTotalsOk] - } + // Add a dummy 5th SV with a fake scan URL. This increases BFT f + // from 0 to 1, requiring 2 agreeing Ok responses. + addDummySvWithFakeScanUrl() // SV2's scan should also have the data once the BFT fix lets it // read from SV1. Currently fails — will pass after implementing // randomSingleCall for the initial round's BFT reads. - eventually(timeUntilSuccess = 60.seconds) { + eventually() { sv2ScanBackend - .getRewardAccountingActivityTotals(initialRound) shouldBe a[RewardAccountingActivityTotalsOk] + .getRewardAccountingActivityTotals(initialRound) shouldBe a[ + RewardAccountingActivityTotalsOk + ] } } private def addDummySvWithFakeScanUrl()(implicit env: SpliceTestConsoleEnvironment ): com.digitalasset.canton.topology.PartyId = { - val dsoParty = sv1Backend.getDsoInfo().svParty + val dsoInfo = sv1Backend.getDsoInfo() + val svParty = dsoInfo.svParty + val dsoParty = dsoInfo.dsoParty // Random suffix avoids collisions with stale parties from // previous runs (databases persist between local test runs). - val dummySvParty = sv1Backend.participantClientWithAdminToken - .ledger_api.parties + val dummySvParty = sv1Backend.participantClientWithAdminToken.ledger_api.parties .allocate(s"dummy-sv5-${scala.util.Random.nextInt().toHexString}") .party @@ -131,7 +132,7 @@ class NonZeroRoundBootstrapBftIntegrationTest "sv1 creates vote request to add dummy SV5", eventuallySucceeds() { sv1Backend.createVoteRequest( - dsoParty.toProtoPrimitive, + svParty.toProtoPrimitive, addSvAction, "url", "Add dummy SV5 for BFT threshold test", @@ -160,15 +161,26 @@ class NonZeroRoundBootstrapBftIntegrationTest }, ) + // Pause ALL delegate-based triggers to prevent DsoRules contract + // churn during the SetSynchronizerNodeConfig submission. + // Sleep briefly after pausing to let in-flight commands complete. + env.svs.local.foreach( + _.dsoDelegateBasedAutomation.triggers[Trigger].foreach(_.pause().futureValue) + ) + Threading.sleep(2000) clue("Set fake scan URL on dummy SV") { - setDummySvScanUrl(dummySvParty) + setDummySvScanUrl(dsoParty, dummySvParty) } + env.svs.local.foreach( + _.dsoDelegateBasedAutomation.triggers[Trigger].foreach(_.resume()) + ) dummySvParty } private def setDummySvScanUrl( - dummySvParty: com.digitalasset.canton.topology.PartyId + dsoParty: com.digitalasset.canton.topology.PartyId, + dummySvParty: com.digitalasset.canton.topology.PartyId, )(implicit env: SpliceTestConsoleEnvironment): Unit = { val synchronizerId = decentralizedSynchronizerId.toProtoPrimitive @@ -188,35 +200,27 @@ class NonZeroRoundBootstrapBftIntegrationTest Optional.empty(), // physicalSynchronizers ) - // SV1's ledger API user needs actAs rights for the dummy party to - // submit SetSynchronizerNodeConfig (controller = sv party) via - // SV1's SpliceLedgerConnection. - sv1Backend.participantClientWithAdminToken.ledger_api.users.rights - .grant(sv1Backend.config.ledgerApiUser, actAs = Set(dummySvParty)) - - // Use SV1's SpliceLedgerConnection (not raw submitJava) because it - // has built-in retry for CONTRACT_NOT_FOUND caused by concurrent - // DsoRules churn from SV automation. - // See ValidatorSequencerConnectionIntegrationTest for the same pattern. - - val dsoStore = sv1Backend.appState.dsoStore - val connection = - sv1Backend.appState.svAutomation.connection(SpliceLedgerConnectionPriority.Low) - import scala.concurrent.duration.DurationInt - (for { - rulesAndState <- dsoStore.getDsoRulesWithSvNodeState(dummySvParty) - cmd = rulesAndState.dsoRules.exercise( - _.exerciseDsoRules_SetSynchronizerNodeConfig( - dummySvParty.toProtoPrimitive, - synchronizerId, - nodeConfig, - rulesAndState.svNodeState.contractId, + eventuallySucceeds() { + val info = sv1Backend.getDsoInfo() + val currentDsoRulesCid = info.dsoRules.contractId + val currentNodeStateCid = info.svNodeStates + .getOrElse(dummySvParty, fail("SvNodeState not found for dummy SV")) + .contractId + sv1Backend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitJava( + actAs = Seq(dummySvParty), + readAs = Seq(dsoParty), + commands = currentDsoRulesCid + .exerciseDsoRules_SetSynchronizerNodeConfig( + dummySvParty.toProtoPrimitive, + synchronizerId, + nodeConfig, + currentNodeStateCid, + ) + .commands() + .asScala + .toSeq, ) - ) - _ <- connection - .submit(Seq(dummySvParty), Seq(dsoParty), cmd) - .noDedup - .yieldResult() - } yield ()).futureValue(timeout = PatienceConfiguration.Timeout(60.seconds)) + } } } diff --git a/test-full-class-names.log b/test-full-class-names.log index 46807bd46b..9d168b9a0c 100644 --- a/test-full-class-names.log +++ b/test-full-class-names.log @@ -26,6 +26,7 @@ org.lfdecentralizedtrust.splice.integration.tests.FeaturedAppActivityMarkerInteg org.lfdecentralizedtrust.splice.integration.tests.GcpBucketPeriodicBackupIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.MemberTrafficIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.MultiHostValidatorOperatorIntegrationTest +org.lfdecentralizedtrust.splice.integration.tests.NonZeroRoundBootstrapBftIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.PackageWithDependencyIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ParticipantKmsIdentitiesIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ParticipantPlaintextIdentitiesIntegrationTest From 288d94d9e85f2225733079361c7517153b97a3ee Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 9 Jul 2026 15:22:38 +0900 Subject: [PATCH 05/10] Use randomSingleCall for BFT reads of the initial round's reward values Signed-off-by: Tim Emiola --- .../admin/api/client/BftScanConnection.scala | 15 +++- .../api/client/BftScanConnectionTest.scala | 85 +++++++++++++++++++ .../automation/SvDsoAutomationService.scala | 38 +++++---- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala index 3c9c689cda..bd060672b5 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala @@ -139,6 +139,7 @@ class BftScanConnection( // In the future, before removing disableBackgroundRefresh flag, refactor the bootstrapWithSeedNodes(); // BftCustom should not reuse the AllDsoScansBft class to fetch the initial DsoRules. val disableBackgroundRefresh: Boolean = false, + val initialRound: Option[Long] = None, )(implicit protected val ec: ExecutionContextExecutor, protected val mat: Materializer) extends FlagCloseableAsync with NamedLogging @@ -1015,7 +1016,11 @@ class BftScanConnection( GetRewardAccountingActivityTotalsResponse( RewardAccountingActivityTotalsUndetermined(status = "Undetermined") ) - val callConfig = BftCallConfig.default(scanList.scanConnections) + val callConfig = + if (initialRound.contains(roundNumber)) + BftCallConfig.randomSingleCall(scanList.scanConnections) + else + BftCallConfig.default(scanList.scanConnections) if (!callConfig.enoughAvailableScans) Future.successful(undetermined) else bftCall[RewardAccountingActivityTotalsOk]( @@ -1063,7 +1068,11 @@ class BftScanConnection( GetRewardAccountingRootHashResponse( RewardAccountingRootHashUndetermined(status = "Undetermined") ) - val callConfig = BftCallConfig.default(scanList.scanConnections) + val callConfig = + if (initialRound.contains(roundNumber)) + BftCallConfig.randomSingleCall(scanList.scanConnections) + else + BftCallConfig.default(scanList.scanConnections) if (!callConfig.enoughAvailableScans) Future.successful(undetermined) else bftCall[String]( @@ -2003,6 +2012,7 @@ object BftScanConnection { clock: Clock, retryProvider: RetryProvider, loggerFactory: NamedLoggerFactory, + initialRound: Option[Long] = None, )(implicit ec: ExecutionContextExecutor, tc: TraceContext, @@ -2054,6 +2064,7 @@ object BftScanConnection { clock, retryProvider, loggerFactory, + initialRound = initialRound, ) } diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala index 889fee4c45..eb50e6873d 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala @@ -225,6 +225,7 @@ class BftScanConnectionTest connectionBuilder: Uri => Future[SingleScanConnection] = _ => Future.failed(new RuntimeException("Shouldn't be refreshing!")), initialFailedConnections: Map[Uri, Throwable] = Map.empty, + initialRound: Option[Long] = None, ) = { new BftScanConnection( mock[SpliceLedgerClient], @@ -242,6 +243,7 @@ class BftScanConnectionTest clock, retryProvider, loggerFactory, + initialRound = initialRound, ) } val notFoundFailure = new BaseAppConnection.UnexpectedHttpJsonResponse( @@ -1346,6 +1348,47 @@ class BftScanConnectionTest ) .map(_ => succeed) } + + "uses randomSingleCall for the initial round" in { + val round = 42L + val connections = getMockedConnections(n = 4) + makeMockReturnRootHashOk(connections(0), round, "aabb") + makeMockReturnRootHashCannotProvide(connections(1), round) + makeMockReturnRootHashCannotProvide(connections(2), round) + makeMockReturnRootHashCannotProvide(connections(3), round) + val bft = getBft(connections, initialRound = Some(round)) + + def attempt(remaining: Int): Future[GetRewardAccountingRootHashResponse] = + bft.getRewardAccountingRootHash(round).flatMap { + case ok: GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk => + Future.successful(ok) + case _ if remaining > 1 => attempt(remaining - 1) + case other => Future.successful(other) + } + + for { + resp <- attempt(100) + } yield inside(resp) { + case GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(ok) => + ok.rootHash should be("aabb") + } + } + + "uses default BFT for non-initial rounds even when initialRound is set" in { + val round = 43L + val connections = getMockedConnections(n = 4) + connections.zipWithIndex.foreach { case (c, i) => + makeMockReturnRootHashOk(c, round, s"hash$i") + } + val bft = getBft(connections, initialRound = Some(42L)) + + for { + resp <- bft.getRewardAccountingRootHash(round) + } yield inside(resp) { + case _: GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashUndetermined => + succeed + } + } } "BftScanConnection.getRewardAccountingActivityTotals" should { @@ -1470,6 +1513,48 @@ class BftScanConnectionTest ) .map(_ => succeed) } + + "uses randomSingleCall for the initial round" in { + val round = 42L + val connections = getMockedConnections(n = 4) + makeMockReturnActivityTotalsOk(connections(0), round, 100L, 10L, 5L) + makeMockReturnActivityTotalsCannotProvide(connections(1), round) + makeMockReturnActivityTotalsCannotProvide(connections(2), round) + makeMockReturnActivityTotalsCannotProvide(connections(3), round) + val bft = getBft(connections, initialRound = Some(round)) + + def attempt(remaining: Int): Future[GetRewardAccountingActivityTotalsResponse] = + bft.getRewardAccountingActivityTotals(round).flatMap { + case ok: GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk => + Future.successful(ok) + case _ if remaining > 1 => attempt(remaining - 1) + case other => Future.successful(other) + } + + for { + resp <- attempt(100) + } yield inside(resp) { + case GetRewardAccountingActivityTotalsResponse.members + .RewardAccountingActivityTotalsOk(ok) => + ok.totalAppActivityWeight should be(100L) + } + } + + "uses default BFT for non-initial rounds even when initialRound is set" in { + val round = 43L + val connections = getMockedConnections(n = 4) + connections.zipWithIndex.foreach { case (c, i) => + makeMockReturnActivityTotalsOk(c, round, 100L + i, 10L + i, 5L + i) + } + val bft = getBft(connections, initialRound = Some(42L)) + + for { + resp <- bft.getRewardAccountingActivityTotals(round) + } yield inside(resp) { + case _: GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsUndetermined => + succeed + } + } } "BftScanConnection.getRewardAccountingBatch" should { diff --git a/apps/sv/src/main/scala/org/lfdecentralizedtrust/splice/sv/automation/SvDsoAutomationService.scala b/apps/sv/src/main/scala/org/lfdecentralizedtrust/splice/sv/automation/SvDsoAutomationService.scala index b67dafbebb..9a532dff10 100644 --- a/apps/sv/src/main/scala/org/lfdecentralizedtrust/splice/sv/automation/SvDsoAutomationService.scala +++ b/apps/sv/src/main/scala/org/lfdecentralizedtrust/splice/sv/automation/SvDsoAutomationService.scala @@ -188,21 +188,29 @@ class SvDsoAutomationService( case Some(future) => future case None => - val future = BftScanConnection - .peerScanConnection( - () => - BftScanConnection.Bft.getPeerScansFromDsoRules( - dsoStore, - dsoStore.key.svParty, - )(tc, ec), - ledgerClient, - ScanAppClientConfig.DefaultScansRefreshInterval, - ScanAppClientConfig.DefaultAmuletRulesCacheTimeToLive, - upgradesConfig, - clock, - retryProvider, - loggerFactory, - )(ec, tc, mat, httpClient, templateJsonDecoder) + val future = for { + initialRound <- connection(SpliceLedgerConnectionPriority.Low) + .lookupUserMetadata( + config.ledgerApiUser, + BaseLedgerConnection.INITIAL_ROUND_USER_METADATA_KEY, + ) + bft <- BftScanConnection + .peerScanConnection( + () => + BftScanConnection.Bft.getPeerScansFromDsoRules( + dsoStore, + dsoStore.key.svParty, + )(tc, ec), + ledgerClient, + ScanAppClientConfig.DefaultScansRefreshInterval, + ScanAppClientConfig.DefaultAmuletRulesCacheTimeToLive, + upgradesConfig, + clock, + retryProvider, + loggerFactory, + initialRound = initialRound.map(_.toLong), + )(ec, tc, mat, httpClient, templateJsonDecoder) + } yield bft peerScanConnectionF = Some(future) future } From 013364b5a44c6e3d6ff35f27d2a1606fb1781684 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 9 Jul 2026 16:24:11 +0900 Subject: [PATCH 06/10] [ci] Assert dummy SV raises BFT quorum in bootstrap test Signed-off-by: Tim Emiola --- ...ZeroRoundBootstrapBftIntegrationTest.scala | 66 +++++++++++-------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala index b9abb1bab7..71a7afdf7b 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala @@ -30,16 +30,11 @@ import org.lfdecentralizedtrust.splice.util.{TimeTestUtil, WalletTestUtil} import java.util.Optional import scala.jdk.CollectionConverters.* -/** Tests that non-firstSV scans can provide reward totals for the - * initial round when bootstrapping TBAR at a non-zero round with - * BFT f >= 1. +/** Tests that SV automation triggers can complete the reward + * pipeline for the initial round when bootstrapping TBAR at a + * non-zero round with BFT f >= 1. * * Adds a dummy 5th SV to increase the BFT quorum from 1 to 2. - * Currently fails because only SV1's scan has the initial round's - * data — will pass after implementing randomSingleCall for the - * initial round's BFT reads. - * - * Related: issue #6110 */ class NonZeroRoundBootstrapBftIntegrationTest extends IntegrationTestWithIsolatedEnvironment @@ -69,12 +64,12 @@ class NonZeroRoundBootstrapBftIntegrationTest ) .addConfigTransform((_, config) => ConfigTransforms.withNoVoteCooldown(config)) - "non-firstSV scans can provide reward totals for the initial round" in { implicit env => + "SV triggers complete the reward pipeline for the initial round" in { implicit env => import definitions.GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk // Advance rounds so the reward pipeline runs for the initial round: - // AdvanceOpenMiningRoundTrigger archives oldest round → creates - // CalculateRewardsV2 → RewardComputationTrigger computes totals. + // SV1's scan computes reward totals from seeded activity data, + // and SVs confirm CalculateRewardsV2 using the root hash. advanceTimeForRewardAutomationToRunForCurrentRound actAndCheck( "Advance to next round opening", @@ -92,15 +87,26 @@ class NonZeroRoundBootstrapBftIntegrationTest // from 0 to 1, requiring 2 agreeing Ok responses. addDummySvWithFakeScanUrl() - // SV2's scan should also have the data once the BFT fix lets it - // read from SV1. Currently fails — will pass after implementing - // randomSingleCall for the initial round's BFT reads. - eventually() { - sv2ScanBackend - .getRewardAccountingActivityTotals(initialRound) shouldBe a[ - RewardAccountingActivityTotalsOk - ] - } + // With BFT f=1, default BFT would need 2 agreeing Ok responses + // for root hash and activity totals — but only SV1's scan has + // the initial round's data. The randomSingleCall override for + // the initial round lets each SV read from a single peer, + // so the pipeline completes despite the higher quorum. + // Advancing another round proves this: if the initial round's + // pipeline had stalled, no further rounds could open. SV2's + // scan has local data for the next round and can serve totals. + advanceTimeForRewardAutomationToRunForCurrentRound + actAndCheck( + "Advance past the initial round", + advanceRoundsToNextRoundOpening, + )( + "SV2's scan has reward totals for the next round", + _ => + sv2ScanBackend + .getRewardAccountingActivityTotals(initialRound + 1) shouldBe a[ + RewardAccountingActivityTotalsOk + ], + ) } private def addDummySvWithFakeScanUrl()(implicit @@ -154,10 +160,10 @@ class NonZeroRoundBootstrapBftIntegrationTest } }, )( - "dummy SV5 is in DsoRules", + "dummy SV5 is in DsoRules and raises BFT f from 0 to 1", _ => { - val svs = sv1Backend.getDsoInfo().dsoRules.payload.svs - svs.asScala should contain key dummySvParty.toProtoPrimitive + val info = sv1Backend.getDsoInfo() + info.dsoRules.payload.svs.size() shouldBe 5 }, ) @@ -168,9 +174,17 @@ class NonZeroRoundBootstrapBftIntegrationTest _.dsoDelegateBasedAutomation.triggers[Trigger].foreach(_.pause().futureValue) ) Threading.sleep(2000) - clue("Set fake scan URL on dummy SV") { - setDummySvScanUrl(dsoParty, dummySvParty) - } + actAndCheck( + "Set fake scan URL on dummy SV", + setDummySvScanUrl(dsoParty, dummySvParty), + )( + "Dummy SV's scan URL is in the BFT peer list", + _ => { + val nodeState = sv1Backend.getDsoInfo().svNodeStates(dummySvParty) + nodeState.payload.state.synchronizerNodes.values.asScala + .exists(_.scan.isPresent) shouldBe true + }, + ) env.svs.local.foreach( _.dsoDelegateBasedAutomation.triggers[Trigger].foreach(_.resume()) ) From fc1254bc03dc4b18d7c7d447a376679ebe176b48 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 15 Jul 2026 11:08:53 +0900 Subject: [PATCH 07/10] [ci] Reorder BFT bootstrap test to run initial round under f=1 quorum Signed-off-by: Tim Emiola --- ...ZeroRoundBootstrapBftIntegrationTest.scala | 98 +++++++++++++------ 1 file changed, 68 insertions(+), 30 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala index 71a7afdf7b..4f32c19aed 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala @@ -24,7 +24,7 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ SpliceTestConsoleEnvironment, } import org.lfdecentralizedtrust.splice.automation.Trigger -import org.lfdecentralizedtrust.splice.sv.config.InitialRewardConfig +import org.lfdecentralizedtrust.splice.sv.automation.singlesv.onboarding.SvOnboardingUnlimitedTrafficTrigger import org.lfdecentralizedtrust.splice.util.{TimeTestUtil, WalletTestUtil} import java.util.Optional @@ -48,15 +48,6 @@ class NonZeroRoundBootstrapBftIntegrationTest EnvironmentDefinition .simpleTopology4SvsWithSimTime(this.getClass.getSimpleName) .withoutAutomaticRewardsCollectionAndAmuletMerging - .addConfigTransform((_, config) => - ConfigTransforms.withRewardConfig( - InitialRewardConfig( - mintingVersion = "RewardVersion_TrafficBasedAppRewards", - dryRunVersion = None, - appRewardCouponThreshold = BigDecimal("0"), - ) - )(config) - ) .addConfigTransforms((_, config) => ConfigTransforms.updateAllSvAppFoundDsoConfigs_( _.copy(initialRound = initialRound) @@ -66,41 +57,88 @@ class NonZeroRoundBootstrapBftIntegrationTest "SV triggers complete the reward pipeline for the initial round" in { implicit env => import definitions.GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk + import definitions.GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsCannotProvide + + // Add a dummy 5th SV with a fake scan URL BEFORE any round + // advancement. This raises BFT f from 0 to 1, so the initial + // round's reward pipeline runs under the higher quorum. + addDummySvWithFakeScanUrl() + + // The dummy SV's participant doesn't exist on the sequencer, so + // this trigger would poll indefinitely logging "does not yet have + // a traffic state". Pause it to avoid log noise on CI. + env.svs.local.foreach( + _.dsoAutomation.trigger[SvOnboardingUnlimitedTrafficTrigger].pause().futureValue + ) - // Advance rounds so the reward pipeline runs for the initial round: - // SV1's scan computes reward totals from seeded activity data, - // and SVs confirm CalculateRewardsV2 using the root hash. + // With f=1, BFT reads need 2 agreeing Ok responses — but only + // SV1's scan has the initial round's data. The randomSingleCall + // override for the initial round lets each SV read from a single + // peer, so the pipeline completes despite the higher quorum. advanceTimeForRewardAutomationToRunForCurrentRound actAndCheck( "Advance to next round opening", advanceRoundsToNextRoundOpening, )( - "SV1's scan has reward totals for the initial round", - _ => + "SV1's scan has reward totals and the initial round is issuing", + _ => { sv1ScanBackend .getRewardAccountingActivityTotals(initialRound) shouldBe a[ RewardAccountingActivityTotalsOk - ], + ] + + // The round becoming IssuingMiningRound is DSO-level proof: + // under BFT f=1, the SummarizingMiningRoundTrigger on each + // SV must obtain reward accounting totals. For the initial + // round, only SV1 has local data; other SVs fall back to + // BFT read (randomSingleCall). If that path failed, fewer + // than f+1=2 SVs could submit summaries and the round would + // not advance. + val (_, issuingRounds) = sv1ScanBackend.getOpenAndIssuingMiningRounds() + issuingRounds.exists( + _.payload.round.number == initialRound + ) shouldBe true + + // SV2's scan does NOT have local reward activity data for + // the initial round — only SV1's scan seeded it. SV2's SV + // trigger obtained the totals via BFT (randomSingleCall), + // but the scan HTTP endpoint queries the local store, so it + // returns CannotProvide. + sv2ScanBackend + .getRewardAccountingActivityTotals(initialRound) shouldBe a[ + RewardAccountingActivityTotalsCannotProvide + ] + }, ) - // Add a dummy 5th SV with a fake scan URL. This increases BFT f - // from 0 to 1, requiring 2 agreeing Ok responses. - addDummySvWithFakeScanUrl() + // Advance two ticks so that verdict ingestion processes batches + // that see OpenMiningRound(initialRound+1) already archived. + // A single tick archives the round, but the verdict batch for + // that tick may be processed before the rewards reference store + // has indexed the archival — so lookupLatestArchivedOpenMiningRound + // returns None and last_archived_round stays at initialRound. + // The second tick generates new verdicts that find the archival + // already indexed, bumping last_archived_round. + advanceTimeAndWaitForRoundOpening + advanceTimeAndWaitForRoundOpening - // With BFT f=1, default BFT would need 2 agreeing Ok responses - // for root hash and activity totals — but only SV1's scan has - // the initial round's data. The randomSingleCall override for - // the initial round lets each SV read from a single peer, - // so the pipeline completes despite the higher quorum. - // Advancing another round proves this: if the initial round's - // pipeline had stalled, no further rounds could open. SV2's - // scan has local data for the next round and can serve totals. - advanceTimeForRewardAutomationToRunForCurrentRound + // Wait for RewardComputationTrigger to compute totals for the + // next round now that last_archived_round covers it. + eventually() { + sv1ScanBackend + .getRewardAccountingActivityTotals(initialRound + 1) shouldBe a[ + RewardAccountingActivityTotalsOk + ] + } + + // Advance another round to confirm the pipeline continues past + // the initial round under normal BFT (f+1=2) without + // randomSingleCall. actAndCheck( "Advance past the initial round", advanceRoundsToNextRoundOpening, )( - "SV2's scan has reward totals for the next round", + "SV2's scan has local reward totals for the next round", _ => sv2ScanBackend .getRewardAccountingActivityTotals(initialRound + 1) shouldBe a[ @@ -128,7 +166,7 @@ class NonZeroRoundBootstrapBftIntegrationTest dummySvParty.toProtoPrimitive, "Dummy-SV5", 1000L, - "dummy-participant-id", + "PAR::dummy-sv5::dummy", new Round(initialRound), ) ) From ff2e89f9d7c6eeec8d13d14197669dbf6ca6f031 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 15 Jul 2026 18:34:40 +0900 Subject: [PATCH 08/10] [ci] Properly name the test as a SimTime test Signed-off-by: Tim Emiola --- ...a => NonZeroRoundBootstrapBftTimeBasedIntegrationTest.scala} | 2 +- test-full-class-names-sim-time.log | 1 + test-full-class-names.log | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) rename apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/{NonZeroRoundBootstrapBftIntegrationTest.scala => NonZeroRoundBootstrapBftTimeBasedIntegrationTest.scala} (99%) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftTimeBasedIntegrationTest.scala similarity index 99% rename from apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala rename to apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftTimeBasedIntegrationTest.scala index 4f32c19aed..4e97bfd53f 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/NonZeroRoundBootstrapBftTimeBasedIntegrationTest.scala @@ -36,7 +36,7 @@ import scala.jdk.CollectionConverters.* * * Adds a dummy 5th SV to increase the BFT quorum from 1 to 2. */ -class NonZeroRoundBootstrapBftIntegrationTest +class NonZeroRoundBootstrapBftTimeBasedIntegrationTest extends IntegrationTestWithIsolatedEnvironment with HasExecutionContext with WalletTestUtil diff --git a/test-full-class-names-sim-time.log b/test-full-class-names-sim-time.log index 6007a6c429..3432f76200 100644 --- a/test-full-class-names-sim-time.log +++ b/test-full-class-names-sim-time.log @@ -1,6 +1,7 @@ org.lfdecentralizedtrust.splice.integration.tests.DisabledWalletTimeBasedIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ExternallySignedTxsTimeBasedIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.FollowAmuletConversionRateFeedTimeBasedIntegrationTest +org.lfdecentralizedtrust.splice.integration.tests.NonZeroRoundBootstrapBftTimeBasedIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ScanWithGradualStartsTimeBasedIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.SvExpiredRewardsCollectionTimeBasedIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.SvTimeBasedAmuletPriceIntegrationTest diff --git a/test-full-class-names.log b/test-full-class-names.log index 9d168b9a0c..46807bd46b 100644 --- a/test-full-class-names.log +++ b/test-full-class-names.log @@ -26,7 +26,6 @@ org.lfdecentralizedtrust.splice.integration.tests.FeaturedAppActivityMarkerInteg org.lfdecentralizedtrust.splice.integration.tests.GcpBucketPeriodicBackupIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.MemberTrafficIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.MultiHostValidatorOperatorIntegrationTest -org.lfdecentralizedtrust.splice.integration.tests.NonZeroRoundBootstrapBftIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.PackageWithDependencyIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ParticipantKmsIdentitiesIntegrationTest org.lfdecentralizedtrust.splice.integration.tests.ParticipantPlaintextIdentitiesIntegrationTest From 6921aa83b67aaa9ef9e7f3508b4ef6019fc4fc1a Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 16 Jul 2026 15:36:16 +0900 Subject: [PATCH 09/10] [ci] Add ignore pattern for dummy SV scan warnings Signed-off-by: Tim Emiola --- project/ignore-patterns/canton_network_test_log.ignore.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/project/ignore-patterns/canton_network_test_log.ignore.txt b/project/ignore-patterns/canton_network_test_log.ignore.txt index fd66e29b6a..cc23710368 100644 --- a/project/ignore-patterns/canton_network_test_log.ignore.txt +++ b/project/ignore-patterns/canton_network_test_log.ignore.txt @@ -175,7 +175,6 @@ api/sv/v0/onboard/validator \(POST\) resulted in a timeout # so it skips it with this WARN. It contains no migrations, so this is harmless and unrelated to DB migrations. Skipping unloadable jar file:.*FlywayExecutor -# Expected in UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest: the test dynamically unvets all -# v2-capable amulet packages from alice, leaving no common vetted amulet version between alice and SV1 -# (minimumInitialization is 0.1.19, and all versions >= 0.1.19 are v2-capable). -No vetted splice-amulet version for.*UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest +# The BFT bootstrap test adds a dummy 5th SV with an unreachable scan URL (http://localhost:1) +# to raise f from 0 to 1. All BFT peers attempt to connect to this URL and log warnings. +Failed to connect to scan of.*NonZeroRoundBootstrapBftTimeBasedIntegrationTest From d88545f0a3a660a4a94d01e9ef66e57688dc086a Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 16 Jul 2026 16:03:11 +0900 Subject: [PATCH 10/10] [ci] Use the updated version of UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest Signed-off-by: Tim Emiola --- ...wardCouponV2TimeBasedIntegrationTest.scala | 88 ++++++++++++++++--- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala index ca705431b8..a52052a1a9 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest.scala @@ -1,6 +1,7 @@ package org.lfdecentralizedtrust.splice.integration.tests import com.digitalasset.canton.HasExecutionContext +import com.digitalasset.canton.config.CantonRequireTypes.InstanceName import com.digitalasset.canton.config.NonNegativeFiniteDuration import com.digitalasset.canton.logging.SuppressionRule import com.digitalasset.canton.metrics.MetricValue @@ -9,6 +10,7 @@ import com.digitalasset.canton.topology.transaction.VettedPackage import com.digitalasset.canton.topology.{ForceFlag, ForceFlags, ParticipantId, PartyId} import com.digitalasset.daml.lf.data.Ref.{PackageId, PackageVersion} import java.time.Duration +import org.lfdecentralizedtrust.splice.codegen.java.splice.amulet.FeaturedAppRight import org.lfdecentralizedtrust.splice.codegen.java.splice.api.rewardassignmentv1.{ RewardBeneficiary, RewardCoupon, @@ -27,7 +29,10 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ IntegrationTestWithIsolatedEnvironment, SpliceTestConsoleEnvironment, } -import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.ExpireRewardCouponV2Trigger +import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.{ + ExpireRewardCouponV2Trigger, + UnhideRewardCouponV2Trigger, +} import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, TimeTestUtil, @@ -61,15 +66,27 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest DarResources.amulet_0_1_19.metadata.version, ) + private val minV2AmuletPackageId = + DarResources.amulet.getPackageIdWithVersion(minV2AmuletVersion.toString).value + + private val latestAmuletDar: DarResource = DarResources.amulet.latest + private val v2CapableAmuletPackageIds: Seq[String] = DarResources.amulet.all .filter(_.metadata.version >= minV2AmuletVersion) .map(_.packageId) .distinct - // DARs that need to be uploaded and vetted when re-vetting alice for v2 - private val v2CapableDarsUnvettedOnAlice: Seq[DarResource] = { - val v2CapableAmuletIds = v2CapableAmuletPackageIds.toSet + private val amuletVersionsAboveOldestV2: Seq[String] = + DarResources.amulet.all + .filter(_.metadata.version > minV2AmuletVersion) + .map(_.packageId) + .distinct + + // Only the latest is unvetted, as this would still cause + // ProcessRewardsTrigger to create hidden coupons + private val darsUnvettedOnAliceAtStart: Seq[DarResource] = { + val latestAmuletIds = Set(latestAmuletDar.packageId) Seq( DarResources.amulet, DarResources.amuletNameService, @@ -87,11 +104,32 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest override def environmentDefinition: SpliceEnvironmentDefinition = EnvironmentDefinition .simpleTopology1SvWithSimTime(this.getClass.getSimpleName) + .withNoVettedPackages(implicit env => Seq(aliceValidatorBackend.participantClient)) + .addConfigTransforms((_, config) => { + val aliceValidator = InstanceName.tryCreate("aliceValidator") + config.copy( + validatorApps = config.validatorApps + + (aliceValidator -> config + .validatorApps(aliceValidator) + .copy( + additionalPackagesToUnvet = darsUnvettedOnAliceAtStart + .groupBy(_.metadata.name) + .map { case (name, resources) => + name -> resources.map(_.metadata.version).toSet + } + )) + ) + }) .addConfigTransform((_, config) => updateAutomationConfig(ConfigurableApp.Validator)( _.withPausedTrigger[AcceptedTransferOfferTrigger] )(config) ) + .addConfigTransform((_, config) => + updateAutomationConfig(ConfigurableApp.Sv)( + _.withPausedTrigger[UnhideRewardCouponV2Trigger] + )(config) + ) .addConfigTransform((_, config) => ConfigTransforms.updateAllSvAppConfigs_(svConfig => svConfig.copy( @@ -106,6 +144,7 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest "Unhide and expire of RewardCouponV2" in { implicit env => val aliceParticipantId = aliceValidatorBackend.appState.participantAdminConnection.getParticipantId().futureValue + assertAliceVettedBelowLatest(aliceParticipantId) val (aliceParty, bobParty) = onboardAliceAndBobWithFeaturedRights() @@ -139,11 +178,6 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest assertOldestOpenRound(round.toLong) } // FA right now effective from round 4 - - // Unvet v2-capable amulet packages from alice after onboarding so - // ProcessRewardsTrigger sees her as having wrong vetting state. - unvetV2AmuletOnAlice(aliceParticipantId) - doTransfer() advanceRoundsToNextRoundOpening @@ -446,6 +480,22 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest .futureValue .flatMap(_.mapping.packages.map(_.packageId)) + private def assertAliceVettedBelowLatest( + aliceParticipantId: ParticipantId + )(implicit env: SpliceTestConsoleEnvironment): Unit = + clue("Alice's validator vets the second-latest amulet version but not the latest") { + eventually() { + val vetted = vettedPackagesOnSv1View(aliceParticipantId) + vetted should contain( + DarResources.amulet.others + .filter(_.metadata.version < latestAmuletDar.metadata.version) + .maxBy(_.metadata.version) + .packageId + ) + vetted should not contain latestAmuletDar.packageId + } + } + private def onboardAliceAndBobWithFeaturedRights()(implicit env: SpliceTestConsoleEnvironment ): (PartyId, PartyId) = { @@ -454,7 +504,25 @@ class UnhideAndExpireRewardCouponV2TimeBasedIntegrationTest bobWalletClient.tap(100) grantFeaturedAppRight(bobWalletClient) - grantFeaturedAppRight(aliceWalletClient) + + // Alice can't self-grant while unvetted, so use dso directly + actAndCheck( + "DSO directly creates Alice's FeaturedAppRight", + sv1Backend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitWithResult( + userId = sv1Backend.config.ledgerApiUser, + actAs = Seq(dsoParty), + readAs = Seq.empty, + update = new FeaturedAppRight( + dsoParty.toProtoPrimitive, + aliceParty.toProtoPrimitive, + java.util.Optional.empty(), + ).create, + ), + )( + "Alice's featured app right is visible in scan", + _ => sv1ScanBackend.lookupFeaturedAppRight(aliceParty) should not be empty, + ) (aliceParty, bobParty) }