Integrate exponential cantonbft backoff

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ReconcileBftSequencingParametersIntegrationTest.scala

@@ -1,10 +1,15 @@
 package org.lfdecentralizedtrust.splice.integration.tests
 
 import com.digitalasset.canton.config.CantonRequireTypes.InstanceName
-import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.framework.data.topology.SequencingParameters
+import com.digitalasset.canton.config.PositiveFiniteDuration
+import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.framework.data.topology.{
+  BlacklistLeaderSelectionPolicyConfig,
+  SequencingParameters,
+}
 import org.lfdecentralizedtrust.splice.config.ConfigTransforms
 import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
 import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.IntegrationTest
+import org.lfdecentralizedtrust.splice.sv.config.BftSequencingParameters
 import org.lfdecentralizedtrust.splice.util.StandaloneCanton
 
 class SvReconcileBftSequencingParametersIntegrationTest
@@ -27,13 +32,26 @@ class SvReconcileBftSequencingParametersIntegrationTest
               (InstanceName.tryCreate("sv1Local") ->
                 c.svApps(InstanceName.tryCreate("sv1"))
                   .copy(
-                    cantonBftSequencingParameters = None
+                    cantonBftSequencingParameters = Some(
+                      BftSequencingParameters(
+                        pbftViewChangeTimeout = PositiveFiniteDuration.ofSeconds(5),
+                        segmentLength = SequencingParameters.DefaultSegmentLength.length,
+                        blacklistLeaderSelectionPolicyConfig =
+                          SequencingParameters.DefaultLeaderSelectionPolicyConfig.copy(
+                            howLongToBlacklist =
+                              BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential(
+                                initialValue = 1L,
+                                maximumEpochBlacklisted = Some(250L),
+                              )
+                          ),
+                      )
+                    )
                   ))
           ),
       )
       .withManualStart
 
-  "SV automation can set and unset bft sequencing parameters" in { implicit env =>
+  "SV automation can modify bft sequencing parameters" in { implicit env =>
     withCantonSvNodes(
       (
         Some(sv1Backend),
@@ -56,12 +74,26 @@ class SvReconcileBftSequencingParametersIntegrationTest
         .value
       bftParameters.pbftViewChangeTimeout shouldBe com.digitalasset.canton.time.PositiveFiniteDuration
         .tryOfSeconds(5)
+      bftParameters.blacklistLeaderSelectionPolicyConfig.howLongToBlacklist shouldBe a[
+        BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Linear
+      ]
       sv1Backend.stop()
-      actAndCheck("Restart with sequencing parameters unset", sv1LocalBackend.startSync())(
+      actAndCheck(
+        "Restart with modified bft sequencing parameters unset",
+        sv1LocalBackend.startSync(),
+      )(
         "sequencing parameters are unset",
         _ => {
-          sv1LocalBackend.participantClient.topology.sequencing_parameters
-            .list(decentralizedSynchronizerId) shouldBe empty
+          val parameters = sv1Backend.participantClient.topology.sequencing_parameters
+            .list(decentralizedSynchronizerId)
+            .loneElement
+          val bytes = parameters.item.payload.value
+          val bftParameters = SequencingParameters
+            .fromByteString(sv1Backend.config.localSynchronizerNodes.current.protocolVersion, bytes)
+            .value
+          bftParameters.blacklistLeaderSelectionPolicyConfig.howLongToBlacklist shouldBe a[
+            BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential
+          ]
         },
       )
       sv1LocalBackend.stop()

canton/community/app-base/src/main/scala/com/digitalasset/canton/config/CantonConfig.scala

@@ -1226,6 +1226,16 @@ object CantonConfig {
     lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistLinearConfigReader
         : ConfigReader[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Linear] =
       deriveReader[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Linear]
+    lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistLinearWithParametersConfigReader
+        : ConfigReader[
+          BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.LinearWithParameters
+        ] =
+      deriveReader[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.LinearWithParameters]
+    lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistExponentialConfigReader
+        : ConfigReader[
+          BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential
+        ] =
+      deriveReader[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential]
     lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistNoBlacklistingConfigReader
         : ConfigReader[
           BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.NoBlacklisting.type
@@ -2046,6 +2056,16 @@ object CantonConfig {
     lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistLinearConfigWriter
         : ConfigWriter[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Linear] =
       deriveWriter[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Linear]
+    lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistLinearWithParametersConfigWriter
+        : ConfigWriter[
+          BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.LinearWithParameters
+        ] =
+      deriveWriter[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.LinearWithParameters]
+    lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistExponentialConfigWriter
+        : ConfigWriter[
+          BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential
+        ] =
+      deriveWriter[BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.Exponential]
     lazy implicit val bftBlockOrdererLeaderSelectionPolicyHowLongToBlacklistNoBlacklistingConfigWriter
         : ConfigWriter[
           BlacklistLeaderSelectionPolicyConfig.HowLongToBlacklist.NoBlacklisting.type

canton/community/synchronizer/src/main/protobuf/com/digitalasset/canton/synchronizer/sequencing/sequencer/bftordering/v31/bft_ordering_sequencing_parameters.proto

@@ -20,6 +20,8 @@ message BlacklistLeaderSelectionPolicy {
   oneof how_long_to_blacklist {
     HowLongLinear how_long_linear = 1;
     HowLongNoBlacklisting how_long_no_blacklisting = 2;
+    HowLongLinearWithParameters how_long_linear_with_parameters = 5;
+    HowLongExponential how_long_exponential = 6;
   }
   oneof how_many_can_we_blacklist {
     HowManyNumFaultsTolerated how_many_num_faults_tolerated = 3;
@@ -31,6 +33,15 @@ message HowLongLinear {
   optional int64 maximum_epoch_length_blacklisted = 1;
 }
 message HowLongNoBlacklisting {}
+message HowLongLinearWithParameters {
+  int64 slope = 1;
+  int64 initial_value = 2;
+  optional int64 maximum_epoch_length_blacklisted = 3;
+}
+message HowLongExponential {
+  int64 initial_value = 1;
+  optional int64 maximum_epoch_length_blacklisted = 2;
+}
 
 message HowManyNumFaultsTolerated {}
 message HowManyNoBlacklisting {}

canton/community/synchronizer/src/main/scala/com/digitalasset/canton/synchronizer/sequencer/block/bftordering/core/modules/output/leaders/BlacklistLeaderSelectionInitializer.scala

@@ -51,7 +51,7 @@ class BlacklistLeaderSelectionInitializer[E <: Env[E]](
       state: BlacklistLeaderSelectionPolicyState,
       orderingTopology: OrderingTopology,
   ): Seq[BftNodeId] =
-    BlacklistLeaderSelectionPolicyStateWithTopology(state, orderingTopology)
+    BlacklistLeaderSelectionPolicyStateWithTopology(state, orderingTopology, protocolVersion)
       .computeLeaders()
 
   def blacklistedNodesFromState(
@@ -61,6 +61,7 @@ class BlacklistLeaderSelectionInitializer[E <: Env[E]](
     BlacklistLeaderSelectionPolicyStateWithTopology(
       state,
       orderingTopology,
+      protocolVersion,
     ).computeBlacklistedNodes()
 
   def leaderSelectionPolicy(
@@ -69,6 +70,7 @@ class BlacklistLeaderSelectionInitializer[E <: Env[E]](
   ): LeaderSelectionPolicy[E] = BlacklistLeaderSelectionPolicy.create(
     blacklistLeaderSelectionPolicyState,
     orderingTopology,
+    protocolVersion,
     store,
     metrics,
     loggerFactory,

canton/community/synchronizer/src/main/scala/com/digitalasset/canton/synchronizer/sequencer/block/bftordering/core/modules/output/leaders/BlacklistLeaderSelectionPolicy.scala

@@ -15,13 +15,15 @@ import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.framewor
   FutureContext,
 }
 import com.digitalasset.canton.tracing.TraceContext
+import com.digitalasset.canton.version.ProtocolVersion
 
 import scala.collection.mutable
 
 @SuppressWarnings(Array("org.wartremover.warts.Var"))
 class BlacklistLeaderSelectionPolicy[E <: Env[E]](
     initialState: BlacklistLeaderSelectionPolicyState,
     initialOrderingTopology: OrderingTopology,
+    protocolVersion: ProtocolVersion,
     store: OutputMetadataStore[E],
     metrics: BftOrderingMetrics,
     override val loggerFactory: NamedLoggerFactory,
@@ -30,7 +32,11 @@ class BlacklistLeaderSelectionPolicy[E <: Env[E]](
     with NamedLogging {
 
   private var state =
-    BlacklistLeaderSelectionPolicyStateWithTopology(initialState, initialOrderingTopology)
+    BlacklistLeaderSelectionPolicyStateWithTopology(
+      initialState,
+      initialOrderingTopology,
+      protocolVersion,
+    )
 
   private var blockToLeader: Map[BlockNumber, BftNodeId] =
     state.computeBlockToLeader()
@@ -161,13 +167,15 @@ object BlacklistLeaderSelectionPolicy {
   def create[E <: Env[E]](
       state: BlacklistLeaderSelectionPolicyState,
       orderingTopology: OrderingTopology,
+      protocolVersion: ProtocolVersion,
       store: OutputMetadataStore[E],
       metrics: BftOrderingMetrics,
       loggerFactory: NamedLoggerFactory,
   )(implicit metricsContext: MetricsContext): BlacklistLeaderSelectionPolicy[E] =
     new BlacklistLeaderSelectionPolicy(
       state,
       orderingTopology,
+      protocolVersion,
       store,
       metrics,
       loggerFactory,

canton/community/synchronizer/src/main/scala/com/digitalasset/canton/synchronizer/sequencer/block/bftordering/core/modules/output/leaders/BlacklistLeaderSelectionPolicyState.scala

@@ -13,6 +13,7 @@ import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.core.mod
 import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.framework.data.BftOrderingIdentifiers.{
   BftNodeId,
   BlockNumber,
+  EpochLength,
   EpochNumber,
 }
 import com.digitalasset.canton.synchronizer.sequencer.block.bftordering.framework.data.topology.OrderingTopology
@@ -30,6 +31,7 @@ import scala.collection.immutable.SortedSet
 final case class BlacklistLeaderSelectionPolicyStateWithTopology(
     state: BlacklistLeaderSelectionPolicyState,
     topology: OrderingTopology,
+    protocolVersion: ProtocolVersion,
 ) {
   def epochNumber: EpochNumber = state.epochNumber
   def startBlock: BlockNumber = state.startBlock
@@ -48,12 +50,9 @@ final case class BlacklistLeaderSelectionPolicyStateWithTopology(
   ): BlacklistLeaderSelectionPolicyStateWithTopology = {
     val newBlacklist = updateBlacklist(newTopology, blockToLeader, nodesToPunish)
     BlacklistLeaderSelectionPolicyStateWithTopology(
-      BlacklistLeaderSelectionPolicyState.create(
-        EpochNumber(epochNumber + 1),
-        BlockNumber(startBlock + topology.epochLength),
-        newBlacklist,
-      )(state.representativeProtocolVersion.representative),
+      state.update(topology.epochLength, newBlacklist, protocolVersion),
       newTopology,
+      protocolVersion,
     )
   }
   private def updateBlacklist(
@@ -124,6 +123,17 @@ final case class BlacklistLeaderSelectionPolicyState(
       blacklist.view.mapValues(_.toProto30).toMap,
     )
 
+  def update(
+      epochLength: EpochLength,
+      newBlacklist: Blacklist,
+      protocolVersion: ProtocolVersion,
+  ): BlacklistLeaderSelectionPolicyState =
+    BlacklistLeaderSelectionPolicyState.create(
+      EpochNumber(epochNumber + 1),
+      BlockNumber(startBlock + epochLength),
+      newBlacklist,
+    )(protocolVersion)
+
   override protected val companionObj: BlacklistLeaderSelectionPolicyState.type =
     BlacklistLeaderSelectionPolicyState
 }

canton/community/synchronizer/src/main/scala/com/digitalasset/canton/synchronizer/sequencer/block/bftordering/framework/data/topology/BlacklistLeaderSelectionPolicyConfig.scala

@@ -58,6 +58,23 @@ object BlacklistLeaderSelectionPolicyConfig {
         ParsingResult.pure(HowLongToBlacklist.Linear(value.maximumEpochLengthBlacklisted))
       case v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist.HowLongNoBlacklisting(value) =>
         ParsingResult.pure(HowLongToBlacklist.NoBlacklisting)
+      case v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist
+            .HowLongLinearWithParameters(value) =>
+        ParsingResult.pure(
+          HowLongToBlacklist.LinearWithParameters(
+            slope = value.slope,
+            initialValue = value.initialValue,
+            maximumEpochBlacklisted = value.maximumEpochLengthBlacklisted,
+          )
+        )
+      case v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist
+            .HowLongExponential(value) =>
+        ParsingResult.pure(
+          HowLongToBlacklist.Exponential(
+            initialValue = value.initialValue,
+            maximumEpochBlacklisted = value.maximumEpochLengthBlacklisted,
+          )
+        )
     }
     howManyCanWeBlacklist <- proto.howManyCanWeBlacklist match {
       case v31.BlacklistLeaderSelectionPolicy.HowManyCanWeBlacklist.Empty =>
@@ -104,6 +121,78 @@ object BlacklistLeaderSelectionPolicyConfig {
         maximumEpochBlacklisted.getOrElse(epochsLeftUntilNextTrial).min(epochsLeftUntilNextTrial)
     }
 
+    // X |-> min(slope*X+initialValue, maximumEpochBlacklisted)
+    final case class LinearWithParameters(
+        maximumEpochBlacklisted: Option[Long],
+        slope: Long,
+        initialValue: Long,
+    ) extends HowLongToBlacklist {
+      override protected def pretty: Pretty[LinearWithParameters] = prettyOfClass(
+        param("maximumEpochBlacklisted", _.maximumEpochBlacklisted),
+        param("slope", _.slope),
+        param("initialValue", _.initialValue),
+      )
+
+      override def punishNodeThatFailed(failedEpochSoFar: Long): BlacklistStatus =
+        BlacklistStatus.Blacklisted.create(
+          failedAttemptsBefore = failedEpochSoFar,
+          epochsLeftUntilNewTrial = updateLeftUntilNextTrial(
+            slope * failedEpochSoFar + initialValue
+          ),
+        )
+
+      override def updateLeftUntilNextTrial(epochsLeftUntilNextTrial: Long): Long =
+        maximumEpochBlacklisted.getOrElse(epochsLeftUntilNextTrial).min(epochsLeftUntilNextTrial)
+
+      override def toProto: v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist =
+        v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist.HowLongLinearWithParameters(
+          v31.HowLongLinearWithParameters(
+            slope = slope,
+            initialValue = initialValue,
+            maximumEpochLengthBlacklisted = maximumEpochBlacklisted,
+          )
+        )
+    }
+
+    final case class Exponential(
+        maximumEpochBlacklisted: Option[Long],
+        initialValue: Long,
+    ) extends HowLongToBlacklist {
+      override protected def pretty: Pretty[Exponential] = prettyOfClass(
+        param("maximumEpochBlacklisted", _.maximumEpochBlacklisted),
+        param("initialValue", _.initialValue),
+      )
+
+      override def punishNodeThatFailed(failedEpochSoFar: Long): BlacklistStatus =
+        BlacklistStatus.Blacklisted.create(
+          failedAttemptsBefore = failedEpochSoFar,
+          epochsLeftUntilNewTrial = updateLeftUntilNextTrial(
+            fixOverflow(
+              pow(failedEpochSoFar) + initialValue
+            )
+          ),
+        )
+
+      private def fixOverflow(x: Long): Long =
+        if (x < 0) Long.MaxValue else x
+
+      private def pow(x: Long): Long = {
+        val temp = scala.math.pow(2, x.toDouble)
+        if (temp < 0.0 || temp > Long.MaxValue.toDouble) Long.MaxValue else temp.toLong
+      }
+
+      override def updateLeftUntilNextTrial(epochsLeftUntilNextTrial: Long): Long =
+        maximumEpochBlacklisted.getOrElse(epochsLeftUntilNextTrial).min(epochsLeftUntilNextTrial)
+
+      override def toProto: v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist =
+        v31.BlacklistLeaderSelectionPolicy.HowLongToBlacklist.HowLongExponential(
+          v31.HowLongExponential(
+            initialValue = initialValue,
+            maximumEpochLengthBlacklisted = maximumEpochBlacklisted,
+          )
+        )
+    }
+
     case object NoBlacklisting extends HowLongToBlacklist {
       override final def pretty: Pretty[this.type] = prettyOfObject[this.type]
       override def punishNodeThatFailed(failedEpochSoFar: Long): BlacklistStatus =

canton/community/synchronizer/src/test/scala/com/digitalasset/canton/synchronizer/sequencer/block/bftordering/bindings/canton/topology/CantonOrderingTopologyProviderTest.scala

@@ -85,6 +85,10 @@ class CantonOrderingTopologyProviderTest
                 case HowLongToBlacklist.Linear(_) =>
                   HowLongToBlacklist.NoBlacklisting
                 case HowLongToBlacklist.NoBlacklisting => HowLongToBlacklist.Linear(Some(1L))
+                case _: HowLongToBlacklist.LinearWithParameters =>
+                  HowLongToBlacklist.NoBlacklisting
+                case _: HowLongToBlacklist.Exponential =>
+                  HowLongToBlacklist.NoBlacklisting
               }
             )
           Table[Option[Long], Option[Long], SegmentLength, Option[