From b25cb7a7730d0eddba373847556f3f9b6065eb50 Mon Sep 17 00:00:00 2001 From: Clebert Suconic Date: Fri, 23 Jan 2026 15:51:45 -0500 Subject: [PATCH] ARTEMIS-5852 Lock coordination for acceptors I'm adding a LockCoordinator to the broker, that will use DistributedLock to help start and stop acceptors. You can associate the LockCoordinator with acceptors and an acceptor serving only clients would then be activated in only one of the brokers. --- .../api/core/TransportConfiguration.java | 11 + .../artemis/lockmanager/DistributedLock.java | 1 + .../lockmanager/DistributedLockManager.java | 12 +- .../DistributedLockManagerFactory.java | 47 +++ .../artemis/lockmanager/Registry.java | 82 +++++ .../file/FileBasedLockManager.java | 10 +- .../file/FileBasedLockManagerFactory.java | 68 ++++ .../lockmanager/file/FileDistributedLock.java | 4 +- .../CuratorDistributedLockManager.java | 55 +-- .../CuratorDistributedLockManagerFactory.java | 85 +++++ .../CuratorDistributedPrimitiveManager.java | 31 -- ....lockmanager.DistributedLockManagerFactory | 2 + .../artemis/lockmanager/RegistryTest.java | 116 +++++++ .../file/FileDistributedLockTest.java | 5 +- .../artemis/core/config/Configuration.java | 5 + .../config/LockCoordinatorConfiguration.java | 95 ++++++ .../core/config/impl/ConfigurationImpl.java | 13 + .../impl/FileConfigurationParser.java | 55 ++- .../remoting/impl/netty/NettyAcceptor.java | 31 ++ .../server/impl/RemotingServiceImpl.java | 9 + .../artemis/core/server/ActiveMQServer.java | 3 + .../core/server/ActiveMQServerLogger.java | 12 + .../core/server/impl/ActiveMQServerImpl.java | 42 +++ .../core/server/lock/LockCoordinator.java | 295 ++++++++++++++++ .../artemis/spi/core/remoting/Acceptor.java | 9 + .../schema/artemis-configuration.xsd | 76 +++++ .../config/impl/ConfigurationImplTest.java | 76 +++++ .../impl/FileConfigurationParserTest.java | 44 +++ .../impl/HAPolicyConfigurationTest.java | 38 +++ docs/user-manual/_book.adoc | 1 + .../_diagrams/lock-coordination-example.odg | Bin 0 -> 25555 bytes .../images/lock-coordination-example.png | Bin 0 -> 28759 bytes docs/user-manual/lock-coordination.adoc | 163 +++++++++ docs/user-manual/restart-sequence.adoc | 12 +- ...ettyNoGroupNameReplicatedFailoverTest.java | 4 +- .../dualMirrorSingleAcceptor/ZK/A/broker.xml | 205 +++++++++++ .../dualMirrorSingleAcceptor/ZK/B/broker.xml | 205 +++++++++++ .../file/A/broker.xml | 186 ++++++++++ .../file/B/broker.xml | 187 ++++++++++ .../DualMirrorSingleAcceptorRunningTest.java | 275 +++++++++++++++ .../lockmanager/LockCoordinatorTest.java | 320 ++++++++++++++++++ .../smoke/lockmanager/ZookeeperCluster.java | 77 +++++ .../ZookeeperLockManagerSinglePairTest.java | 41 +-- 43 files changed, 2864 insertions(+), 144 deletions(-) create mode 100644 artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManagerFactory.java create mode 100644 artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/Registry.java create mode 100644 artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManagerFactory.java create mode 100644 artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManagerFactory.java delete mode 100644 artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/quorum/zookeeper/CuratorDistributedPrimitiveManager.java create mode 100644 artemis-lockmanager/artemis-lockmanager-ri/src/main/resources/META-INF/services/org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory create mode 100644 artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/RegistryTest.java create mode 100644 artemis-server/src/main/java/org/apache/activemq/artemis/core/config/LockCoordinatorConfiguration.java create mode 100644 artemis-server/src/main/java/org/apache/activemq/artemis/core/server/lock/LockCoordinator.java create mode 100644 docs/user-manual/_diagrams/lock-coordination-example.odg create mode 100644 docs/user-manual/images/lock-coordination-example.png create mode 100644 docs/user-manual/lock-coordination.adoc create mode 100644 tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/A/broker.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/B/broker.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/A/broker.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/B/broker.xml create mode 100644 tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/DualMirrorSingleAcceptorRunningTest.java create mode 100644 tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/LockCoordinatorTest.java create mode 100644 tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperCluster.java diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/core/TransportConfiguration.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/core/TransportConfiguration.java index eb6f25025c5..4a852344172 100644 --- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/core/TransportConfiguration.java +++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/core/TransportConfiguration.java @@ -51,6 +51,8 @@ public class TransportConfiguration implements Serializable { private String name; + private String lockCoordinator; + private String factoryClassName = "null"; private Map params; @@ -413,6 +415,15 @@ public void decode(final ActiveMQBuffer buffer) { } } + public String getLockCoordinator() { + return lockCoordinator; + } + + public TransportConfiguration setLockCoordinator(String lockCoordinator) { + this.lockCoordinator = lockCoordinator; + return this; + } + private static String replaceWildcardChars(final String str) { return str.replace('.', '-'); } diff --git a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLock.java b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLock.java index 11f73ebdd23..a624c0004b5 100644 --- a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLock.java +++ b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLock.java @@ -24,6 +24,7 @@ public interface DistributedLock extends AutoCloseable { String getLockId(); + // TODO: A better name for this method would be isLockValid boolean isHeldByCaller() throws UnavailableStateException; boolean tryLock() throws UnavailableStateException, InterruptedException; diff --git a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManager.java b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManager.java index d42f8e985fa..dafdf32216c 100644 --- a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManager.java +++ b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManager.java @@ -21,16 +21,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.activemq.artemis.utils.ClassloadingUtil; - public interface DistributedLockManager extends AutoCloseable { static DistributedLockManager newInstanceOf(String className, Map properties) throws Exception { - return (DistributedLockManager) ClassloadingUtil.getInstanceForParamsWithTypeCheck(className, - DistributedLockManager.class, - DistributedLockManager.class.getClassLoader(), - new Class[]{Map.class}, - properties); + DistributedLockManagerFactory factory = Registry.getInstance().getFactoryWithClassName(className); + if (factory == null) { + throw new IllegalArgumentException(className + " not found"); + } + return factory.build(properties); } @FunctionalInterface diff --git a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManagerFactory.java b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManagerFactory.java new file mode 100644 index 00000000000..b9fb4de039e --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/DistributedLockManagerFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.lockmanager; + +import java.util.Map; +import java.util.Set; + +public interface DistributedLockManagerFactory { + DistributedLockManager build(Map properties); + + String getName(); + + String getImplName(); + + default Map validateParameters(Map config) { + config.forEach((parameterName, ignore) -> validateParameter(parameterName)); + return config; + } + + default String getParameterListAsString() { + return String.join(", ", getValidParametersList()); + } + + Set getValidParametersList(); + + default void validateParameter(String parameterName) { + Set validList = getValidParametersList(); + if (!validList.contains(parameterName)) { + throw new IllegalArgumentException("Invalid parameter '" + parameterName + "'. Accepted parameters: " + String.join(", ", validList)); + } + } +} diff --git a/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/Registry.java b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/Registry.java new file mode 100644 index 00000000000..af479d4f18b --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-api/src/main/java/org/apache/activemq/artemis/lockmanager/Registry.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.lockmanager; + +import java.util.HashMap; +import java.util.ServiceLoader; + +public class Registry { + + private final HashMap factories = new HashMap<>(); + private final HashMap factoriesWithImpl = new HashMap<>(); + + private volatile boolean serviceLoaded = false; + + private static final Registry INSTANCE = new Registry(); + + private Registry() { + } + + public static Registry getInstance() { + return INSTANCE; + } + + public synchronized void register(DistributedLockManagerFactory factory) { + factories.put(factory.getName().toLowerCase(), factory); + factoriesWithImpl.put(factory.getImplName(), factory); + } + + public synchronized void unregisterWithType(String type) { + unregister(factories.get(type.toLowerCase())); + } + + public synchronized void unregisterWithClassName(String name) { + unregister(factoriesWithImpl.get(name)); + } + + private void unregister(DistributedLockManagerFactory factory) { + if (factory != null) { + factories.remove(factory.getName()); + factoriesWithImpl.remove(factory.getImplName()); + } + } + + public synchronized DistributedLockManagerFactory getFactoryWithClassName(String className) { + checkService(); + DistributedLockManagerFactory factory = factoriesWithImpl.get(className); + if (factory == null) { + throw new IllegalArgumentException("factory " + className + " not found"); + } + return factory; + } + + public synchronized DistributedLockManagerFactory getFactory(String type) { + checkService(); + return factories.get(type.toLowerCase()); + } + + public synchronized void checkService() { + if (serviceLoaded) { + return; + } + ServiceLoader services = ServiceLoader.load(DistributedLockManagerFactory.class); + services.forEach(this::register); + serviceLoaded = true; + } + +} diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManager.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManager.java index dcd5b19d887..52005a7861c 100644 --- a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManager.java +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManager.java @@ -31,21 +31,13 @@ import org.apache.activemq.artemis.lockmanager.MutableLong; import org.apache.activemq.artemis.lockmanager.UnavailableStateException; -/** - * This is an implementation suitable to be used just on unit tests and it won't attempt to manage nor purge existing - * stale locks files. It's part of the tests life-cycle to properly set-up and tear-down the environment. - */ public class FileBasedLockManager implements DistributedLockManager { private final File locksFolder; private final Map locks; private boolean started; - public FileBasedLockManager(Map args) { - this(new File(args.get("locks-folder"))); - } - - public FileBasedLockManager(File locksFolder) { + FileBasedLockManager(File locksFolder) { Objects.requireNonNull(locksFolder); if (!locksFolder.exists()) { throw new IllegalStateException(locksFolder + " is supposed to already exists"); diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManagerFactory.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManagerFactory.java new file mode 100644 index 00000000000..c68e55d7ca3 --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileBasedLockManagerFactory.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.lockmanager.file; + +import java.io.File; +import java.util.Map; +import java.util.Set; + +import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory; + +/** + * Factory for creating file-based distributed lock managers. + *

+ * This implementation uses the file system to manage distributed locks + *

+ * Valid configuration parameters: + *

    + *
  • locks-folder (required): Path to the directory where lock files will be created and managed. + * The directory must be created in advance before using this lock manager.
    • + *
+ */ +public class FileBasedLockManagerFactory implements DistributedLockManagerFactory { + + private static final String LOCK_FOLDER = "locks-folder"; + + private static final Set VALID_PARAMS = Set.of(LOCK_FOLDER); + + @Override + public String getName() { + return "file"; + } + + @Override + public DistributedLockManager build(Map config) { + config = validateParameters(config); + String folder = config.get(LOCK_FOLDER); + if (folder == null) { + throw new IllegalArgumentException("folder not passed as a parameter"); + } + return new FileBasedLockManager(new File(folder)); + } + + @Override + public Set getValidParametersList() { + return VALID_PARAMS; + } + + @Override + public String getImplName() { + return FileBasedLockManager.class.getName(); + } +} diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLock.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLock.java index 60ac64f235a..9f57f736443 100644 --- a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLock.java +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLock.java @@ -65,7 +65,7 @@ public boolean isHeldByCaller() { } @Override - public boolean tryLock() { + public synchronized boolean tryLock() { checkNotClosed(); final FileLock fileLock = this.fileLock; if (fileLock != null) { @@ -88,7 +88,7 @@ public boolean tryLock() { } @Override - public void unlock() { + public synchronized void unlock() { checkNotClosed(); final FileLock fileLock = this.fileLock; if (fileLock != null) { diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManager.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManager.java index 6b5fd5f2104..c92864a1a01 100644 --- a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManager.java +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManager.java @@ -20,13 +20,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.activemq.artemis.lockmanager.DistributedLock; import org.apache.activemq.artemis.lockmanager.DistributedLockManager; @@ -42,7 +39,6 @@ import org.apache.curator.utils.DebugUtils; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; public class CuratorDistributedLockManager implements DistributedLockManager, ConnectionStateListener { @@ -97,41 +93,6 @@ public int hashCode() { } } - private static final String CONNECT_STRING_PARAM = "connect-string"; - private static final String NAMESPACE_PARAM = "namespace"; - private static final String SESSION_MS_PARAM = "session-ms"; - private static final String SESSION_PERCENT_PARAM = "session-percent"; - private static final String CONNECTION_MS_PARAM = "connection-ms"; - private static final String RETRIES_PARAM = "retries"; - private static final String RETRIES_MS_PARAM = "retries-ms"; - private static final Set VALID_PARAMS = Stream.of( - CONNECT_STRING_PARAM, - NAMESPACE_PARAM, - SESSION_MS_PARAM, - SESSION_PERCENT_PARAM, - CONNECTION_MS_PARAM, - RETRIES_PARAM, - RETRIES_MS_PARAM).collect(Collectors.toSet()); - private static final String VALID_PARAMS_ON_ERROR = VALID_PARAMS.stream().collect(joining(",")); - // It's 9 times the default ZK tick time ie 2000 ms - private static final String DEFAULT_SESSION_TIMEOUT_MS = Integer.toString(18_000); - private static final String DEFAULT_CONNECTION_TIMEOUT_MS = Integer.toString(8_000); - private static final String DEFAULT_RETRIES = Integer.toString(1); - private static final String DEFAULT_RETRIES_MS = Integer.toString(1000); - // why 1/3 of the session? https://cwiki.apache.org/confluence/display/CURATOR/TN14 - private static final String DEFAULT_SESSION_PERCENT = Integer.toString(33); - - private static Map validateParameters(Map config) { - config.forEach((parameterName, ignore) -> validateParameter(parameterName)); - return config; - } - - private static void validateParameter(String parameterName) { - if (!VALID_PARAMS.contains(parameterName)) { - throw new IllegalArgumentException("non existent parameter " + parameterName + ": accepted list is " + VALID_PARAMS_ON_ERROR); - } - } - private CuratorFramework client; private final Map primitives; private List listeners; @@ -146,21 +107,7 @@ private static void validateParameter(String parameterName) { } } - public CuratorDistributedLockManager(Map config) { - this(validateParameters(config), true); - } - - private CuratorDistributedLockManager(Map config, boolean ignore) { - this(config.get(CONNECT_STRING_PARAM), - config.get(NAMESPACE_PARAM), - Integer.parseInt(config.getOrDefault(SESSION_MS_PARAM, DEFAULT_SESSION_TIMEOUT_MS)), - Integer.parseInt(config.getOrDefault(SESSION_PERCENT_PARAM, DEFAULT_SESSION_PERCENT)), - Integer.parseInt(config.getOrDefault(CONNECTION_MS_PARAM, DEFAULT_CONNECTION_TIMEOUT_MS)), - Integer.parseInt(config.getOrDefault(RETRIES_PARAM, DEFAULT_RETRIES)), - Integer.parseInt(config.getOrDefault(RETRIES_MS_PARAM, DEFAULT_RETRIES_MS))); - } - - private CuratorDistributedLockManager(String connectString, + CuratorDistributedLockManager(String connectString, String namespace, int sessionMs, int sessionPercent, diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManagerFactory.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManagerFactory.java new file mode 100644 index 00000000000..4dc07155eaf --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/lockmanager/zookeeper/CuratorDistributedLockManagerFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.lockmanager.zookeeper; + +import java.util.Map; +import java.util.Set; + +import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory; + +/** + * Factory for creating ZooKeeper-based distributed lock managers using Apache Curator. + *

+ * Valid configuration parameters: + *

    + *
  • connect-string (required): ZooKeeper connection string (e.g., "localhost:2181" or "host1:2181,host2:2181,host3:2181")
    • + *
  • namespace (required): Namespace prefix for all ZooKeeper paths to isolate lock manager data
    • + *
  • session-ms (optional, default: 18000): Session timeout in milliseconds
    • + *
  • session-percent (optional, default: 33): Percentage of session timeout to use for lock operations
    • + *
  • connection-ms (optional, default: 8000): Connection timeout in milliseconds
    • + *
  • retries (optional, default: 1): Number of retry attempts for failed operations
    • + *
  • retries-ms (optional, default: 1000): Delay in milliseconds between retry attempts
    • + *
+ */ +public class CuratorDistributedLockManagerFactory implements DistributedLockManagerFactory { + + private static final String CONNECT_STRING_PARAM = "connect-string"; + private static final String NAMESPACE_PARAM = "namespace"; + private static final String SESSION_MS_PARAM = "session-ms"; + private static final String SESSION_PERCENT_PARAM = "session-percent"; + private static final String CONNECTION_MS_PARAM = "connection-ms"; + private static final String RETRIES_PARAM = "retries"; + private static final String RETRIES_MS_PARAM = "retries-ms"; + private static final Set VALID_PARAMS = Set.of(CONNECT_STRING_PARAM, NAMESPACE_PARAM, SESSION_MS_PARAM, SESSION_PERCENT_PARAM, CONNECTION_MS_PARAM, RETRIES_PARAM, RETRIES_MS_PARAM); + + // It's 9 times the default ZK tick time ie 2000 ms + private static final String DEFAULT_SESSION_TIMEOUT_MS = Integer.toString(18_000); + private static final String DEFAULT_CONNECTION_TIMEOUT_MS = Integer.toString(8_000); + private static final String DEFAULT_RETRIES = Integer.toString(1); + private static final String DEFAULT_RETRIES_MS = Integer.toString(1000); + // why 1/3 of the session? https://cwiki.apache.org/confluence/display/CURATOR/TN14 + private static final String DEFAULT_SESSION_PERCENT = Integer.toString(33); + + @Override + public Set getValidParametersList() { + return VALID_PARAMS; + } + + @Override + public DistributedLockManager build(Map config) { + validateParameters(config); + return new CuratorDistributedLockManager(config.get(CONNECT_STRING_PARAM), + config.get(NAMESPACE_PARAM), + Integer.parseInt(config.getOrDefault(SESSION_MS_PARAM, DEFAULT_SESSION_TIMEOUT_MS)), + Integer.parseInt(config.getOrDefault(SESSION_PERCENT_PARAM, DEFAULT_SESSION_PERCENT)), + Integer.parseInt(config.getOrDefault(CONNECTION_MS_PARAM, DEFAULT_CONNECTION_TIMEOUT_MS)), + Integer.parseInt(config.getOrDefault(RETRIES_PARAM, DEFAULT_RETRIES)), + Integer.parseInt(config.getOrDefault(RETRIES_MS_PARAM, DEFAULT_RETRIES_MS))); + } + + @Override + public String getName() { + return "ZK"; + } + + @Override + public String getImplName() { + return CuratorDistributedLockManager.class.getName(); + } +} diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/quorum/zookeeper/CuratorDistributedPrimitiveManager.java b/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/quorum/zookeeper/CuratorDistributedPrimitiveManager.java deleted file mode 100644 index b4f008313ae..00000000000 --- a/artemis-lockmanager/artemis-lockmanager-ri/src/main/java/org/apache/activemq/artemis/quorum/zookeeper/CuratorDistributedPrimitiveManager.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.activemq.artemis.quorum.zookeeper; - -import java.util.Map; - -import org.apache.activemq.artemis.lockmanager.zookeeper.CuratorDistributedLockManager; - -/** - * This is for backwards compatibility - */ -@Deprecated(forRemoval = true) -public class CuratorDistributedPrimitiveManager extends CuratorDistributedLockManager { - public CuratorDistributedPrimitiveManager(Map config) { - super(config); - } -} diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/main/resources/META-INF/services/org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory b/artemis-lockmanager/artemis-lockmanager-ri/src/main/resources/META-INF/services/org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory new file mode 100644 index 00000000000..587525afa2e --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/main/resources/META-INF/services/org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory @@ -0,0 +1,2 @@ +org.apache.activemq.artemis.lockmanager.zookeeper.CuratorDistributedLockManagerFactory +org.apache.activemq.artemis.lockmanager.file.FileBasedLockManagerFactory diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/RegistryTest.java b/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/RegistryTest.java new file mode 100644 index 00000000000..667b149b0a8 --- /dev/null +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/RegistryTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.lockmanager; + +import java.lang.invoke.MethodHandles; +import java.util.Map; +import java.util.Set; + +import org.apache.activemq.artemis.lockmanager.file.FileBasedLockManager; +import org.apache.activemq.artemis.lockmanager.file.FileBasedLockManagerFactory; +import org.apache.activemq.artemis.lockmanager.zookeeper.CuratorDistributedLockManager; +import org.apache.activemq.artemis.lockmanager.zookeeper.CuratorDistributedLockManagerFactory; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RegistryTest { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Test + public void testDiscovery() { + + DistributedLockManagerFactory factory; + + factory = Registry.getInstance().getFactory("file"); + assertTrue(factory instanceof FileBasedLockManagerFactory); + assertEquals(FileBasedLockManager.class.getName(), factory.getImplName()); + + factory = Registry.getInstance().getFactory("FILE"); + assertTrue(factory instanceof FileBasedLockManagerFactory); + assertEquals(FileBasedLockManager.class.getName(), factory.getImplName()); + + factory = Registry.getInstance().getFactoryWithClassName(FileBasedLockManager.class.getName()); + assertTrue(factory instanceof FileBasedLockManagerFactory); + assertEquals(FileBasedLockManager.class.getName(), factory.getImplName()); + + factory = Registry.getInstance().getFactory("ZK"); + assertTrue(factory instanceof CuratorDistributedLockManagerFactory); + assertEquals(CuratorDistributedLockManager.class.getName(), factory.getImplName()); + + factory = Registry.getInstance().getFactory("zk"); + assertTrue(factory instanceof CuratorDistributedLockManagerFactory); + assertEquals(CuratorDistributedLockManager.class.getName(), factory.getImplName()); + + factory = Registry.getInstance().getFactoryWithClassName(CuratorDistributedLockManager.class.getName()); + assertTrue(factory instanceof CuratorDistributedLockManagerFactory); + assertEquals(CuratorDistributedLockManager.class.getName(), factory.getImplName()); + } + + @Test + public void testUnregister() { + Registry.getInstance().register(new FakeDistributedLockManagerFactory()); + assertInstanceOf(FakeDistributedLockManagerFactory.class, Registry.getInstance().getFactory("fake")); + assertInstanceOf(FakeDistributedLockManagerFactory.class, Registry.getInstance().getFactoryWithClassName("Fake")); + Registry.getInstance().unregisterWithType("fake"); + assertNull(Registry.getInstance().getFactory("fake")); + assertThrows(IllegalArgumentException.class, () -> Registry.getInstance().getFactoryWithClassName("Fake")); + Registry.getInstance().register(new FakeDistributedLockManagerFactory()); + assertInstanceOf(FakeDistributedLockManagerFactory.class, Registry.getInstance().getFactory("fake")); + assertInstanceOf(FakeDistributedLockManagerFactory.class, Registry.getInstance().getFactoryWithClassName("Fake")); + Registry.getInstance().unregisterWithClassName("Fake"); + assertNull(Registry.getInstance().getFactory("fake")); + assertNull(Registry.getInstance().getFactory("Fake")); + assertThrows(IllegalArgumentException.class, () -> Registry.getInstance().getFactoryWithClassName("Fake")); + assertDoesNotThrow(() -> Registry.getInstance().unregisterWithType("dontExist")); + assertDoesNotThrow(() -> Registry.getInstance().unregisterWithClassName("dontExist")); + } + + public static class FakeDistributedLockManagerFactory implements DistributedLockManagerFactory { + + @Override + public DistributedLockManager build(Map properties) { + return null; + } + + @Override + public String getName() { + return "fake"; + } + + @Override + public String getImplName() { + return "Fake"; + } + + @Override + public Set getValidParametersList() { + return Set.of(); + } + } + + +} diff --git a/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLockTest.java b/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLockTest.java index bcddde55158..60ea70d5068 100644 --- a/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLockTest.java +++ b/artemis-lockmanager/artemis-lockmanager-ri/src/test/java/org/apache/activemq/artemis/lockmanager/file/FileDistributedLockTest.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.Map; @@ -63,14 +62,14 @@ public void reflectiveManagerCreation() throws Exception { @Test public void reflectiveManagerCreationFailWithoutLocksFolder() throws Exception { - assertThrows(InvocationTargetException.class, () -> { + assertThrows(IllegalArgumentException.class, () -> { DistributedLockManager.newInstanceOf(managerClassName(), Collections.emptyMap()); }); } @Test public void reflectiveManagerCreationFailIfLocksFolderIsNotFolder() throws Exception { - assertThrows(InvocationTargetException.class, () -> { + assertThrows(IllegalStateException.class, () -> { DistributedLockManager.newInstanceOf(managerClassName(), Collections.singletonMap("locks-folder", File.createTempFile("junit", null, tmpFolder).toString())); }); } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java index dfbc499c081..440abab3767 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java @@ -18,6 +18,7 @@ import java.io.File; import java.net.URL; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Properties; @@ -1368,6 +1369,10 @@ default boolean isJDBC() { void unRegisterBrokerPlugin(ActiveMQServerBasePlugin plugin); + Collection getLockCoordinatorConfigurations(); + + void addLockCoordinatorConfiguration(LockCoordinatorConfiguration configuration); + List getBrokerPlugins(); List getBrokerConnectionPlugins(); diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/LockCoordinatorConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/LockCoordinatorConfiguration.java new file mode 100644 index 00000000000..7c31e96fb5b --- /dev/null +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/LockCoordinatorConfiguration.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.core.config; + +import java.util.HashMap; +import java.util.Map; + +public class LockCoordinatorConfiguration { + + String name; + String lockId; + String type; + int checkPeriod; + Map properties; + + public LockCoordinatorConfiguration() { + properties = new HashMap<>(); + } + + public LockCoordinatorConfiguration(Map properties) { + this.properties = properties; + } + + public String getName() { + return name; + } + + public LockCoordinatorConfiguration setName(String name) { + if (name == null || name.trim().isEmpty()) { + throw new IllegalArgumentException("LockCoordinator name cannot be null or empty"); + } + this.name = name; + return this; + } + + public String getLockId() { + return lockId; + } + + public LockCoordinatorConfiguration setLockId(String lockId) { + if (lockId == null || lockId.trim().isEmpty()) { + throw new IllegalArgumentException("LockCoordinator lockId cannot be null or empty"); + } + this.lockId = lockId; + return this; + } + + public String getLockType() { + return type; + } + + public LockCoordinatorConfiguration setLockType(String type) { + if (type == null || type.trim().isEmpty()) { + throw new IllegalArgumentException("LockCoordinator type cannot be null or empty"); + } + this.type = type; + return this; + } + + public int getCheckPeriod() { + return checkPeriod; + } + + public LockCoordinatorConfiguration setCheckPeriod(int checkPeriod) { + if (checkPeriod <= 0) { + throw new IllegalArgumentException("LockCoordinator checkPeriod must be positive, got: " + checkPeriod); + } + this.checkPeriod = checkPeriod; + return this; + } + + public Map getProperties() { + return properties; + } + + @Override + public String toString() { + return "LockCoordinatorConfiguration{" + "name='" + name + '\'' + ", lockId='" + lockId + '\'' + ", type='" + type + '\'' + ", checkPeriod=" + checkPeriod + ", properties=" + properties + '}'; + } +} diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java index b74f6596dfa..4bb27eeef8e 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java @@ -88,6 +88,7 @@ import org.apache.activemq.artemis.core.config.FederationConfiguration; import org.apache.activemq.artemis.core.config.HAPolicyConfiguration; import org.apache.activemq.artemis.core.config.JaasAppConfiguration; +import org.apache.activemq.artemis.core.config.LockCoordinatorConfiguration; import org.apache.activemq.artemis.core.config.MetricsConfiguration; import org.apache.activemq.artemis.core.config.StoreConfiguration; import org.apache.activemq.artemis.core.config.WildcardConfiguration; @@ -227,6 +228,8 @@ public class ConfigurationImpl extends javax.security.auth.login.Configuration i private boolean persistIDCache = ActiveMQDefaultConfiguration.isDefaultPersistIdCache(); + private Set lockCoordinatorConfigurations = new HashSet<>(); + private List incomingInterceptorClassNames = new ArrayList<>(); private List outgoingInterceptorClassNames = new ArrayList<>(); @@ -3509,6 +3512,16 @@ public Configuration setMirrorPageTransaction(boolean ignorePageTransactions) { return this; } + @Override + public Set getLockCoordinatorConfigurations() { + return lockCoordinatorConfigurations; + } + + @Override + public void addLockCoordinatorConfiguration(LockCoordinatorConfiguration configuration) { + lockCoordinatorConfigurations.add(configuration); + } + // extend property utils with ability to auto-fill and locate from collections // collection entries are identified by the name() property private static class CollectionAutoFillPropertiesUtil extends PropertyUtilsBean { diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java index f6ad71f36d2..25210b0de1c 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java @@ -51,6 +51,7 @@ import org.apache.activemq.artemis.core.config.CoreAddressConfiguration; import org.apache.activemq.artemis.core.config.DivertConfiguration; import org.apache.activemq.artemis.core.config.FederationConfiguration; +import org.apache.activemq.artemis.core.config.LockCoordinatorConfiguration; import org.apache.activemq.artemis.core.config.MetricsConfiguration; import org.apache.activemq.artemis.core.config.ScaleDownConfiguration; import org.apache.activemq.artemis.core.config.TransformerConfiguration; @@ -97,6 +98,7 @@ import org.apache.activemq.artemis.core.server.SecuritySettingPlugin; import org.apache.activemq.artemis.core.server.cluster.impl.MessageLoadBalancingType; import org.apache.activemq.artemis.core.server.group.impl.GroupingHandlerConfiguration; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.metrics.ActiveMQMetricsPlugin; import org.apache.activemq.artemis.core.server.plugin.ActiveMQServerPlugin; import org.apache.activemq.artemis.core.server.routing.KeyType; @@ -737,6 +739,22 @@ public void parseMainConfig(final Element e, final Configuration config) throws } } + NodeList lockCoordinators = e.getElementsByTagName("lock-coordinators"); + + if (lockCoordinators != null) { + for (int i = 0; i < lockCoordinators.getLength(); i++) { + Element lockCoordinatorElement = (Element) lockCoordinators.item(i); + + for (int j = 0; j < lockCoordinatorElement.getChildNodes().getLength(); ++j) { + Node node = lockCoordinatorElement.getChildNodes().item(j); + + if (node.getNodeName().equalsIgnoreCase("lock-coordinator")) { + parseLockCoordinator((Element) node, config); + } + } + } + } + // Persistence config config.setLargeMessagesDirectory(getString(e, "large-messages-directory", config.getLargeMessagesDirectory(), NOT_NULL_OR_EMPTY)); @@ -916,6 +934,31 @@ public void parseMainConfig(final Element e, final Configuration config) throws } } + private void parseLockCoordinator(final Element lockCoordinatorElement, final Configuration mainConfig) throws Exception { + String name = lockCoordinatorElement.getAttribute("name"); + String lockId = getString(lockCoordinatorElement, "lock-id", name, NO_CHECK); + String lockType = getString(lockCoordinatorElement, "type", null, NOT_NULL_OR_EMPTY); + int checkPeriod = getInteger(lockCoordinatorElement, "check-period", LockCoordinator.DEFAULT_CHECK_PERIOD, NO_CHECK); + + HashMap properties = new HashMap<>(); + + if (parameterExists(lockCoordinatorElement, "properties")) { + final NodeList propertyNodeList = lockCoordinatorElement.getElementsByTagName("property"); + final int propertiesCount = propertyNodeList.getLength(); + properties = new HashMap<>(propertiesCount); + for (int i = 0; i < propertiesCount; i++) { + final Element propertyNode = (Element) propertyNodeList.item(i); + final String propertyName = propertyNode.getAttributeNode("key").getValue(); + final String propertyValue = propertyNode.getAttributeNode("value").getValue(); + properties.put(propertyName, propertyValue); + } + } + + LockCoordinatorConfiguration lockCoordinatorConfiguration = new LockCoordinatorConfiguration(properties).setName(name).setLockId(lockId).setLockType(lockType).setCheckPeriod(checkPeriod); + mainConfig.addLockCoordinatorConfiguration(lockCoordinatorConfiguration); + } + + private void parseJournalRetention(final Element e, final Configuration config) { NodeList retention = e.getElementsByTagName("journal-retention-directory"); @@ -1621,13 +1664,21 @@ private TransportConfiguration parseAcceptorTransportConfiguration(final Element final Configuration mainConfig) throws Exception { Node nameNode = e.getAttributes().getNamedItem("name"); + String lockCoordinator = e.getAttribute("lock-coordinator"); + + String name = nameNode != null ? nameNode.getNodeValue() : null; String uri = e.getChildNodes().item(0).getNodeValue(); List configurations = ConfigurationUtils.parseAcceptorURI(name, uri); + TransportConfiguration transportConfiguration = configurations.get(0); - Map params = configurations.get(0).getParams(); + Map params = transportConfiguration.getParams(); + + if (lockCoordinator != null) { + transportConfiguration.setLockCoordinator(lockCoordinator); + } if (mainConfig.isMaskPassword() != null) { params.put(ActiveMQDefaultConfiguration.getPropMaskPassword(), mainConfig.isMaskPassword()); @@ -1637,7 +1688,7 @@ private TransportConfiguration parseAcceptorTransportConfiguration(final Element } } - return configurations.get(0); + return transportConfiguration; } private TransportConfiguration parseConnectorTransportConfiguration(final Element e, diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/impl/netty/NettyAcceptor.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/impl/netty/NettyAcceptor.java index e4fb9fd2188..a0d48cc2a3b 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/impl/netty/NettyAcceptor.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/impl/netty/NettyAcceptor.java @@ -85,6 +85,7 @@ import org.apache.activemq.artemis.core.server.ActiveMQMessageBundle; import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; import org.apache.activemq.artemis.core.server.cluster.ClusterConnection; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.management.Notification; import org.apache.activemq.artemis.core.server.management.NotificationService; import org.apache.activemq.artemis.core.server.metrics.MetricsManager; @@ -125,6 +126,8 @@ public class NettyAcceptor extends AbstractAcceptor { } } + LockCoordinator lockCoordinator; + //just for debug private final String protocolsString; @@ -441,6 +444,17 @@ private Object loadSSLContext() { } } + @Override + public LockCoordinator getLockCoordinator() { + return lockCoordinator; + } + + @Override + public NettyAcceptor setLockCoordinator(LockCoordinator lockCoordinator) { + this.lockCoordinator = lockCoordinator; + return this; + } + public int getTcpReceiveBufferSize() { return tcpReceiveBufferSize; } @@ -451,6 +465,15 @@ public SSLContextConfig getSSLContextConfig() { @Override public synchronized void start() throws Exception { + if (lockCoordinator == null) { + internalStart(); + } else { + lockCoordinator.onLockAcquired(this::internalStart); + lockCoordinator.onLockReleased(this::internalStop); + } + } + + private void internalStart() throws Exception { if (channelClazz != null) { // Already started return; @@ -770,6 +793,14 @@ public Map getConfiguration() { @Override public void stop() throws Exception { + if (lockCoordinator != null) { + lockCoordinator.stop(); + } else { + internalStop(); + } + } + + private void internalStop() throws Exception { CountDownLatch latch = new CountDownLatch(1); asyncStop(latch::countDown); latch.await(); diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/server/impl/RemotingServiceImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/server/impl/RemotingServiceImpl.java index c4ec118c1c8..932a5fb2567 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/server/impl/RemotingServiceImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/remoting/server/impl/RemotingServiceImpl.java @@ -71,6 +71,7 @@ import org.apache.activemq.artemis.core.server.ServiceRegistry; import org.apache.activemq.artemis.core.server.cluster.ClusterConnection; import org.apache.activemq.artemis.core.server.cluster.ClusterManager; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.management.ManagementService; import org.apache.activemq.artemis.core.server.reload.ReloadManager; import org.apache.activemq.artemis.logs.AuditLogger; @@ -284,6 +285,14 @@ public Acceptor createAcceptor(TransportConfiguration info) { } acceptor = factory.createAcceptor(info.getName(), clusterConnection, info.getParams(), new DelegatingBufferHandler(), this, threadPool, scheduledThreadPool, selectedProtocols, server.getThreadGroupName("remoting-" + info.getName()), server.getMetricsManager()); + if (info.getLockCoordinator() != null) { + LockCoordinator lockCoordinator = server.getLockCoordinator(info.getLockCoordinator()); + if (lockCoordinator == null) { + ActiveMQServerLogger.LOGGER.lockCoordinatorNotFoundOnAcceptor(info.getLockCoordinator(), acceptor.getName()); + } else { + acceptor.setLockCoordinator(lockCoordinator); + } + } if (defaultInvmSecurityPrincipal != null && acceptor.isUnsecurable()) { acceptor.setDefaultActiveMQPrincipal(defaultInvmSecurityPrincipal); diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServer.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServer.java index d6b0662a7b7..3158483591a 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServer.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServer.java @@ -56,6 +56,7 @@ import org.apache.activemq.artemis.core.server.impl.Activation; import org.apache.activemq.artemis.core.server.impl.AddressInfo; import org.apache.activemq.artemis.core.server.impl.ConnectorsService; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.management.ManagementService; import org.apache.activemq.artemis.core.server.metrics.MetricsManager; import org.apache.activemq.artemis.core.server.mirror.MirrorController; @@ -177,6 +178,8 @@ enum SERVER_STATE { CriticalAnalyzer getCriticalAnalyzer(); + LockCoordinator getLockCoordinator(String name); + void updateStatus(String component, String statusJson); /** diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java index fa75e685d40..f30452053b4 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java @@ -1518,4 +1518,16 @@ void slowConsumerDetected(String sessionID, @LogMessage(id = 224153, value = "Unable to find page {} on Address {} while reloading ACKNOWLEDGE_CURSOR, deleting record {}.", level = LogMessage.Level.INFO) void cannotFindPageFileDuringPageAckReload(long pageNr, Object address, long id); + + @LogMessage(id = 224154, value = "Invalid type configured on LockCoordinator {}, type {} does not exist", level = LogMessage.Level.WARN) + void invalidTypeLockCoordinator(String name, String type); + + @LogMessage(id = 224155, value = "LockCoordinator {} not found on acceptor {}", level = LogMessage.Level.WARN) + void lockCoordinatorNotFoundOnAcceptor(String lockName, String acceptorName); + + @LogMessage(id = 224156, value = "LockCoordinator {} starting with type={} and lockID={} with checkPeriod={} milliseconds", level = LogMessage.Level.INFO) + void lockCoordinatorStarting(String lockName, String type, String lockID, int checkPeriod); + + @LogMessage(id = 224157, value = "At least one of the components failed to start under the lockCoordinator {}. A retry will be executed", level = LogMessage.Level.INFO) + void retryLockCoordinator(String name); } \ No newline at end of file diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java index 0d434133239..6a83ff7242e 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java @@ -73,6 +73,7 @@ import org.apache.activemq.artemis.core.config.DivertConfiguration; import org.apache.activemq.artemis.core.config.FederationConfiguration; import org.apache.activemq.artemis.core.config.HAPolicyConfiguration; +import org.apache.activemq.artemis.core.config.LockCoordinatorConfiguration; import org.apache.activemq.artemis.core.config.amqpBrokerConnectivity.AMQPFederationBrokerPlugin; import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl; import org.apache.activemq.artemis.core.config.impl.LegacyJMSConfiguration; @@ -161,6 +162,7 @@ import org.apache.activemq.artemis.core.server.group.impl.LocalGroupingHandler; import org.apache.activemq.artemis.core.server.group.impl.RemoteGroupingHandler; import org.apache.activemq.artemis.core.server.impl.jdbc.JdbcNodeManager; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.management.ManagementService; import org.apache.activemq.artemis.core.server.management.impl.ManagementServiceImpl; import org.apache.activemq.artemis.core.server.metrics.BrokerMetricNames; @@ -195,6 +197,9 @@ import org.apache.activemq.artemis.core.transaction.ResourceManager; import org.apache.activemq.artemis.core.transaction.impl.ResourceManagerImpl; import org.apache.activemq.artemis.core.version.Version; +import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory; +import org.apache.activemq.artemis.lockmanager.Registry; import org.apache.activemq.artemis.logs.AuditLogger; import org.apache.activemq.artemis.spi.core.protocol.ProtocolManagerFactory; import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection; @@ -291,6 +296,8 @@ public class ActiveMQServerImpl implements ActiveMQServer { private ReplayManager replayManager; + private ConcurrentHashMap lockCoordinators = new ConcurrentHashMap<>(); + /** * Certain management operations shouldn't use more than one thread. this semaphore is used to guarantee a single * thread used. @@ -552,6 +559,10 @@ public boolean isRebuildCounters() { return this.rebuildCounters; } + @Override + public LockCoordinator getLockCoordinator(String name) { + return lockCoordinators.get(name); + } @Override public void replay(Date start, Date end, String address, String target, String filter) throws Exception { @@ -735,6 +746,8 @@ private void internalStart() throws Exception { ActiveMQServerLogger.LOGGER.serverStarting((haPolicy.isBackup() ? "Backup" : "Primary"), configuration); + startLockCoordinators(); + final boolean wasPrimary = !haPolicy.isBackup(); if (!haPolicy.isBackup()) { activation = haPolicy.createActivation(this, false, activationParams, ioCriticalErrorListener); @@ -793,6 +806,33 @@ private void internalStart() throws Exception { } } + private void startLockCoordinators() { + for (LockCoordinatorConfiguration lockCoordinatorConfiguration : configuration.getLockCoordinatorConfigurations()) { + String lockType = lockCoordinatorConfiguration.getLockType(); + String name = lockCoordinatorConfiguration.getName(); + String lockId = lockCoordinatorConfiguration.getLockId(); + int checkPeriod = lockCoordinatorConfiguration.getCheckPeriod(); + + DistributedLockManagerFactory distributedLockManagerFactory = Registry.getInstance().getFactory(lockType); + if (distributedLockManagerFactory == null) { + ActiveMQServerLogger.LOGGER.invalidTypeLockCoordinator(lockCoordinatorConfiguration.getName(), lockType); + continue; + } + + DistributedLockManager lockManager = distributedLockManagerFactory.build(lockCoordinatorConfiguration.getProperties()); + LockCoordinator lockCoordinator = new LockCoordinator(scheduledPool, executorFactory.getExecutor(), checkPeriod, lockManager, lockId, name); + lockCoordinators.put(name, lockCoordinator); + ActiveMQServerLogger.LOGGER.lockCoordinatorStarting(name, lockType, lockId, checkPeriod); + lockCoordinator.start(); + } + } + + private void stopLockCoordinators() { + if (lockCoordinators != null) { + lockCoordinators.values().forEach(LockCoordinator::stop); + lockCoordinators.clear(); + } + } private void takingLongToStart(Object criticalComponent) { ActiveMQServerLogger.LOGGER.tooLongToStart(criticalComponent); @@ -1381,6 +1421,8 @@ private void stop(boolean failoverOnServerShutdown, ActiveMQServerLogger.LOGGER.errorStoppingComponent(remotingService.getClass().getName(), t); } + stopLockCoordinators(); + stopComponent(pagingManager); if (!criticalIOError && pagingManager != null) { diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/lock/LockCoordinator.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/lock/LockCoordinator.java new file mode 100644 index 00000000000..b614cb6ce81 --- /dev/null +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/lock/LockCoordinator.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.core.server.lock; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.activemq.artemis.core.server.ActiveMQScheduledComponent; +import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; +import org.apache.activemq.artemis.lockmanager.DistributedLock; +import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.utils.RunnableEx; +import org.apache.activemq.artemis.utils.SimpleFutureImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages distributed locks a pluggable distributed lock mechanism. + *

+ * The LockMonitor periodically attempts to acquire a distributed lock. When the lock + * is acquired, registered "acquired" callbacks are executed. When the lock is lost + * or released, "released" callbacks are executed. + * + * @see org.apache.activemq.artemis.lockmanager.DistributedLockManager + */ +public class LockCoordinator extends ActiveMQScheduledComponent { + + /** Default period (in milliseconds) for checking lock status */ + public static final int DEFAULT_CHECK_PERIOD = 5000; + + String debugInfo; + + public String getDebugInfo() { + return debugInfo; + } + + public LockCoordinator setDebugInfo(String debugInfo) { + this.debugInfo = debugInfo; + return this; + } + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final ArrayList lockAcquiredCallback = new ArrayList<>(); + private final ArrayList lockReleasedCallback = new ArrayList<>(); + private final long checkPeriod; + private final String name; + private final String lockID; + + DistributedLockManager lockManager; + DistributedLock distributedLock; + volatile boolean locked; + + public DistributedLockManager getLockManager() { + return lockManager; + } + + /** + * Registers a callback to be executed when lock is acquired. + * If the lock is already held when this method is called, the callback + * will be executed immediately (on the executor thread). + * + * Also In case the runnable throws any exceptions, the lock will be released, any previously added callback will be called for stop + * and the monitor will retry the locks + * + * @param runnable the callback to execute when lock is acquired + */ + public void onLockAcquired(RunnableEx runnable) { + this.lockAcquiredCallback.add(runnable); + // if it's locked we run the runnable being added, + // however we must check this inside the executor + // or within a global locking + executor.execute(() -> runIfLocked(runnable)); + } + + /** + * Registers a callback to be executed when lock is released or lost. + * + * @param runnable the callback to execute when lock is released + */ + public void onLockReleased(RunnableEx runnable) { + this.lockReleasedCallback.add(runnable); + } + + /** + * Stops the lock coordinator, releasing any held locks and cleaning up resources. + * This method blocks until all cleanup is complete. + */ + @Override + public void stop() { + super.stop(); + SimpleFutureImpl simpleFuture = new SimpleFutureImpl<>(); + executor.execute(() -> { + if (locked) { + fireLockChanged(false); + } + if (distributedLock != null) { + try { + distributedLock.unlock(); + } catch (Exception e) { + logger.debug("Error unlocking during stop", e); + } + try { + distributedLock.close(); + } catch (Exception e) { + logger.debug("Error closing lock during stop", e); + } + distributedLock = null; + } + if (lockManager != null) { + try { + lockManager.stop(); + } catch (Exception e) { + logger.debug("Error stopping lock manager during stop", e); + } + lockManager = null; + } + simpleFuture.set(null); + }); + try { + simpleFuture.get(); + } catch (Exception e) { + logger.debug("Error waiting for stop to complete", e); + } + } + + /** + * Returns whether this instance currently holds the lock. + * + * @return true if lock is currently held, false otherwise + */ + public boolean isLocked() { + return locked; + } + + /** + * Constructs a new LockCoordinator. + * + * @param scheduledExecutor the executor for scheduling periodic lock checks + * @param executor the executor for running callbacks + * @param checkPeriod how often to check lock status (in milliseconds) + * @param lockManager the distributed lock manager implementation to use + * @param lockID the unique identifier for the lock + * @param name a descriptive name for this lock coordinator + */ + public LockCoordinator(ScheduledExecutorService scheduledExecutor, Executor executor, long checkPeriod, DistributedLockManager lockManager, String lockID, String name) { + super(scheduledExecutor, executor, checkPeriod, checkPeriod, TimeUnit.MILLISECONDS, false); + assert executor != null; + this.lockManager = lockManager; + this.checkPeriod = checkPeriod; + this.lockID = lockID; + this.name = name; + } + + private void fireLockChanged(boolean locked) { + this.locked = locked; + if (locked) { + AtomicBoolean treatErrors = new AtomicBoolean(false); + lockAcquiredCallback.forEach(r -> doRunTreatingErrors(r, treatErrors)); + if (treatErrors.get()) { + retryLock(); + } + } else { + lockReleasedCallback.forEach(this::doRunWithLogException); + } + } + + private void retryLock() { + ActiveMQServerLogger.LOGGER.retryLockCoordinator(name); + // Release lock and retry on next scheduled run if callbacks failed + executor.execute(this::executeRetryLock); + } + + // to be used as a runnable on the executor + private void executeRetryLock() { + if (locked) { + logger.debug("Unlocking to retry the callback"); + fireLockChanged(false); + if (distributedLock != null) { + try { + distributedLock.unlock(); + distributedLock.close(); + } catch (Exception e) { + logger.debug(e.getMessage(), e); + } + distributedLock = null; + } + if (lockManager != null) { + try { + lockManager.stop(); + } catch (Exception e) { + logger.debug(e.getMessage(), e); + } + } + } + } + + private void runIfLocked(RunnableEx checkBeingAdded) { + if (locked) { + try { + doRun(checkBeingAdded); + } catch (Throwable e) { + logger.warn(e.getMessage(), e); + retryLock(); + } + } + } + + private void doRunTreatingErrors(RunnableEx r, AtomicBoolean errorOnStart) { + try { + r.run(); + } catch (Throwable e) { + logger.warn(e.getMessage(), e); + errorOnStart.set(true); + } + } + + private void doRun(RunnableEx r) throws Exception { + r.run(); + } + + private void doRunWithLogException(RunnableEx r) { + try { + r.run(); + } catch (Throwable e) { + logger.warn(e.getMessage(), e); + } + } + + @Override + public void run() { + try { + if (!locked) { + if (!lockManager.isStarted()) { + lockManager.start(); + } + DistributedLock lock = lockManager.getDistributedLock(lockID); + if (lock.tryLock(1, TimeUnit.SECONDS)) { + logger.debug("Succeeded on locking {}, lockID={}", name, lockID); + this.distributedLock = lock; + fireLockChanged(true); + } else { + logger.debug("Not able to lock {}, lockID={}", name, lockID); + lock.close(); + lockManager.stop(); + } + } else { + if (!distributedLock.isHeldByCaller()) { + fireLockChanged(false); + distributedLock.close(); + distributedLock = null; + lockManager.stop(); + } + } + } catch (Exception e) { + fireLockChanged(false); + if (distributedLock != null) { + try { + distributedLock.close(); + } catch (Exception closeEx) { + logger.debug("Error closing lock", closeEx); + } + distributedLock = null; + } + if (lockManager != null) { + try { + lockManager.stop(); + } catch (Exception stopEx) { + logger.debug("Error stopping lock manager", stopEx); + } + } + logger.warn(e.getMessage(), e); + } + } +} + diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/spi/core/remoting/Acceptor.java b/artemis-server/src/main/java/org/apache/activemq/artemis/spi/core/remoting/Acceptor.java index 5914c8c59e7..baa6fd7b223 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/spi/core/remoting/Acceptor.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/spi/core/remoting/Acceptor.java @@ -24,6 +24,7 @@ import org.apache.activemq.artemis.core.security.ActiveMQPrincipal; import org.apache.activemq.artemis.core.server.ActiveMQComponent; import org.apache.activemq.artemis.core.server.cluster.ClusterConnection; +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; import org.apache.activemq.artemis.core.server.management.NotificationService; import static org.apache.activemq.artemis.core.remoting.impl.netty.TransportConstants.DEFAULT_AUTO_START; @@ -39,6 +40,14 @@ public interface Acceptor extends ActiveMQComponent { */ String getName(); + default Acceptor setLockCoordinator(LockCoordinator lockCoordinator) { + return this; + } + + default LockCoordinator getLockCoordinator() { + return null; + } + /** * Pause the acceptor and stop it from receiving client requests. */ diff --git a/artemis-server/src/main/resources/schema/artemis-configuration.xsd b/artemis-server/src/main/resources/schema/artemis-configuration.xsd index 0cf7e94fa3a..795cada91d9 100644 --- a/artemis-server/src/main/resources/schema/artemis-configuration.xsd +++ b/artemis-server/src/main/resources/schema/artemis-configuration.xsd @@ -551,6 +551,8 @@ + + @@ -3224,6 +3226,78 @@ + + + + + + + The lockID the implementation will use to reach the lock provider. + This is different from the lock-coordinator name, but if lock-id is omitted, we will use name of the lock-coordinator as a value. + Notice this feature is in tech preview and its configuration is subjected to changes. + + + + + + + The type of lock being used for coordination. Options: file, ZK. + Notice this is a pluggable functionality so a provider may introduce additional options. + + + + + + + A period used to verify if the lock still valid and renew it if needed. + + + + + + + A list of options for the distributed-primitive-manager + + + + + + + + A key-value pair option for the distributed-primitive-manager + + + + + + + + + + + + unique name for the lock coordinator + + + + + + + + + + a list of lock coordinators + + + + + + + + + + + @@ -4901,6 +4975,8 @@ + + diff --git a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImplTest.java b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImplTest.java index 8987868d028..a0bbe97a41d 100644 --- a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImplTest.java +++ b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImplTest.java @@ -29,8 +29,10 @@ import static org.junit.jupiter.api.Assertions.fail; import java.beans.PropertyDescriptor; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.PrintWriter; import java.io.StringReader; @@ -62,6 +64,7 @@ import org.apache.activemq.artemis.core.config.Configuration; import org.apache.activemq.artemis.core.config.ConfigurationUtils; import org.apache.activemq.artemis.core.config.HAPolicyConfiguration; +import org.apache.activemq.artemis.core.config.LockCoordinatorConfiguration; import org.apache.activemq.artemis.core.config.ScaleDownConfiguration; import org.apache.activemq.artemis.core.config.amqpBrokerConnectivity.AMQPBridgeAddressPolicyElement; import org.apache.activemq.artemis.core.config.amqpBrokerConnectivity.AMQPBridgeBrokerConnectionElement; @@ -3044,6 +3047,79 @@ public void testExportInvalidPropertyOnAcceptor() throws Exception { assertDoesNotThrow(() -> configuration.exportAsProperties(fileOutput)); } + /** + * Verifies the lock coordinator configuration parsing and export process: + *

    + *
  • Creates a configuration from broker properties
    • + *
  • Validates the configuration output
    • + *
  • Exports the configuration back to a new {@link java.util.Properties}
    • + *
  • Verifies that the new output contains all initially specified properties
    • + *
+ */ + @Test + public void testParseLockCoordinator() throws Exception { + Properties properties = new Properties(); + + properties.put("lockCoordinatorConfigurations.hello.checkPeriod", "123"); + properties.put("lockCoordinatorConfigurations.hello.lockId", "lock-id"); + properties.put("lockCoordinatorConfigurations.hello.name", "hello"); + properties.put("lockCoordinatorConfigurations.hello.lockType", "someLock"); + for (int i = 0; i < 10; i++) { + properties.put("lockCoordinatorConfigurations.hello.properties.k" + i, "v" + i); + } + + properties.put("acceptorConfigurations.netty.factoryClassName", "netty"); + properties.put("acceptorConfigurations.netty.lockCoordinator", "hello"); + properties.put("acceptorConfigurations.netty.name", "netty"); + properties.put("acceptorConfigurations.netty.params.port", "8888"); + properties.put("acceptorConfigurations.netty.params.host", "localhost"); + + ConfigurationImpl configuration = new ConfigurationImpl(); + configuration.parsePrefixedProperties(properties, null); + + assertEquals(1, configuration.getAcceptorConfigurations().size()); + TransportConfiguration acceptorConfig = null; + for (TransportConfiguration t :configuration.getAcceptorConfigurations()) { + acceptorConfig = t; + } + // I am not going to validate all the parameters from netty since this is already tested elsewhere + assertEquals("hello", acceptorConfig.getLockCoordinator()); + assertEquals("netty", acceptorConfig.getFactoryClassName()); + + assertEquals(1, configuration.getLockCoordinatorConfigurations().size()); + + LockCoordinatorConfiguration lockCoordinatorConfiguration = null; + + for (LockCoordinatorConfiguration t : configuration.getLockCoordinatorConfigurations()) { + lockCoordinatorConfiguration = t; + } + Map lockProperties = lockCoordinatorConfiguration.getProperties(); + for (int i = 0; i < 10; i++) { + assertEquals("v" + i, lockProperties.get("k" + i)); + } + + assertEquals(123, lockCoordinatorConfiguration.getCheckPeriod()); + assertEquals("lock-id", lockCoordinatorConfiguration.getLockId()); + assertEquals("hello", lockCoordinatorConfiguration.getName()); + assertEquals("someLock", lockCoordinatorConfiguration.getLockType()); + + File outputProperty = new File(getTestDirfile(), "broker.properties"); + configuration.exportAsProperties(outputProperty); + + Properties brokerProperties = new Properties(); + + try (FileInputStream is = new FileInputStream(outputProperty)) { + BufferedInputStream bis = new BufferedInputStream(is); + brokerProperties.load(bis); + } + + properties.forEach((k, v) -> { + logger.debug("Validating {} = {}", k, v); + assertEquals(v, brokerProperties.get(k)); + }); + + } + /** * To test ARTEMIS-926 */ diff --git a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/FileConfigurationParserTest.java b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/FileConfigurationParserTest.java index 53b07421d76..315a77d93c9 100644 --- a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/FileConfigurationParserTest.java +++ b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/FileConfigurationParserTest.java @@ -26,7 +26,9 @@ import java.io.ByteArrayInputStream; import java.io.PrintStream; +import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,6 +42,7 @@ import org.apache.activemq.artemis.core.config.FederationConfiguration; import org.apache.activemq.artemis.core.config.FileDeploymentManager; import org.apache.activemq.artemis.core.config.HAPolicyConfiguration; +import org.apache.activemq.artemis.core.config.LockCoordinatorConfiguration; import org.apache.activemq.artemis.core.config.ScaleDownConfiguration; import org.apache.activemq.artemis.core.config.WildcardConfiguration; import org.apache.activemq.artemis.core.config.federation.FederationQueuePolicyConfiguration; @@ -55,10 +58,14 @@ import org.apache.activemq.artemis.utils.PasswordMaskingUtil; import org.apache.activemq.artemis.utils.StringPrintStream; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.SAXParseException; public class FileConfigurationParserTest extends ServerTestBase { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String PURGE_FOLDER_FALSE = """ @@ -90,6 +97,7 @@ public class FileConfigurationParserTest extends ServerTestBase { tcp://localhost:5545 tcp://localhost:5545 vm://0 + tcp://localhost:5545 @@ -130,6 +138,21 @@ public class FileConfigurationParserTest extends ServerTestBase { """; + + private static final String LOCK_COORDINATOR_PART = """ + + + sausage-factory + etcd + 333 + + + + + + """; + + /** * These "InvalidConfigurationTest*.xml" files are modified copies of {@literal ConfigurationTest-full-config.xml}, * so just diff it for changes, e.g. @@ -423,6 +446,27 @@ public void testParsingDefaultServerConfigWithENCMaskedPwd() throws Exception { assertEquals("helloworld", bconfig.getPassword()); } + @Test + public void testLockCoordinatorParse() throws Exception { + FileConfigurationParser parser = new FileConfigurationParser(); + String configStr = FIRST_PART + LOCK_COORDINATOR_PART + LAST_PART; + Configuration configuration = parser.parseMainConfig(new ByteArrayInputStream(configStr.getBytes(StandardCharsets.UTF_8))); + + Collection lockConfigurations = configuration.getLockCoordinatorConfigurations(); + lockConfigurations.forEach(f -> logger.info("lockConfiguration={}", f)); + assertEquals(1, lockConfigurations.size()); + for (LockCoordinatorConfiguration lockConfiguration : lockConfigurations) { + assertEquals("my-lock", lockConfiguration.getName()); + assertEquals("sausage-factory", lockConfiguration.getLockId()); + assertEquals("etcd", lockConfiguration.getLockType()); + Map properties = lockConfiguration.getProperties(); + assertEquals(2, properties.size()); + assertEquals("value1", properties.get("test1")); + assertEquals("value2", properties.get("test2")); + } + configuration.getAcceptorConfigurations().stream().filter(f -> f.getName().equals("netty-with-lock")).forEach(f -> assertEquals("my-lock", f.getLockCoordinator())); + } + @Test public void testDefaultBridgeProducerWindowSize() throws Exception { FileConfigurationParser parser = new FileConfigurationParser(); diff --git a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/HAPolicyConfigurationTest.java b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/HAPolicyConfigurationTest.java index 16a82a64c6f..5d42df0378e 100644 --- a/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/HAPolicyConfigurationTest.java +++ b/artemis-server/src/test/java/org/apache/activemq/artemis/core/config/impl/HAPolicyConfigurationTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -57,14 +58,28 @@ import org.apache.activemq.artemis.core.server.impl.SharedStorePrimaryActivation; import org.apache.activemq.artemis.lockmanager.DistributedLock; import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory; import org.apache.activemq.artemis.lockmanager.MutableLong; +import org.apache.activemq.artemis.lockmanager.Registry; import org.apache.activemq.artemis.lockmanager.UnavailableStateException; import org.apache.activemq.artemis.tests.util.ServerTestBase; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; public class HAPolicyConfigurationTest extends ServerTestBase { + @BeforeAll + public static void register() { + Registry.getInstance().register(new FakeDistributedLockManagerFactory()); + } + + @AfterAll + public static void unregister() { + Registry.getInstance().unregisterWithType("fake"); + } + @Override @AfterEach public void tearDown() throws Exception { @@ -166,6 +181,29 @@ public void liveOnlyTest() throws Exception { } } + public static class FakeDistributedLockManagerFactory implements DistributedLockManagerFactory { + + @Override + public DistributedLockManager build(Map properties) { + return new FakeDistributedLockManager(properties); + } + + @Override + public String getName() { + return "fake"; + } + + @Override + public String getImplName() { + return FakeDistributedLockManager.class.getName(); + } + + @Override + public Set getValidParametersList() { + return Set.of(); + } + } + public static class FakeDistributedLockManager implements DistributedLockManager { private final Map config; diff --git a/docs/user-manual/_book.adoc b/docs/user-manual/_book.adoc index d32db5faa83..bf048d4c29e 100644 --- a/docs/user-manual/_book.adoc +++ b/docs/user-manual/_book.adoc @@ -65,6 +65,7 @@ include::network-isolation.adoc[leveloffset=1] include::restart-sequence.adoc[leveloffset=1] include::activation-tools.adoc[leveloffset=1] include::amqp-broker-connections.adoc[leveloffset=1] +include::lock-coordination.adoc[leveloffset=1] include::federation.adoc[leveloffset=1] include::federation-address.adoc[leveloffset=1] include::federation-queue.adoc[leveloffset=1] diff --git a/docs/user-manual/_diagrams/lock-coordination-example.odg b/docs/user-manual/_diagrams/lock-coordination-example.odg new file mode 100644 index 0000000000000000000000000000000000000000..d82f5b5c729e460b43a73c63b7bf59d9ffed8a0c GIT binary patch literal 25555 zcmbSz1yCH{_GS<6I=B-cxC99#xVyW%%i!)V!QI_GxVsZHxCITaK>~rz@9qA#cDLTX zs@+?2Z=LF{@0{*?dS=eK=ew;a0}TTP00aQgapu*{gJvFL007{h{`U}IXJu#R;t4S` zf`j>+jGV2UnH?Zz_NERdu6AbjF3c8AMi5IY6KBQ$P2pc1|Fhx$ zl|&ru&8;k4o&KfD`2&l&t%H$^nG?(Z9Yy$WP|hw!F0RhTM*j)-@2t5vIN1JY)W5U# z4+>&rZ)W=+uJYem`#UYJ|L@iy4iHz!KhXbRjriZp3B<|4!pY3}f0-2o1cZM%^?y3x zukilgAdO5+%xum6y2QbW#l+Rg=^t-;*x5n@iZby3JOlvLKf~Yle|Hku-?RUxYpv{z zEXU$ z{j5tjIybKPgNVd-A{3wHK&nDE%E|WU(=zw1oafCyd0VqoLYUdZ9NJ~^5SAlv6laSx zi4++7d3(>H~unNFsS^U$c@`0rO%lF#i{2rg=aSY?#|e zf?yshUAOJ#&z)a3HIk3s@w;neTYUWi>D)wXDFT^I{wNDCD8J?2xGpHrZ{!|vT`aha z3!5}T;lAQRub1%`#M-e9CMQvf-7sV#~J7r$za+SK!4=efnvhc~oDR zs5(Zu!;Zvt5+l4?kwbjHL2ItCMUew~T|EKHpVpF5zEEABGN7pNHjzVzoiTFP5`hgI zbTnv)0X}40>(61MY+*d~6<(%NcNY5r?@nMUsS0Aunx^MdrD=x{rxys zU)93NG8>eQP!DibkD{uz7Gw25+ByNJ8oF~qL;4($cz&*F&aT<9owHFijW&AKcT%0Y zfV3=pq0wT^#e&GWa8&{{CJ{^YGrc1qW0qWBwW}^%WNhBCx?N6J<|z(z!-&pC8-?6H ze~gMNTv3`aUTJ?$r3lb_zs-m$6m`|Z*qRkZ=zxf5N}JgkQyQR3`(dc)`^F{^w$Di9 zhQIcdk{>ZL`*hV17A+7QbLjt>O0}eB>fkwh@Q!YyTLO8kxPqGuG)|Q9Isde8Jt^=x z!)fiCs*t0aTQ=G2wZhRJdRhB?h{l`=&(AmrG=e5n*e8f4mO<#Gf}xm-;-H4ob#+k@ z`0(2d|N5QN8;aky`wheVsT2jM)eQ2SfiB!HC5}7Oi|YkLOGfr({~K>DXIIr9rJ>B3*Qw` zgy%~cv+{#m?D6KNi^X+>JiM{P43b)ssqmIl;R^FwNRi%aR8@t4_1Tg0!Crpqv%-hD zVwg`(-M`^i1kEX^yoKQ}*kv`+gZ1uk2FIJk4Wh5la|B*1@2IKU+j=-Kj((o*A(^t) zodonpKsV~{31iOLU&tfX7k2+qRZ}*5Gi0b}-1A6GtSu^X`qNt-JMl-lCRVwRl;SJr zT`C)!)KCN^=`-KvwBGi0j^c~j!S%);@?6M{cflph?ptp(1;>uJS-!0&tC)*D`opT$YG0Pk@jQn3)n9j2(8AV5Ja!$ znPw*!6=N%o0`J(W5&JiCMm}njwj9(uf0VGI+4Zc{e^hYjC-=zi_$8M72kI==X)XFY?Sue5?9xo@vQj2C-Q@7zi=RtsY>R(o+NMrUz6 zL8e=cCQT`wZvnhCkZ#;({P6N*(O%Y`r4lnSt@ERCjU*D5rp?xYRw7J39SK$W=>Dt zm{EzGbd;oU$Ptr>5EGVZ18m`%o(L^*i~+UqM?4YBJNoPP zZesqrMTKoWl!9x9UkeM!ly{&0xa=696pk++l-F4M787yhOlj}oAmYGhmtbg;fbZyp z@ET8~MAIsRC7Y|cgMS(mXt2H~zNG&^d zJhGT~#CXf&^}2T4#b4IEF_v<4AAi;(q+%?coJ_dp!j~O3(_lq>@80UZpr2GGRp4}^ zE3srx{H99sdjscXC{ef6#zi+VF7fab@9=0W!+(Cl0xR&__frLni{w}?8ye7vc&}tj z#nqDQFxh&Us&lojBC)%~V6jSGbl722)|_j+?yV!uwi%MsCb)K=JdYvYygWMKS0lKq z+3J?n>WqAm(nN%hbuOceDPNVgHYSSjxuU*{Hf~Ww0{x)L=G)aD%HN)f*w<^~HjSn_ ztQCp61h{;U-{YUeVAb*e1lWUW>xCkE4RUFRs)vr{6yH?{9ph@1Q+w`J;!Naya+9QVg9# z&3eLlpu*XJi&o+TxOM__OpBcLD0xE%Vnm*)Q_*Vu`#Oj&gYYpZh@*j z)-<`}4!r2G+e#M*$n`VBwSh^RWAHI&JlYm6<@`Po4JVnYSMu+mgyl)|HaI=i z$3GLp+`s?&=La0M>$PWi(eK1QZfq?oD< zczBlhbzGgbufO^cXXb04c;OBD!6y+Fv742b5OsuPiE##(%ekO_Xk#h$f+&!8wcpPo zUJznzf2hQDaKTiH5@1cteJ1hxdEE-YJ9D*Bx zqtD6}+V3Lfck*aN;tN)bXqDC;&KtA3UL2gd=j<6`ubO3&D{CADy8}=o`X*#aQozN2 znON8;oIVrdMj|J)%#q7ZaM&+0`QA5ye^Rs5p(H;VQ~>bM`>)jO-(^?Gf3qtS2YZ+Q zk=32+t|zXyVfxN!)VFOd)~a-!3ezo>6FZ!XeAruCYpBZZ5~7kcLdDVrI?fgL+1}Dv zq(EI!V7iIiU`vwsiih_N1KN*JS=)Y`ry{z@02#G#_Wk&D%bku^ddWINGKCuNv@D=Wje z6EUR1qq#Ol8S?B#UY{vZ2m@*At6!57rU+(B+k7~$olrTSG@1Ir?mF)Ifiq^>yvTt# zXLk1Ss*1vrFkHue|7ZMfe*Taaxdk@iL3myV2gK{tacEAON#jHDi%=7aGRvvg z$3I@@bOsPCY0Zg;NEzE-_H<#n$vKRfJk}-0c~;&l+03jX*g#NC^z$<$lm@(YGzzQ2 zFhm{>a98LZU>+OAMr*HuM`b_WjiblK>F3x~rl<*zka%P6{&-5KbCj67_H3=~gccMV zkkz%9@K~wyBB~hi{Jw{?v#YMfwbgRD+%*Y3Jnqf5BJ?UnY z%=L*fVE*XYS|7Wy(P$d3oSR^a1XbdmeNOEwW( zXkAr=eT#u`0qCc`fN<@JnH zK29Wr_WJ{}Z36UV&fgrcGe%<$YJLk1hTD?yDOd$n1+cMOS&~!Mdxx0gMf0Q}3FRs(nVk;O_O>awb7=Z9YZ&lw62h0@0 zRy>wfkt|BSB|;V9J(E z9)}w*y{$wlG^HI{^}}n-tz~x$TfQTd9zz31!t>$5pEm9?Rwe~hXFTdOo3}$aL=FS~ z6<-fyAU1(+C?zVk9$!Vyd;F6&NY)em<3wEsJF&U~syp4$hczgTB{cQy2;a1_alfbt z8i5Cz-S_Hjnlzs31eS;h3|^LvVNLyn%JGZ|X&>JQM4cD;%qr6~*j$5teec?i5tE^n zB0oU*Acnw-iZQ~Tf*Iq}Y&*vrgqTyY?U{Q7D^m@CO{~3~E(`tI$1%K4bct5o zQ3f9w@SboQ=JgNRVbwikH7WKobUw;~9l}EqaECkqtIVQS)8tM|TxeX(9KHH6nv;pk zDQsrh_#3u1Bh(#xG2ve0g|7_VYPpO14nS}GH#!Trq5;e71;B{TDz!xi5S9y(kp(#~=4}$dH zD;Wq$K4N90`~^2N=uGWtIbc1;O+vB==2ZSh_?eF#F+KYH2tgBskJ94ot`-|j85Ig} zdq*~3uaCLjMQ`VievDulk3wd+lS6$E_b5*5Eswd~9k!L*0Ox0wd>R^8nWImHL8m8o zHn2c#N5=d>jq(R%aUx1a!%LbA<@XpS_=uEmGc?5TtxkzCo3r4AVRMUM{VLXT}iP)2L?rrF)bdurmcoa z1mxYzsPMB7o0~+QE>yLPCE=1*PfWQ?oV^^%J?_b^H;oT9F(UMryJk(JH-9=sJs3u$ ziW{gU4Zptkl&bUSZ+mc3>FQ4eqqyyEalw6V+&%V9nIeVWBrbI?TKdt{C2_FknYH5l z^Pkd8D>tPA|k~>90ayi~y9pmQ)<+#=+(M&m! zPPeyUGS=^2I_MDoct?ofdAB{2{`C}zk{V0~QCv5~N7@ke3BaH2cXQI__sw!+a z>x3Dr7KdSoJG(oFg=;*Ycwx!>oB{JZl%}7yFJXt3&Z-%J& zR*Gi#$oW1=r4e2$1(2Juf2S*)fK%ie z{kBh4|1F+;&}o{?|LkI@*8KxI<3Y2@7h)4$C-KcG`5c15fHyPJ73q)L`QJvR=7a4t zEsAc6FS+;1zqOI;4$bdn?*+5_n2{Peqsv{)y#)q=XT!xNeOZrPLH{6YD#-WQBE!k#EUp)r=T-SaqhcCQP{*0gTqzMM}1;O z$||^vI2;KrR7Z-`FVpWd#SilxlTA-9Jen>^s4d^?PVn!NUSK}pSGYfT$2!8u%7eVi z0tE6jOYYNGXB4>Dy^u6{U|%l4to*Ps@}NPLzH2%|Kcq+8rf+E4cEjUsoSHad`nWZe z2rE&-bC?V^$x@suTI{abBZY;+4YJ*hUs}cL3FG3xw04ah7855+rqiNyra2s<&2f=l z4bla4*H*uRI2-Pf;JkhI>-W{!c=MuuSnC@vcMY|Bj!(-F>VGKr9KSlG{>WnGF ztn4lRRdLwVwsYA1g!|glYaDVR{b2o_Lh;jjzE$0T3@wSi2o|k@I1LYs0rBU?oYK4H zI%3N*mHa`uG3(|(f)tT?M~{yye%~B>UZ22;3UPg~=69v4Pt_^>pkL{6RuB~@+uPla zyQ6b~p2h<#Z^efUyl}yTP=JmNb*QW?UQua#%RFW?SzTHDms5tAV5^1zJ85!NW@%b$ zuS_Rb#O_ro*jd{`(om_47Dg#eUysw5qVmlHS8B?VuNmU46q25el%LyurHM}wi3hVa zAW96Qm@$xt^w2V`!x%S5jV3R)%KxY34|c!HpOZlsz%wP_2TEOCBkXKSA= z9Q<>vSL~Zq2Hm!|LnLclp0Utbj3Cdh%W3Dm$cz-KY&TZ!S_zxU_$I$3P{wWUlF^!< z+rLDI60!zEH)^SRwbOmX+(>3kDOr!6oH$s%;s3ThaM7XG|Gu08Cz=o@+pB4+ zHhLBgHf-a3_U{TdU=;`E=FhP(L7rD~-+p?N~CxOv3&7nbFbvp2NfR;dFz*M`< zy1Bll9y2*_H^Q99M2BNzdw1 z5C#mleCi5F&&D_l#s{@(I(cAN6D-S+O+vqmul@1_`{S)l@f$6=0`7#Do9BvY1TwdM zcdBdaGw3zp+;dh=HjXIQGtEzk-`Vs( zatCL>TBZy2ttyl$Q&pOnx|x|p)8wG{1n#1r_HjKzO@~2d#Dv7NYnswKf6^soRcBb^ z89{D4yX$Mi4eOmwV?W?w#uiTX*OA#2N+!1H0_1o_^V2KV;i-(#&^8w=Kf5L=F*gmu zKFZ;W!{pMCjwYyrv_uamL_sm!jDgNqIz<-*zf{*ENfXm?*FuxcCC5iFcl^f^-t%ct zh4q!M^xWa!FbVP(?f;}qE5eiCe}_n8QGjYB8qY`-_ym_-T1z;*nB=HbF^u2{hJb5m z9Px3*oWhG+8^4#HX?DL$S>E!6)~zui0}x*6o>fhbNM~<;r6|)T&IPK zE=p{?F|N>?BpV{xo#2_PALe{v_0un+5s0~QQjHj4qtxulQ}SGDIYe^Ym)fyTI3!;j zkL`~9re`TxQ!Z|Vb&gu#8au1 zG{2cEg>IF^sstkiJ&7R}DmNv@JlevKA?O(rIRLj+!e5RD9WX(79=1uGC_#@?)iDf?ew{d6mIdXnQiEUS-@vrn@n%87PoHa4Oq=(qn)<@jR-)GNp zkoT!tRZk(5@98=0@@AmVV%`|{S% z>o`pEc_!gFJvc$mWr4FUFP!Oph8o-uGU9dOvP4ica%X*mXRfXfUQR2M;8Kk>pQZ0itw!r>-{+lb%-(p)@8mXfXD*7Ks^9D!J;|oaG}Z>EqHw=w zUbx(S@ptbP{(9XSUYSel|K+UZH#07YO4fSHmC^4zUNVUrBR7S)J8RBMQJl;9L?jDS zqQY%i%iavCX~wj)O5*R7c(Y?Tdb7dab(>`A+IF*-1F)?AZ)#pTV*AEmM~1!OyC0I?${}0Y6;#=%o*x(VXRL1z#o#{Pz0&!{jN$`7;bR zT!Q1t0M@W5H8!^ABYwSd_ou`;5+IGH;5z`+qGZ1yLF^yuks0GW~!0ylDs2h6}ondp9~E~z))1C6Cs zaW(tgAKt9!y1cnIB;5hG6Pb3hi5#(g2n;C|E!Pj55>1h0lkTB>q$vPRjin7M=sSjo zOKtPDEqCLR{azjg2a3sh06}A*#-8@W8^ElT`NL8@#ap%a7Td{rb?ZZjs zh1?bH-1!;1$aO6@4}RM$ZqwwfS?Epx<8!!B#YqWYh49&!`E7R)jNcMJlwse!ndtTq z>|t!`477>(V(vZIUZ*5AfzLWX>6#=z}Y9G~S3VjC&rLCv$y({FZ$ zkozFOwaTv}&OX%Rc#jtmpuyq$I7`ojNn)pu10O!Qe!pq|oG-d~bY6fhIM704Ya(o- zi~DG;{J6AT@PhnL6ealV*s~1!O9+DbuZQ8k=s~kKy=1tnvm9t#0LOX=!O}>}Bib@8;$vXBnho7pm%RYV3C2P>LaKg7@A)F8BBqr{rrS0J23KId+ zq=6hoAVU>gA_J7E0L|J!j}g!zDcPy6-eqOg;p#etj68~sy$l64NKZe&$$1C>uYW&y zcz-^_yuE+_#?3vZ2aK8mGZ5gr4Y2M6Jc}YdN#Q;!l0C>#-mB2u=yDyo0xv$mwGnTh zghao$_nU;owSnN9hQ@`3^qH;dsiV%7spzr0`H`>FtGW4`hsR|g@EQUTidg<2SZDS1FM%q8a7iZ`vdECBHNCly07DV z&x?wNi;5;|Yx~N}r>d%!Dq98{Yri)&^)+@5HT8^jcMmr-Y<6^<^z>ZE^<91$xKHT6 zNS}PonS1RR7%y0Ut=fNUJA3O72^maJ-%82c$ST{a%Nwbx8f$G`Ev#NCY1?Qh-Hi<2 ziA&f?NO=njdrMD0EDznSN zcYAYn^>+LE`tt9y`Z*Ol`&EVv}5+X!W%NYQWvH$5HAT#^#mx~;b5*1SQ zSUdMc)l{))xrn8L%0&56malu7;)Imrlx@|mk+Gh7q$oU7;RRbJKPoT86uhGeDOISE z=1dC4IT`l)hEU@YQA}PeCoyUYy;{m%QEGLACocvflrt)BTm1nrWn@lhS%Nq zGa}W#!}}0Zi5^(0RRP3Ro9=DxOYQBMQd?2;CPh(L7(f&Oh`<0aC;)O{KrjRdKmY-l zPyprs?}?390G}u_H70gA*?N?+w4$K_0c2SH<{={Tnc*bj&DhG415MMU{~mheBV{pd zxN$6Dox*3}H&Kdp4kFBtys@`ke>odYREfIljhE#(u5wqGA&z0|u%$|(*zI(A;O={s z)es>?C6s|s89zs#$t^lZ)2S>iQWTrbdfZkzGoV|ikyYs6t3E{|T16z`Y5h7k;`Yfp zO8C=(<7pGH)0QZ1j_nU~eDdsJ_4ewer@D?fzhsaF2M{a2!`>F%91?jZN|}3zo9KZ+ z8&>O23Or}UmnOL1JM0RH5PYEfVgv`UoX258k}d89zM)G*u6;3r6?`Xfj*eH!bu;nm zEA2-+5tE!YhyWC2mfgd{&_1bRw&{-Q(+o6WUmiI`H@L8Z8eC}%1?2mmfcgdwh{DfK#UGV{%D5hT-EDONLQ-IH z!JK*fOBflMt8quzy?X1Sl?w%Ks@}HQ(ls?UHz}{0cs^0iN7bJkYZbVg2}}660E`oG zUzBPl+@MKSA)?%Opp9h6`^R{UcjZqCj*G)9hRha+{9;cL z8ofmDH-g3o)&hzo&ljq$a1(4nZNKD@?q)FhUcTAaVZ!^h9V{kJ!HL|7Rn}F#)L)k8da=3 z`+4bRhgBPbD>~0N-1R>YxWOR$CH8}D;Dh!d(_G`B{c8!1`t7a z6#77Z!eNlQqn&nXsJL~cX!RljEJ1!&6Jd0uymIFJF11zgnN;DyM@&MfO-8);Qpvbx zUgkbtxP279?bHnViN)6yMDC|QGY5bGzwmd$TdapjKqRb|rBIDj{1x@)v zWI zvVFF=_=D@}6?2W?ZJ{FaH(Zyms*{0KoM9k9+wd8-s~4paRz zPafSRHz{YDClFxcK)Frp>gN#6aa$^;YEpp{q~IW-rQBxT<~1t^hh=)|!h&>maV<9c zMb$(qX`Zm=Z?G0>rl@J5Mbjer!Egn>qy{G)10HLZ|C1}g0Y^oC&oEg$zp#)x7a^aQ zLsOERnz=gg_^DV18;KnRXj4^|z-lv5Cr}N`oWC<;l+<{iVV8(dl1P=%fo2$pYoTp! z@h*XuP714T61NDluxV8TWi#d1Z0bG%WhGv!XsyK@#7ty5o~)g(HprP3aw0^}FGZQK zHmVcIuJ)IAQvsZcSKtZ;rR3UUvV5Jwm7<8i`gPh{{#@W9h4!|!W#eO z@*rDI<_kuy|4|NHGA=j`k+EtUi-{pb)IHA{>o;&|TV8a^otJVHt7|Mag z(rOsil#L9W1rNpEQXnoC1Q2wYRY0_s=4C-&>XwSxEsW9gTTDy7M`1Eqx}Supqh-jY z3P~f2k9H^_fdE&^FOoKxEX_MDNle1T6Y!D?0%kxAr`HqYk{A#gEM<}zMqn!8I6sEK zLb--vZOY30jqsAO#+^(PO*Ln4%+iOgX+4BxU`x$}*9zrLK)?$*-6qsEnbygCAOP#e zkA^!IF5RcY5ssB_xhiMcAlYbHxirs=KQ&+{YzGoYa7!r-H2PX^-xO2=x%d!B&`Kkh zDHRDlsF+AaQoxvi$W1YU>qwUjnT~q_88&Ug-)4otoq^8YRFbLma%t3MhOG)w1Yf#} zJ#x^}krR=~Sr#KolreP}f#~Rx-PPIYcr*)hEo+NCQQBx_kszv+Vxdn7OhP@U0@b!fWtkEyF~X2}+OhklU8;(5T$?#%2CVnP!MTbZ2sTP| zgae3FB_WG~gn4id)d}QiDM0aOOgI3-Fa}Ck2%NoADS~;cEjqpWDy5==+_|-Q497{x zEqQ8Pa0BZN_oJb<6pa+#%IwS9wcks`Oy%U#E*}TqPz~iu9~nR{P(ZMnQoK;nV7!&i*!hECGv6XP;E_!IP^5|G*C!f17s zl<=+RTfk8h$Fs_(=TXwp_1HX=QM%(;m9zIh-tBo?+H=$=7?AeiAt>5q>LDIi8()cs z`K~j|`_@aa&(CZADJkQ@ndJXFWM2607Z7p6qZ^{K13p23C9<9W4WH_lk)o85nUb7N zJ~$3JKg{~RrLkU&ok6B7xu#ZoaGN@z0n7PqhXDKP41)c}*|ilih;*v&Y%rS?38CR~ zRHIsi7gjL7+shnpGd*}NcUw5%8@DU^RhWZhQT_die2GSV!VEFF6OwR}epo0>rBZS& z4gL>4_^X~7UO93>Y^?xSFJVC!p`GJs<8WwE^#M$%wS+fq5g63iE|)tg`b3_T0%RNv4TBoA(^&h{35W&v<-yH@{|LNs3({&3n%ip8k<^?`tZQdJY-*1O zG3~1B0ZgcP`kGSsecWO5@ppj!kt_1A%x`<30eq;^+3Y_e2v1e!&M$Xxxz51;gW8IA z{AiX@bv7rKD}W{f=aaud)FEsLYUl$gwjo>4)_0;zM&tv;z!Qv6x-eAK^fi#pOy+pH zfS3XMGgiUCKrmu6cMSY3%Kgj<7;31=uW*46`3?;^0EpRP>R82fHGy4Eg!G930DMRd zMfn{spfsN1u`noG3)q%46r`rG7xI~43^+$?5s!op;iMC7MTHlnXaIU;1|FH1Ma1a%rpQuL;K|$OtNPQ}AND@8PfJS6dE3Q$vG8Zjd zvKG2GAOkwvTfw{3_0VK?Sh0O%7(fFK^p1VjxrT7n zxfx~`bW3P25;8pGWO%=n;3J!-^{mOHUi)j7I@Kuj-RrY|+vNWF&cXIuu+75G2*|c= zI*tc`v|5gnf=)qwu=t@x03Zn(TG_z|_hN~Jk;DCBl4=wJ>g98weRy3%J?JgRystzP zj(|e})i47;o}rSR=?ZX+NMCwQBq4r3fN|Lkx4u+N%?Q}qo;n0^`74fv0q_7CO&2eHoEiRa`79v?@504bzI zZ(4vX*;`xdqBB$%k%NlScn>t?@8RV~*OQz0l(ipeGh#>Qt{aryy^^=|_Kp>yi-ks8TSyfEzieck1x0Z%5MfIJMHb!ZisRI}X)-{#TqhPO#CoFWh;2qYZV+fpL4N?SEi= zQpU4%w{BQmW{C}l6h`cuZA@Iiy&>-z&aUdnPBqk@iF|jhV;KJGv*JvNV&O=kS5!SS zPuZ);SG5M&=!E*bj32Osh_85Gl{~1*WCKU7H4R{hDlgrjoqGOShrA@BeX!16b zx+x9ftf<1FMpni>Pfe?(Pe*^0r?mIfiBx!Wo2mG~j(*MTZ3F(UD25Q>v zjZ`H~JI9M^H}kM``C)R^LG)P2SdPd6zEwL`V$$^Liq;cRo7hO128Ei?^w6+4)xRAy zUOvj}?T^4f0m_LQ6FAX_Gjz$LIO-$QdYTD|@!HDLeA?XZv0VWJcny}dl#V63`h>0J z0onzUxjO0Didw=ow6ePSnqt!|l2R+(iQ1YCnc8ac@s*mQ({I}%#FIi_&~BsYPBdET zwbFvm3TVfLT0)59pmU1`5PlWVlN})c( z#KXkg7AhM9-yJKWo+3kUE2OLYdFZ0P=aa8!iF?@f zeZ1rYc?;?jwJYLh{ zDODAlQJapTIQ8s~u*9YG8dPI%W)UWHMRQhuQEN8Q#b^C*iJ^jmXjf$tNyyj-H$0)v z8sq(lg2;-+Q%bbTm#Sm#Dhg@J0-@;{ZPH5dniZPd8U?gku+}PP`bum9Y6XBEPLTP& z<5f={{h?usQm2p^h#QF7*$@N-oy-PG6Gc4>H=&cq4xI7E89=k0KfLl{p=S&vacb%E zrCG-q+!>ItH2IU#(|O&Q)wJV*Vsb|_v5nG+HepX{z-Uvmk2O>T!D-TY9 z2Lu5qAwjfyCwntvDLUG6!Gr*!NZs6Q`n2tK#h%U&j%rrKLo-;F`7p`FZO))GmFh^7 z_QmXwcITvu#2}am#7GAEJ?VoT0YRZGb?OO!&tZQDLGI7NP|gAEQTsT^cyxcsiurv& zK&pXa5GX!7p7%Y0?mDs0^vwWw>Lv{{y`G9P1^f)K+=_)_fXG_65duFTFET&y$pgEQvRX({5flPg5~W~3UC5f5Q2CGo zeN*sghVKL2g`In_eLnTHNl$aK=eKhE22qNyg3zus4^(>KDHMn5BnH*KXXL&VVolfk#e%F~( zVpuzfm>rs5I*cg?T>}IRfBSS@1!}sJXr1WU_&UOpcQ^41vlT-zX_Q9;YD4bWWB(8i z?f%n&$^Gc-MEiBaEruUtyQP=+jp%-Cl$i{w_X0fH3Z3~Al^49470$uwsQ=C~zWXMQ zB<--v?YVaA8k+30>(%um!F}Fs@6(D{do21Rci+pY{0|d)ILAw zoDmNt3%Y!E2Ee#EMV1ht*WClzzC?Ct)nQ~pyFak3EhGgdU$O=h9LRb7XNDk(8ZBU8 z>h2{p9D;_7o&qhqr)|WmM+p~t_M>|$dcO;%YWcR*MAKvJI)VWn!OY8u(FqVh+SB$z zL+}3&2*yQYm4g;8oCI~n2dq>==VLHV(tSg z=xYVFl#PHi1sDsjgh@p>h@i|>IGC^)v5{RqNhebk47q%w9?Pc=DzREr4q2-(YgQ%k z!!<{M`J^pr7C}pmr81dF`$bApYR6WOhB{q>rHBd=hKmSG*?=aN3ggw-P@^_j-AUG? z$p3P|KKR)UElWVhzoPT(>yMcUZvlqq$t(8ngM%V@s0Q?xBp0jJq+8m5I)6lWZ)Syu z3w*ZPy18KOP~_6>HuHFtUM*3{`oXt3ICzY5xAHHonGm+?fQ9J0cB4C$gZhb1Y_$%?Zq zLMuIzM+~MAIEs?)46*N;9kaD|J`MGpL$+RUv;wD(Bq?{pfGc5_Wk?%WTp2+Zg`TyC2jaeUq!tZ&&k8- z)2decehY_fEp@%-p2`mR+&s_H-@(UwfPw$2|4Ghvo$t$$z|(k=rr%9coTH5jSF1v@3A@6E=g56zH_lfed~E7Z`lMKV8_ z23>c~SSol)z_fwUY@31;afn7x377XBXV+wP!68N9*o;TO9_9%${)Yb2rti&u=!v`c zL`M~u!AjSkf#*#;7S)vAZQsV~Bhyv*`Z>L3KMVJ#CA-fpc3Ys=?q^f?hi}uH?|;PH zw z(0d6*KaP{$7`*;U{8%OfmrZ{DN2z+IW3Fg# z<>$jk*+i4mK3d4Xa26I4nO=n65GRjqSaw~?Xc)=3gp#xQ{3>tPalHG4lI}Gd?}4hF z^c8%_wdyx})328u($@CPcRCH{#xZx(YEY#TSe6vG08{fVSJScHPnNIKWlFKb<5t(I zD~kh2JIfAzW1%K907_8j>L<X_NOP`p@;UVlGX$kdmW zd~-ZNS*om?9*7b4KGi?N*?&nU#2td5JBQ)Jj~_oyN>5Ku%Fc$c@}@7W9%0nFv=mm- z-CfMocuMR`6T%zkAy6<~5WCqN9wFCtAibCNE<04qc;*yy?BE3E>7-$5?>5moEXUec zGeN0WP0@e3VRhrVb;ryx?~T1V2+U6m`C(R69%{vpM%cIHE!+( z6w0@yh@XVF!A>&(mxi{DhO2g`E#hbos#gCTnsy{|HbeytmPegn`OG@*bLUHT#M%tO zM@6^j!GT72x5M_+35RB}FFi;Z>dI-|5brFWJMv_{q7f$xnU;gM!-KOje22vV&HPm))nB&xMy6&s!sT5j%CE><#Buynqd{CIHRd%f*=$ zw9+kY3*+z03TSaWyP9X&RD$*SLX3RC9ro-hJZ;!%O~WSQQyL~@Qd;gE;(__h{YG-q8EkZtzp zqhV5PEML1fz}>g2PVUGFs60565tNQ@zf+CS+I_e?q1|iJ!iEoEfU}~7cS`p&8~lkA zrO!NsgMCX2N#CN(Y(`y6@5~I{%F)4buXr8GYujxKfaY-vcM!PQ9x3yT^#Q9rCUuX-ErWM+ON%VM&ViaJcZ1#z;cuCL*VkHfi#U?1PONqu$IBm$o#9AyWy;lK zKMA|9IHfhJgSkO_$Q=iQk7$>#M~KnJqPHmRe%jGkzk%cNr|!ZOp734lI-Ar&eYR^9 zTYE8;KFAt_sV|4v3O20`GM68q&lsab7?KTH9WE6*Cdb?s6iIm?7TuO2e|pFdk`vVj z-6M5XH04>BXe~sti_Ui5u~5dW%~LO;tu;CJTxO6OU+=YXdxzFW?M!oPM*N10tUND_ zqyy-VV5dfBW~2en^n04fs+&!IwI1%`GRk23mG3zWuZX6&Hilh?^6ig993gIB4{-qm zk9dM$`3Wbbn1S_W590L3sLQUwUONO$5Z%TX!JJvCk#qW&c-)qF@V9ShsHFKw4Kran$6ap?-L4sk#9&XU^wdVlQzVBQWrD|%P*IS=#VV?$~UoyDJ=A8c$cb!;A*- z0Akn^2l*NtE0#<8M*EnaW9DIwAaHaO!O66Tzsaw(mK4LiB#M(1 z;(&z;@4`i3)L94G*$Z~Lel$(jsr{L$)z`*_6pvwF%z86%tG!U zVOz=Fd}M!dTZr+wM7NhMiQf~QR5U4LlX21$VDKM##BX!4MY8M6#Mo{;g2F0=+pZ*N<0yE z;5b~9_*PY>Wzs%!9+6NIgu*#lvdQP_h?a6$!F3y|dHFaFEx)KTRgp~{C+_Zp*F^e0p;L2;SJvc7IF;__r-=J}d ze`Hc@0>_vwM{f(@jH6HK#PN!oh-0vEY~t6`jekMG7#(f!?BDFJxXxjzM5FXqViq@t6ApQ(!Bv5rQ_XnjPg@ zQptU=u{;nq(BD$w3c;woVy6a?E8AlvG2v=*OyuNO*x*7~9s6xr$=sV5E=6#i^+@cl3PVC$KTFgUFRGz3QZ;4(sj!&AtGFLcq zKe(f90SAgHW%!K!3f6vCAy(kjktdR=<;Z{@q-sGiirxlswUb2oBLW~jtg+jm)9n~O zrYsLyL(XM{L|>!55PK3?u(e|YeL#-0Zfk+qsx<8e^K04XTDO(Z{po^N)=)2uH4G9T z{iyt|dNim?hkNUOV&+$rZ2TBW!mgcGnhh<`FW~*IkUtE?&4E`a6L#tEuO)n*>+oue zXxu05Zjsgwh!;+XG;SWr=^>zjwanq20f_M^Z3NAo*o-3`LwTl3#Hd8B-&<~&QwC_O zX@^x49^W~GIxW!bT)k#Z8?RJFHs!D?$Zva@M%1&rv;XX>Yk_5R{e+`BGE?$I><#ao z6a?aO6V`jWb}f&>SMl!Q1kZe%%~;1Hfe`lW1YsYOIr`=gYk7@`_Jc0bk+&!Llf(@u z_DzAwvJlKSuhb!m+Ku3n-lx0>=xL-)=%N#!hwvZr2aBb&PTFTKnOnVup1Uu`<5H6j z7df1AI*tu=-`-hIi37IBW8Bcx&n}*vT{${%L`FF1ERd}{iQ|EQP6$0PU@-u)V)hX> zy5@Ai!TtMQn};?H`n_R@@Q)JO?%Fiq59nc#D3A!yvuGGlI1b;VE`$5Nc@%yWh^UDC zU#0(61ldJrD+ba!KaS{ApSk~+a2yQ_8yGdv5`m!Oy8k7OF-l%kj|bFhRjJ>kO;Kyp zKIe|POb;BqVdej7mhr!&*x=uvxq z0LUx4PGf@2gS(P0y~!R>V4CrL6=RJZ`*=hx5Lc3*his;*VHDk`xAu3WMmKHmy_rP1 z2bq~U!cSRQmG{}o%4X@WZ?FiEH(z@I1%4oQf{1?+KX3l2@?vvQwfW=JRI@rpmpbXSspbY{2JM#QUrMtI)Dd>~R%b>>HOG&BOR14{^wzhc3>F)1m@JdQbHH8-A*W-baHugwYkExf}OCrC@<{FWo z9{8TLc_ZNg3)V|0!Oor5bbz?*$Sbbso<^sX^9tSeh5JSanx0^T=$Zv1tz(+(L$Eav zL5w&A2-{IAg7gK5$gZwVf+wF%8LB7BAh96KIM>IALP}xjW_8Q$4;M>pAzy_b@L_rc zYS_W*<=yRSWy#4uGREbR`|*Kvt359MZx|U}xw+sj648)XijjAFdYpPfdQb>dy$--M zf2BO^p4zi?-%!mwS`ICKf_=Je@9)E7`yma=GvD3iq`4Hu9IrS=2S~@EWQ%1*R9BKp zPl8?j%jB3fEA9RxT}caSqQ?a)+ud1|KLOWUHU(dL{Zw37uvo}cq{>>4o9r0{_{AF1ZY7 z?@yMl{jwd06jW|ANZZx`Zme&FuqK zT56Rv=+x_mowwsp+x|gzwD4H}9LNc^^)YiSdbo^8nbJIK0Lw5fvGN}|n0Y;s$FmLlid zx?B%sbNx%%n%l{v-@bQ&XDrHPFlpuv1-T^LvC7hJ`J$%C^l?UV6dd}^Dl#fgpN2;n z6OU-qy&!-*YV-AWaJv%>1bzUnhtN0uw5mdU)N0fxrl0fW63X#AnuylYwUwzrkx=I& z!2Dcq_{c3ucqTiwk`7vs4&h%B^`Q`o85*kh6(>|CUf>_IwwmK4+}D6YA;V;d)N6MP zF9dDpOdsvC)2-_gduk`3_8smy-)i*B82Wslw0B1F#hiM~9}Xa+*qVzqbins0d5nq8 zF__rlR(VG2blpy+d61cLyD*%EN%GnPvx|&XP0+n6>Vc+D>$=_{ubrv^aEaab9!Y+J(TjVatYsU5XR@9eO zwC&z7FHHhyrFEs%O(UT}kT&sv3A}7w*~A*E@;J}8$%;82~U2oWb3tc^T2~;@sgwu&+$hvp5A&IGTB&XpfF)$N6QGy{g zBiXX3!94oz`byF#=)Ve6*26u zjFo*9O+Y~Ff?%K#fX(5#vmw}Xu=x*eowRDgNPC*OUF*+p9Kq^d3XcxP2nYN|moG;I zj`){zlgY?Ep?rrul~C$0%Pi<-ZBbe7_KmoW0~Ry*YT^f6%|4tHrap0Q#hwOT*j2E@ z17v-3);xhx{D=O9o?s)9PrFg=q>iPAOMuOV7qXr@-dQ!PR;F(mJV2inS69V`liV0Jf z+6&I4W4h66x-=bAN5z~D>09ouGrzi~TPe2g^VMkdc+MKOvdu_(h2#UD9H>I20Fiq6 z`kcGHN~hHx)7}wG04^$%lI<`-*xoCBafW1my>nW$p0j)c0>JEkmdcZd^n(Y`Ef&|Q z1SYH%W|aBk^hagtGkS zb%vA>!6vN{bEtB+E8k|P?}?~bf<|(sPW}cb4O+Y_Zoj98WtRX$zY)M`9KgSAId1A~ z@hR-ZLlkcLlOF6%l-?L&16Ae8Yl`u0T}!SS)&irstZy!hQ%tDtFOt^6%2|x4%W<{Q}bFMh$N5PEdcW2%|t>pZoBu1 zRHm0!Ue2SuX5BLhBAH9)+geN;c^FI=qbV>*@v?_-yq?# z9T+~dcX~9s+{Gu6qy_@N?Qz>vZ6rw^BY^XwLC<$#n@&NwF{6g9Gg~4y#{`EBja5+L z&u?~%`=ZBsBBLBb^hxNaTk-G}wjj8x=w6exs+#o?#!N~1s1g_$NbSPjZ^6WBkOBb4 zP+dt~+*O)h{#1cs|Ad6^XrfHw!V5numr5boZYqWhrQSB-ksxX-hgMPl4PfA~Jzzo+ z2;dr>LRGPnZEHgwduaKBms>6766q2LLVAf#vJ-B7<7-&T7igu>H{T5UiHq6*gQz2F z*~bsG7+Lerr~u3tWF*|N!Y-P(odr!W@d?Doli-^)ymqxS1Z+GdTZa-fP)sP~A)tKh zshJq+Ntc_8)&ATnIb@`f0nu|8MUx%V^b@t@h27m9DwL!7sGet{;QMWSf(@w+Tw0bGjaikETl`Cl)m$7n;K$b9B z)2#nR3=xD^s}y2TFDu+I{eyP>`H?#fdkI;xoP+kZiOr?zx+j>9!Jsvsa>`qT#3^%_ zuiJZ;LNR<>q|xxgbq5Ug>aXYsKMk}Lp z|E}C&uu2ah3(;^TjXdE32eg~8wH9JVmvCg)R@gpdoabW zYbibBMW37N7(o^~jEm*pTVcvf_hAjG8%q4t9#PMi*?bAs-V41kA~r$cgaASy#xSeI z&nWl2=2^bOD{Hjb33M6hpggYWt@;F~uRUm2I^M`izx@^=Mot0fDEFX{m{@w{0U6sO zRwihZ8akZ;jAMy&o^nlg+R?ZBq0-$4&D3*@NmH0}c{05;E$O<-;I{fr-7ZT-8qlOoLp%sPL&R))4igNEnomg1|c&PIb!qTtO|e#mt)UMXp))$X~_ z>cg_R)-*QNkj$cY{4R3tpV(+Xnnre4iLk8pN!P$n{jUj>vG>ekw;s?n4z{{JAY~#Y zp)==gc#kQ?y?Qq&E4OGVj~de!mqXtj-+mA`aYmAO?<{;BI?4F3O|y1?)+ukcJxOkX zc9&QU6PRg%S%@oB9y!T*e}Mr6oAMMbWToxAzJP(8}B2}gn#-B z`Q@Dc?Xny3El$UQYZOMm?1M#EQ=G`@gRlD+ZQjjtB7o6j{u4xD&5sMO8Pc$9n0YnR zD4YHFe0iyc0QZjhhMG67{3W7{1t9{{{)FlZ*&k|38hvw}2K2oIi||ZZ8X5W zeuSdcRB>Axg2Jss_E#69SesegaYHrnyjpKG{R3_YL(n~h+7>rne*Smz4%L&^so(EH z?*m&2Wa$5Qrq%yW+Bzah5eE0V^O=1#Y^fP&;KKP!=St7ohW(TA{l7qRoYYbm&tKJ% zx^PKP#^omRu7kbDKS1Fn$u0NVWSJrfK@V{O(=JQ0M#(}90bwf;b^Qy6UOPM2{fW{# z{6x{Ns@M{BSRPBcfZK9#eXXo*H2;FDZzx$D9g9lZwFq3A>{6<@d(EkUUd^ER>ocCr z=blKXB(alah~wu!>s)b8Vbv~C8iQ2q9#~(zfJch1N?f|Lcr&f8!c3uXBi*8Bet`p~ zi;kW#u$*gPVJwg;~BVSCWs$DPG9#k}wWA z2FqH}iT4;Q9$xRLK{fpYt={S$I)0j(XDj!QJX;722lzFJ|N9XAu)nbU#)g0Q`cdMa z`ME##f2VH#C!qZYd;ixe-+A~yEQ-4MpDI7I_td=RsPLK- zp8ZVo`A)I?VI8c0V_N>FG^Z&`#)ZN@Nd_{N2qVZ_!s-& zFRx#v#sAw02p|1*lKfvPe|7igaPU3W{xCM{UqoJG!$VYP0|3+z4-~0UB+BOd-v0xQ Cp!~=H literal 0 HcmV?d00001 diff --git a/docs/user-manual/images/lock-coordination-example.png b/docs/user-manual/images/lock-coordination-example.png new file mode 100644 index 0000000000000000000000000000000000000000..752c2909d3920a4e9702646715ef82c68b8414d8 GIT binary patch literal 28759 zcmdSB1ydYd7p^_HOK`UYcXxMpcXxLuxI4k!-Ccu(0Kwe}?jBsee%`9{7rr`YYEm!* z({%4%d+oLF`&x-oQjkP~$AbrfKuFS3Vk#gII4TGPW)BMvytB$NtOtC7bCS|_1%Z%< z|N94)MvIIO0+E2E#e~(oa?UqB_0Xm9VXh5n;Cdg+NXSeee21dpSCo|PR#3Yw9m~tN zRxj=9^{4Ob3NVx#$clnYz?7AvWgKO$$Vo5WzY_RpF_b5QIJZ#(zh$pxQj(@vdU|;N zAVZ51LWLAY{S2i!1HlH9L>_+sh8Ro&D?;|o_`mnjNZ~?AV1pB3rE!2)p<+^lz$=n( zB52UUsF0*zl9_;4uu+Mjz$;;?(Epo0s=oxbFuGURg~vzq*;v5nA6dO1vtRnwx(L@(t?v2q0XJg@`Rk+n%l`-rQ^1NhevN(s#a6{ zS-!Fu%VunS)!^PJT)9FnEc~s8n%c=+X?|&GZB$fPe0)+>Rap)}%$OZCv5D;9fkt|I z`u)_mU?_8Rv`Z6mqLmxj7Z=w~0Rc@0!#*5L_vJq| z{^bh(Yi$=MCUY9HvWC*q>I(7nXE@pI(+@1X-NGr9ig=t3h%t+9zL|f}B;47His#!M z^xx05pH><#0zN>S1XP7mm8g_6dYJ|7op0kA$LcP?@xPty6a#>MA`rD z(~E1M&itov0SGD7wJZn~L9jjsd2n#h?eUzbN~eR!klyT&8+dTCNKl8HwU_+%jUqX+oBy(eznfFj}mN>+HPp zxVR->Eh0}!F}+y%o06K^?7SzTJu32M3k9XsAAp%VrPcewG=?RQ9wwsQdPLeYuB)%E#H~YIhPMpMg*T0;6!<|;!WivwAJrrqj%JhFHroG;GCk~s?>^7pIi2VLOj_Vxl zD=RA$EORn`xl*C!iW3s?cKz;W-nzQH+7Y5}&6d-H?GCwk*pzz=OC7#AdJG=tE338Z zB+2wTZFTxE9+!Mx%XRuq$f1c+21M~ho2$LAsLAwM2>96_`dx^tjiSHwYn&gMi1=kZ zosLgWp7kV08*X$+vw}A^YOLqG_qJ2y(mUJ|j*-Qp^oS1AD#9Z%?n=L`Mqt#LgH@O|eI3m-+mXEqLvpyc93frpj)aa$_H z>&YM`ABQby!J>_Yl}oAkAuKBT$XosIjyEfZpxa{duVWo41;y^mzvk7|N;R6rm5z(2 zn>(FWo5x&bc4`8Je=Ci(LxY!Xw#!9ZnzCeQKeqyEDy<{D@7iQ)l^NQ7MxvR4@Kg+qiBzFDGr@5>EgYt2B zd`y)B!lN~Y4a;CQ&CJP3X6sKkVYRq#X=%Y_HlYa`A7=^t+>@5B@98xhdx+hlR1DUv zC+c@QslQnFJaVaa-0s7{@mgDdP0KTg!5iRqHc9F6d+zgpH6w(*@&5Vq(Q+#1?@{`I zk<^vWM75@3KL4w(hlh)iQC8XEGRGzX0b*~jX}b9q%HzK{O(m-bO7qZggPZ#eJVHYH6xKrgvjCifgQ9HCUd0?(+me@l1;yzW)$*;~ z(Y7`Z2<~m({Sy)qm#0QOpXXz;T zWgwR~02sKMDjH5Uwrn=*sLINNe`iNPdg0>kyjWp&oX!Jzdy9j@!nn_tyDv9Ric3lk z_gtA+SRjQSFGcnAux3vJU+un!!G~&c$LTPn#}vMtpiW3_iTTCtmolTz!&g~ezG2o?}dO+TwHL%#8A@Lcm8(g3qPZ zZqUwt<>s##X1i+^a1NEPtoUeCBc(Li8$ocr3K0ng_nKU$=J{&90#Ni*&(oPJ^ zO2p?B);q%QG@oH%^dKPerKP9yn+#(R5}E>&V7FC92HwvmnX#K&M*6Y;2*`q%fes6aI(rKg>r{=To$ zb9QVD3^e5A4l7K{v&EK%eQ(`6^}G-E+X8|V6efQM?f*M+uaBRc+;A||I_T|wKA(fb zWB1a++8P}f7Y+Yat7rV*nPpn)!m(TUx6O+s`uc_jA>?v+Q0|nI$AgmUJa~i6;?w#0 zLK^Mo(P7)#yz5mM!Y9P_rT%FqYd;MQ4GIpEgu43OS}W_W4;s3uh$uvsnrE>RRSE?? zkH>k2TUK>>BigT_2$XKOlVz#2&*F`ZKOG&vf#>hviE;SlxVV|_rwb7Z<-R@w*{qhu zb#==50$&jkbw)=eA*dJ`HI!%Bn5^t=c}bXBT&SdFFjS~n zHz-r0`YIuZfc)@95DC6(XP>ROs)~Z+>B z66X&=z{J~ccJZglG`HvfHewDnI|37g3#@2BtN zBy}|9Kf*hzvUlcdcC3U=PQD1tSSPNr_(eBv?iRzV@cmC7d)_GO}F>ms?s8Vt7tz>BY;xQ5-~I zu(|wwmV13ababHQD1drKl?ns2(l%+k-Ivz?**Eg5tiO2g9=J@PYIJ*hg1WqX2QJFz zp(UTqdTjI&+Q}`4!`UPvI=U%2`E7VO{ow)K)s+Za*et9Z99Ysf~8`^7~fz$l== zcXb#K-IOhDVqmCsd0e0q5b!x{a8Yw^Qo)5xh)YPAY=VmJfeY&BtgUNj;(I#K~ zPi<8doYC9U8mqURJxq?^XG;Uq(chu$_4euN%D>3?9AiVK&E2nMB_-)Vs9R2+`X9)s z=<@?P?8guN!9j2r7ZzwloZ6BS8Y1(xw#hTva$^kaqlvM`g9A)VOm!fKFI6u$0M$1T zlhgU1cj-&-PfS4}LoGW-R|5V7XlO-`mq?H#jP)z^`SUs4f!V{-J=h|p<1S?|=;$%m z*9a_#K~KpY9UaQW;j+ym0|Tnj(N)v=+kqc%eO@%ZqX{8R&8k{-P@u&{On7)1!}>yC zHeVal+v(edyo1{R0u%y~wxm3GO5pK3J! zekbv=>AdInbzMr7sz}CWpb?x8Ssb)6p2-RtvyCTWBt-P|oR@i^ z={h_2b#=9I>5554SXh?BLcS#@5BB<|-+AdFA#Emowe&dSprSrm9ExbKGvJpmRiUTj zVbp2qZE0y}V0zrEA|NP>2-Al{ejws@p#TvQeiapME+~lE)g%@~76=Sb>|HvV#J^v6 zj5&jL0`j+t%I^I?UFjx9~dG&JV#lE}NW}mIAqqD-5 zztLY@YMznyFH>GzkMREP@W<(UwexO-c@-*@}pU+dRaR-Rw;XBOF)=2_uT zP%yBw1p^!Mx0Pwq$OK&P_hc&6)*`>CUT)m<`Ys3VOL0nf;ipez@^#)-s;-|ZfYH;8 zPfRJWGcmQcv>3B!11STGvEO_C?pK^^zQpAY6Fnav6L24Z!oXR;%4TzNZSC-2B7M%W zUHO6O$uBh(ECw^%)@+m=(i{1Q)y9r)&46%z72Qyb>y`EauvB~!q^^xGM@l8Va3{OI4eXMW#D z=*V=r%%->1=4)zIJ39LA;bFe>P2bSy=y0zT$+pxqb_94cV(6R7w>Lg__mj=&Ltwg$ zj65SFs|p%$1-!lKcVqc@*{ycm?1v+%UT*jQNKc=%pan9&Ig7TA&I_$(d@h&s&1f|j zk3u$2YjvqPHhM~1vKuEi_mX43{#ujyQmd^cu+hKuVvmYq238$ZGPDGR~N`n&mquMSFfvgq}fuh@;tHjdsD@V+3a$E?HaeW>5!IYq{fgrTe|3X zn;h--v<9{=ZahOP9dS>n+v!V_V`FrTj5hOSs`{D0zNe<9ghBtszXu^FDM?E&$I)m; zMpadjz(Y<>ECRVVLk`#P)`XpX0}LXyQ?ITDoTM*wEENGI-Jt%@xJ+FYO?$)f+l|PL zW(%-%dV80io{pE4Xxg;}XJ)oOB?ajzb|{w5x|~(1 zSD1=Y!6HdE^%$^_tE<#=*0P7kZe4u=w3Y1Twe)Pvh0DC0a`1R(gZww`%{Mfk$+5BR z`Gl!&C?+fiSd2e@Zv5ES;FHf}`{dPgXsy3$pT3ywp`xyw%zaH+$SPJcYnES%^uRIn4;(`Hxl13jIj~p9oHZ$@24e9>{0q*wpmXDwR^FoK2nc4Gd%RVO( zwuqVxF2w2=@V5N_9=feIi>0#ZnXHyTeg1@hDlH?EB^db8uTLu=@bU5fx;;Z6%%6DXesOrh}nytQLQMPfoJ4vm+rQwzRbce7t$PyZ>8jwFP2rq0ir!le7K&+ztc=2FAd^ zAo%&#+}hfTAxIPaWB}-^^wB9PDS3HYwYu$~lJfF9FTR`GqN1YUU~rIR0ud7Ye-EFJ zw@#qE(Ud5Xk(EUu;yXS)eF4FdcZba`YAPx(AD=p%RvA&z zkP(ww!+rr478ZK?IFUj%H8s2KzFya(DWsPrxFYA)0aF||2V`|6`ZFrC*y#ndIZSiH zFuR_>PXm4ZciDP_{(#NR&0gUCt*x!?CRNv~mDbkM2MCqV1MAi)5dvhe4|!Z%oM3>B znHgjOFjzG;e+CAGdXzLZv4glWPfkvlabg|)W8|@OvlTJbe)GWNqBAW(A!2Gv&Qj?Z z4Kn#Ni;@dUEh+!jvPr)ahgLNs3`6Aexu>G1w+F6rx7_`?0H}oDj7Fj{ZI@~Q4U;+u zIC=Yxjz-}%%qy*hLX}F;iXBJF7#%hB7e*kQpbDz0veMGj>-}PkGa6LBp4HS8CpOAe z^Ly8(Pw+%b2#1$M{@4S;zpk!MHk|>`zkt&4^z;-C4h{s5fDjUF(@x4o5D&igBA4&n+9JP#q27&lTauAli1=5u$5I6 zuy}Z><%;Ghkdyg{k6d{BgfW^3Mz!z8&*7+c{Rsuu4V*(CqsIktKWp*JM&1(|&#?lC zqqy6URR5;u_=Q&z{!AvBnDCIQvR#&VHt?_N>FH@`PEZTNyW(EY50V=AhUj<5(H8eET)k~|Z zb=w<&gMCA@wy^<6g@w$_%mlKF8_)Y*)stUee0=@rRl8Ub$VU}9<;M{Hc_!i2z*NkmDk zx)aOGkm`3-OVfgSwb%QIJfsJI%qS-|c+>U2$MfNp*Rr8xHHhF5+a0xRZ9r}Vv`vF9 zw@P6fRTUMM_n5l6%W@d#)YR1B33WBK(|Da_ddPx`ii)bLag)YVa|GuqR7-@L?-5-P zJAf9xJGAVMz|O`tyeh(nXl`Tk;@pOcT`_?L!oa}5k*ANV#y%^hP*J12L6M`+vc}6m-bFX zUIQ1bC;oY0>nWYRW;m|R%-od%%hec`qeloMq&FGWL}Z9uue@SU?3y0LO5lz3C&+g)xzlyyuHjVet8g*B3Q*3_lR=NQ30T zq@g<}4F90;%7}{Ub7~)Ha|+EpWI^nE`ukfeW`SInG$((skVEtIIJpn2;aQR(jydX8oD}!_mw-tN|l)1^Y|(TkT_M zn$ZV)4+99QVQ#Q7iAn9(!)xS(s4_c?%$Igl!xR5L=cgM68Lk3 ziM#OOr1Hq=4Do}qT#2Qk;|&kR!q^&xsUmVRq?Lo^$9Kws3WN-!u>Df6W*l;Y8ss|q zEA)ikp`9}o-Jv9^KgWnj-f}I!W-HdhNU5L55g#qd9N7uzcXlC?VAltev_kjEf9u4{ z5G`$i8Ad8&GmYU@3hrE(q&hb;K!T3do+x(`A&CAAR?cRiM-I|FDMAK~UgFqC(HncE z#dL&8=>?nfGar)C*bEdDRp1c_Y}JSUAXAkgZh-`rtR)~(RH;`34upo<+Y#p+-s;7A z`|JG)tKA9{?xlSLF3uuWx#5BU|33nUT#_fVr%>UM5zP*^FTUrWORu1gm!C2S)Urt} z7|`Pxm`b`I7HAghJF_`!cmnIM2G~X#pf|n5Zg|_5fmM9| zUdM~4@@~=nVy2XP5Y>UhQ3H7;g)9)yj1<8A>kD5rL5B|F=4gE#c~0v{hfaUUrxqcP~h{5}Z^1OL)c0<*B<^@7Zed3_8IlEbs``1!BB^`QH-I3I3U@HcL0#{-tD9cW+^ zYpao}4jS|c$)z=*1O-~U|GCFqptO@8bdoz@Arz0i4Y$IJIKBO~6o(C7p?@Gu3x{gC zjciK--7$AHGgy8!<-9-4pB${&j$|4HT1MVMxTZqKfUok>TE#!LQ>v8Tt&vZrF%v(u zHjPL6@Ip?4NL!HsIW8?&UH}6d}gxc$uu)}4mXaXbYhqtL&3xk)7WW`Mh|I^g3lQg=~kGx?$9%v=5 zja-`3Kcc6ERND`)M6G06+<<3eH&dx(`Pr>lndHX@B0tmm=yuo!B>e~bOhlE{%pmv; zSy5`3i`E&Pi@OmZ28KmP-M5{PI3USVUCz=!Rjz0&f?o^?<76t|%^g>^9-Y54zH_xm z(Q_M9F)@$f9U1JpfX8zMG_H)o$ea<`juodOAoo9@N(CH5)H~|qG7wYd8;iWIT@Zry zYMtJw1CHV7z4tW&s2<8tFIqp^8Y=L@)Mqrr;y|>%!Z#^q8d~bFLR2DO$^-Cben@T^ zs>6j+z0d}-IWa)?vIul|EUH^6PZ%*g>=|E+ z>v@v_F4|mpaI2g;K{8ODni}n|IuRWkUiG8@$!KAz&6udlF8{FVw9;20$Kt@0dbacb z!veGy)e0b{Ilocc^k+@~-KY!5l)V8p1akd&ta|5V0@pV0$c|?I0Yqw47|4kUZLe3j z)F_r*L83S(1`q5Ia;F{EZC0F?{oG5b`$*suukF*_H;lJZiXJ*>U>OyDcBsK4u_Y7& zv{&PF%3bA!n!i{#@V4yO&`)SF6bb6?iYix2Wgy41rq8w%f>>|$ByRey@gwEaATWDD z84D=ASA*4SB`%zpC#5(JTd4V(t~1GxOvC-JS@2G+AIq8f3gSVuSO%j%DN!StO4!>_ z>I@H^Fb-GzaN58N1_$?%xgIe37*_Z|qEkg#2}@Pi*|6|Xf)A3ym~_a&jKd(+b3AxF zOilg9YiPNL?;^sEU*A|N+Gs{Ow4Cwl5tOt;=1mL6gQXySS>3q9?L!(11*5BGKbWl4 zj;VIm%o?&+dc+X0=rY)Cmnv1uf$hcf3C3Rge^;!CZKfCBRylWwLb+ltkNZ$K z(%~V?lk~j0m?Fe6wf{u=yeq+>e%LBlq}Xv)LQ3%|_LI*QFF*7=V{t5kJ=Bw8DA}Da zum)2TGM|EDd5%tpBq~Qc^oJ;X2S@)vH!g?`u6p^>CPjZY*n23DzOYekH0x|M+`c^H z#;~#H#Fbwb$7(s17&_Z%tRb5?O9UWSRhK60j ztxh6^O#J-4Kuu~${S{b)M0m{Dzb%44GDT9vPg*pu%J11EVTH<+2Jq{m4nKY!{r?pt znL|SlR~i_PQkC;3Nzae$cVHSEk75@*0R_E(q&*WW#4WHCpXaK$P=!rfm%P9g!BtgI z#l^%#M|o>$WvI3T3hwBT=|Bj~GML>a&%}^vkPwMg4-SBZ0JcByJUAeaTZW&VLksu0 z?~E73&-&yZ-+5VUi+E}IY|Rxpg#fWBWSRh9_{cYd9aul@TmKB2y=M!4m@*d!5w8&t zS3L|K=?preFzarw7wLAT7czd%0j6&_%Sz{=u zD=-{<-HdF{5~UCKcacITz@<$ff_gU{17vSsFU0$6oZ}V1E86gQqFZ`1i)yQq3Nm&4gB@Gy;UEvJ2%&G0Kl@!T-SDyJ=xV3K7y z{Xs<$q3lKA;~g(z1KCT40;+9NV!c%?-B%jfpIK{-p0jy05*VcNEl0!8V6}={Dt3XNO(R4GPud~Fd=L73xQGGlYCv1_)8pdhWd}rS!Oz?$Yk4G29v(Js zZUrg~U|rk2SjS0{f+b4xP8XuS4 zvB{z>lQ4_rR@xXzOGCH)7?p}w;KeIX)zD8q{_;n4X*rs!#P@O8g_YOB881yUjqM1_ zP7q0)TcH z>woXP+q0UVEv*7bK_1`VSt~ce{^Hk%S|gSNt{fUbr-eb@SXkoV^7HZ+8;)bQMF#c) zY9u?r#xkGJTacdK`0=t7G?l-dopj@Iw&V&#NOUZsapOLBPF0(I_xX09ysRv-kb!}R znVD0)!4TjK_x7Uv8WGpgeMIH-KV5i0r)&f02O%@#S?}#WAwt69(9mP`>Ks=W7SQhr zTKg3zKfsd(WKZ`qJu)(fsw$fwsTdty4ElY3hT`It5u-u?e4@ud$-toUM)Y{q@A3Ae z`1kMkAgeL72*ZGzb0Dr01QmJ%KWT)(#10P$170V$fB$}au)hMblU#1s;Dl{|%KW{2 zw!+wP;G&Vv0ueOttG2f8Yd?LF5?9w!K7Gx{pOp7Y@wZ!S>pII_&0kby8=6tw@IqpR zDqiYQt|7^mP+0C+6<_@0i%&pX7o+s+ml%vR?Sijh>$Y#^HvtF@050R=+VYd zww_bHC?#&DTD)G%T8<$qkr#OM`0HP%p}U!Q0TL>ltBO|Hcx#rpkk0i}_*{~RPRzG$ z7a2CPB}R*f&1I;k=dGB5j#$VaVFkmaP0>AXYj{dmGIwFQs*pTrboFHA@9J zQ(*Tedn8j>CV=UT%Wi{}LjAa^4nP;@YpoRkx8-?>@$}k1IW8mM~vUOp0H#rTl+XpE_}a>iI?k;LiQD3D>`1#=ZJY12s) zfv@eJNiKpD*6B~qt-{z~y7B_7(d9KLr2Q*v&-j9)pJh!f`$u0HQA$g9Tz(z|4}qm4 zB*D^QWyt*=6IMW$k(rqbZsS;3xJD|zrI(UNf&pdITPHxE;^G1qo077JMTbLYfQISb zez|BpuK}P13HY4bqeo7g1xkee{;Kd{E&v4s_z;9V_Ztm=L&L-Co&cs`3P4a`5OKaT z)>qVT(@-7nHktco%5_FY;)B9b@&`@E9#8gAfpIKU0zgsVX>WJF)^Y+nFn@=eKW*BG zK_rL_?euprK(PO7TG|1qsOYlQ-WFou&z~+!MPE>Syo0i`=iV+^1hV@xSKG`Cts0H} zc&a%d@l83TCM9{x{Tb>Sy80?Pg~L`6=JxAH3R8x-H5t76=h81c_jyIb()1&hX99FX zPakjOFird2v$|9Qnc7N|{(3@u%-_|`+r3OT?&Y_gYbeMSqh1h+(>@Z_G%PpnmX?C$ znphc6fRJR>t}chn`d|nu7%z}bE2IaNmiS^2jou0=f`TG-RE>eu8`o8-;!+``Fjm4q z4~7AjK(>Fj^{P%%uhE{EhA#Q#`ZO4f#H2yaZ&SPThE0LG`CRm97UcyFHEJ6i1XK(b z3;E+wq^)Up+&q6@q3Fz&Dt>8VLF`mz9wM&J;1GMcs;jErzCBkTO&cB^QczM@0Didu zYjZ_Uj~Yn@Bo_!9L;`tA_-6dkDmVqw0%eOR8#k({skiF3FP+W&)H2+j8z}If&Z}K- zEpE9N1H?a6A|kJ|AJ{>usT~Q1GP1t4b#=8hH6N>()z!3M0I}N~_^Axa$vujCq#s^b zY&~O1Jgy(|a$jq0=6BjD>;H@bz$AlCm*lb3^}hFA0EXDzD^^k61ANIJjKtIqMKv|Y zAMe#6FbUYi2$izd@bKdx8k%8*gAoM4`E+qP?^*%Gd%(aSX2iwMjv)p-pJ-wr?}K(+ zs?SC=w6%X3F=fgv8IE9psc9|~5za0wyp|<&QrmFDg_N5pt@!^?F_8+t`%2 zZ0_OVce;Pqa*77rnX7R7r0GaC8RS4tyBK6u15#QZbDaxZ`PezTye0uEj8;nnDE>n= zTz9Af^tOn!>}!V=tc&Z<#_iD_{5B}EFST7-I}M_5^Ns;?-;2$a&5eq`c+;47)d%Un zj;KWSm!SpH+t186rZTfaQ604H@8;e*cjJr1jWX#FH)mh9{h>BJm*rxdMk!71RxBbjH7N zZ1FljzS&#f4M8=e6d-OdDFIe`=Vb;%nzR=HwLY2eetjh-W-|H%_`4tg)51K$hk`n| zzOG)b&nM*b>gnKimf3UR7JO&Z*If2G{Nt`hL(k4GOogFRsirvX&uAz?r~i`O=$xIJ z%B4E*!NKtk;NhjECniv}-gZLX=)k!%l!_dYkGbsoA(RDMwMKoo(#=Oiai z?NXoJrT$YhFf7;H7p7`?^4EF^-L7`$Wr~?UNrO@kZ>wTtOw&eYJh?WmZMMywyYse|nKG5*Pj(-JZ)l}~u)>^L<6CTqo7+-NW`on;8K4Yo(7L_3QS|1!jK+K&>EFyh-Sic!*0`uM z6s#^PGEhZSca1=FZfv_u%FU%iy4JP4BRAX>v83|5O~7e-J1jx&a=g_uAos$Zkn zB!lY_OS79kC|ctkvSy7q6pvjsS_xE85d-W_zP{_?&r{^eXZk3!-li^vg!n#kczTuoo}@9s%K-kx?sXpC8k9jwwT=3p27+bMXtLjd9qcP}2y%kdN4P+6DJtUFR< z(&eSKBZfAp{qE`{^pc~mNo+tRfb;EWSD`he1d4wHQudV$`2czg9Kjjdcy`E*z8_tXqxNZy}DnHkGIY@RAhKDn<@TMx1|D)N4Evc++@d0BqO478Z zrmL!=0s@0V^>R3hNpZTj-cS%Q>b3xkZiPwnMmQJ%rf+6YQ^uNFThT3(NJS}zN5sp@ z`{@~`0Q6asLzk3}j)ZKtD>?ZzLGVMsX5mk0s3ovZ=yJ!Wo5I>9_T+3|4YZ)=mWcCFJsY{1B zpNIqbI^0f%W3W*Gh7&mIe^aIxrO>=!f!DHA7z0ySH01W)dT9C0aQ-mVUwi#;XCR@D zBSV*med^WXvU5!I6cN{TKLm_TbWAx|@pojQ`Df#J2yigQg1KR{x`>F^jZQo~JfL+% z!PQHZ}R}3^cyKCY-OiE5&SR{SdRZp56J0m&^0-c`pG5!0R1b zT8-`KLJXE%(8BeElm9NJ&Dkl#W6TlVrnzo)(i|Rc z(CImyFH(q1#J4r&pcaA5f5y=ITC?8H2o0rg_s7dTVfsj1Y(q&y<7%#S`*9ht@W-ce zUpd6|z8U)oCPk9%%eIaj0z0)Jnv3s}C*+8QOlS8enbjI;n3xw3K7pc0{;vSiSN<`H zsXi1jG>KHV&CWaFYlOd3T-0Pd@gocYJHU@xbgwcUV7Cn1+^n>;1_0VEEmaN3NhjNZ zsQ}6ZXNwLG<(_WSuI49jwOB0)4^IQY|9$-&G-(p09Dhn3n`Ch4k8z;G$cmWF2q8(u zwY8I2E#o^~ElC@&o_+}f%;x>cU1n%udBmKGn);$Z>1;u1SP$RAYZM!chyt=#PvCkD zkrS+znoieo1TjE~Oy}}0x7n{d`oer-rB8U@e&Yf{n2t_KbJ^-xDSbN-g_G@V*|!m( zw`J`AimJQu6zjJCY>F>}vlfO6d19>W@pMeDK5N|AP!)nu6tJ_2mQ7Dg1i89Cy_ifB zxn0Hzeu}A}{rSTb@D`CwM1;czn?m95s`^8S$%&1oD!=xkE(FGM>TP-6M5eB(32mUu zJtdE?bF(UKai)*~Xd42+Qv7=Hq645Y&MF|!*H+l=)2^iHN1IqXumVvSK7V)+ z0kVpUkwbhu;ms{Mu-im-bc}>)^7GsI3&zkcyk>_W26{vxSm|&!4N=QTPxpL#x*-(} z&&bGldVN*QU?OE_#~c7Uy1wb6ZpWS*qfU-hmn%{#=ABcbu@7+58mg#592#okGi6p( z3dlft0!k>dY{aZ3kvo>J$Xq= zjsiFhG&J=1!~~~Wq8K{qzv9i--$c-}cGu5-?U}5eH(qfW^q`Gy7)#4Y&#x^l*DvgY zgFCz6Ai?)gdR@@YiK^WuJO{pm1JX*<520(ym{zM z3S(?+jN9`vCng4A3`-2+Kj?_#&EFr>#1#J)8&60a3L0@>y}dP1G{R(=fr^^+8)O8E z>D;$#FRfO1|CgOH^o;DiWkY3secvy?a=0@%QFM^F2U!8nozUrKSI}^q1T890FtE>h zo`RN^*4Gy>J+=W7=?>`kqg2Sj;dc3!B7KPZ70&>n3pvD!1@D9>R=HS0zR&01`n0qS zg&#egLUy)cf53ZlJpeNSUOcd^7}?&=mnn@W77SRZQ0Di2NZp704D6Z9wqj^$F)|UU z8zPBhjPbkO&!{m?4DQRyipdkGZBHEC3kwg2hJ@7X^V3#WUmO`Zm@Adl?~?LKA{1tW zO=gO;D6Z&u>URoA2+z@TcD}gS<|a^Za&i(95;8I>NJ>hIiIMg4>d0JeSHu)W-46MN zShNa&G1JqNX+KLY+*2fRaY zO$rre!{6-oL`lEK`IR5|adEO#EA{?H?=Mj7YT3~tq)#X<&E<>b?cLB^oy&K6Z;gl# z-3O`q0~arNL_{UIBrf1vumoUXSWf4;dw2-A9VeER9dfG8kpUE+np&!jE&gww#7_p7 z9~iTCazJ`!BAGSrJDI_b7tXapwZb+ndiD zyf?wYg30t(gk51Z0>`VPIyJShIMi-6=i#598O%y5;2n$oqnE|ILc<=H|O82K_0%KmpO@ITT{xTFY6*F4kc{T0! zMDq5=WPu2e@C7%@VUx*$U~X=vjs91>xWupJV&v9#MXzh4InO_ei)16eB5WSuj}%@=?$7p8(d@UZiXK(UjEn+XO^{I|iGU?F z2Nr02LNHb}>S8k{oAV16w4mTd*S|EK7+w!II)*$J_fyVkgt^;Du_D#-He%kk#>-%c zWG1s1AZRmj`Mj=cudgdr%k*TWRz6CfVZ_N0318aD^QTau78Fs&#+aCb@`$$j>-Fav zzrrJ&X-*yN-dl@_gAM$k_j*JYdVe)r;C6+^W*nSC&gKwSML(JjY+Jymt!~7`{Q4{Q z`S~dS^916l9@vO|+{U~DfW3f$6k51lQplFr-8=v>{;r_9JDBKJtz7=5cQlV%B%rPz z7XtJ50?E}?sj{Itq+d{g|J?79;pLyG4&L~9gOjoSWu+tuC>seLic%3+{@NYCGNIb9 z`RM@tvV^TI=jrL~r=!G==d1G#uGyT10EUc!ja-{f4~#W;Q-m;^EWlUjKAt<{@%#K) zX^e@EURvnhGw|_w$Mm=u?DL0(1Ist$6;%`d8G(PgxY`2edbB$&SXlTy6jClHG6Scg zYAMe$EfX#Yw%c>BecR~@=#4(#M%fWn2M23v2HQDRf4d5-s2Z%*?Qkje*uYt}1c+9LKEObgn;)QGv`R&HL0jp1t3~8*AqdhZ)omU z3Mp%wd919gmK~kWmz(P-ZYc^WNQGLeci7@1tOo~Y%k_p$W>IdJCjdqN^(&NokLKZ_ zg+i{4u3paPs;2Kli=Z2o)TuB#cUIQmQY|j{ccJkM4gl3pgQrlfD=Ra;q!(pl^b?S* z+uHn!my(pEe4We73gqF^(gTS?zlk)LxinLwhF=RUaY6&`Zn1H3fTG^%)&w=f=Ti!F zC62_@^YOL$J%IB!MUVnIOxd@-amg+`a^sm3AnVaHGCIVLM^V&ORf&oFy=S#3RjPS< zK5|M++qXGfl-3FSknty>Twb6s#~ho8(AH6CZm=@c*FV_Dz$ElPo*~uO*SG!!Y^nh0 z?As((e_VKYJkW9x6qIVe&H(5H?@#2(Ov78f6NzJL%yCWDUnybe=>bgBGb_sn4TDlA zwW!GR<%K2VT3TECzgA-)(Lb*?TW)?n3Yu9?GkB-J{vQ^gP)X04?cF5Pp@WLH|7vNn zyz10U_gtJaSO9Ga;NSH0xNn|)U?D#VIP6pu6acR_K05jr(67cv zl>QB|F*Fo=X=w=n`5PaYbXr*y6$9sS($MAgY)-Q|@_7lRcmTTeXfhj{m^kolt(6m< z(xYfLTuVC_Xo#`8HnOsnrG_OZN5oa>b2&KSb`4)!X^D&Vv$61a+8+aYa>-?X{HDrb#k0;bMx2ftf9)i|}0l&H50ix*f_O=UHoW6hmo>pej-NaIzmDN^N zWx>vFQ&~wx{k+AIA(xT8oRLw|xEkf+QjZgxu6H@HvT}d#2J{7LYHCVKN_zBGKb!6U z*Q4*ix7zdEQUj5sW_6s_Y#GS!emX?Q$Y+2Qa%0$^NfI`QFR($v!ZNpTf0s_mp7)Dq zZK+zb-gY^_u4Fu_?0m5rR&&|Vj8L2&D~8=+qr-Xc*W$uLkN+zNDJdyHH`|B|=CwNl z&@V1xcxG!WZjAZB-U&Qpms%A=c5QJ%!5<^4Su47v$f-SMDh#5KpO#D=YBt;nnhY3N zu#lZ1#_>|Al7;&XTGs07W7O1Fms>!ihShvoTx;vU>v%%Vdj0h}eO_y#xAf57-uE(j zL4+`HnluVw=heFhf&Ja4<2~|kx*)IHZ@)J-^j=nrzM%gK0`pH!*#lfmV8@472{_%m zcQNQ_yFQPX^k+h+rYxC2Q2Yz`zXJ^DloWtWY-(<9c6BLob#Zb1?*VYbKyAlOfSf;l zbocDWldDiZF*XKpnY1)C09kzcR+=3@<%JH&;jHNB=)VNzG09y4V~&CX8W6jE?{R2^ zopEL5;}qrNhylH<2Ey8fL63_K_}>IF**3M{M3t560zZWTzYAy}iCci2h2-}_XO{;zzCT!`qCM zWj9;u+4A)LF>^_04j3K#^YbbxXdWM?^&QO_vQpLj&vb z(lvp=x`=Ro`SCgw4nex@|7-56zoPp7uSZH$Bn0V{H=RnibeEKXG)RYZ4Jjd=B8_x| zbax}&jda&AbbJq=_52aftoenzhI7w7^{T!1&EM|&ArISf>toeszad5Osi^p1X}J&I zS9ICu%vUCQXB;3QsxT~o5lH~%=##qoPbcp0VluLW#U&-)cc*eQrHCdZgDb|Sc@6bW zt1i<8M+-7hI^Zj&)+}YQ5;h5E7H5Ih@1(n8oqKvXv*rf`(>&A}<}E zu;wS|9C9q(=Vo`ug_^C~%fSbvcXS})Rsvl8$>QR(R6gj&Mp9Q3ZdQ(26+&j8Fji?r0gl(yBL3bX?l<5rRC0wiJs&Fbxa@NsGZ)_lt`S zlHvhUB|4eybaWJ4T#jv!1>I(MH+T0BiNjJo-zRB|NBpMfHejXa8xxk zI77{435fy_UWWt9VIE*pHG=c^FNV%!^j;uEyv+TR*L1Z$JDgfsYn2FuG?Li0XyUZw$ZgrK<5tS?x%*V!%5^I2KK>`JgcuSJH=`lU%>(D#H@_g7i7BVe**~~I z9;MYCZh@gGokPN7z)V2_)}wE&ZI+;Ca5}Hr{oS<=6_pv!6!z%}CYr=%DBh#ZqL~1+ zA)vTe)!zOLur1t&@d-W$1O=I1pKVG@OXGyZ#m06wI8B3?L|bnpH`xa2IUG(Vr1B-D zq^SK8)6egFhW=(U*JG)ciM6RI1r=3%TwIms^;w!7kqC`dgKXg|IyyQKfi*kr|E#Oq zC^z`2r6pluk#u%8@aBy@0s^4j)V)rCy1E8$ZK2Rt128@-4Gj||B{k=TiM{58wEgfC9@hr((H-pQFa&VDiClTgM$D_gWiL?#NE8D#^%e-}0Vn7k zxX6iw0*UGeWNI+lcXvzGU)akrnWdjW%z@PdaQpG`aj7pjsK0`+z|VRB*QUMwnRz=M zYX<~k4-_rT%*+-WoPe|O>p;tg4&ML*;Xyz^xPk8nRjrbvqCv=ml9Lw(;u-o&5n2$R zntcJt3cXQ zo)i)yV;QUovJr+ZQG^EWj# z%=h#J9*C^=Qh;Vi7MGTEzI<_YJLL-UHy9^^H0S{C1>ima*`Vy~?5Bo_$4Ma#OTb=w z-NQAqk@KE<)1cwtP??mZ1=1P5zRy4(72BV#CPYR?g3QRc-z#!fNQQ?2oqDM+m}nau zGP3m4Q~|d$1BlG49wl^RUz4z7)j z4R8yo8(->EJq_u!tfC?rhlQ$wLImImgQf?cnl5(Pn=I%K#iL}ZLwg!LLnkMv{}G)f zKu18!@lR8R+X#3%@$qlR{cgcC`NV^HZEWQGX}Bo^Zls?BHZB&D0f47CQ?$81U7-SD zN<4$c*~v+i4r)0mb@MYH6Try}d`FmEq;y3cgR zm>IY^JG-iiO6-AFhW7t@_zlp43_1fTnCkGs{C!50{bMiiG_fCzjg3Jko0|pZ0zPty z=tCf{>gwxDKL7Dc!1Ml8M*wcS3e+Ovc6N4*O-!U^Wn*<-Ia1#7K_Ei=01E?XHd3!j zpEk`M4sR-m=A0iN?m<>E3jB%%jPh#U)y)k$F*m097!i2t)x1@k`DMg}Y{+UakRX_y zn=>&reT9np^mD$apZC^-Euvk6W`qU?fSN=7{Y4xb<;< zZjOS<9pmYrBmrH|!rUB$g>GHJSjIC5Rvn7Y&mhFRqUM^0GFmn^y2`@3!d|)t%F^bV zZkldx!2XB|RM*!hE-x-FPSTRocCnB@edy__DVdmPXcjaS)OB^ed@2SYCT}^^-dZyO zM}wzAXebi(<;$43I5DkrdNA^yM5)VfVx!)Wz^OT@>8Y9N1eodz3N+I)sfwhc)s>>v zl%k_laAKpd65`&*N8!G~+Q!CQUB|=0#Idlku=>qa&CbQeIXN|>I<1zVVo7f?5Hb)7 zqYs4*gxbs6PkawRI(Kw@jdmVL@S2O`{q)$_zx8#>TWs%z?=P@(71NJ4HpIYM5ky7v zVX?HbaL2-epNaQ%(u^2py`$R1p=;o-WO7u=#v58PP=R~GENSe!m^UT!7pSZ(WN8N3+UasmTwYXfQ?pOdkpFy9xwHK z8l3h4Od0TYo;s`!7B^6*I}0gXaDmz^o`r;<&2SRr+1gmL+9Skv1}B}xU|bWt?RNX$ z_nPBv_vGG73nw?IwTTg8b-?oC_@bPx%?2Lk+W$rq^9>gEAte(v9Rv7MFt8M5j63Vd zf;nJ@_j^j{sA;YZKL0av|4C2iJsT;f?Zch9N}i0gw9`VQFDO691EiItB#^VPXL!ab zs{gU9hdg+v3vDJ#653>Aflg**$hM%rfh24&#=JbfDCLn(ke=`+HQ}a)mam?hFMjo3 zw8mJRTDH_6!ys*;R3Ck?>_CWg-(8z03R$ai?_+ssA>-9_Kb~%0iA2O~wdMB7zfMgj3u?fh1Ur_P&)EuuxmJQ4 zQRu$UniU*gVO_o3YfJYSf3q^rcxSW`mm~7dF#TA<)@C5=wX7GM_Ij$bqJtRTzC{@L z(;tnAj#mP~uj7EOdfijPyq%9^LJuV}CEwP7g_iMV{J3MCLGo`Pfz?9GP{(4502~tu z>&=_BwKcGtx;i^MYhpz(e8D4U#|!hj#a9{KP8Y{Uk$8P<5UPBk%*gGmQCQ7*nwpQ}GEJZ;V4*A%mky#ep26 zp2uKBvdyMi6_hwPGDF5gL49R2nj^Z^8FUAq(Yv{EXS_Fu&V>=M0Sf$RT8)c{{zqG^ zSlv3KnWNF(hxKh+V%R?Yv){v6-w(R_Fw0cFQCv<2eMYNJ`d~cy-wVy_(#UHS7{QP{ z(|K+WiiM67Pjy3zmX7lEKd88}jrcd+-P>Y}>qO8eoacM=mIQ^nghhLaReDzz=_67e z*u{JuJ+61f<9;rYEM#eW`YtD{!ccI z;*#te%uc}Uz4@QROr}hzVEth5*gWB7$FcAB!_9o5a>VdfY)TiENh3F*#}=EG_72%u z<;xYS&777YY4_A*Vk&xy5+19n+%YqGN(N2wm%^BEj(BB7n^yFu3ksHrK6jLGt>n9L z+nI<%VEwjCZTQcdfu|2DD$uiy{v&Sgb5B&XReNEB_rtQOVmUWM%buUoGN(?p(NHLF z`VV~^Zoi9lAL7|^;q&+gT@D^{OQ(+j^|@}Ig55ES-La90EK#u=soms-J?&XbG$JZ; z+Ak@wgr2ajWJ#{)X6PkFcjPoZ$GdK6o3MX?{EPE)^BT6TrKKfzfc^#5_t}?FnJimS zdLEx!XPYMB_>($e9>dA^{%m**?&v;|z3Bn}d)UGqa2pFFSo+?skMotPCums_?Uidf zL79<-HM>pVgVgOb?hTADFMZkW?HerH;6Qf4eIl+38xTGiM6q|wz)$b19ua`E_Pg*i}NC9n8K_Ooi!1d+D(&X{78 zA2W%CRYpYGk=0STRPo?-SX$P)XJB_66$<4c;_$KJON>uSs&)IDaF~Dz5bI)d*!l~Q zQW0ihs7pw`&la~9;T}ZlEPCbZi>u1%cPxvGQjO~k6q{37$c@Zq_*%~Hc+Cax50P{-Ta@UEG0CXb)h2G02bKd_I_L%^S+k+Lq%ivP> z`K6Vn+h0AknY#5_!41>2rL&@@ke;SPQ}He;AvX=neXvgMM~=YZTjelcDA=_|GiYXQ zPB$RkFYgtbOsp0aubPs&DDoJsyEB;-(y(`ybhZ|f*)$Mycb2&r)8y#auB0@ft~#MM zMw**x#I8TOI&zFbjq0bRGRB$yo5#x0OmVuH|RQ)Gg zkM^iMCVra$dV0&nW%|N&3fd>tiQPJZ`$L$6w&@&?H3^#od|PE@=25G?vscLI5G&ju z%j;_Ld@FZxBmaCSsw&^~_zh)FLkXU{b$8HO%4CO&bUodeTfrDr{ceawgFl+21-=$K zp~al!O>9KXp}g?czEy$=vntxe3q0zNgk zRC$+vPL+k{?tsn2Ma9cT%}z+G#2qDO?l;nmk>H zemoDcNY9o!E0WZ>VB#ou{Iyo0r@Gb%1e0dey92MpgiZ%SZjW?#mh!4|sDuPKDbBqN zUG@Jp8bcix4AzO^LxRCex%X+v)5-WFPzZ9ug%J5j82={D6Ad{!>u2jy*oA)l&a; ze>D(2BRrFM5TNiAI3aa-UVkWBj@LRR6|RTA{v%uaR8bwDgXh8|6@m(2GP6J#B9VU* zM9n}~ffLlMZI}tTAqa@vMBxgb6>s7di;q*7j!0^QZgtR37F(Ao>qlCOLjMM81C+9c~HzT4dX<887f%s zJbBwi2Fm$%-@aXMEyTve^h7>|LQ8BKA;_GRFkdyFbeE04@Pn{P>x44L#a}09zLB@n zuT9fk2drO#OA~GfHNF|T?u6dm zgHvA6ua_Tf^PM%q8elv z89VsZww~!&-hVd})Ozox&gB=ldiY`W7YVR6p|YK%aMOn?b_;8t?G{$O5#@+MYh0_N z$?B7BCq0&PmVpqAOZz1$R{ToNR2t>M2f!hz*_j)%|3xL{-Lg88;JkE5>R1N+_TC%a?hpFt-48F$acJQ~nwTbqf@m#7caq^xl zP_xF9v7acr-&_9BJ67l7I(-gmc6+wkbsOkJ$7KN6e{Ow^B$EK3e$6D;@4ci$*dA|H)Hr?Z1I zWIa-*Q{6MOhbJu@OPJ1kH6jID+`ER&7k(ucxq0W?e!@5oSZ>iabX|>yw!R+4Pf2*= z1lJo;&_VY1^8=V1ToRJ-oX@{Wz<9=7W(995I6R&!*dDX%#w0wXQ&DoIVpx+&;}}zu zYQMjL1+lf{HnDDZlwRNYiNI>o4;Q@2v5&8iOyMGUSg~gNKvMEnp8Ski!W zOKs~|d%FP$ib0~B`j6|5IoOhJ_-Gx5cQ|TUr|N899%=)=dB%sgOWl5}6cW_^m^DrO zMc=B`HE8bpwGQb)R(@}-?Sm=&%8jUG+xlw$gNz;W5@?PUpH zS-01pSWC&tq?eKM5eOphWH0ovjT$7SYA-}g^CFr@C3$#->g+y#SGQm9aiXLI%F@gE z1qIPMNS610=1r6)`NTVxf+d{hUz+gQvU@t zj6Yl*i_I;0NKU99RAK~Ek6m>s!bQv$-fkw3c3J^stzj_AM&_MwU?L1YIJ^o%C-B@* zmR?|zLqjpTbUbuh7_mSnTB`AnT$r987)R!79Sh~-qMEl}>XIJ3&{ELu*(lJQdev-d zaU8&$m9eO*vv%w<0@7hWE`?dnNMb!80`0NAkxp_w)hj1Srx(a%t@<8XstIocZc>~+ z*W~T{8+Qk!e=!nr34I=GE?iR3Eam=`LyM+y2s$OWA#L7Hl(q>=j-m6oR9%rmO;)b)+8rk5%NlNY=+ z*Wp!nJ{Jn>??*i2s)OJvq?6qXWe0kN)@QPL@oX*mncR<$C*A74Dt57#J88md|2k_S={Rn5b)p}Tw^nL4xb;&Ue%Yl_6E(f=_SEuVB17X||;?ZW%b#rc> zG5s=B8WTxbMV)b~%zNP?C3~jOT@PwQE+oJVENGdA%*^vhw5Ba@=aE3R_P>`R$5b0} zlbmVL7VrA_y1J|F=F#5?Ajirb=GWxuq-n5VNm!j*vQICJdA<9GeqKV)BO!9(=L=aE z=jH~BPhwFC`#n>=WaLaOQWCAP4d@HsIu8&0=JF$+m6= zuLkb|=%Z7G(wZ$Si&#S32hx`*TE`3-nZyK^F zu-*eDBV#-StAI8LI_0}}h6XliIHY2j-!qdGlrMUT(s$&;5%d~%&bR)(#%`?FAs&4^ zSZ@D`O~<=Zw}i|42J?+>IqYta!xAVYfwb}LbKhqvra=6?451Vg>34RiUexIyKGN0< zB?kwiJOLu2e4c(=M>((S`QIRMri=U~t;x`vE8%azWKe`E*PqTp&t^ab;T{RwjhOrP z8zjM!IwkU2uIp@KGRmc3kEA=632!eyX%3d(_n(Xa3&G`n6C(KGzMLKDo^3;H8T>SP(>Y=xDWOsHq%drl4 z&4YS0umnrywQDKt$RQVak)vWo7?-W=U%zUEnznI)v*y1%o1OJ_Q82~mV2V|5FYgmi z4kF4wr{lV-+$?MlBn7(Bjip``l+Z?BtymDGcD0^pOp!cPo8J4l^{kt|V|9N4mOVWn z*@=4JJGrfl>ur@{a9`#XrhQX5OIEmLSF#lEy74?cj24~=v)(Sx`NV9>YQF{?b0Q+X zl}waHFpg{;KFmL9viGZ*K(lCW4QBVh_3XoSh6rngaC3efiQ>j6W^m z(MJpg+^p2ppvsW301g%Z(~}d|G`?6om~n>hoX(8=d}r@}-U2C5jK*#qCziKh#vK!a-ph zRC^kpJgBs^(wfU9KBRVV^!A6nxQ|Modnjpp+6YRJkBxI)`S|prE)_Ord6uKJ2<;r?wp=cPrZYRT*lpz0D}d98UODlb-p?E zbGz7*Ih)VfIXL+n52ssn%%g;b)jPIt@wT(+^S!Q}7mlLt_Qv3%%(56?Rt~Q1Pg{zT zS(+RLJuip zvVP=ymh@x~w=Tk8MS$zk6WB0~>#pS}WxP7I(bs!UU85j8NUL;q#w&z`_%y9w3`6|y zOPPp_H{$gc1ynR+R%)U%jp9;I_F$=&1EEy67pB_Qilt<1$>kqc0JtqQt^P2&mn^Ot zZK2k)Z?Mj*FMu2sB@4b-GspEkM?PV z>b;6{0(8M*pBE`Bxf-1x-)pe`PMd=j>57uvP8VUu#sGq#@xrbT5KOuk!QsHbuW5Ff+#$>)cS)>AMNw4kt^60a=;0(gso2-#0Hi;afi_zKy&$619LC- zi7NjwbP((_n2I3!@pIJ=iMP11`ijzL^hx#o%GBwCG_$iG?+uBTe&RM%{QT#ay1h_~ ziW=(aNpaPREad9S+Z_5-u>RLL_anL=3j?}XGrniZm5yPgji>9bk`*dONX zSPQie1|P(3r-fRvkqj(^jt^DJsI^3UNyNHtz)3NuKGN4O+vzz{TLu@y6f=>27%m{B zD@*rmsRU+1TDPI<4x&k=rb#t*(_kU!)m|Dg3|?1bR$Rv5p1?oaX0=wa1U$tb!LmL4 zMtY4AAep8-#==kj;tLAMXzlLB4^sW0z@Qk zGE}B74sR3e6=vXJnbuFm?w5id2j1#2Leq0H*@}~erXQ7y26A$8=A3!SU#$SGvcA%1 zK5$U`0Ma_JAQ*8q1umFy-Cj!nkNnjmkS>i)J)!6Or=}u$Uo{n^ z>J&Z)+eP*3Q}mE!ko#O`N3k%Bn!Rekif=pv8Px%Lc3+1RTQ4Rjv>%H(s*^3>yGbux zJWqR0xK~Kx-Fuc_osEHr@*^-XP|FH3}`RGum1`@6nvg;_5h2v)(F8BuL zD5z*VZ*vt=-B(r=)g^Am)5NVKKPJx)L$MuG4;t!(2L^8e9L^Ozi|Zpu_5)qe>l?Tw z0oK~Zm=lsP8tGSbsW zALGjwH@|~SZldb{*tj@Qe8*(TSuU zs*(cYGf$@cgc#*q0>B$oaut#vdK0g%cIrQG!a-HA5BLU3AlVb-w zS9eMW2Uy$lXUmV@enzRd9yvACtRc?5{gzEW(hL+*MYYaJ!Gq_7g@wKA=&bB$a&@~t zN?is)LrNYsCF70wC`i5b16Kwd+06-{>MkbseGGG{+#{`?PT4r2Q~LXh`NSnz#T0G| zD)^>Jh7W@QI-)xtZ|c#(!5ufh^L~y-ljmoOt%WPsU`ZMsSMzu-n?>?QTz!Tx){NTq z?QJ1I+Rzd@rJ2#`FAZSBX=)-QdYJ6&^asznbn=J;Y47Y@-`zF0GUPPN@BZkIWt=dP z#j!2kaHvg2Obqljr9n;G22@tTE2#2|iu78YHxGK~#5I2y&x|{~P&GMlNnG3AEhsB{ z^1%_1P}(E`bdV;JOyK(VJBFJ%_!ah#$?e12p|79{jgF2EqKAz(DmtIXWIh)g6_p8z z?LQ2WOJo@V7e$XWy6Mf(c2EYJVb-j8D*vfP7p;fH;`yLR#v?9K3Z<(Geoi>MU-<#^bDw| zu$PhwE~x47~t&}T>4xjm9zl%F4^gLRHm_aE|!HDG)IdC=zdb)mz{M>H~_ zWNlN9D_2x$t%&W1c;sW)ND7a?FR~-Zo9L?>8*czX(ZZyw+_HPqsDHE-g9h)-CEQM? zT93apmjn4%kg$IN3gMt}Zcu$b9-W^D$gd&5dI_dCo9D4sBeD{97H1?M(>k5MJddv| z1thuz{q*dt>%-kOrDR0#x!O}XVX!L+o8{9frkJd`MPCVaBi4PG9XxA4L69W^p*tjkEzMPeqt-H-W=6f;!oD=j}bs;<5UlxikKBldl z@bDo&wAXWU58wUZ=ryDfT(a}LF;C>D&t>?ulzWzfTXiU!u zv_3h8At43KDM9?(Y5V1eJc(im;ruXbDIrZdZEbBpNCI@D7q2R_AgiEQ?qj2>r3Dy} zPalAi`W3gRA+9365S(e0&90;(!s5mZ*wFXJIRG*Q z?rsJO1%Rj&QwaeOjM?uJS>51O8uik`nyxr_RmVM>G5CWrrRVj1E zcFRDwySMk$RU)F~U%$@xdy{Apc0kGQV-jG*fpZBcz=F@+-5sd?g18D~)CLA*=;P-B z5CeQ>XKDZY?>~UVprxgyrltnzJO6#dHk`Ku00c;tFD=BKm&+2XX3J A4FCWD literal 0 HcmV?d00001 diff --git a/docs/user-manual/lock-coordination.adoc b/docs/user-manual/lock-coordination.adoc new file mode 100644 index 00000000000..bbae74f4d01 --- /dev/null +++ b/docs/user-manual/lock-coordination.adoc @@ -0,0 +1,163 @@ += Lock Coordination +:idprefix: +:idseparator: - +:docinfo: shared + +The Lock Coordinator provides pluggable distributed lock mechanism monitoring. +It allows multiple broker instances to coordinate the activation of specific configuration elements, ensuring that only one broker instance activates a particular element at any given time. + +When a broker acquires a lock through a distributed lock, the associated configuration elements are activated. +If the lock is lost or released, those elements are deactivated. + +In the current version, the Lock Coordinator can be applied to control the startup and shutdown of acceptors. +When an acceptor is associated with a lock coordinator, it will only start accepting connections when the broker successfully acquires the distributed lock. +If lock is lost for any reason, the acceptor automatically stops accepting new connections. + +The same pattern used on acceptors may eventually be applied to other configuration elements. +If you have ideas for additional use cases where this pattern could be applied, please file a JIRA issue. + +WARNING: This feature is in technical preview and its configuration elements are subject to possible modifications. + +== Configuration + +It is possible to specify multiple lock-coordinators and associate them with other broker elements. + +The broker element associated with a lock-coordinator (e.g., an acceptor) will only be started if the distributed lock can be acquired. +If the lock cannot be acquired or is lost, the associated element will be stopped. + +This pattern can be used to ensure clients connect to only one of your mirrored brokers at a time, preventing split-brain scenarios and duplicate message processing. + +Depending on the provider selector, multiple configuration options can be provided. +Please consult the javadoc for your lock implementation. +A simple table will be provided in this chapter for the two reference implementations we provide, but this could be a plugin being added to your broker. + +In this next example, we configure a broker with: + +* Two acceptors: one for mirroring traffic (`for-mirroring-only`) and one for client connections (`for-clients-only`) +* A File-based lock-coordinator named `clients-lock` +* The client acceptor associated with the lock-coordinator, so it only activates when the distributed lock is acquired +* A mirror connection to another broker for data replication + +[,xml] +---- + + tcp://0.0.0.0:61001?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + tcp://0.0.0.0:61616?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + + + + + file + mirror-cluster-clients + 1000 + + + + + + + + + + + + + + + +---- + +In the previous configuration, the broker will use a file lock, and the acceptor will only be active if it can hold the distributed lock between the mirrored brokers. + +image:images/lock-coordination-example.png[HA with mirroring] + +You can find a https://github.com/apache/artemis-examples/tree/main/examples/features/broker-connection/ha-with-mirroring[working example] on how to run HA with Mirroring. + +== Configuration Options + +=== Common Configuration + +The following elements are configured on lock-coordinator + +[cols="1,1,1,3"] +|=== +|Element |Required |Default |Description + +|name +|Yes +|None +|Unique identifier for this lock-coordinator instance, used to reference it from other configuration elements + +|type +|Yes +|None +|The lock provider type (e.g., "FILE" or "ZK") + +|lock-id +|Yes +|None +|Unique identifier for the distributed lock. All brokers competing for the same distributed lock must use the same lock-id + +|check-period +|No +|5000 +|How often to check if the lock is still valid, in milliseconds +|=== + +=== File + +The file-based lock uses the file system to manage distributed locks. + +[cols="1,1,1,3"] +|=== +|Property |Required |Default |Description + +|locks-folder +|Yes +|None +|Path to the directory where lock files will be created and managed. The directory must be created in advance before using this lock. +|=== + +=== ZooKeeper + +The ZooKeeper-based lock uses Apache Curator to manage distributed locks via ZooKeeper. + +[cols="1,1,1,3"] +|=== +|Property |Required |Default |Description + +|connect-string +|Yes +|None +|ZooKeeper connection string (e.g., "localhost:2181" or "host1:2181,host2:2181,host3:2181") + +|namespace +|Yes +|None +|Namespace prefix for all ZooKeeper paths to isolate data + +|session-ms +|No +|18000 +|Session timeout in milliseconds + +|session-percent +|No +|33 +|Percentage of session timeout to use for lock operations + +|connection-ms +|No +|8000 +|Connection timeout in milliseconds + +|retries +|No +|1 +|Number of retry attempts for failed operations + +|retries-ms +|No +|1000 +|Delay in milliseconds between retry attempts +|=== diff --git a/docs/user-manual/restart-sequence.adoc b/docs/user-manual/restart-sequence.adoc index fa535aee76e..dc1f90e8d77 100644 --- a/docs/user-manual/restart-sequence.adoc +++ b/docs/user-manual/restart-sequence.adoc @@ -1,11 +1,15 @@ -= Restart Sequence += Restart Sequence if using Journal replication :idprefix: :idseparator: - :docinfo: shared -{project-name-full} ships with 2 architectures for providing HA features. -The primary and backup brokers can be configured either using network replication or using shared storage. -This document will share restart sequences for the brokers under various circumstances when the client applications are connected to it. +{project-name-full} ships with 3 possibilities for providing HA: + +- Shared storage +- Network Journal Replication +- AMQP Broker Connection Mirroring + +This page will cover steps to restart the broker while using journal replication. == Restarting 1 broker at a time diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/lockmanager/LockManagerNettyNoGroupNameReplicatedFailoverTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/lockmanager/LockManagerNettyNoGroupNameReplicatedFailoverTest.java index 92830e25127..f89f54e9023 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/lockmanager/LockManagerNettyNoGroupNameReplicatedFailoverTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/lockmanager/LockManagerNettyNoGroupNameReplicatedFailoverTest.java @@ -38,6 +38,7 @@ import org.apache.activemq.artemis.dto.WebServerDTO; import org.apache.activemq.artemis.lockmanager.MutableLong; import org.apache.activemq.artemis.lockmanager.file.FileBasedLockManager; +import org.apache.activemq.artemis.lockmanager.file.FileBasedLockManagerFactory; import org.apache.activemq.artemis.tests.integration.cluster.failover.FailoverTest; import org.apache.activemq.artemis.tests.integration.cluster.util.TestableServer; import org.apache.activemq.artemis.tests.util.Wait; @@ -226,7 +227,8 @@ public static void doDecrementActivationSequenceForForceRestartOf(Logger log, No nodeManager.start(); long localActivation = nodeManager.readNodeActivationSequence(); // file based - FileBasedLockManager fileBasedPrimitiveManager = new FileBasedLockManager(DistributedLockManagerConfiguration.getProperties()); + FileBasedLockManagerFactory factory = new FileBasedLockManagerFactory(); + FileBasedLockManager fileBasedPrimitiveManager = (FileBasedLockManager) factory.build(DistributedLockManagerConfiguration.getProperties()); fileBasedPrimitiveManager.start(); try { MutableLong mutableLong = fileBasedPrimitiveManager.getMutableLong(nodeManager.getNodeId().toString()); diff --git a/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/A/broker.xml b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/A/broker.xml new file mode 100644 index 00000000000..08580a0eea8 --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/A/broker.xml @@ -0,0 +1,205 @@ + + + + + + + + 0.0.0.0 + + true + + + NIO + + ./data/paging + + ./data/bindings + + ./data/journal + + ./data/large-messages + + true + + 2 + + -1 + + 1000 + + false + + + + + + + + + + + + + + + + + + + + + + 5000 + + + 90 + + + + + + + + + + + + tcp://0.0.0.0:61000?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + tcp://0.0.0.0:61616?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + + + + + ZK + fail + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + DLQ + ExpiryQueue + 0 + + -1 + 1 + 10 + PAGE + true + true + + + + +
+ + + +
+
+ + + +
+
+ + + + + +
+ + + + diff --git a/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/B/broker.xml b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/B/broker.xml new file mode 100644 index 00000000000..c515bf7bbdc --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/B/broker.xml @@ -0,0 +1,205 @@ + + + + + + + + 0.0.0.0 + + true + + + NIO + + ./data/paging + + ./data/bindings + + ./data/journal + + ./data/large-messages + + true + + 2 + + -1 + + 1000 + + false + + + + + + + + + + + + + + + + + + + + + + 5000 + + + 90 + + + + + + + + + + + + tcp://0.0.0.0:61001?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + tcp://0.0.0.0:61616?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + + + + + ZK + fail + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + DLQ + ExpiryQueue + 0 + + -1 + 1 + 10 + PAGE + true + true + + + + +
+ + + +
+
+ + + +
+
+ + + + + +
+ + + + diff --git a/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/A/broker.xml b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/A/broker.xml new file mode 100644 index 00000000000..f5f50f78360 --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/A/broker.xml @@ -0,0 +1,186 @@ + + + + + + + + 0.0.0.0 + + true + + + NIO + + ./data/paging + + ./data/bindings + + ./data/journal + + ./data/large-messages + + true + + 2 + + -1 + + 1000 + + false + + + + + + + + + + + + + + + + + + + 5000 + + + 90 + + + + + tcp://0.0.0.0:61000?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + DLQ + ExpiryQueue + 0 + + -1 + 1 + 10 + PAGE + true + true + + + + +
+ + + +
+
+ + + +
+
+ + + + + +
+ + + + diff --git a/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/B/broker.xml b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/B/broker.xml new file mode 100644 index 00000000000..8b9b054733a --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/B/broker.xml @@ -0,0 +1,187 @@ + + + + + + + + 0.0.0.0 + + true + + + NIO + + ./data/paging + + ./data/bindings + + ./data/journal + + ./data/large-messages + + true + + 2 + + -1 + + 1000 + + false + + + + + + + + + + + + + + + + + + + + + + 5000 + + + 90 + + + + + tcp://0.0.0.0:61001?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + + + DLQ + ExpiryQueue + 0 + + -1 + 1 + 10 + PAGE + true + true + + + + +
+ + + +
+
+ + + +
+
+ + + + + +
+ + + + diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/DualMirrorSingleAcceptorRunningTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/DualMirrorSingleAcceptorRunningTest.java new file mode 100644 index 00000000000..d05bde63686 --- /dev/null +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/DualMirrorSingleAcceptorRunningTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.activemq.artemis.tests.smoke.lockmanager; + +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.JMSException; +import javax.jms.MessageConsumer; +import javax.jms.MessageProducer; +import javax.jms.Queue; +import javax.jms.Session; +import javax.jms.TextMessage; +import java.io.File; +import java.io.FileOutputStream; +import java.lang.invoke.MethodHandles; +import java.util.Properties; +import java.util.function.Consumer; + +import org.apache.activemq.artemis.api.core.management.SimpleManagement; +import org.apache.activemq.artemis.cli.commands.helper.HelperCreate; +import org.apache.activemq.artemis.tests.smoke.common.SmokeTestBase; +import org.apache.activemq.artemis.tests.util.CFUtil; +import org.apache.activemq.artemis.utils.FileUtil; +import org.apache.activemq.artemis.utils.Wait; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DualMirrorSingleAcceptorRunningTest extends SmokeTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String SERVER_NAME_WITH_ZK_A = "lockmanager/dualMirrorSingleAcceptor/ZK/A"; + public static final String SERVER_NAME_WITH_ZK_B = "lockmanager/dualMirrorSingleAcceptor/ZK/B"; + + public static final String SERVER_NAME_WITH_FILE_A = "lockmanager/dualMirrorSingleAcceptor/file/A"; + public static final String SERVER_NAME_WITH_FILE_B = "lockmanager/dualMirrorSingleAcceptor/file/B"; + + // Test constants + private static final int ALTERNATING_TEST_ITERATIONS = 2; + private static final int MESSAGES_SENT_PER_ITERATION = 100; + private static final int MESSAGES_CONSUMED_PER_ITERATION = 17; + private static final int MESSAGES_REMAINING_PER_ITERATION = MESSAGES_SENT_PER_ITERATION - MESSAGES_CONSUMED_PER_ITERATION; + private static final int EXPECTED_FINAL_MESSAGE_COUNT = ALTERNATING_TEST_ITERATIONS * MESSAGES_REMAINING_PER_ITERATION; + + private static final int ZK_BASE_PORT = 2181; + + Process processA; + Process processB; + + private static void customizeFileServer(File serverLocation, File fileLock) { + try { + FileUtil.findReplace(new File(serverLocation, "/etc/broker.xml"), "CHANGEME", fileLock.getAbsolutePath()); + } catch (Throwable e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + private static void createServerPair(String serverNameA, String serverNameB, + String configPathA, String configPathB, + Consumer customizeServer) throws Exception { + File serverLocationA = getFileServerLocation(serverNameA); + File serverLocationB = getFileServerLocation(serverNameB); + deleteDirectory(serverLocationB); + deleteDirectory(serverLocationA); + + createSingleServer(serverLocationA, configPathA, "A", customizeServer); + createSingleServer(serverLocationB, configPathB, "B", customizeServer); + } + + private static void createSingleServer(File serverLocation, String configPath, + String userAndPassword, Consumer customizeServer) throws Exception { + HelperCreate cliCreateServer = helperCreate(); + cliCreateServer.setAllowAnonymous(true) + .setUser(userAndPassword) + .setPassword(userAndPassword) + .setNoWeb(true) + .setConfiguration(configPath) + .setArtemisInstance(serverLocation); + cliCreateServer.createServer(); + + if (customizeServer != null) { + customizeServer.accept(serverLocation); + } + } + + @BeforeEach + public void prepareServers() throws Exception { + + } + + @Test + public void testAlternatingZK() throws Throwable { + { + createServerPair(SERVER_NAME_WITH_ZK_A, SERVER_NAME_WITH_ZK_B, + "./src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/A", + "./src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/ZK/B", + null); + + cleanupData(SERVER_NAME_WITH_ZK_A); + cleanupData(SERVER_NAME_WITH_ZK_B); + } + + // starting zookeeper + ZookeeperCluster zkCluster = new ZookeeperCluster(temporaryFolder, 1, ZK_BASE_PORT, 100); + zkCluster.start(); + runAfter(zkCluster::stop); + + testAlternating(SERVER_NAME_WITH_ZK_A, SERVER_NAME_WITH_ZK_B, null, null); + } + + @Test + public void testAlternatingFile() throws Throwable { + File fileLock = new File("./target/serverLock"); + fileLock.mkdirs(); + + { + createServerPair(SERVER_NAME_WITH_FILE_A, SERVER_NAME_WITH_FILE_B, + "./src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/A", + "./src/main/resources/servers/lockmanager/dualMirrorSingleAcceptor/file/B", + s -> customizeFileServer(s, fileLock)); + + cleanupData(SERVER_NAME_WITH_FILE_A); + cleanupData(SERVER_NAME_WITH_FILE_B); + } + + Properties properties = new Properties(); + + properties.put("acceptorConfigurations.artemis.extraParams.amqpCredits", "1000"); + properties.put("acceptorConfigurations.artemis.extraParams.amqpLowCredits", "300"); + properties.put("acceptorConfigurations.artemis.factoryClassName", "org.apache.activemq.artemis.core.remoting.impl.netty.NettyAcceptorFactory"); + properties.put("acceptorConfigurations.artemis.lockCoordinator", "failover"); + properties.put("acceptorConfigurations.artemis.name", "artemis"); + properties.put("acceptorConfigurations.artemis.params.scheme", "tcp"); + properties.put("acceptorConfigurations.artemis.params.tcpReceiveBufferSize", "1048576"); + properties.put("acceptorConfigurations.artemis.params.port", "61616"); + properties.put("acceptorConfigurations.artemis.params.host", "localhost"); + properties.put("acceptorConfigurations.artemis.params.protocols", "CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE"); + properties.put("acceptorConfigurations.artemis.params.useEpoll", "true"); + properties.put("acceptorConfigurations.artemis.params.tcpSendBufferSize", "1048576"); + + properties.put("lockCoordinatorConfigurations.failover.checkPeriod", "5000"); + properties.put("lockCoordinatorConfigurations.failover.lockType", "file"); + properties.put("lockCoordinatorConfigurations.failover.lockId", "fail"); + properties.put("lockCoordinatorConfigurations.failover.name", "failover"); + properties.put("lockCoordinatorConfigurations.failover.properties.locks-folder", fileLock.getAbsolutePath()); + + try (FileOutputStream fileOutputStream = new FileOutputStream(new File(getServerLocation(SERVER_NAME_WITH_FILE_A), "broker.properties"))) { + properties.store(fileOutputStream, null); + } + + try (FileOutputStream fileOutputStream = new FileOutputStream(new File(getServerLocation(SERVER_NAME_WITH_FILE_B), "broker.properties"))) { + properties.store(fileOutputStream, null); + } + + // I'm using broker properties in one of the tests, to help validating it + File propertiesA = new File(getServerLocation(SERVER_NAME_WITH_FILE_A), "broker.properties"); + File propertiesB = new File(getServerLocation(SERVER_NAME_WITH_FILE_B), "broker.properties"); + + testAlternating(SERVER_NAME_WITH_FILE_A, SERVER_NAME_WITH_FILE_B, propertiesA, propertiesB); + } + + public void testAlternating(String nameServerA, String nameServerB, File brokerPropertiesA, File brokerPropertiesB) throws Throwable { + processA = startServer(nameServerA, 0, -1, brokerPropertiesA); + waitForXToStart(); + processB = startServer(nameServerB, 0, -1, brokerPropertiesB); + ConnectionFactory cfX = CFUtil.createConnectionFactory("amqp", "tcp://localhost:61616"); + + for (int i = 0; i < ALTERNATING_TEST_ITERATIONS; i++) { + logger.info("Iteration {}: Server {} active", i, (i % 2 == 0) ? "A" : "B"); + + if (i % 2 == 0) { + // Even iteration: Server A active, kill Server B + killServer(processB); + waitForXToStart(); + } else { + // Odd iteration: Server B active, kill Server A + killServer(processA); + waitForXToStart(); + } + + // Send messages through the shared acceptor + cfX = CFUtil.createConnectionFactory("amqp", "tcp://localhost:61616"); + sendMessages(cfX, MESSAGES_SENT_PER_ITERATION); + + // Consume some messages + receiveMessages(cfX, MESSAGES_CONSUMED_PER_ITERATION); + + // Restart the killed server + if (i % 2 == 0) { + processB = startServer(nameServerB, 0, -1, brokerPropertiesB); + } else { + processA = startServer(nameServerA, 0, -1, brokerPropertiesA); + } + } + + // Verify they both have the expected message count (iterations × (sent - consumed)) + assertMessageCount("tcp://localhost:61000", "myQueue", EXPECTED_FINAL_MESSAGE_COUNT); + assertMessageCount("tcp://localhost:61001", "myQueue", EXPECTED_FINAL_MESSAGE_COUNT); + } + + private static void sendMessages(ConnectionFactory cfX, int nmessages) throws JMSException { + try (Connection connectionX = cfX.createConnection("A", "A")) { + Session sessionX = connectionX.createSession(true, Session.SESSION_TRANSACTED); + Queue queue = sessionX.createQueue("myQueue"); + MessageProducer producerX = sessionX.createProducer(queue); + for (int i = 0; i < nmessages; i++) { + producerX.send(sessionX.createTextMessage("hello " + i)); + } + sessionX.commit(); + } + } + + private static void receiveMessages(ConnectionFactory cfX, int nmessages) throws JMSException { + try (Connection connectionX = cfX.createConnection("A", "A")) { + connectionX.start(); + Session sessionX = connectionX.createSession(true, Session.SESSION_TRANSACTED); + Queue queue = sessionX.createQueue("myQueue"); + MessageConsumer consumerX = sessionX.createConsumer(queue); + for (int i = 0; i < nmessages; i++) { + TextMessage message = (TextMessage) consumerX.receive(5000); + assertNotNull(message, "Expected message " + i + " but got null"); + } + sessionX.commit(); + } + } + + private void waitForXToStart() { + for (int i = 0; i < 20; i++) { + try { + ConnectionFactory factory = CFUtil.createConnectionFactory("AMQP", "tcp://localhost:61616"); + Connection connection = factory.createConnection(); + connection.close(); + return; + } catch (Throwable e) { + logger.debug(e.getMessage(), e); + try { + Thread.sleep(500); + } catch (Throwable ignored) { + } + } + } + } + + protected void assertMessageCount(String uri, String queueName, int count) throws Exception { + SimpleManagement simpleManagement = new SimpleManagement(uri, null, null); + Wait.assertEquals(count, () -> { + try { + return simpleManagement.getMessageCountOnQueue(queueName); + } catch (Throwable e) { + return -1; + } + }); + } + +} \ No newline at end of file diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/LockCoordinatorTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/LockCoordinatorTest.java new file mode 100644 index 00000000000..6937c01b6cd --- /dev/null +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/LockCoordinatorTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.smoke.lockmanager; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.apache.activemq.artemis.core.server.lock.LockCoordinator; +import org.apache.activemq.artemis.lockmanager.DistributedLockManager; +import org.apache.activemq.artemis.lockmanager.DistributedLockManagerFactory; +import org.apache.activemq.artemis.lockmanager.MutableLong; +import org.apache.activemq.artemis.lockmanager.Registry; +import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; +import org.apache.activemq.artemis.utils.RandomUtil; +import org.apache.activemq.artemis.utils.Wait; +import org.apache.activemq.artemis.utils.actors.OrderedExecutorFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * This test needs external dependencies. It follows the same pattern described at {@link DualMirrorSingleAcceptorRunningTest}. + * please read the documentation from that test for more detail on how to run this test. + */ +public class LockCoordinatorTest extends ActiveMQTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final int ZK_BASE_PORT = 2181; + private static final String ZK_ENDPOINTS = "127.0.0.1:2181"; + private static final long KEEP_ALIVE_INTERVAL_MS = 200; + private static final int NUM_THREADS = 10; + + private ExecutorService executorService; + private ScheduledExecutorService scheduledExecutor; + private AtomicInteger lockHolderCount; + private AtomicInteger lockChanged; + private OrderedExecutorFactory executorFactory; + + @BeforeEach + @Override + public void setUp() { + disableCheckThread(); + scheduledExecutor = Executors.newScheduledThreadPool(NUM_THREADS); + executorService = Executors.newFixedThreadPool(NUM_THREADS * 2); + executorFactory = new OrderedExecutorFactory(executorService); + lockHolderCount = new AtomicInteger(0); + lockChanged = new AtomicInteger(0); + } + + @AfterEach + @Override + public void tearDown() { + scheduledExecutor.shutdownNow(); + executorService.shutdownNow(); + } + + @Test + public void testWithFile() throws Exception { + internalTest(i -> getFileCoordinators(i)); + } + + @Test + public void testWithZK() throws Exception { + ZookeeperCluster zkCluster = new ZookeeperCluster(temporaryFolder, 1, ZK_BASE_PORT, 100); + zkCluster.start(); + runAfter(zkCluster::stop); + assertEquals(ZK_ENDPOINTS, zkCluster.getConnectString()); + internalTest(i -> getZKCoordinators(i, zkCluster.getConnectString())); + } + + private void internalTest(Function> lockCoordinatorSupplier) throws Exception { + testOnlyOneLockHolderAtATime(lockCoordinatorSupplier.apply(NUM_THREADS)); + testAddAfterLocked(lockCoordinatorSupplier.apply(1).get(0)); + testRetryAfterError(lockCoordinatorSupplier.apply(1).get(0)); + testRetryAfterErrorWithDelayAdd(lockCoordinatorSupplier.apply(1).get(0)); + + { + List list = lockCoordinatorSupplier.apply(2); + testNoRetryWhileNotAcquired(list.get(0), list.get(1)); + } + } + + private void testAddAfterLocked(LockCoordinator lockCoordinator) throws Exception { + lockHolderCount.set(0); + lockChanged.set(0); + + try { + lockCoordinator.start(); + Wait.assertEquals(1, () -> lockHolderCount.get(), 15000, 100); + + AtomicInteger afterRunning = new AtomicInteger(0); + assertTrue(lockCoordinator.isLocked()); + lockCoordinator.onLockAcquired(afterRunning::incrementAndGet); + + Wait.assertEquals(1, afterRunning::get); + + assertEquals(1, lockHolderCount.get()); + } finally { + lockCoordinator.stop(); + } + } + + private void testRetryAfterError(LockCoordinator lockCoordinator) throws Exception { + lockHolderCount.set(0); + lockChanged.set(0); + + AtomicBoolean succeeded = new AtomicBoolean(false); + AtomicInteger numberOfTries = new AtomicInteger(0); + try { + lockCoordinator.onLockAcquired(() -> { + if (numberOfTries.incrementAndGet() < 5) { + throw new IOException("please retry"); + } + succeeded.set(true); + }); + lockCoordinator.start(); + + Wait.assertTrue(succeeded::get, 5000, 100); + Wait.assertEquals(1, lockHolderCount::get); + } finally { + lockCoordinator.stop(); + } + } + + private void testRetryAfterErrorWithDelayAdd(LockCoordinator lockCoordinator) throws Exception { + lockHolderCount.set(0); + lockChanged.set(0); + + AtomicBoolean succeeded = new AtomicBoolean(false); + AtomicInteger numberOfTries = new AtomicInteger(0); + try { + lockCoordinator.start(); + Wait.assertEquals(1, lockHolderCount::get); + + lockCoordinator.onLockAcquired(() -> { + if (numberOfTries.incrementAndGet() < 5) { + throw new RuntimeException("please retry"); + } + succeeded.set(true); + }); + + Wait.assertTrue(succeeded::get, 5000, 100); + Wait.assertEquals(1, lockHolderCount::get); + } finally { + lockCoordinator.stop(); + } + } + + // validate that no retry would happen since the lock wasn't held in the secondLock + private void testNoRetryWhileNotAcquired(LockCoordinator firstLock, LockCoordinator secondLock) throws Exception { + lockHolderCount.set(0); + lockChanged.set(0); + AtomicBoolean throwError = new AtomicBoolean(true); + AtomicBoolean errorHappened = new AtomicBoolean(false); + + AtomicBoolean succeeded = new AtomicBoolean(false); + try { + firstLock.start(); + Wait.assertEquals(1, lockHolderCount::get); + assertTrue(firstLock.isLocked()); + secondLock.start(); + assertFalse(secondLock.isLocked()); + + secondLock.onLockAcquired(() -> { + if (throwError.get()) { + errorHappened.set(true); + throw new RuntimeException("please retry"); + } + succeeded.set(true); + }); + + assertFalse(succeeded.get()); + assertFalse(errorHappened.get()); + firstLock.stop(); + Wait.assertTrue(errorHappened::get, 5000, 100); + throwError.set(false); + Wait.assertTrue(succeeded::get, 5000, 100); + Wait.assertEquals(1, lockHolderCount::get); + } finally { + firstLock.stop(); + secondLock.stop(); + } + } + + + private void testOnlyOneLockHolderAtATime(List lockCoordinators) throws Exception { + try { + + lockCoordinators.forEach(LockCoordinator::start); + + Wait.assertEquals(1, () -> lockHolderCount.get(), 15000, 100); + + long value = RandomUtil.randomPositiveLong(); + + boolean first = true; + + for (LockCoordinator lockCoordinator : lockCoordinators) { + MutableLong mutableLong = lockCoordinator.getLockManager().getMutableLong("mutableLong"); + if (first) { + mutableLong.set(value); + first = false; + } else { + assertEquals(value, mutableLong.get()); + } + mutableLong.close(); + } + + logger.info("Stopping ********************************************************************************"); + + // We keep stopping lockManager that is holding the lock + // we do this until we stop every one of the locks + while (!lockCoordinators.isEmpty()) { + if (!Wait.waitFor(() -> lockHolderCount.get() == 1, 15000, 100)) { + for (LockCoordinator lock : lockCoordinators) { + logger.info("lock {} is holdingLock={}", lock.getDebugInfo(), lock.isLocked()); + } + } + Wait.assertEquals(1, () -> lockHolderCount.get(), 15000, 100); + for (LockCoordinator lock : lockCoordinators) { + if (lock.isLocked()) { + long changed = lockChanged.get(); + lock.stop(); + lockCoordinators.remove(lock); + //Wait.assertTrue(() -> lockChanged.get() != changed, 5000, 100); + break; + } + } + } + + // Verify that no locks are held after stopping + Wait.assertEquals(0, () -> lockHolderCount.get(), 15000, 100); + } finally { + try { + lockCoordinators.forEach(LockCoordinator::stop); + } catch (Throwable ignored) { + } + } + } + + private List getFileCoordinators(int numberOfCoordinators) { + File file = new File(getTemporaryDir() + "/lockFolder"); + file.mkdirs(); + HashMap parameters = new HashMap<>(); + parameters.put("locks-folder", file.getAbsolutePath()); + return getLockCoordinators(numberOfCoordinators, "file", parameters); + } + + private List getZKCoordinators(int numberOfCoordinators, String connectString) { + HashMap parameters = new HashMap<>(); + parameters.put("connect-string", connectString); + return getLockCoordinators(numberOfCoordinators, "ZK", parameters); + } + + private List getLockCoordinators(int numberOfCoordinators, String factoryName, HashMap parameters) { + return getLockCoordinators(numberOfCoordinators, () -> { + DistributedLockManagerFactory factory = Registry.getInstance().getFactory(factoryName); + return factory.build(parameters); + }); + } + + private List getLockCoordinators(int numberOfCoordinators, Supplier lockManagerSupplier) { + List locks = new ArrayList<>(); + String lockName = "lock-test-" + RandomUtil.randomUUIDString(); + for (int i = 0; i < numberOfCoordinators; i++) { + DistributedLockManager lockManager = lockManagerSupplier.get(); + + LockCoordinator lockCoordinator = new LockCoordinator(scheduledExecutor, executorFactory.getExecutor(), KEEP_ALIVE_INTERVAL_MS, lockManager, lockName, lockName); + lockCoordinator.onLockAcquired(() -> lock(lockCoordinator)); + lockCoordinator.onLockReleased(() -> unlock(lockCoordinator)); + lockCoordinator.onLockReleased(() -> lockChanged.incrementAndGet()); + lockCoordinator.onLockAcquired(() -> lockChanged.incrementAndGet()); + lockCoordinator.setDebugInfo("ID" + i); + locks.add(lockCoordinator); + } + return locks; + } + + private void lock(LockCoordinator lockCoordinator) { + logger.info("++Lock {} lock", lockCoordinator.getDebugInfo()); + lockHolderCount.incrementAndGet(); + } + + private void unlock(LockCoordinator lockCoordinator) { + logger.info("--Lock {} unlocking", lockCoordinator.getDebugInfo()); + lockHolderCount.decrementAndGet(); + } + +} diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperCluster.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperCluster.java new file mode 100644 index 00000000000..2634b06060d --- /dev/null +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperCluster.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.tests.smoke.lockmanager; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.activemq.artemis.tests.extensions.ThreadLeakCheckExtension; +import org.apache.curator.test.InstanceSpec; +import org.apache.curator.test.TestingCluster; +import org.apache.curator.test.TestingZooKeeperServer; + +/** + * This is encapsulating Zookeeper instances for tests + * */ +public class ZookeeperCluster { + private TestingCluster testingServer; + private InstanceSpec[] clusterSpecs; + private int nodes; + private final File root; + + public ZookeeperCluster(File root, int nodes, int basePort, int serverTickMS) throws IOException { + this.root = root; + this.nodes = nodes; + clusterSpecs = new InstanceSpec[nodes]; + for (int i = 0; i < nodes; i++) { + clusterSpecs[i] = new InstanceSpec(newFolder(root, "node" + i), basePort + i, -1, -1, true, -1, serverTickMS, -1); + } + testingServer = new TestingCluster(clusterSpecs); + } + + public void start() throws Exception { + testingServer.start(); + } + + public void stop() throws Exception { + ThreadLeakCheckExtension.addKownThread("ListenerHandler-"); + testingServer.stop(); + } + + private static File newFolder(File root, String... subDirs) throws IOException { + String subFolder = String.join("/", subDirs); + File result = new File(root, subFolder); + if (!result.mkdirs()) { + throw new IOException("Couldn't create folders " + root); + } + return result; + } + + public String getConnectString() { + return testingServer.getConnectString(); + } + + public List getServers() { + return testingServer.getServers(); + } + + public int getNodes() { + return nodes; + } +} diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperLockManagerSinglePairTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperLockManagerSinglePairTest.java index 634c6a80f0f..1fea82cda4b 100644 --- a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperLockManagerSinglePairTest.java +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/lockmanager/ZookeeperLockManagerSinglePairTest.java @@ -22,10 +22,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.activemq.artemis.tests.extensions.ThreadLeakCheckExtension; import org.apache.activemq.artemis.tests.extensions.parameterized.ParameterizedTestExtension; -import org.apache.curator.test.InstanceSpec; -import org.apache.curator.test.TestingCluster; import org.apache.curator.test.TestingZooKeeperServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -33,8 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; import java.lang.invoke.MethodHandles; //Parameters set in super class @@ -46,34 +41,27 @@ public class ZookeeperLockManagerSinglePairTest extends LockManagerSinglePairTes // Beware: the server tick must be small enough that to let the session to be correctly expired private static final int SERVER_TICK_MS = 100; - private TestingCluster testingServer; - private InstanceSpec[] clusterSpecs; - private int nodes; + ZookeeperCluster zookeeperCluster; @BeforeEach @Override public void setup() throws Exception { super.setup(); - nodes = 3; - clusterSpecs = new InstanceSpec[nodes]; - for (int i = 0; i < nodes; i++) { - clusterSpecs[i] = new InstanceSpec(newFolder(temporaryFolder, "node" + i), BASE_SERVER_PORT + i, -1, -1, true, -1, SERVER_TICK_MS, -1); - } - testingServer = new TestingCluster(clusterSpecs); - testingServer.start(); - assertEquals("127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668", testingServer.getConnectString()); - logger.info("Cluster of {} nodes on: {}", 3, testingServer.getConnectString()); + + zookeeperCluster = new ZookeeperCluster(temporaryFolder, 3, BASE_SERVER_PORT, SERVER_TICK_MS); + zookeeperCluster.start(); + assertEquals("127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668", zookeeperCluster.getConnectString()); + logger.info("Cluster of {} nodes on: {}", 3, zookeeperCluster.getConnectString()); } @Override @AfterEach public void after() throws Exception { // zk bits that leak from servers - ThreadLeakCheckExtension.addKownThread("ListenerHandler-"); try { super.after(); } finally { - testingServer.close(); + zookeeperCluster.stop(); } } @@ -88,8 +76,8 @@ protected boolean awaitAsyncSetupCompleted(long timeout, TimeUnit unit) { @Override protected int[] stopMajority() throws Exception { - List followers = testingServer.getServers(); - final int quorum = (nodes / 2) + 1; + List followers = zookeeperCluster.getServers(); + final int quorum = (zookeeperCluster.getNodes() / 2) + 1; final int[] stopped = new int[quorum]; for (int i = 0; i < quorum; i++) { followers.get(i).stop(); @@ -100,18 +88,9 @@ protected int[] stopMajority() throws Exception { @Override protected void restart(int[] nodes) throws Exception { - List servers = testingServer.getServers(); + List servers = zookeeperCluster.getServers(); for (int nodeIndex : nodes) { servers.get(nodeIndex).restart(); } } - - private static File newFolder(File root, String... subDirs) throws IOException { - String subFolder = String.join("/", subDirs); - File result = new File(root, subFolder); - if (!result.mkdirs()) { - throw new IOException("Couldn't create folders " + root); - } - return result; - } }