Agent connection improvements

Author: sureshanaparti

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -1040,15 +1040,19 @@ protected Status getNextStatusOnDisconnection(Host host, final Status.Event even
 
     protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attache, final Status.Event event, final boolean transitState, final boolean removeAgent) {
         final long hostId = attache.getId();
-
+        final HostVO host = _hostDao.findById(hostId);
         boolean result = false;
         GlobalLock joinLock = getHostJoinLock(hostId);
-        if (joinLock.lock(60)) {
+        try {
+            if (!joinLock.lock(60)) {
+                logger.debug("Unable to acquire lock on host {} to process agent disconnection", host != null? host : hostId);
+                return result;
+            }
+
+            logger.debug("Acquired lock on host {}, to process agent disconnection", host != null? host : hostId);
             try {
-                logger.info("Host {} is disconnecting with event {}",
-                        attache, event);
+                logger.info("Host {} is disconnecting with event {}", attache, event);
                 Status nextStatus;
-                final HostVO host = _hostDao.findById(hostId);
                 if (host == null) {
                     logger.warn("Can't find host with {} ({})", hostId, attache);
                     nextStatus = Status.Removed;
@@ -1068,8 +1072,10 @@ protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attach
                 joinLock.unlock();
             }
             result = true;
+        } finally {
+            joinLock.releaseRef();
         }
-        joinLock.releaseRef();
+
         return result;
     }
 
@@ -1357,29 +1363,34 @@ private AgentAttache sendReadyAndGetAttache(HostVO host, ReadyCommand ready, Lin
         ready.setArch(host.getArch().getType());
         AgentAttache attache;
         GlobalLock joinLock = getHostJoinLock(host.getId());
-        if (joinLock.lock(60)) {
-            try {
+        try {
+            if (!joinLock.lock(60)) {
+                throw new ConnectionException(true, String.format("Unable to acquire lock on host %s, to process agent connection", host));
+            }
 
-                if (!indirectAgentLB.compareManagementServerList(host.getId(), host.getDataCenterId(), agentMSHostList, lbAlgorithm)) {
+            logger.debug("Acquired lock on host {}, to process agent connection", host);
+            try {
+                if (!indirectAgentLB.compareManagementServerListAndLBAlorithm(host.getId(), host.getDataCenterId(), agentMSHostList, lbAlgorithm)) {
                     final List<String> newMSList = indirectAgentLB.getManagementServerList(host.getId(), host.getDataCenterId(), null);
                     ready.setMsHostList(newMSList);
-                    final List<String> avoidMsList = _mshostDao.listNonUpStateMsIPs();
-                    ready.setAvoidMsHostList(avoidMsList);
-                    ready.setLbAlgorithm(indirectAgentLB.getLBAlgorithmName());
-                    ready.setLbCheckInterval(indirectAgentLB.getLBPreferredHostCheckInterval(host.getClusterId()));
-                    logger.debug("Agent's management server host list is not up to date, sending list update: {}", newMSList);
+                    String newLBAlgorithm = indirectAgentLB.getLBAlgorithmName();
+                    ready.setLbAlgorithm(newLBAlgorithm);
+                    logger.debug("Agent's management server host list or lb algorithm is not up to date, sending list and algorithm update: {}, {}", newMSList, newLBAlgorithm);
                 }
 
+                final List<String> avoidMsList = _mshostDao.listNonUpStateMsIPs();
+                ready.setAvoidMsHostList(avoidMsList);
+                ready.setLbCheckInterval(indirectAgentLB.getLBPreferredHostCheckInterval(host.getClusterId()));
+
                 attache = createAttacheForConnect(host, link);
                 attache = notifyMonitorsOfConnection(attache, startup, false);
             } finally {
                 joinLock.unlock();
             }
-        } else {
-            throw new ConnectionException(true,
-                    String.format("Unable to acquire lock on host %s", host));
+        } finally {
+            joinLock.releaseRef();
         }
-        joinLock.releaseRef();
+
         return attache;
     }
 
@@ -1821,11 +1832,11 @@ protected boolean isHostOwnerSwitched(final HostVO host) {
         return false;
     }
 
-    private void disconnectInternal(final long hostId, final Status.Event event, final boolean invstigate) {
+    private void disconnectInternal(final long hostId, final Status.Event event, final boolean investigate) {
         final AgentAttache attache = findAttache(hostId);
 
         if (attache != null) {
-            if (!invstigate) {
+            if (!investigate) {
                 disconnectWithoutInvestigation(attache, event);
             } else {
                 disconnectWithInvestigation(attache, event);

framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java

@@ -48,13 +48,14 @@ public interface IndirectAgentLB {
     List<String> getManagementServerList(Long hostId, Long dcId, List<Long> orderedHostIdList, String lbAlgorithm);
 
     /**
-     * Compares received management server list against expected list for a host in a zone.
+     * Compares received management server list against expected list for a host in a zone and LB algorithm.
      * @param hostId host id
      * @param dcId zone id
      * @param receivedMSHosts received management server list
-     * @return true if mgmtHosts is up to date, false if not
+     * @param lbAlgorithm received LB algorithm
+     * @return true if mgmtHosts and LB algorithm are up to date, false if not
      */
-    boolean compareManagementServerList(Long hostId, Long dcId, List<String> receivedMSHosts, String lbAlgorithm);
+    boolean compareManagementServerListAndLBAlorithm(Long hostId, Long dcId, List<String> receivedMSHosts, String lbAlgorithm);
 
     /**
      * Returns the configure LB algorithm

server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java

@@ -149,7 +149,7 @@ public List<String> getManagementServerList(final Long hostId, final Long dcId,
     }
 
     @Override
-    public boolean compareManagementServerList(final Long hostId, final Long dcId, final List<String> receivedMSHosts, final String lbAlgorithm) {
+    public boolean compareManagementServerListAndLBAlorithm(final Long hostId, final Long dcId, final List<String> receivedMSHosts, final String lbAlgorithm) {
         if (receivedMSHosts == null || receivedMSHosts.isEmpty()) {
             return false;
         }