Merge branch '4.12'

- Fixes PR #3146 db cleanup to the correct 4.12->4.13 upgrade path
- Fixes failing unit test due to jdk specific changes after forward
  merging

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

api/src/main/java/com/cloud/storage/Volume.java

@@ -52,7 +52,7 @@ enum State {
         UploadInProgress("Volume upload is in progress"),
         UploadError("Volume upload encountered some error"),
         UploadAbandoned("Volume upload is abandoned since the upload was never initiated within a specificed time"),
-        Attaching("The volume is attaching to a VM");
+        Attaching("The volume is attaching to a VM from Ready state.");
 
         String _description;
 
@@ -72,6 +72,8 @@ public String getDescription() {
         static {
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Allocated, Event.CreateRequested, Creating, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Allocated, Event.DestroyRequested, Destroy, null));
+            s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Allocated, Event.OperationFailed, Allocated, null));
+            s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Allocated, Event.OperationSucceeded, Allocated, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Creating, Event.OperationRetry, Creating, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Creating, Event.OperationFailed, Allocated, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Creating, Event.OperationSucceeded, Ready, null));

engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java

@@ -266,13 +266,14 @@ public long countMemoryAllocatedToAccount(long accountId) {
     }
 
     private long executeSqlCountComputingResourcesForAccount(long accountId, String sqlCountComputingResourcesAllocatedToAccount) {
-        try (TransactionLegacy tx = TransactionLegacy.currentTxn()) {
+        TransactionLegacy tx = TransactionLegacy.currentTxn();
+        try {
             PreparedStatement pstmt = tx.prepareAutoCloseStatement(sqlCountComputingResourcesAllocatedToAccount);
             pstmt.setLong(1, accountId);
 
             ResultSet rs = pstmt.executeQuery();
             if (!rs.next()) {
-                throw new CloudRuntimeException(String.format("An unexpected case happened while counting allocated computing resources for account: " + accountId));
+                return 0L;
             }
             return rs.getLong("total");
         } catch (SQLException e) {

engine/schema/src/main/resources/META-INF/db/schema-41120to41200-cleanup.sql

@@ -25,8 +25,3 @@ DROP TABLE IF EXISTS `cloud`.`iam_group_policy_map`;
 DROP TABLE IF EXISTS `cloud`.`iam_group`;
 DROP TABLE IF EXISTS `cloud`.`iam_policy_permission`;
 DROP TABLE IF EXISTS `cloud`.`iam_policy`;
-
--- Cleanup Nuage VSP
-DELETE FROM `cloud`.`host` WHERE id in (SELECT vsp.host_id FROM `cloud`.`external_nuage_vsp_devices` vsp);
-DROP TABLE IF EXISTS `cloud`.`external_nuage_vsp_devices`;
-delete from `cloud`.`role_permissions` where rule = 'issueNuageVspResourceRequest';

engine/schema/src/main/resources/META-INF/db/schema-41200to41300-cleanup.sql

@@ -19,3 +19,7 @@
 -- Schema upgrade cleanup from 4.12.0.0 to 4.13.0.0
 --;
 
+-- Cleanup Nuage VSP
+DELETE FROM `cloud`.`host` WHERE id in (SELECT vsp.host_id FROM `cloud`.`external_nuage_vsp_devices` vsp);
+DROP TABLE IF EXISTS `cloud`.`external_nuage_vsp_devices`;
+DELETE FROM `cloud`.`role_permissions` WHERE rule = 'issueNuageVspResourceRequest';

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -217,6 +217,9 @@ public boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore
     @Override
     public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) {
         DataStoreDriver dataStoreDriver = dataStore != null ? dataStore.getDriver() : null;
+        if (dataStoreDriver == null) {
+            return;
+        }
 
         if (dataStoreDriver instanceof PrimaryDataStoreDriver) {
             ((PrimaryDataStoreDriver)dataStoreDriver).revokeAccess(dataObject, host, dataStore);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -201,7 +201,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     private String _modifyVlanPath;
     private String _versionstringpath;
-    private String _patchViaSocketPath;
+    private String _patchScriptPath;
     private String _createvmPath;
     private String _manageSnapshotPath;
     private String _resizeVolumePath;
@@ -694,9 +694,9 @@ public boolean configure(final String name, final Map<String, Object> params) th
             throw new ConfigurationException("Unable to find versions.sh");
         }
 
-        _patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/", "patchviasocket.py");
-        if (_patchViaSocketPath == null) {
-            throw new ConfigurationException("Unable to find patchviasocket.py");
+        _patchScriptPath = Script.findScript(kvmScriptsDir, "patch.sh");
+        if (_patchScriptPath == null) {
+            throw new ConfigurationException("Unable to find patch.sh");
         }
 
         _heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@@ -1391,13 +1391,13 @@ private boolean checkOvsNetwork(final String networkName) {
     }
 
     public boolean passCmdLine(final String vmName, final String cmdLine) throws InternalErrorException {
-        final Script command = new Script(_patchViaSocketPath, 5 * 1000, s_logger);
+        final Script command = new Script(_patchScriptPath, 30 * 1000, s_logger);
         String result;
         command.add("-n", vmName);
-        command.add("-p", cmdLine.replaceAll(" ", "%"));
+        command.add("-c", cmdLine);
         result = command.execute();
         if (result != null) {
-            s_logger.error("passcmd failed:" + result);
+            s_logger.error("Passing cmdline failed:" + result);
             return false;
         }
         return true;
@@ -2186,12 +2186,6 @@ So if getMinSpeed() returns null we fall back to getSpeed().
         final SerialDef serial = new SerialDef("pty", null, (short)0);
         devices.addDevice(serial);
 
-        /* Add a VirtIO channel for SystemVMs for communication and provisioning */
-        if (vmTO.getType() != VirtualMachine.Type.User) {
-            File virtIoChannel = Paths.get(_qemuSocketsPath.getPath(), vmTO.getName() + ".agent").toFile();
-            devices.addDevice(new ChannelDef(vmTO.getName() + ".vport", ChannelDef.ChannelType.UNIX, virtIoChannel));
-        }
-
         if (_rngEnable) {
             final RngDef rngDevice = new RngDef(_rngPath, _rngBackendModel, _rngRateBytes, _rngRatePeriod);
             devices.addDevice(rngDevice);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java

@@ -108,17 +108,18 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
 
             // pass cmdline info to system vms
             if (vmSpec.getType() != VirtualMachine.Type.User) {
-                //wait and try passCmdLine for 5 minutes at most for CLOUDSTACK-2823
                 String controlIp = null;
                 for (final NicTO nic : nics) {
                     if (nic.getType() == TrafficType.Control) {
                         controlIp = nic.getIp();
                         break;
                     }
                 }
-                for (int count = 0; count < 30; count++) {
+                // try to patch and SSH into the systemvm for up to 5 minutes
+                for (int count = 0; count < 10; count++) {
+                    // wait and try passCmdLine for 30 seconds at most for CLOUDSTACK-2823
                     libvirtComputingResource.passCmdLine(vmName, vmSpec.getBootArgs());
-                    //check router is up?
+                    // check router is up?
                     final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource();
                     final boolean result = virtRouterResource.connect(controlIp, 1, 5000);
                     if (result) {

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParserTest.java

@@ -22,14 +22,14 @@
 import java.io.File;
 import java.util.List;
 
-import junit.framework.TestCase;
-
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.WatchDogDef;
 
+import junit.framework.TestCase;
+
 public class LibvirtDomainXMLParserTest extends TestCase {
 
     public void testDomainXMLParser() {
@@ -45,9 +45,6 @@ public void testDomainXMLParser() {
         InterfaceDef.GuestNetType ifType = InterfaceDef.GuestNetType.BRIDGE;
 
         ChannelDef.ChannelType channelType = ChannelDef.ChannelType.UNIX;
-        ChannelDef.ChannelState channelState = ChannelDef.ChannelState.DISCONNECTED;
-        String ssvmAgentPath =  "/var/lib/libvirt/qemu/s-2970-VM.agent";
-        String ssvmAgentName = "s-2970-VM.vport";
         String guestAgentPath = "/var/lib/libvirt/qemu/guest-agent.org.qemu.guest_agent.0";
         String guestAgentName = "org.qemu.guest_agent.0";
 
@@ -155,12 +152,6 @@ public void testDomainXMLParser() {
                      "<target type='serial' port='0'/>" +
                      "<alias name='serial0'/>" +
                      "</console>" +
-                     "<channel type='unix'>" +
-                     "<source mode='bind' path='/var/lib/libvirt/qemu/s-2970-VM.agent'/>" +
-                     "<target type='virtio' name='s-2970-VM.vport' state='disconnected'/>" +
-                     "<alias name='channel0'/>" +
-                     "<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
-                     "</channel>" +
                      "<input type='tablet' bus='usb'>" +
                      "<alias name='input0'/>" +
                      "</input>" +
@@ -215,14 +206,9 @@ public void testDomainXMLParser() {
             assertEquals(channelType, channels.get(i).getChannelType());
         }
 
-        /* SSVM provisioning port/channel */
-        assertEquals(channelState, channels.get(0).getChannelState());
-        assertEquals(new File(ssvmAgentPath), channels.get(0).getPath());
-        assertEquals(ssvmAgentName, channels.get(0).getName());
-
         /* Qemu Guest Agent port/channel */
-        assertEquals(new File(guestAgentPath), channels.get(1).getPath());
-        assertEquals(guestAgentName, channels.get(1).getName());
+        assertEquals(new File(guestAgentPath), channels.get(0).getPath());
+        assertEquals(guestAgentName, channels.get(0).getName());
 
         List<InterfaceDef> ifs = parser.getInterfaces();
         for (int i = 0; i < ifs.size(); i++) {

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java

@@ -261,7 +261,7 @@ public void testRngDef() {
     public void testChannelDef() {
         ChannelDef.ChannelType type = ChannelDef.ChannelType.UNIX;
         ChannelDef.ChannelState state = ChannelDef.ChannelState.CONNECTED;
-        String name = "v-136-VM.vport";
+        String name = "v-136-VM.org.qemu.guest_agent.0";
         File path = new File("/var/lib/libvirt/qemu/" + name);
 
         ChannelDef channelDef = new ChannelDef(name, type, state, path);

scripts/vm/hypervisor/kvm/patch.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Get the VM name and cmdline
+while getopts "n:c:h" opt; do
+  case ${opt} in
+    n )
+      name=$OPTARG
+      ;;
+    c )
+      cmdline=$(echo $OPTARG | base64 -w 0)
+      ;;
+    h )
+      echo "Usage: $0 -n [VM name] -c [command line]"
+      exit 0
+      ;;
+  esac
+done
+
+SSHKEY_FILE="/root/.ssh/id_rsa.pub.cloud"
+if [ ! -e $SSHKEY_FILE ]; then
+    echo "SSH public key file $SSHKEY_FILE not found!"
+    exit 1
+fi
+
+if ! which virsh > /dev/null; then
+    echo "Libvirt CLI 'virsh' not found"
+    exit 1
+fi
+
+# Read the SSH public key
+sshkey=$(cat $SSHKEY_FILE | base64 -w 0)
+
+# Method to send and write payload inside the VM
+send_file() {
+    local name=${1}
+    local path=${2}
+    local content=${@:3}
+    local fd=$(virsh qemu-agent-command $name "{\"execute\":\"guest-file-open\", \"arguments\":{\"path\":\"$path\",\"mode\":\"w+\"}}" | sed 's/[^:]*:\([^}]*\).*/\1/')
+    virsh qemu-agent-command $name "{\"execute\":\"guest-file-write\", \"arguments\":{\"handle\":$fd,\"buf-b64\":\"$content\"}}" > /dev/null
+    virsh qemu-agent-command $name "{\"execute\":\"guest-file-close\", \"arguments\":{\"handle\":$fd}}" > /dev/null
+}
+
+# Wait for the guest agent to come online
+while ! virsh qemu-agent-command $name '{"execute":"guest-ping"}' >/dev/null 2>&1
+do
+    sleep 0.1
+done
+
+# Test guest agent sanity
+while [ "$(virsh qemu-agent-command $name '{"execute":"guest-sync","arguments":{"id":1234567890}}' 2>/dev/null)" != '{"return":1234567890}' ]
+do
+    sleep 0.1
+done
+
+# Write ssh public key
+send_file $name "/root/.ssh/authorized_keys" $sshkey
+
+# Fix ssh public key permission
+virsh qemu-agent-command $name '{"execute":"guest-exec","arguments":{"path":"chmod","arg":["go-rwx","/root/.ssh/authorized_keys"]}}' > /dev/null
+
+# Write cmdline payload
+send_file $name "/var/cache/cloud/cmdline" $cmdline