Merge branch '4.8'

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

debian/control

@@ -15,7 +15,7 @@ Description: A common package which contains files which are shared by several C
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6, sudo, jsvc, python-mysqldb, libmysql-java, augeas-tools, mysql-client, adduser
+Depends: ${misc:Depends}, ${python:Depends}, cloudstack-common (= ${source:Version}), tomcat6, sudo, jsvc, python-mysqldb, libmysql-java, augeas-tools, mysql-client, adduser, bzip2
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -2909,12 +2909,12 @@ public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffe
                     newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory");
         }
 
-        // Check that the service offering being upgraded to has storage tags subset of the current service offering storage tags, since volume is not migrated.
+        // Check that the service offering being upgraded to has all the tags of the current service offering.
         final List<String> currentTags = StringUtils.csvTagsToList(currentServiceOffering.getTags());
         final List<String> newTags = StringUtils.csvTagsToList(newServiceOffering.getTags());
-        if (!currentTags.containsAll(newTags)) {
-            throw new InvalidParameterValueException("Unable to upgrade virtual machine; the new service offering " + " should have tags as subset of " +
-                    "current service offering tags. Current service offering tags: " + currentTags + "; " + "new service " + "offering tags: " + newTags);
+        if (!newTags.containsAll(currentTags)) {
+            throw new InvalidParameterValueException("Unable to upgrade virtual machine; the current service offering " + " should have tags as subset of " +
+                    "the new service offering tags. Current service offering tags: " + currentTags + "; " + "new service " + "offering tags: " + newTags);
         }
     }
 

engine/orchestration/test/com/cloud/vm/VirtualMachineManagerImplTest.java

@@ -30,7 +30,9 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.ArrayList;
 
+import com.cloud.service.dao.ServiceOfferingDao;
 import junit.framework.Assert;
 
 import org.junit.Before;
@@ -136,6 +138,8 @@ public class VirtualMachineManagerImplTest {
     @Mock
     VMInstanceDao _vmInstanceDao;
     @Mock
+    ServiceOfferingDao _offeringDao;
+    @Mock
     VMTemplateDao _templateDao;
     @Mock
     VolumeDao _volsDao;
@@ -150,6 +154,8 @@ public class VirtualMachineManagerImplTest {
     @Mock
     VMInstanceVO _vmInstance;
     @Mock
+    ServiceOfferingVO _serviceOfferingMock;
+    @Mock
     HostVO _host;
     @Mock
     VMTemplateVO _templateMock;
@@ -228,6 +234,8 @@ public void setup() {
         _vmMgr._uservmDetailsDao = _vmDetailsDao;
         _vmMgr._entityMgr = _entityMgr;
         _vmMgr._configDepot = _configDepot;
+        _vmMgr._offeringDao = _offeringDao;
+        _vmMgr.hostAllocators = new ArrayList<>();
 
         when(_vmMock.getId()).thenReturn(314l);
         when(_vmInstance.getId()).thenReturn(1L);
@@ -514,4 +522,24 @@ public void testExeceuteInSequence() {
         assertTrue(_vmMgr.getExecuteInSequence(HypervisorType.VMware) == HypervisorGuru.VmwareFullClone.value());
         assertTrue(_vmMgr.getExecuteInSequence(HypervisorType.Ovm3) == VirtualMachineManager.ExecuteInSequence.value());
     }
+
+    @Test
+    public void testCheckIfCanUpgrade() throws Exception {
+        when(_vmInstance.getState()).thenReturn(State.Stopped);
+        when(_serviceOfferingMock.isDynamic()).thenReturn(true);
+        when(_vmInstance.getServiceOfferingId()).thenReturn(1l);
+        when(_serviceOfferingMock.getId()).thenReturn(2l);
+
+        ServiceOfferingVO mockCurrentServiceOffering = mock(ServiceOfferingVO.class);
+
+        when(_offeringDao.findByIdIncludingRemoved(anyLong(), anyLong())).thenReturn(mockCurrentServiceOffering);
+        when(mockCurrentServiceOffering.getUseLocalStorage()).thenReturn(true);
+        when(_serviceOfferingMock.getUseLocalStorage()).thenReturn(true);
+        when(mockCurrentServiceOffering.getSystemUse()).thenReturn(true);
+        when(_serviceOfferingMock.getSystemUse()).thenReturn(true);
+        when(mockCurrentServiceOffering.getTags()).thenReturn("x,y");
+        when(_serviceOfferingMock.getTags()).thenReturn("z,x,y");
+
+        _vmMgr.checkIfCanUpgrade(_vmInstance, _serviceOfferingMock);
+    }
 }

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -237,7 +237,7 @@
 import com.cloud.hypervisor.vmware.mo.NetworkDetails;
 import com.cloud.hypervisor.vmware.mo.TaskMO;
 import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType;
-import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfo;
+import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
 import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder;
 import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.mo.VirtualSwitchType;
@@ -1452,7 +1452,7 @@ protected StartAnswer execute(StartCommand cmd) {
                 s_logger.error(msg);
                 throw new Exception(msg);
             }
-
+            String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value();
             DiskTO[] disks = validateDisks(vmSpec.getDisks());
             assert (disks.length > 0);
             NicTO[] nics = vmSpec.getNics();
@@ -1565,7 +1565,7 @@ protected StartAnswer execute(StartCommand cmd) {
                         tearDownVm(vmMo);
                     }else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(),
                             getReservedCpuMHZ(vmSpec), vmSpec.getLimitCpuUse(), (int)(vmSpec.getMaxRam() / (1024 * 1024)), getReservedMemoryMb(vmSpec),
-                            translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(), rootDiskDataStoreDetails.first(), false, controllerInfo, systemVm)) {
+                            guestOsId, rootDiskDataStoreDetails.first(), false, controllerInfo, systemVm)) {
                         throw new Exception("Failed to create VM. vmName: " + vmInternalCSName);
                     }
                 }
@@ -1589,7 +1589,6 @@ protected StartAnswer execute(StartCommand cmd) {
             }
 
             VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
-            String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value();
 
             VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(),
                     getReservedCpuMHZ(vmSpec), (int)(vmSpec.getMaxRam() / (1024 * 1024)), getReservedMemoryMb(vmSpec),

plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java

@@ -33,6 +33,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
@@ -88,7 +89,7 @@
 import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO;
 import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
 import com.cloud.hypervisor.vmware.mo.NetworkDetails;
-import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfo;
+import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
 import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost;
 import com.cloud.hypervisor.vmware.resource.VmwareResource;
@@ -1366,24 +1367,15 @@ private Answer attachVolume(Command cmd, DiskTO disk, boolean isAttach, boolean
             AttachAnswer answer = new AttachAnswer(disk);
 
             if (isAttach) {
-                String dataDiskController = controllerInfo.get(VmDetailConstants.DATA_DISK_CONTROLLER);
-                String rootDiskController = controllerInfo.get(VmDetailConstants.ROOT_DISK_CONTROLLER);
-                DiskControllerType rootDiskControllerType = DiskControllerType.getType(rootDiskController);
-
-                if (dataDiskController == null) {
-                    dataDiskController = getLegacyVmDataDiskController();
-                } else if ((rootDiskControllerType == DiskControllerType.lsilogic) ||
-                           (rootDiskControllerType == DiskControllerType.lsisas1068) ||
-                           (rootDiskControllerType == DiskControllerType.pvscsi) ||
-                           (rootDiskControllerType == DiskControllerType.buslogic)) {
-                    //TODO: Support mix of SCSI controller types for single VM. If root disk is already over
-                    //a SCSI controller then use the same for data volume as well. This limitation will go once mix
-                    //of SCSI controller types for single VM.
-                    dataDiskController = rootDiskController;
-                } else if (DiskControllerType.getType(dataDiskController) == DiskControllerType.osdefault) {
-                    dataDiskController = vmMo.getRecommendedDiskController(null);
+                String diskController = getLegacyVmDataDiskController();
+                if (controllerInfo != null &&
+                        !Strings.isNullOrEmpty(controllerInfo.get(VmDetailConstants.DATA_DISK_CONTROLLER))) {
+                    diskController = controllerInfo.get(VmDetailConstants.DATA_DISK_CONTROLLER);
                 }
-                vmMo.attachDisk(new String[] {datastoreVolumePath}, morDs, dataDiskController);
+                if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) {
+                    diskController = vmMo.getRecommendedDiskController(null);
+                }
+                vmMo.attachDisk(new String[] {datastoreVolumePath}, morDs, diskController);
             } else {
                 vmMo.removeAllSnapshots();
                 vmMo.detachDisk(datastoreVolumePath, false);

server/src/com/cloud/api/query/QueryManagerImpl.java

@@ -2620,11 +2620,11 @@ private List<ServiceOfferingJoinVO> filterOfferingsOnCurrentTags(List<ServiceOff
         if(currentVmOffering == null) return offerings;
         List<String> currentTagsList = StringUtils.csvTagsToList(currentVmOffering.getTags());
 
-        // New offerings should be a subset of existing storage tags. Discard offerings who are not.
+        // New service offering should have all the tags of the current service offering.
         List<ServiceOfferingJoinVO> filteredOfferings = new ArrayList<>();
         for (ServiceOfferingJoinVO offering : offerings){
-            List<String> tags = StringUtils.csvTagsToList(offering.getTags());
-            if(currentTagsList.containsAll(tags)){
+            List<String> newTagsList = StringUtils.csvTagsToList(offering.getTags());
+            if(newTagsList.containsAll(currentTagsList)){
                 filteredOfferings.add(offering);
             }
         }

server/src/com/cloud/storage/VolumeApiServiceImpl.java

@@ -30,15 +30,18 @@
 
 import com.cloud.utils.EncryptionUtil;
 import com.cloud.utils.db.TransactionCallbackWithException;
+import com.google.common.base.Strings;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
 
 import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd;
 import org.apache.cloudstack.api.response.GetUploadParamsResponse;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
 import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
+import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
 import org.apache.log4j.Logger;
 import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
@@ -112,12 +115,14 @@
 import com.cloud.hypervisor.HypervisorCapabilitiesVO;
 import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
 import com.cloud.org.Grouping;
+import com.cloud.serializer.GsonHelper;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.dao.DiskOfferingDao;
 import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.dao.VolumeDetailsDao;
 import com.cloud.storage.snapshot.SnapshotApiService;
 import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.template.TemplateManager;
@@ -146,6 +151,7 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.fsm.NoTransitionException;
 import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.vm.UserVmManager;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
@@ -191,6 +197,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     @Inject
     VolumeDao _volsDao;
     @Inject
+    VolumeDetailsDao _volDetailDao;
+    @Inject
     HostDao _hostDao;
     @Inject
     SnapshotDao _snapshotDao;
@@ -240,6 +248,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     VmWorkJobDao _workJobDao;
     @Inject
     ClusterDetailsDao _clusterDetailsDao;
+    @Inject
+    UserVmManager _userVmMgr;
+    protected Gson _gson;
 
     private List<StoragePoolAllocator> _storagePoolAllocators;
 
@@ -253,6 +264,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
     protected VolumeApiServiceImpl() {
         _volStateMachine = Volume.State.getStateMachine();
+        _gson = GsonHelper.getGsonLogger();
     }
 
     /*
@@ -1835,6 +1847,26 @@ private Volume orchestrateDetachVolumeFromVM(long vmId, long volumeId) {
         }
     }
 
+    public void updateMissingRootDiskController(final VMInstanceVO vm, final String rootVolChainInfo) {
+        if (vm == null || !VirtualMachine.Type.User.equals(vm.getType()) || Strings.isNullOrEmpty(rootVolChainInfo)) {
+            return;
+        }
+        String rootDiskController = null;
+        try {
+            final VirtualMachineDiskInfo infoInChain = _gson.fromJson(rootVolChainInfo, VirtualMachineDiskInfo.class);
+            if (infoInChain != null) {
+                rootDiskController = infoInChain.getControllerFromDeviceBusName();
+            }
+            final UserVmVO userVmVo = _userVmDao.findById(vm.getId());
+            if ((rootDiskController != null) && (!rootDiskController.isEmpty())) {
+                _userVmDao.loadDetails(userVmVo);
+                _userVmMgr.persistDeviceBusInfo(userVmVo, rootDiskController);
+            }
+        } catch (JsonParseException e) {
+            s_logger.debug("Error parsing chain info json: " + e.getMessage());
+        }
+    }
+
     @DB
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_VOLUME_MIGRATE, eventDescription = "migrating volume", async = true)
@@ -1924,6 +1956,7 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) {
                                 throw new InvalidParameterValueException("Cannot migrate ROOT volume of a stopped VM to a storage pool in a different VMware datacenter");
                             }
                         }
+                        updateMissingRootDiskController(vm, vol.getChainInfo());
                     }
                 }
             }
@@ -2475,6 +2508,7 @@ private VolumeVO sendAttachVolumeCommand(UserVmVO vm, VolumeVO volumeToAttach, L
             controllerInfo.put(VmDetailConstants.ROOT_DISK_CONTROLLER, vm.getDetail(VmDetailConstants.ROOT_DISK_CONTROLLER));
             controllerInfo.put(VmDetailConstants.DATA_DISK_CONTROLLER, vm.getDetail(VmDetailConstants.DATA_DISK_CONTROLLER));
             cmd.setControllerInfo(controllerInfo);
+            s_logger.debug("Attach volume id:" + volumeToAttach.getId() +  " on VM id:" + vm.getId() + " has controller info:" + controllerInfo);
 
             try {
                 answer = (AttachAnswer)_agentMgr.send(hostId, cmd);

server/src/com/cloud/vm/UserVmManager.java

@@ -114,4 +114,6 @@ UserVm updateVirtualMachine(long id, String displayName, String group, Boolean h
     public void removeCustomOfferingDetails(long vmId);
 
     void generateUsageEvent(VirtualMachine vm, boolean isDisplay, String eventType);
+
+    void persistDeviceBusInfo(UserVmVO paramUserVmVO, String paramString);
 }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -89,6 +89,7 @@
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
@@ -5452,6 +5453,17 @@ private void encryptAndStorePassword(UserVmVO vm, String password) {
         }
     }
 
+    public void persistDeviceBusInfo(UserVmVO vm, String rootDiskController) {
+        String existingVmRootDiskController = vm.getDetail(VmDetailConstants.ROOT_DISK_CONTROLLER);
+        if (StringUtils.isEmpty(existingVmRootDiskController) && !StringUtils.isEmpty(rootDiskController)) {
+            vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, rootDiskController);
+            _vmDao.saveDetails(vm);
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Persisted device bus information rootDiskController=" + rootDiskController + " for vm: " + vm.getDisplayName());
+            }
+        }
+    }
+
     @Override
     public String getConfigComponentName() {
         return UserVmManager.class.getSimpleName();

server/test/com/cloud/storage/VolumeApiServiceImplTest.java

@@ -18,7 +18,11 @@
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.lang.reflect.Field;
@@ -28,7 +32,10 @@
 
 import javax.inject.Inject;
 
+import com.cloud.serializer.GsonHelper;
 import com.cloud.user.User;
+import com.cloud.vm.UserVmManager;
+import com.cloud.vm.VirtualMachine;
 import junit.framework.Assert;
 import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
 import org.junit.After;
@@ -101,6 +108,8 @@ public class VolumeApiServiceImplTest {
     VolumeService volService;
     @Mock
     CreateVolumeCmd createVol;
+    @Mock
+    UserVmManager _userVmMgr;
 
     DetachVolumeCmd detachCmd = new DetachVolumeCmd();
     Class<?> _detachCmdClass = detachCmd.getClass();
@@ -118,6 +127,8 @@ public void setup() throws Exception {
         _svc._jobMgr = _jobMgr;
         _svc.volFactory = _volFactory;
         _svc.volService = volService;
+        _svc._userVmMgr = _userVmMgr;
+        _svc._gson = GsonHelper.getGsonLogger();
 
         // mock caller context
         AccountVO account = new AccountVO("admin", 1L, "networkDomain", Account.ACCOUNT_TYPE_NORMAL, "uuid");
@@ -383,6 +394,20 @@ public void testNonEmptyGetVolumeNameFromCmd() {
         Assert.assertSame(_svc.getVolumeNameFromCommand(createVol), "abc");
     }
 
+    @Test
+    public void testUpdateMissingRootDiskControllerWithNullChainInfo() {
+        _svc.updateMissingRootDiskController(null, null);
+        verify(_svc._userVmMgr, times(0)).persistDeviceBusInfo(any(UserVmVO.class), anyString());
+    }
+
+    @Test
+    public void testUpdateMissingRootDiskControllerWithValidChainInfo() {
+        UserVmVO vm = _svc._userVmDao.findById(1L);
+        assert vm.getType() == VirtualMachine.Type.User;
+        _svc.updateMissingRootDiskController(vm, "{\"diskDeviceBusName\":\"scsi0:0\",\"diskChain\":[\"[somedatastore] i-3-VM-somePath/ROOT-1.vmdk\"]}");
+        verify(_svc._userVmMgr, times(1)).persistDeviceBusInfo(any(UserVmVO.class), eq("scsi"));
+    }
+
     @After
     public void tearDown() {
         CallContext.unregister();