Skip to content

Commit fc285e2

Browse files
sureshanapartinvazquez
sureshanaparti
and
nvazquez
authored
vmware: Cancel the pending tasks for worker VM before destroying it (#5374)
Co-authored-by: nicolas <nicovazquez90@gmail.com>
1 parent 557dc5e commit fc285e2

6 files changed

Lines changed: 106 additions & 45 deletions

File tree

‎plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -380,8 +380,7 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd)
380380
try {
381381
if (workerVm != null) {
382382
// detach volume and destroy worker vm
383-
workerVm.detachAllDisks();
384-
workerVm.destroy();
383+
workerVm.detachAllDisksAndDestroy();
385384
}
386385
} catch (Throwable e) {
387386
s_logger.warn("Failed to destroy worker VM: " + workerVMName);
@@ -670,8 +669,7 @@ private Ternary<String, Long, Long> createTemplateFromVolume(VirtualMachineMO vm
670669

671670
} finally {
672671
if (clonedVm != null) {
673-
clonedVm.detachAllDisks();
674-
clonedVm.destroy();
672+
clonedVm.detachAllDisksAndDestroy();
675673
}
676674

677675
vmMo.removeSnapshot(templateUniqueName, false);
@@ -923,8 +921,7 @@ private void restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto
923921
clonedVm.detachAllDisks();
924922
} finally {
925923
if (clonedVm != null) {
926-
clonedVm.detachAllDisks();
927-
clonedVm.destroy();
924+
clonedVm.detachAllDisksAndDestroy();
928925
}
929926
}
930927
}
@@ -980,8 +977,7 @@ private void exportVolumeToSecondaryStorage(VirtualMachineMO vmMo, String volume
980977
}
981978
} finally {
982979
if (clonedVm != null) {
983-
clonedVm.detachAllDisks();
984-
clonedVm.destroy();
980+
clonedVm.detachAllDisksAndDestroy();
985981
}
986982
}
987983
}
@@ -1037,8 +1033,7 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
10371033
}
10381034
if (workerVm != null) {
10391035
//detach volume and destroy worker vm
1040-
workerVm.detachAllDisks();
1041-
workerVm.destroy();
1036+
workerVm.detachAllDisksAndDestroy();
10421037
}
10431038
}
10441039
}

‎plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6099,9 +6099,9 @@ private void gcAndKillHungWorkerVMs() {
60996099
if (recycle) {
61006100
s_logger.info("Recycle pending worker VM: " + vmMo.getName());
61016101

6102+
vmMo.cancelPendingTasks();
61026103
vmMo.powerOff();
6103-
vmMo.detachAllDisks();
6104-
vmMo.destroy();
6104+
vmMo.detachAllDisksAndDestroy();
61056105
}
61066106
}
61076107
}

‎plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java‎

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -429,8 +429,7 @@ private String cleanUpDatastore(Command cmd, HostDatastoreSystemMO hostDatastore
429429
virtualDeviceBackingInfo = backingInfo.getParent();
430430
}
431431

432-
vmMo.detachAllDisks();
433-
vmMo.destroy();
432+
vmMo.detachAllDisksAndDestroy();
434433

435434
VmwareStorageLayoutHelper.moveVolumeToRootFolder(dcMo, backingFiles);
436435

@@ -820,8 +819,7 @@ public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd) {
820819
if (volume.getDeviceId().equals(0L)) {
821820
if (existingVm != null) {
822821
s_logger.info("Found existing VM " + vmName + " before cloning from template, destroying it");
823-
existingVm.detachAllDisks();
824-
existingVm.destroy();
822+
existingVm.detachAllDisksAndDestroy();
825823
}
826824
s_logger.info("ROOT Volume from deploy-as-is template, cloning template");
827825
cloneVMFromTemplate(hyperHost, template.getPath(), vmName, primaryStore.getUuid());
@@ -853,8 +851,7 @@ public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd) {
853851
s_logger.info("Destroy dummy VM after volume creation");
854852
if (vmMo != null) {
855853
s_logger.warn("Unable to destroy a null VM ManagedObjectReference");
856-
vmMo.detachAllDisks();
857-
vmMo.destroy();
854+
vmMo.detachAllDisksAndDestroy();
858855
}
859856
}
860857
} else {
@@ -926,8 +923,7 @@ private String cloneVMforVvols(VmwareContext context, VmwareHypervisorHost hyper
926923
String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0);
927924
if (volume.getVolumeType() == Volume.Type.DATADISK) {
928925
s_logger.info("detach disks from volume-wrapper VM " + vmName);
929-
vmMo.detachAllDisks();
930-
vmMo.destroy();
926+
vmMo.detachAllDisksAndDestroy();
931927
}
932928
return vmdkFileBaseName;
933929
}
@@ -961,11 +957,8 @@ private String createVMFolderWithVMName(VmwareContext context, VmwareHypervisorH
961957
dsMo.moveDatastoreFile(vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
962958
}
963959

964-
s_logger.info("detach disks from volume-wrapper VM " + vmdkName);
965-
vmMo.detachAllDisks();
966-
967-
s_logger.info("destroy volume-wrapper VM " + vmdkName);
968-
vmMo.destroy();
960+
s_logger.info("detach disks from volume-wrapper VM and destroy" + vmdkName);
961+
vmMo.detachAllDisksAndDestroy();
969962

970963
String srcFile = dsMo.getDatastorePath(vmdkName, true);
971964

@@ -1134,8 +1127,7 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
11341127
}
11351128
if (workerVm != null) {
11361129
//detach volume and destroy worker vm
1137-
workerVm.detachAllDisks();
1138-
workerVm.destroy();
1130+
workerVm.detachAllDisksAndDestroy();
11391131
}
11401132
}
11411133
}
@@ -1272,8 +1264,7 @@ private Ternary<String, Long, Long> createTemplateFromVolume(VmwareContext conte
12721264

12731265
} finally {
12741266
if (clonedVm != null) {
1275-
clonedVm.detachAllDisks();
1276-
clonedVm.destroy();
1267+
clonedVm.detachAllDisksAndDestroy();
12771268
}
12781269
}
12791270
}
@@ -1354,8 +1345,7 @@ public Answer createTemplateFromVolume(CopyCommand cmd) {
13541345
} finally {
13551346
try {
13561347
if (volume.getVmName() == null && workerVmMo != null) {
1357-
workerVmMo.detachAllDisks();
1358-
workerVmMo.destroy();
1348+
workerVmMo.detachAllDisksAndDestroy();
13591349
}
13601350
} catch (Throwable e) {
13611351
s_logger.error("Failed to destroy worker VM created for detached volume");
@@ -1665,8 +1655,7 @@ private void exportManagedStorageSnapshotToTemplate(CopyCommand cmd, String inst
16651655

16661656
workerVM.exportVm(installFullPath, exportName, false, false);
16671657

1668-
workerVM.detachAllDisks();
1669-
workerVM.destroy();
1658+
workerVM.detachAllDisksAndDestroy();
16701659
}
16711660

16721661
private String getTemplateVmdkName(String installFullPath, String exportName) {
@@ -1843,8 +1832,7 @@ private Pair<String, String[]> exportVolumeToSecondaryStorage(VmwareContext cont
18431832
return new Pair<>(diskDevice, disks);
18441833
} finally {
18451834
if (clonedVm != null) {
1846-
clonedVm.detachAllDisks();
1847-
clonedVm.destroy();
1835+
clonedVm.detachAllDisksAndDestroy();
18481836
}
18491837
}
18501838
}
@@ -2013,8 +2001,7 @@ public Answer backupSnapshot(CopyCommand cmd) {
20132001
try {
20142002
if (workerVm != null) {
20152003
// detach volume and destroy worker vm
2016-
workerVm.detachAllDisks();
2017-
workerVm.destroy();
2004+
workerVm.detachAllDisksAndDestroy();
20182005
}
20192006
} catch (Throwable e) {
20202007
s_logger.warn("Failed to destroy worker VM: " + workerVMName);
@@ -2556,8 +2543,7 @@ public Answer createVolume(CreateObjectCommand cmd) {
25562543
} finally {
25572544
s_logger.info("Destroy dummy VM after volume creation");
25582545
if (vmMo != null) {
2559-
vmMo.detachAllDisks();
2560-
vmMo.destroy();
2546+
vmMo.detachAllDisksAndDestroy();
25612547
}
25622548
}
25632549
}
@@ -3768,8 +3754,7 @@ private Long restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto
37683754
return _storage.getSize(srcOVFFileName);
37693755
} finally {
37703756
if (clonedVm != null) {
3771-
clonedVm.detachAllDisks();
3772-
clonedVm.destroy();
3757+
clonedVm.detachAllDisksAndDestroy();
37733758
}
37743759
}
37753760
}

‎vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2115,8 +2115,7 @@ public static void createOvfFile(VmwareHypervisorHost host, String diskFileName,
21152115
throw e;
21162116
}
21172117
} finally {
2118-
workerVmMo.detachAllDisks();
2119-
workerVmMo.destroy();
2118+
workerVmMo.detachAllDisksAndDestroy();
21202119
}
21212120
}
21222121

‎vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
import com.cloud.utils.exception.CloudRuntimeException;
4040
import com.vmware.vim25.InvalidStateFaultMsg;
4141
import com.vmware.vim25.RuntimeFaultFaultMsg;
42+
import com.vmware.vim25.TaskInfo;
43+
import com.vmware.vim25.TaskInfoState;
4244
import com.vmware.vim25.VirtualMachineTicket;
4345
import org.apache.commons.collections.CollectionUtils;
4446
import org.apache.commons.lang.StringUtils;
@@ -1466,6 +1468,11 @@ public List<Pair<String, ManagedObjectReference>> detachDisk(String vmdkDatastor
14661468
return chain;
14671469
}
14681470

1471+
public void detachAllDisksAndDestroy() throws Exception {
1472+
detachAllDisks();
1473+
destroy();
1474+
}
1475+
14691476
public void detachAllDisks() throws Exception {
14701477
if (s_logger.isTraceEnabled())
14711478
s_logger.trace("vCenter API trace - detachAllDisk(). target MOR: " + _mor.getValue());
@@ -2016,8 +2023,7 @@ public VirtualMachineMO cloneFromDiskChain(String clonedVmName, int cpuSpeedMHz,
20162023
return clonedVmMo;
20172024
} finally {
20182025
if (!bSuccess) {
2019-
clonedVmMo.detachAllDisks();
2020-
clonedVmMo.destroy();
2026+
clonedVmMo.detachAllDisksAndDestroy();
20212027
}
20222028
}
20232029
}
@@ -3568,6 +3574,30 @@ public String acquireVncTicket() throws InvalidStateFaultMsg, RuntimeFaultFaultM
35683574
return ticket.getTicket();
35693575
}
35703576

3577+
public void cancelPendingTasks() throws Exception {
3578+
String vmName = getVmName();
3579+
s_logger.debug("Checking for pending tasks of the VM: " + vmName);
3580+
3581+
ManagedObjectReference taskmgr = _context.getServiceContent().getTaskManager();
3582+
List<ManagedObjectReference> tasks = _context.getVimClient().getDynamicProperty(taskmgr, "recentTask");
3583+
3584+
int vmTasks = 0, vmPendingTasks = 0;
3585+
for (ManagedObjectReference task : tasks) {
3586+
TaskInfo info = (TaskInfo) (_context.getVimClient().getDynamicProperty(task, "info"));
3587+
if (info.getEntityName().equals(vmName)) {
3588+
vmTasks++;
3589+
if (!(info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR))) {
3590+
String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown";
3591+
s_logger.debug(taskName + " task pending for the VM: " + vmName + ", cancelling it");
3592+
vmPendingTasks++;
3593+
_context.getVimClient().cancelTask(task);
3594+
}
3595+
}
3596+
}
3597+
3598+
s_logger.debug(vmPendingTasks + " pending tasks for the VM: " + vmName + " found, out of " + vmTasks + " recent VM tasks");
3599+
}
3600+
35713601
public void tagAsWorkerVM() throws Exception {
35723602
setCustomFieldValue(CustomFieldConstants.CLOUD_WORKER, "true");
35733603
String workerTag = String.format("%d-%s", System.currentTimeMillis(), getContext().getStockObject("noderuninfo"));

‎vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
import com.vmware.pbm.PbmPortType;
4343
import com.vmware.pbm.PbmService;
4444
import com.vmware.pbm.PbmServiceInstanceContent;
45+
46+
import org.apache.commons.lang.StringUtils;
4547
import org.apache.log4j.Logger;
4648
import org.w3c.dom.Element;
4749

@@ -779,4 +781,54 @@ public int getVcenterSessionTimeout() {
779781
return vCenterSessionTimeout;
780782
}
781783

784+
public void cancelTask(ManagedObjectReference task) throws Exception {
785+
TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info"));
786+
if (info == null) {
787+
s_logger.warn("Unable to get the task info, so couldn't cancel the task");
788+
return;
789+
}
790+
791+
String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown";
792+
taskName += "(" + info.getKey() + ")";
793+
794+
String entityName = StringUtils.isNotBlank(info.getEntityName()) ? info.getEntityName() : "";
795+
796+
if (info.getState().equals(TaskInfoState.SUCCESS)) {
797+
s_logger.debug(taskName + " task successfully completed for the entity " + entityName + ", can't cancel it");
798+
return;
799+
}
800+
801+
if (info.getState().equals(TaskInfoState.ERROR)) {
802+
s_logger.debug(taskName + " task execution failed for the entity " + entityName + ", can't cancel it");
803+
return;
804+
}
805+
806+
s_logger.debug(taskName + " task pending for the entity " + entityName + ", trying to cancel");
807+
if (!info.isCancelable()) {
808+
s_logger.warn(taskName + " task will continue to run on vCenter because it can't be cancelled");
809+
return;
810+
}
811+
812+
s_logger.debug("Cancelling task " + taskName + " of the entity " + entityName);
813+
getService().cancelTask(task);
814+
815+
// Since task cancellation is asynchronous, wait for the task to be cancelled
816+
Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"},
817+
new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}});
818+
819+
if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error
820+
if (result[0].equals(TaskInfoState.SUCCESS)) {
821+
s_logger.warn("Failed to cancel" + taskName + " task of the entity " + entityName + ", the task successfully completed");
822+
}
823+
824+
if (result[1] instanceof LocalizedMethodFault) {
825+
MethodFault fault = ((LocalizedMethodFault)result[1]).getFault();
826+
if (fault instanceof RequestCanceled) {
827+
s_logger.debug(taskName + " task of the entity " + entityName + " was successfully cancelled");
828+
}
829+
} else {
830+
s_logger.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage());
831+
}
832+
}
833+
}
782834
}

0 commit comments

Comments
 (0)