Skip to content

Commit 3f69c83

Browse files
SudharmaJainyadvr
SudharmaJain
authored and
yadvr
committed
CLOUDSTACK-9956: File search on the vmware datastore may select wrong file if there are multiple files with same name (#2153)
If there are multiple files with the same name on vmware datastore, search operation may select any one file during volume related operations. This involves volume attach/detach, volume download, volume snapshot etc. While using NetApp as the backup solution. This has .snapshot folder on the datastore and sometimes files from this folder gets selected during volume operations and the operation fails. Because of wrong selection of file following exception can be observed while volume deletion. 2017-02-23 19:39:05,750 ERROR [c.c.s.r.VmwareStorageProcessor] (DirectAgent-304:ctx-a1dbf5d8 ac.local) delete volume failed due to Exception: java.lang.RuntimeException Message: Cannot delete file [4cbcd46d44c53f5c8244c0aad26a97e1] .snapshot/hourly.2017-02-23_1605/r-97-VM/ROOT-97.vmdk To fix this behavior I have added a global configuration by name vmware.search.exclude.folders which can be comma separated list of folder paths. I have also added a unit test to test the new method.
1 parent 3bc2341 commit 3f69c83

9 files changed

Lines changed: 199 additions & 32 deletions

File tree

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
531531
// FIXME: Fix long checkPointId2 = _checkPointMgr.pushCheckPoint(new VmwareCleanupMaid(hostDetails.get("guid"), workerName2));
532532
cmd.setContextParam("worker2", workerName2);
533533
cmd.setContextParam("checkpoint2", String.valueOf(checkPointId2));
534+
cmd.setContextParam("searchexludefolders", _vmwareMgr.s_vmwareSearchExcludeFolder.value());
534535
}
535536

536537
return new Pair<Boolean, Long>(Boolean.TRUE, cmdTarget.first().getId());

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ public interface VmwareManager {
3939
public static final ConfigKey<Boolean> s_vmwareCleanOldWorderVMs = new ConfigKey<Boolean>("Advanced", Boolean.class, "vmware.clean.old.worker.vms", "false",
4040
"If a worker vm is older then twice the 'job.expire.minutes' + 'job.cancel.threshold.minutes' , remove it.", true, ConfigKey.Scope.Global);
4141

42+
static final ConfigKey<String> s_vmwareSearchExcludeFolder = new ConfigKey<String>("Advanced", String.class, "vmware.search.exclude.folders", null,
43+
"Comma seperated list of Datastore Folders to exclude from VMWare search", true, ConfigKey.Scope.Global);
44+
4245
String composeWorkerName();
4346

4447
String getSystemVMIsoFileNameOnDatastore();

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ public String getConfigComponentName() {
229229

230230
@Override
231231
public ConfigKey<?>[] getConfigKeys() {
232-
return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval};
232+
return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder};
233233
}
234234

235235
@Override

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,7 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd)
307307
String snapshotUuid = cmd.getSnapshotUuid(); // not null: Precondition.
308308
String prevSnapshotUuid = cmd.getPrevSnapshotUuid();
309309
String prevBackupUuid = cmd.getPrevBackupUuid();
310+
String searchExcludedFolders = cmd.getContextParam("searchexludefolders");
310311
VirtualMachineMO workerVm = null;
311312
String workerVMName = null;
312313
String volumePath = cmd.getVolumePath();
@@ -344,7 +345,7 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd)
344345
workerVm = vmMo;
345346

346347
// attach volume to worker VM
347-
String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk");
348+
String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk", searchExcludedFolders);
348349
vmMo.attachDisk(new String[] {datastoreVolumePath}, morDs);
349350
}
350351
}
@@ -986,6 +987,8 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
986987
VirtualMachineMO workerVm = null;
987988
VirtualMachineMO vmMo = null;
988989
String exportName = UUID.randomUUID().toString();
990+
String searchExcludedFolders = cmd.getContextParam("searchexludefolders");
991+
989992

990993
try {
991994
ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId);
@@ -1009,7 +1012,7 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
10091012
}
10101013

10111014
//attach volume to worker VM
1012-
String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk");
1015+
String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk", searchExcludedFolders);
10131016
workerVm.attachDisk(new String[] {datastoreVolumePath}, morDs);
10141017
vmMo = workerVm;
10151018
}
@@ -1030,8 +1033,8 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
10301033
}
10311034
}
10321035

1033-
private String getVolumePathInDatastore(DatastoreMO dsMo, String volumeFileName) throws Exception {
1034-
String datastoreVolumePath = dsMo.searchFileInSubFolders(volumeFileName, true);
1036+
private String getVolumePathInDatastore(DatastoreMO dsMo, String volumeFileName, String searchExcludeFolders) throws Exception {
1037+
String datastoreVolumePath = dsMo.searchFileInSubFolders(volumeFileName, true, searchExcludeFolders);
10351038
if (datastoreVolumePath == null) {
10361039
throw new CloudRuntimeException("Unable to find file " + volumeFileName + " in datastore " + dsMo.getName());
10371040
}

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -645,7 +645,7 @@ private void registerVm(String vmName, DatastoreMO dsMo) throws Exception{
645645
ManagedObjectReference vmFolderMor = dataCenterMo.getVmFolder();
646646

647647
//2nd param
648-
String vmxFilePath = dsMo.searchFileInSubFolders(vmName + ".vmx", false);
648+
String vmxFilePath = dsMo.searchFileInSubFolders(vmName + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value());
649649

650650
// 5th param
651651
ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool();
@@ -1683,7 +1683,7 @@ protected StartAnswer execute(StartCommand cmd) {
16831683
assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null);
16841684

16851685
boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter);
1686-
String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false);
1686+
String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value());
16871687
if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present.
16881688
registerVm(vmNameOnVcenter, dsRootVolumeIsOn);
16891689
vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
@@ -2360,7 +2360,7 @@ private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, Virtual
23602360
DatastoreFile file = new DatastoreFile(disks[i]);
23612361
if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) {
23622362
s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder");
2363-
disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName());
2363+
disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value());
23642364
}
23652365
}
23662366
return disks;
@@ -2370,13 +2370,13 @@ private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, Virtual
23702370

23712371
if (isManaged) {
23722372
if (volumeTO.getVolumeType() == Volume.Type.ROOT) {
2373-
datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getName());
2373+
datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getName(), VmwareManager.s_vmwareSearchExcludeFolder.value());
23742374
}
23752375
else {
23762376
datastoreDiskPath = dsMo.getDatastorePath(dsMo.getName() + ".vmdk");
23772377
}
23782378
} else {
2379-
datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath());
2379+
datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value());
23802380
}
23812381

23822382
if (!dsMo.fileExists(datastoreDiskPath)) {
@@ -2802,7 +2802,7 @@ else if (file.getType().equals("config"))
28022802
vmFolder = new DatastoreFile(fileInDatastore.getDatastoreName(), fileInDatastore.getDir());
28032803
DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName()));
28042804
s_logger.debug("Deleting file: " + file.getName());
2805-
dsMo.deleteFile(file.getName(), dcMo.getMor(), true);
2805+
dsMo.deleteFile(file.getName(), dcMo.getMor(), true, VmwareManager.s_vmwareSearchExcludeFolder.value());
28062806
}
28072807
// Delete files that are present in the VM folder - this will take care of the VM disks as well.
28082808
DatastoreMO vmFolderDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(vmFolder.getDatastoreName()));
@@ -2811,7 +2811,7 @@ else if (file.getType().equals("config"))
28112811
for (String file : files) {
28122812
String vmDiskFileFullPath = String.format("%s/%s", vmFolder.getPath(), file);
28132813
s_logger.debug("Deleting file: " + vmDiskFileFullPath);
2814-
vmFolderDsMo.deleteFile(vmDiskFileFullPath, dcMo.getMor(), true);
2814+
vmFolderDsMo.deleteFile(vmDiskFileFullPath, dcMo.getMor(), true, VmwareManager.s_vmwareSearchExcludeFolder.value());
28152815
}
28162816
}
28172817
// Delete VM folder

‎plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageLayoutHelper.java‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ public static String[] getVmdkFilePairDatastorePath(DatastoreMO dsMo, String vmN
6767
}
6868

6969
public static String findVolumeDatastoreFullPath(DatastoreMO dsMo, String vmName, String vmdkFileName) throws Exception {
70+
return findVolumeDatastoreFullPath(dsMo, vmName, vmdkFileName, null);
71+
}
72+
73+
public static String findVolumeDatastoreFullPath(DatastoreMO dsMo, String vmName, String vmdkFileName, String excludeFolders) throws Exception {
74+
7075
if (vmName != null) {
7176
String path = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkFileName);
7277
if (!dsMo.fileExists(path)) {
@@ -80,7 +85,7 @@ public static String findVolumeDatastoreFullPath(DatastoreMO dsMo, String vmName
8085
String path = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName);
8186
if (!dsMo.fileExists(path)) {
8287
// Datastore file movement is not atomic operations, we need to sync and repair
83-
path = dsMo.searchFileInSubFolders(vmdkFileName, false);
88+
path = dsMo.searchFileInSubFolders(vmdkFileName, false, excludeFolders);
8489

8590
// to save one call to vCenter, we won't check file existence for this round, so the caller
8691
// may still fail with exception, but if that's case, we will let it fail anyway
@@ -90,6 +95,10 @@ public static String findVolumeDatastoreFullPath(DatastoreMO dsMo, String vmName
9095
}
9196

9297
public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmName, DatastoreMO ds, String vmdkName) throws Exception {
98+
return syncVolumeToVmDefaultFolder(dcMo, vmName, ds, vmdkName, null);
99+
}
100+
101+
public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmName, DatastoreMO ds, String vmdkName, String excludeFolders) throws Exception {
93102

94103
assert (ds != null);
95104
if (!ds.folderExists(String.format("[%s]", ds.getName()), vmName)) {
@@ -109,7 +118,7 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam
109118
// be left over in its previous owner VM. We will do a fixup synchronization here by moving it to root
110119
// again.
111120
//
112-
syncVolumeToRootFolder(dcMo, ds, vmdkName, vmName);
121+
syncVolumeToRootFolder(dcMo, ds, vmdkName, vmName, excludeFolders);
113122
}
114123

115124
if (ds.fileExists(vmdkFullCloneModeLegacyPair[1])) {
@@ -134,7 +143,11 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam
134143
}
135144

136145
public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, String vmdkName, String vmName) throws Exception {
137-
String fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false);
146+
syncVolumeToRootFolder(dcMo, ds, vmdkName, null);
147+
}
148+
149+
public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, String vmdkName, String vmName, String excludeFolders) throws Exception {
150+
String fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false, excludeFolders);
138151
if (fileDsFullPath == null)
139152
return;
140153

@@ -259,33 +272,37 @@ public static Pair<String, String> decodeTemplateRelativePathAndNameFromUrl(Stri
259272
}
260273

261274
public static void deleteVolumeVmdkFiles(DatastoreMO dsMo, String volumeName, DatacenterMO dcMo) throws Exception {
275+
deleteVolumeVmdkFiles(dsMo, volumeName, dcMo, null);
276+
}
277+
278+
public static void deleteVolumeVmdkFiles(DatastoreMO dsMo, String volumeName, DatacenterMO dcMo, String excludeFolders) throws Exception {
262279

263280
String fileName = volumeName + ".vmdk";
264281
String fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName);
265282
if (!dsMo.fileExists(fileFullPath))
266-
fileFullPath = dsMo.searchFileInSubFolders(fileName, false);
283+
fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders);
267284
if (fileFullPath != null) {
268-
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true);
285+
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders);
269286
} else {
270287
s_logger.warn("Unable to locate VMDK file: " + fileName);
271288
}
272289

273290
fileName = volumeName + "-flat.vmdk";
274291
fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName);
275292
if (!dsMo.fileExists(fileFullPath))
276-
fileFullPath = dsMo.searchFileInSubFolders(fileName, false);
293+
fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders);
277294
if (fileFullPath != null) {
278-
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true);
295+
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders);
279296
} else {
280297
s_logger.warn("Unable to locate VMDK file: " + fileName);
281298
}
282299

283300
fileName = volumeName + "-delta.vmdk";
284301
fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName);
285302
if (!dsMo.fileExists(fileFullPath))
286-
fileFullPath = dsMo.searchFileInSubFolders(fileName, false);
303+
fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders);
287304
if (fileFullPath != null) {
288-
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true);
305+
dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders);
289306
} else {
290307
s_logger.warn("Unable to locate VMDK file: " + fileName);
291308
}

0 commit comments

Comments
 (0)