Skip to content

Commit 090c222

Browse files
abh1sarabh1sar
committed
Added timeout for mount operation in backup restore configurable via global setting
1 parent 4eac151 commit 090c222

4 files changed

Lines changed: 57 additions & 33 deletions

File tree

‎core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public class RestoreBackupCommand extends Command {
3636
private Boolean vmExists;
3737
private String restoreVolumeUUID;
3838
private VirtualMachine.State vmState;
39+
private Integer mountTimeout;
3940

4041
protected RestoreBackupCommand() {
4142
super();
@@ -136,4 +137,12 @@ public List<String> getBackupVolumesUUIDs() {
136137
public void setBackupVolumesUUIDs(List<String> backupVolumesUUIDs) {
137138
this.backupVolumesUUIDs = backupVolumesUUIDs;
138139
}
140+
141+
public Integer getMountTimeout() {
142+
return this.mountTimeout;
143+
}
144+
145+
public void setMountTimeout(Integer mountTimeout) {
146+
this.mountTimeout = mountTimeout;
147+
}
139148
}

‎plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,18 @@
7070
import java.util.UUID;
7171
import java.util.stream.Collectors;
7272

73+
import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled;
74+
7375
public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable {
7476
private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class);
7577

78+
ConfigKey<Integer> NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class,
79+
"nas.backup.restore.mount.timeout",
80+
"30",
81+
"Timeout in seconds after which backup repository mount for restore fails.",
82+
true,
83+
BackupFrameworkEnabled.key());
84+
7685
@Inject
7786
private BackupDao backupDao;
7887

@@ -299,6 +308,7 @@ private Pair<Boolean, String> restoreVMBackup(VirtualMachine vm, Backup backup)
299308
restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes));
300309
restoreCommand.setVmExists(vm.getRemoved() == null);
301310
restoreCommand.setVmState(vm.getState());
311+
restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value());
302312

303313
BackupAnswer answer;
304314
try {
@@ -542,6 +552,7 @@ public Boolean isDraasEnabled(BackupOffering backupOffering) {
542552
@Override
543553
public ConfigKey<?>[] getConfigKeys() {
544554
return new ConfigKey[]{
555+
NASBackupRestoreMountTimeout
545556
};
546557
}
547558

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -61,19 +61,21 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser
6161
List<String> backedVolumeUUIDs = command.getBackupVolumesUUIDs();
6262
List<String> restoreVolumePaths = command.getRestoreVolumePaths();
6363
String restoreVolumeUuid = command.getRestoreVolumeUUID();
64+
Integer mountTimeout = command.getMountTimeout() * 1000;
6465

6566
String newVolumeId = null;
6667
try {
68+
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions, mountTimeout);
6769
if (Objects.isNull(vmExists)) {
6870
String volumePath = restoreVolumePaths.get(0);
6971
int lastIndex = volumePath.lastIndexOf("/");
7072
newVolumeId = volumePath.substring(lastIndex + 1);
71-
restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid,
72-
new Pair<>(vmName, command.getVmState()), mountOptions);
73+
restoreVolume(backupPath, volumePath, diskType, restoreVolumeUuid,
74+
new Pair<>(vmName, command.getVmState()), mountDirectory);
7375
} else if (Boolean.TRUE.equals(vmExists)) {
74-
restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions);
76+
restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, mountDirectory);
7577
} else {
76-
restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions);
78+
restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, mountDirectory);
7779
}
7880
} catch (CloudRuntimeException e) {
7981
String errorMessage = e.getMessage() != null ? e.getMessage() : "";
@@ -92,10 +94,9 @@ private void verifyBackupFile(String backupPath, String volUuid) {
9294
}
9395
}
9496

95-
private void restoreVolumesOfExistingVM(List<String> restoreVolumePaths, List<String> backedVolumesUUIDs, String backupPath,
96-
String backupRepoType, String backupRepoAddress, String mountOptions) {
97+
private void restoreVolumesOfExistingVM(List<String> restoreVolumePaths, List<String> backedVolumesUUIDs,
98+
String backupPath, String mountDirectory) {
9799
String diskType = "root";
98-
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
99100
try {
100101
for (int idx = 0; idx < restoreVolumePaths.size(); idx++) {
101102
String restoreVolumePath = restoreVolumePaths.get(idx);
@@ -113,9 +114,7 @@ private void restoreVolumesOfExistingVM(List<String> restoreVolumePaths, List<St
113114
}
114115
}
115116

116-
private void restoreVolumesOfDestroyedVMs(List<String> volumePaths, String vmName, String backupPath,
117-
String backupRepoType, String backupRepoAddress, String mountOptions) {
118-
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
117+
private void restoreVolumesOfDestroyedVMs(List<String> volumePaths, String vmName, String backupPath, String mountDirectory) {
119118
String diskType = "root";
120119
try {
121120
for (int i = 0; i < volumePaths.size(); i++) {
@@ -133,9 +132,8 @@ private void restoreVolumesOfDestroyedVMs(List<String> volumePaths, String vmNam
133132
}
134133
}
135134

136-
private void restoreVolume(String backupPath, String backupRepoType, String backupRepoAddress, String volumePath,
137-
String diskType, String volumeUUID, Pair<String, VirtualMachine.State> vmNameAndState, String mountOptions) {
138-
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
135+
private void restoreVolume(String backupPath, String volumePath, String diskType, String volumeUUID,
136+
Pair<String, VirtualMachine.State> vmNameAndState, String mountDirectory) {
139137
Pair<String, String> bkpPathAndVolUuid;
140138
try {
141139
bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID);
@@ -155,36 +153,42 @@ private void restoreVolume(String backupPath, String backupRepoType, String back
155153
}
156154

157155

158-
private String mountBackupDirectory(String backupRepoAddress, String backupRepoType, String mountOptions) {
156+
private String mountBackupDirectory(String backupRepoAddress, String backupRepoType, String mountOptions, Integer mountTimeout) {
159157
String randomChars = RandomStringUtils.random(5, true, false);
160158
String mountDirectory = String.format("%s.%s",BACKUP_TEMP_FILE_PREFIX , randomChars);
159+
161160
try {
162161
mountDirectory = Files.createTempDirectory(mountDirectory).toString();
163-
String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory);
164-
if ("cifs".equals(backupRepoType)) {
165-
if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) {
166-
mountOptions = "nobrl";
167-
} else {
168-
mountOptions += ",nobrl";
169-
}
170-
}
171-
if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) {
172-
mount += " -o " + mountOptions;
162+
} catch (IOException e) {
163+
logger.error(String.format("Failed to create the tmp mount directory {} for restore", mountDirectory), e);
164+
throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host");
165+
}
166+
167+
String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory);
168+
if ("cifs".equals(backupRepoType)) {
169+
if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) {
170+
mountOptions = "nobrl";
171+
} else {
172+
mountOptions += ",nobrl";
173173
}
174-
Script.runSimpleBashScript(mount);
175-
} catch (Exception e) {
176-
logger.error(String.format("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory), e);
174+
}
175+
if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) {
176+
mount += " -o " + mountOptions;
177+
}
178+
179+
int exitValue = Script.runSimpleBashScriptForExitValue(mount, mountTimeout, false);
180+
if (exitValue != 0) {
181+
logger.error(String.format("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory));
177182
throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host");
178183
}
179184
return mountDirectory;
180185
}
181186

182187
private void unmountBackupDirectory(String backupDirectory) {
183-
try {
184-
String umountCmd = String.format(UMOUNT_COMMAND, backupDirectory);
185-
Script.runSimpleBashScript(umountCmd);
186-
} catch (Exception e) {
187-
logger.error(String.format("Failed to unmount backup directory {}", backupDirectory), e);
188+
String umountCmd = String.format(UMOUNT_COMMAND, backupDirectory);
189+
int exitValue = Script.runSimpleBashScriptForExitValue(umountCmd);
190+
if (exitValue != 0) {
191+
logger.error(String.format("Failed to unmount backup directory {}", backupDirectory));
188192
throw new CloudRuntimeException("Failed to unmount the backup directory");
189193
}
190194
}

‎server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1340,7 +1340,7 @@ public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws Re
13401340
}
13411341

13421342
if (result != null && !result.first()) {
1343-
String error_msg = String.format("Failed to create Instance [%s] from backup [%s] due to: [%s].", vm.getInstanceName(), backupDetailsInMessage, result.second());
1343+
String error_msg = String.format("Failed to create Instance [%s] from backup [%s] due to: %s.", vm.getInstanceName(), backupDetailsInMessage, result.second());
13441344
logger.error(error_msg);
13451345
processRestoreBackupToVMFailure(vm, backup, eventId);
13461346
throw new CloudRuntimeException(error_msg);

0 commit comments

Comments
 (0)