Skip to content

Commit 19d6578

Browse files
smeetsryadvr
smeetsr
authored and
yadvr
committed
CLOUDSTACK-10303 : Refactor test data to nuage_test_data.py runnable against simulator (#2483)
* Refactored nuage tests Added simulator support for ConfigDrive Allow all nuage tests to run against simulator Refactored nuage tests to remove code duplication * Move test data from test_data.py to nuage_test_data.py Nuage test data is now contained in nuage_test_data.py instead of test_data.py Removed all nuage test data from nuage_test_data.py * CLOUD-1252 fixed cleanup of vpc tier network * Import libVSD into the codebase * CLOUDSTACK-1253: Volumes are not expunged in simulator * Fixed some merge issues in test_nuage_vsp_mngd_subnets test * Implement GetVolumeStatsCommand in Simulator * Add vspk as marvin nuagevsp dependency, after removing libVSD dependency * correct libVSD files for license purposes pep8 pyflakes compliant
1 parent 7112aff commit 19d6578

24 files changed

Lines changed: 1957 additions & 3405 deletions

‎engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,14 +128,14 @@ protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId)
128128
String sql = sbuilder.toString();
129129
HostVO host = null;
130130
TransactionLegacy txn = TransactionLegacy.currentTxn();
131-
try(PreparedStatement pstmt = txn.prepareStatement(sql);) {
131+
try (PreparedStatement pstmt = txn.prepareStatement(sql)) {
132132
pstmt.setLong(1, poolId);
133133
try(ResultSet rs = pstmt.executeQuery();) {
134134
while (rs.next()) {
135135
long id = rs.getLong(1);
136136
host = hostDao.findById(id);
137137
}
138-
}catch (SQLException e) {
138+
} catch (SQLException e) {
139139
s_logger.warn("can't find endpoint", e);
140140
}
141141
} catch (SQLException e) {

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManager.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
import com.cloud.agent.api.DeleteStoragePoolCommand;
3434
import com.cloud.agent.api.GetStorageStatsAnswer;
3535
import com.cloud.agent.api.GetStorageStatsCommand;
36+
import com.cloud.agent.api.GetVolumeStatsAnswer;
37+
import com.cloud.agent.api.GetVolumeStatsCommand;
38+
import com.cloud.agent.api.HandleConfigDriveIsoCommand;
3639
import com.cloud.agent.api.ManageSnapshotCommand;
3740
import com.cloud.agent.api.ModifyStoragePoolCommand;
3841
import com.cloud.agent.api.SecStorageSetupCommand;
@@ -77,6 +80,8 @@ public interface MockStorageManager extends Manager {
7780

7881
public Answer DownloadProcess(DownloadProgressCommand cmd);
7982

83+
GetVolumeStatsAnswer getVolumeStats(GetVolumeStatsCommand cmd);
84+
8085
public GetStorageStatsAnswer GetStorageStats(GetStorageStatsCommand cmd);
8186

8287
public Answer ManageSnapshot(ManageSnapshotCommand cmd);
@@ -107,4 +112,5 @@ public interface MockStorageManager extends Manager {
107112

108113
public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd);
109114

115+
Answer handleConfigDriveIso(HandleConfigDriveIsoCommand cmd);
110116
}

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
import java.util.List;
2727
import java.util.Map;
2828
import java.util.UUID;
29+
import java.util.function.Function;
30+
import java.util.stream.Collectors;
2931

3032
import javax.inject.Inject;
3133
import javax.naming.ConfigurationException;
@@ -38,6 +40,7 @@
3840
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
3941
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
4042
import org.apache.cloudstack.storage.command.UploadStatusCommand;
43+
import org.apache.cloudstack.storage.to.VolumeObjectTO;
4144

4245
import com.cloud.agent.api.Answer;
4346
import com.cloud.agent.api.AttachIsoCommand;
@@ -52,6 +55,9 @@
5255
import com.cloud.agent.api.DeleteStoragePoolCommand;
5356
import com.cloud.agent.api.GetStorageStatsAnswer;
5457
import com.cloud.agent.api.GetStorageStatsCommand;
58+
import com.cloud.agent.api.GetVolumeStatsAnswer;
59+
import com.cloud.agent.api.GetVolumeStatsCommand;
60+
import com.cloud.agent.api.HandleConfigDriveIsoCommand;
5561
import com.cloud.agent.api.ManageSnapshotAnswer;
5662
import com.cloud.agent.api.ManageSnapshotCommand;
5763
import com.cloud.agent.api.ModifyStoragePoolAnswer;
@@ -60,6 +66,7 @@
6066
import com.cloud.agent.api.SecStorageSetupCommand;
6167
import com.cloud.agent.api.SecStorageVMSetupCommand;
6268
import com.cloud.agent.api.StoragePoolInfo;
69+
import com.cloud.agent.api.VolumeStatsEntry;
6370
import com.cloud.agent.api.storage.CopyVolumeAnswer;
6471
import com.cloud.agent.api.storage.CopyVolumeCommand;
6572
import com.cloud.agent.api.storage.CreateAnswer;
@@ -578,6 +585,37 @@ public DownloadAnswer DownloadProcess(DownloadProgressCommand cmd) {
578585
}
579586
}
580587

588+
@Override
589+
public GetVolumeStatsAnswer getVolumeStats(final GetVolumeStatsCommand cmd) {
590+
HashMap<String, VolumeStatsEntry> volumeStats =
591+
cmd.getVolumeUuids()
592+
.stream()
593+
.collect(Collectors.toMap(Function.identity(),
594+
this::getVolumeStat,
595+
(v1, v2) -> v1, HashMap::new));
596+
597+
return new GetVolumeStatsAnswer(cmd, "", volumeStats);
598+
}
599+
600+
private VolumeStatsEntry getVolumeStat(final String volumeUuid) {
601+
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
602+
603+
try {
604+
txn.start();
605+
MockVolumeVO volume = _mockVolumeDao.findByUuid(volumeUuid);
606+
txn.commit();
607+
return new VolumeStatsEntry(volumeUuid, volume.getSize(), volume.getSize());
608+
} catch (Exception ex) {
609+
txn.rollback();
610+
throw new CloudRuntimeException("Error when finding volume " + volumeUuid, ex);
611+
} finally {
612+
txn.close();
613+
txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
614+
txn.close();
615+
}
616+
617+
}
618+
581619
@Override
582620
public GetStorageStatsAnswer GetStorageStats(GetStorageStatsCommand cmd) {
583621
String uuid = cmd.getStorageId();
@@ -786,9 +824,13 @@ public Answer Delete(DeleteCommand cmd) {
786824
txn.start();
787825
MockVolumeVO template = _mockVolumeDao.findByStoragePathAndType(cmd.getData().getPath());
788826
if (template == null) {
789-
return new Answer(cmd, false, "can't find object to delete:" + cmd.getData().getPath());
827+
if(!((VolumeObjectTO)cmd.getData()).getName().startsWith("ROOT-")) {
828+
return new Answer(cmd, false, "can't find object to delete:" + cmd.getData()
829+
.getPath());
830+
}
831+
} else {
832+
_mockVolumeDao.remove(template.getId());
790833
}
791-
_mockVolumeDao.remove(template.getId());
792834
txn.commit();
793835
} catch (Exception ex) {
794836
txn.rollback();
@@ -1228,4 +1270,49 @@ public CopyVolumeAnswer CopyVolume(CopyVolumeCommand cmd) {
12281270
public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd) {
12291271
return new UploadStatusAnswer(cmd, UploadStatus.COMPLETED);
12301272
}
1273+
1274+
@Override public Answer handleConfigDriveIso(HandleConfigDriveIsoCommand cmd) {
1275+
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
1276+
MockSecStorageVO sec;
1277+
try {
1278+
txn.start();
1279+
sec = _mockSecStorageDao.findByUrl(cmd.getDestStore().getUrl());
1280+
if (sec == null) {
1281+
return new Answer(cmd, false, "can't find secondary storage");
1282+
}
1283+
1284+
txn.commit();
1285+
} catch (Exception ex) {
1286+
txn.rollback();
1287+
throw new CloudRuntimeException("Error when creating config drive.");
1288+
} finally {
1289+
txn.close();
1290+
txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
1291+
txn.close();
1292+
}
1293+
1294+
MockVolumeVO template = new MockVolumeVO();
1295+
String uuid = UUID.randomUUID().toString();
1296+
template.setName(uuid);
1297+
template.setPath(sec.getMountPoint() + cmd.getIsoFile());
1298+
template.setPoolId(sec.getId());
1299+
template.setSize((long)(Math.random() * 200L) + 200L);
1300+
template.setStatus(Status.DOWNLOADED);
1301+
template.setType(MockVolumeType.ISO);
1302+
txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
1303+
try {
1304+
txn.start();
1305+
template = _mockVolumeDao.persist(template);
1306+
txn.commit();
1307+
} catch (Exception ex) {
1308+
txn.rollback();
1309+
throw new CloudRuntimeException("Encountered " + ex.getMessage() + " when persisting config drive " + template.getName(), ex);
1310+
} finally {
1311+
txn.close();
1312+
txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
1313+
txn.close();
1314+
}
1315+
1316+
return new Answer(cmd);
1317+
}
12311318
}

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@
6464
import com.cloud.agent.api.GetStorageStatsCommand;
6565
import com.cloud.agent.api.GetVmStatsCommand;
6666
import com.cloud.agent.api.GetVncPortCommand;
67+
import com.cloud.agent.api.GetVolumeStatsCommand;
68+
import com.cloud.agent.api.HandleConfigDriveIsoCommand;
6769
import com.cloud.agent.api.MaintainCommand;
6870
import com.cloud.agent.api.ManageSnapshotCommand;
6971
import com.cloud.agent.api.MigrateCommand;
@@ -206,6 +208,7 @@ public List<Class<?>> getCommands() {
206208
@DB
207209
@Override
208210
public Answer simulate(final Command cmd, final String hostGuid) {
211+
s_logger.debug("Simulate command " + cmd);
209212
Answer answer = null;
210213
Exception exception = null;
211214
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
@@ -363,6 +366,8 @@ public Answer simulate(final Command cmd, final String hostGuid) {
363366
answer = _mockStorageMgr.Download((DownloadCommand)cmd);
364367
} else if (cmd instanceof GetStorageStatsCommand) {
365368
answer = _mockStorageMgr.GetStorageStats((GetStorageStatsCommand)cmd);
369+
} else if (cmd instanceof GetVolumeStatsCommand) {
370+
answer = _mockStorageMgr.getVolumeStats((GetVolumeStatsCommand)cmd);
366371
} else if (cmd instanceof ManageSnapshotCommand) {
367372
answer = _mockStorageMgr.ManageSnapshot((ManageSnapshotCommand)cmd);
368373
} else if (cmd instanceof BackupSnapshotCommand) {
@@ -431,8 +436,14 @@ public Answer simulate(final Command cmd, final String hostGuid) {
431436
answer = storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd);
432437
} else if (cmd instanceof FenceCommand) {
433438
answer = _mockVmMgr.fence((FenceCommand)cmd);
434-
} else if (cmd instanceof GetRouterAlertsCommand || cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand || cmd instanceof SetMonitorServiceCommand || cmd instanceof AggregationControlCommand ||
435-
cmd instanceof SecStorageFirewallCfgCommand) {
439+
} else if (cmd instanceof HandleConfigDriveIsoCommand) {
440+
answer = _mockStorageMgr.handleConfigDriveIso((HandleConfigDriveIsoCommand)cmd);
441+
} else if (cmd instanceof GetRouterAlertsCommand
442+
|| cmd instanceof VpnUsersCfgCommand
443+
|| cmd instanceof RemoteAccessVpnCfgCommand
444+
|| cmd instanceof SetMonitorServiceCommand
445+
|| cmd instanceof AggregationControlCommand
446+
|| cmd instanceof SecStorageFirewallCfgCommand) {
436447
answer = new Answer(cmd);
437448
} else {
438449
s_logger.error("Simulator does not implement command of type " + cmd.toString());
@@ -447,6 +458,8 @@ public Answer simulate(final Command cmd, final String hostGuid) {
447458
}
448459
}
449460

461+
s_logger.debug("Finished simulate command " + cmd);
462+
450463
return answer;
451464
} catch (final Exception e) {
452465
s_logger.error("Failed execute cmd: ", e);

‎plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java‎

100755100644
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
8585
public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
8686
TemplateObjectTO template = new TemplateObjectTO();
8787
template.setPath(UUID.randomUUID().toString());
88-
template.setSize(new Long(100));
88+
template.setSize(100L);
8989
template.setFormat(Storage.ImageFormat.RAW);
9090
return new CopyCmdAnswer(template);
9191
}

‎server/src/com/cloud/network/element/ConfigDriveNetworkElement.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,9 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
118118
@Inject
119119
VolumeOrchestrationService _volumeMgr;
120120

121-
public final static String CONFIGDRIVEFILENAME = "configdrive.iso";
122-
public final static String CONFIGDRIVEDIR= "ConfigDrive";
123-
public final static Integer CONFIGDRIVEDISKSEQ= new Integer(4);
121+
private final static String CONFIGDRIVEFILENAME = "configdrive.iso";
122+
private final static String CONFIGDRIVEDIR = "ConfigDrive";
123+
private final static Integer CONFIGDRIVEDISKSEQ = 4;
124124

125125
private boolean canHandle(TrafficType trafficType) {
126126
return trafficType.equals(TrafficType.Guest);
@@ -320,9 +320,10 @@ private boolean updateConfigDriveIso(Network network, VirtualMachineProfile prof
320320
s_logger.debug(String.format("%s config drive ISO for vm %s in host %s",
321321
(update?"update":"create"), profile.getInstanceName(), _hostDao.findById(hostId).getName()));
322322
EndPoint endpoint = _ep.select(secondaryStore);
323-
if (endpoint == null )
324-
throw new ResourceUnavailableException(String.format("%s failed, secondary store not available",
325-
(update?"Update":"Create")),secondaryStore.getClass(),secondaryStore.getId());
323+
if (endpoint == null) {
324+
throw new ResourceUnavailableException(String.format("%s failed, secondary store not available", (update ? "Update" : "Create")), secondaryStore.getClass(),
325+
secondaryStore.getId());
326+
}
326327
String isoPath = CONFIGDRIVEDIR + "/" + profile.getInstanceName() + "/" + CONFIGDRIVEFILENAME;
327328
HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(profile.getVmData(),
328329
profile.getConfigDriveLabel(), secondaryStore.getTO(), isoPath, true, update);

‎test/integration/plugins/nuagevsp/libVSD/__init__.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
from .client import ApiClient
19+
from .helpers import VSDHelpers
20+
21+
__version__ = "1.0"
22+
__all__ = ['ApiClient', 'VSDHelpers']

0 commit comments

Comments
 (0)