Skip to content

Commit 9ff819d

Browse files
yadvryadvr
authored
systemvm: new qemu-guest-agent based patching for KVM (#3278)
This introduces a new patching script for patching systemvms on KVM using qemu-guest-agent that runs inside the systemvm on startup. This also removes the vport device which was previously used by the legacy patching script and instead uses the modern and new uniform guest agent vport for host-guest communication. Also updates the sytemvmtemplate build config to use the latest Debian 9.9.0 iso. Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
1 parent f9b61bc commit 9ff819d

13 files changed

Lines changed: 123 additions & 302 deletions

File tree

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@
4747
import javax.xml.parsers.DocumentBuilderFactory;
4848
import javax.xml.parsers.ParserConfigurationException;
4949

50-
import com.cloud.resource.RequestWrapper;
5150
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
5251
import org.apache.cloudstack.storage.to.TemplateObjectTO;
5352
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -146,6 +145,7 @@
146145
import com.cloud.network.Networks.BroadcastDomainType;
147146
import com.cloud.network.Networks.RouterPrivateIpStrategy;
148147
import com.cloud.network.Networks.TrafficType;
148+
import com.cloud.resource.RequestWrapper;
149149
import com.cloud.resource.ServerResource;
150150
import com.cloud.resource.ServerResourceBase;
151151
import com.cloud.storage.JavaStorageLayer;
@@ -199,7 +199,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
199199

200200
private String _modifyVlanPath;
201201
private String _versionstringpath;
202-
private String _patchViaSocketPath;
202+
private String _patchScriptPath;
203203
private String _createvmPath;
204204
private String _manageSnapshotPath;
205205
private String _resizeVolumePath;
@@ -682,9 +682,9 @@ public boolean configure(final String name, final Map<String, Object> params) th
682682
throw new ConfigurationException("Unable to find versions.sh");
683683
}
684684

685-
_patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/", "patchviasocket.py");
686-
if (_patchViaSocketPath == null) {
687-
throw new ConfigurationException("Unable to find patchviasocket.py");
685+
_patchScriptPath = Script.findScript(kvmScriptsDir, "patch.sh");
686+
if (_patchScriptPath == null) {
687+
throw new ConfigurationException("Unable to find patch.sh");
688688
}
689689

690690
_heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@@ -1362,13 +1362,13 @@ private boolean checkOvsNetwork(final String networkName) {
13621362
}
13631363

13641364
public boolean passCmdLine(final String vmName, final String cmdLine) throws InternalErrorException {
1365-
final Script command = new Script(_patchViaSocketPath, 5 * 1000, s_logger);
1365+
final Script command = new Script(_patchScriptPath, 30 * 1000, s_logger);
13661366
String result;
13671367
command.add("-n", vmName);
1368-
command.add("-p", cmdLine.replaceAll(" ", "%"));
1368+
command.add("-c", cmdLine);
13691369
result = command.execute();
13701370
if (result != null) {
1371-
s_logger.error("passcmd failed:" + result);
1371+
s_logger.error("Passing cmdline failed:" + result);
13721372
return false;
13731373
}
13741374
return true;
@@ -2141,12 +2141,6 @@ So if getMinSpeed() returns null we fall back to getSpeed().
21412141
final SerialDef serial = new SerialDef("pty", null, (short)0);
21422142
devices.addDevice(serial);
21432143

2144-
/* Add a VirtIO channel for SystemVMs for communication and provisioning */
2145-
if (vmTO.getType() != VirtualMachine.Type.User) {
2146-
devices.addDevice(new ChannelDef(vmTO.getName() + ".vport", ChannelDef.ChannelType.UNIX,
2147-
new File(_qemuSocketsPath + "/" + vmTO.getName() + ".agent")));
2148-
}
2149-
21502144
if (_rngEnable) {
21512145
final RngDef rngDevice = new RngDef(_rngPath, _rngBackendModel, _rngRateBytes, _rngRatePeriod);
21522146
devices.addDevice(rngDevice);

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,17 +115,18 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
115115

116116
// pass cmdline info to system vms
117117
if (vmSpec.getType() != VirtualMachine.Type.User) {
118-
//wait and try passCmdLine for 5 minutes at most for CLOUDSTACK-2823
119118
String controlIp = null;
120119
for (final NicTO nic : nics) {
121120
if (nic.getType() == TrafficType.Control) {
122121
controlIp = nic.getIp();
123122
break;
124123
}
125124
}
126-
for (int count = 0; count < 30; count++) {
125+
// try to patch and SSH into the systemvm for up to 5 minutes
126+
for (int count = 0; count < 10; count++) {
127+
// wait and try passCmdLine for 30 seconds at most for CLOUDSTACK-2823
127128
libvirtComputingResource.passCmdLine(vmName, vmSpec.getBootArgs());
128-
//check router is up?
129+
// check router is up?
129130
final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource();
130131
final boolean result = virtRouterResource.connect(controlIp, 1, 5000);
131132
if (result) {

‎plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParserTest.java‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@
2222
import java.io.File;
2323
import java.util.List;
2424

25-
import junit.framework.TestCase;
26-
2725
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
2826
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
2927
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
3028
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
3129
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.WatchDogDef;
3230

31+
import junit.framework.TestCase;
32+
3333
public class LibvirtDomainXMLParserTest extends TestCase {
3434

3535
public void testDomainXMLParser() {
@@ -45,9 +45,6 @@ public void testDomainXMLParser() {
4545
InterfaceDef.GuestNetType ifType = InterfaceDef.GuestNetType.BRIDGE;
4646

4747
ChannelDef.ChannelType channelType = ChannelDef.ChannelType.UNIX;
48-
ChannelDef.ChannelState channelState = ChannelDef.ChannelState.DISCONNECTED;
49-
String ssvmAgentPath = "/var/lib/libvirt/qemu/s-2970-VM.agent";
50-
String ssvmAgentName = "s-2970-VM.vport";
5148
String guestAgentPath = "/var/lib/libvirt/qemu/guest-agent.org.qemu.guest_agent.0";
5249
String guestAgentName = "org.qemu.guest_agent.0";
5350

@@ -155,12 +152,6 @@ public void testDomainXMLParser() {
155152
"<target type='serial' port='0'/>" +
156153
"<alias name='serial0'/>" +
157154
"</console>" +
158-
"<channel type='unix'>" +
159-
"<source mode='bind' path='/var/lib/libvirt/qemu/s-2970-VM.agent'/>" +
160-
"<target type='virtio' name='s-2970-VM.vport' state='disconnected'/>" +
161-
"<alias name='channel0'/>" +
162-
"<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
163-
"</channel>" +
164155
"<input type='tablet' bus='usb'>" +
165156
"<alias name='input0'/>" +
166157
"</input>" +
@@ -215,14 +206,9 @@ public void testDomainXMLParser() {
215206
assertEquals(channelType, channels.get(i).getChannelType());
216207
}
217208

218-
/* SSVM provisioning port/channel */
219-
assertEquals(channelState, channels.get(0).getChannelState());
220-
assertEquals(new File(ssvmAgentPath), channels.get(0).getPath());
221-
assertEquals(ssvmAgentName, channels.get(0).getName());
222-
223209
/* Qemu Guest Agent port/channel */
224-
assertEquals(new File(guestAgentPath), channels.get(1).getPath());
225-
assertEquals(guestAgentName, channels.get(1).getName());
210+
assertEquals(new File(guestAgentPath), channels.get(0).getPath());
211+
assertEquals(guestAgentName, channels.get(0).getName());
226212

227213
List<InterfaceDef> ifs = parser.getInterfaces();
228214
for (int i = 0; i < ifs.size(); i++) {

‎plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
import java.io.File;
2323

24-
import junit.framework.TestCase;
25-
2624
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
2725
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
2826
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SCSIDef;
2927
import com.cloud.utils.Pair;
3028

29+
import junit.framework.TestCase;
30+
3131
public class LibvirtVMDefTest extends TestCase {
3232

3333
public void testInterfaceEtehrnet() {
@@ -180,7 +180,7 @@ public void testRngDef() {
180180
public void testChannelDef() {
181181
ChannelDef.ChannelType type = ChannelDef.ChannelType.UNIX;
182182
ChannelDef.ChannelState state = ChannelDef.ChannelState.CONNECTED;
183-
String name = "v-136-VM.vport";
183+
String name = "v-136-VM.org.qemu.guest_agent.0";
184184
File path = new File("/var/lib/libvirt/qemu/" + name);
185185

186186
ChannelDef channelDef = new ChannelDef(name, type, state, path);

‎scripts/installer/windows/client.wxs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1046,7 +1046,7 @@
10461046
<File Id="fil47212822DDAFFCD71C79615CF4583BAF" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\kvmheartbeat.sh" />
10471047
</Component>
10481048
<Component Id="cmpF2BBDD336FEC0B34B3C744ACF1E4B959" Guid="{56D8ECF7-49F8-4B26-A8F4-662252C0A647}">
1049-
<File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patchviasocket.py" />
1049+
<File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patch.sh" />
10501050
</Component>
10511051
<Component Id="cmp2F4D4D81563D153E86B0A652A83D363A" Guid="{7BFC7637-E33D-4BC4-8B25-3CDEA601110C}">
10521052
<File Id="fil349420D6088A01C9F63E27634623F5BE" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\setup_agent.sh" />

‎scripts/vm/hypervisor/kvm/patch.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
set -e
20+
21+
# Get the VM name and cmdline
22+
while getopts "n:c:h" opt; do
23+
case ${opt} in
24+
n )
25+
name=$OPTARG
26+
;;
27+
c )
28+
cmdline=$(echo $OPTARG | base64 -w 0)
29+
;;
30+
h )
31+
echo "Usage: $0 -n [VM name] -c [command line]"
32+
exit 0
33+
;;
34+
esac
35+
done
36+
37+
SSHKEY_FILE="/root/.ssh/id_rsa.pub.cloud"
38+
if [ ! -e $SSHKEY_FILE ]; then
39+
echo "SSH public key file $SSHKEY_FILE not found!"
40+
exit 1
41+
fi
42+
43+
if ! which virsh > /dev/null; then
44+
echo "Libvirt CLI 'virsh' not found"
45+
exit 1
46+
fi
47+
48+
# Read the SSH public key
49+
sshkey=$(cat $SSHKEY_FILE | base64 -w 0)
50+
51+
# Method to send and write payload inside the VM
52+
send_file() {
53+
local name=${1}
54+
local path=${2}
55+
local content=${@:3}
56+
local fd=$(virsh qemu-agent-command $name "{\"execute\":\"guest-file-open\", \"arguments\":{\"path\":\"$path\",\"mode\":\"w+\"}}" | sed 's/[^:]*:\([^}]*\).*/\1/')
57+
virsh qemu-agent-command $name "{\"execute\":\"guest-file-write\", \"arguments\":{\"handle\":$fd,\"buf-b64\":\"$content\"}}" > /dev/null
58+
virsh qemu-agent-command $name "{\"execute\":\"guest-file-close\", \"arguments\":{\"handle\":$fd}}" > /dev/null
59+
}
60+
61+
# Wait for the guest agent to come online
62+
while ! virsh qemu-agent-command $name '{"execute":"guest-ping"}' >/dev/null 2>&1
63+
do
64+
sleep 0.1
65+
done
66+
67+
# Test guest agent sanity
68+
while [ "$(virsh qemu-agent-command $name '{"execute":"guest-sync","arguments":{"id":1234567890}}' 2>/dev/null)" != '{"return":1234567890}' ]
69+
do
70+
sleep 0.1
71+
done
72+
73+
# Write ssh public key
74+
send_file $name "/root/.ssh/authorized_keys" $sshkey
75+
76+
# Fix ssh public key permission
77+
virsh qemu-agent-command $name '{"execute":"guest-exec","arguments":{"path":"chmod","arg":["go-rwx","/root/.ssh/authorized_keys"]}}' > /dev/null
78+
79+
# Write cmdline payload
80+
send_file $name "/var/cache/cloud/cmdline" $cmdline

‎scripts/vm/hypervisor/kvm/patchviasocket.py‎

Lines changed: 0 additions & 76 deletions
This file was deleted.

0 commit comments

Comments
 (0)