Skip to content

Commit 6baa598

Browse files
skattoju4skattoju4
authored
Clean up inactive iscsi sessions when VMs get moved due to crashes (#3819)
1 parent 82d94a8 commit 6baa598

2 files changed

Lines changed: 178 additions & 0 deletions

File tree

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848

4949
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
5050
import com.cloud.resource.RequestWrapper;
51+
import com.cloud.hypervisor.kvm.storage.IscsiStorageCleanupMonitor;
5152
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
5253
import org.apache.cloudstack.storage.to.TemplateObjectTO;
5354
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -1086,6 +1087,10 @@ public boolean configure(final String name, final Map<String, Object> params) th
10861087
storageProcessor.configure(name, params);
10871088
storageHandler = new StorageSubsystemCommandHandlerBase(storageProcessor);
10881089

1090+
IscsiStorageCleanupMonitor isciCleanupMonitor = new IscsiStorageCleanupMonitor();
1091+
final Thread cleanupMonitor = new Thread(isciCleanupMonitor);
1092+
cleanupMonitor.start();
1093+
10891094
return true;
10901095
}
10911096

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiStorageCleanupMonitor.java‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.hypervisor.kvm.storage;
18+
19+
import com.cloud.hypervisor.kvm.resource.LibvirtConnection;
20+
import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser;
21+
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
22+
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
23+
import org.apache.log4j.Logger;
24+
import org.libvirt.Connect;
25+
import org.libvirt.Domain;
26+
import org.libvirt.LibvirtException;
27+
28+
import java.io.File;
29+
import java.nio.file.Files;
30+
import java.nio.file.Paths;
31+
import java.util.HashMap;
32+
import java.util.List;
33+
import java.util.Map;
34+
35+
public class IscsiStorageCleanupMonitor implements Runnable{
36+
private static final Logger s_logger = Logger.getLogger(IscsiStorageCleanupMonitor.class);
37+
private static final int CLEANUP_INTERVAL_SEC = 60; // check every X seconds
38+
private static final String ISCSI_PATH_PREFIX = "/dev/disk/by-path";
39+
private static final String KEYWORD_ISCSI = "iscsi";
40+
private static final String KEYWORD_IQN = "iqn";
41+
42+
private IscsiAdmStorageAdaptor iscsiStorageAdaptor;
43+
44+
private Map<String, Boolean> diskStatusMap;
45+
46+
public IscsiStorageCleanupMonitor() {
47+
diskStatusMap = new HashMap<>();
48+
s_logger.debug("Initialize cleanup thread");
49+
iscsiStorageAdaptor = new IscsiAdmStorageAdaptor();
50+
}
51+
52+
53+
private class Monitor extends ManagedContextRunnable {
54+
55+
@Override
56+
protected void runInContext() {
57+
Connect conn = null;
58+
try {
59+
conn = LibvirtConnection.getConnection();
60+
61+
//populate all the iscsi disks currently attached to this host
62+
File[] iscsiVolumes = new File(ISCSI_PATH_PREFIX).listFiles();
63+
if (iscsiVolumes == null || iscsiVolumes.length == 0) {
64+
s_logger.debug("No iscsi sessions found for cleanup");
65+
return;
66+
}
67+
68+
// set all status values to false
69+
initializeDiskStatusMap(iscsiVolumes);
70+
71+
// check if iscsi sessions belong to any VM
72+
updateDiskStatusMapWithInactiveIscsiSessions(conn);
73+
74+
// disconnect stale iscsi sessions
75+
disconnectInactiveSessions();
76+
77+
} catch (LibvirtException e) {
78+
s_logger.warn("[ignored] Error trying to cleanup ", e);
79+
}
80+
}
81+
82+
private boolean isIscsiDisk(String path) {
83+
return path.startsWith(ISCSI_PATH_PREFIX) && path.contains(KEYWORD_ISCSI) && path.contains(KEYWORD_IQN);
84+
}
85+
86+
/**
87+
* for each volume if the volume is path is of type iscsi, add to diskstatusmap and set status to false.
88+
* @param iscsiVolumes
89+
*/
90+
private void initializeDiskStatusMap(File[] iscsiVolumes){
91+
diskStatusMap.clear();
92+
for( File v : iscsiVolumes) {
93+
if (isIscsiDisk(v.getAbsolutePath())) {
94+
s_logger.debug("found iscsi disk by cleanup thread, marking inactive: " + v.getAbsolutePath());
95+
diskStatusMap.put(v.getAbsolutePath(), false);
96+
}
97+
}
98+
}
99+
100+
/** Loop over list of VMs or domains, get disks, if disk is in diskStatusMap, set status value to true.
101+
*
102+
* @param conn
103+
*/
104+
private void updateDiskStatusMapWithInactiveIscsiSessions(Connect conn){
105+
try {
106+
int[] domains = conn.listDomains();
107+
s_logger.debug(String.format("found %d domains", domains.length));
108+
for (int domId : domains) {
109+
Domain dm = conn.domainLookupByID(domId);
110+
final String domXml = dm.getXMLDesc(0);
111+
final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
112+
parser.parseDomainXML(domXml);
113+
List<LibvirtVMDef.DiskDef> disks = parser.getDisks();
114+
115+
//check the volume map. If an entry exists change the status to True
116+
for (final LibvirtVMDef.DiskDef disk : disks) {
117+
if (diskStatusMap.containsKey(disk.getDiskPath())) {
118+
diskStatusMap.put(disk.getDiskPath(), true);
119+
s_logger.debug("active disk found by cleanup thread" + disk.getDiskPath());
120+
}
121+
}
122+
}
123+
} catch (LibvirtException e) {
124+
s_logger.warn("[ignored] Error trying to cleanup ", e);
125+
}
126+
127+
}
128+
129+
/**
130+
* When the state is false, the iscsi sessions are stale. They may be
131+
* removed. We go through each volume which is false, check iscsiadm,
132+
* if the volume still exisits, logout of that volume and remove it from the map
133+
134+
* XXX: It is possible that someone had manually added an iSCSI volume.
135+
* we would not be able to detect that
136+
*/
137+
private void disconnectInactiveSessions(){
138+
139+
for (String diskPath : diskStatusMap.keySet()) {
140+
if (!diskStatusMap.get(diskPath)) {
141+
if (Files.exists(Paths.get(diskPath))) {
142+
try {
143+
s_logger.info("Cleaning up disk " + diskPath);
144+
iscsiStorageAdaptor.disconnectPhysicalDiskByPath(diskPath);
145+
} catch (Exception e) {
146+
s_logger.warn("[ignored] Error cleaning up " + diskPath, e);
147+
}
148+
}
149+
}
150+
}
151+
152+
}
153+
}
154+
155+
@Override
156+
public void run() {
157+
while(true) {
158+
try {
159+
Thread.sleep(CLEANUP_INTERVAL_SEC * 1000);
160+
} catch (InterruptedException e) {
161+
s_logger.debug("[ignored] interupted between heartbeats.");
162+
}
163+
164+
Thread monitorThread = new Thread(new Monitor());
165+
monitorThread.start();
166+
try {
167+
monitorThread.join();
168+
} catch (InterruptedException e) {
169+
s_logger.debug("[ignored] interupted joining monitor.");
170+
}
171+
}
172+
}
173+
}

0 commit comments

Comments
 (0)