Skip to content

Commit e1df782

Browse files
nvazquezDaanHoogland
nvazquez
authored and
DaanHoogland
committed
Add retry logic to direct download and checksum, also refactoring
1 parent f008118 commit e1df782

12 files changed

Lines changed: 232 additions & 86 deletions

File tree

‎agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java‎

Lines changed: 47 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,14 @@
2020

2121
import com.cloud.utils.exception.CloudRuntimeException;
2222
import com.cloud.utils.script.Script;
23-
import org.apache.cloudstack.utils.security.ChecksumValue;
23+
import org.apache.cloudstack.utils.security.DigestHelper;
2424
import org.apache.commons.lang.StringUtils;
25+
import org.apache.log4j.Logger;
2526

2627
import java.io.File;
28+
import java.io.FileInputStream;
29+
import java.io.IOException;
30+
import java.security.NoSuchAlgorithmException;
2731
import java.util.UUID;
2832

2933
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
@@ -34,6 +38,7 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
3438
private String downloadedFilePath;
3539
private String installPath;
3640
private String checksum;
41+
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
3742

3843
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
3944
this.url = url;
@@ -67,6 +72,10 @@ public String getUrl() {
6772
return url;
6873
}
6974

75+
public void setUrl(String url) {
76+
this.url = url;
77+
}
78+
7079
public String getDestPoolPath() {
7180
return destPoolPath;
7281
}
@@ -149,37 +158,47 @@ public DirectTemplateInformation getTemplateInformation() {
149158
return new DirectTemplateInformation(installPath, size, checksum);
150159
}
151160

152-
/**
153-
* Return checksum command from algorithm
154-
*/
155-
private String getChecksumCommandFromAlgorithm(String algorithm) {
156-
if (algorithm.equalsIgnoreCase("MD5")) {
157-
return "md5sum";
158-
} else if (algorithm.equalsIgnoreCase("SHA-1")) {
159-
return "sha1sum";
160-
} else if (algorithm.equalsIgnoreCase("SHA-224")) {
161-
return "sha224sum";
162-
} else if (algorithm.equalsIgnoreCase("SHA-256")) {
163-
return "sha256sum";
164-
} else if (algorithm.equalsIgnoreCase("SHA-384")) {
165-
return "sha384sum";
166-
} else if (algorithm.equalsIgnoreCase("SHA-512")) {
167-
return "sha512sum";
168-
} else {
169-
throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
170-
}
171-
}
172-
173161
@Override
174162
public boolean validateChecksum() {
175163
if (StringUtils.isNotBlank(checksum)) {
176-
ChecksumValue providedChecksum = new ChecksumValue(checksum);
177-
String algorithm = providedChecksum.getAlgorithm();
178-
String checksumCommand = "echo '%s %s' | %s -c --quiet";
179-
String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
180-
int result = Script.runSimpleBashScriptForExitValue(cmd);
181-
return result == 0;
164+
int retry = 3;
165+
boolean valid = false;
166+
try {
167+
while (!valid && retry > 0) {
168+
s_logger.debug("Performing checksum validation for downloaded template " + templateId + ", retries left: " + retry);
169+
valid = DigestHelper.check(checksum, new FileInputStream(downloadedFilePath));
170+
retry--;
171+
if (!valid && retry > 0) {
172+
s_logger.debug("Checksum validation failded, re-downloading template");
173+
resetDownloadFile();
174+
downloadTemplate();
175+
}
176+
}
177+
return valid;
178+
} catch (IOException e) {
179+
throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath, e);
180+
} catch (NoSuchAlgorithmException e) {
181+
throw new CloudRuntimeException("Unknown checksum algorithm: " + checksum, e);
182+
}
182183
}
183184
return true;
184185
}
186+
187+
/**
188+
* Delete and create download file
189+
*/
190+
private void resetDownloadFile() {
191+
File f = new File(getDownloadedFilePath());
192+
s_logger.debug("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it");
193+
try {
194+
if (f.exists()) {
195+
f.delete();
196+
}
197+
f.createNewFile();
198+
} catch (IOException e) {
199+
s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage());
200+
throw new CloudRuntimeException("Failed to create download file for direct download");
201+
}
202+
}
203+
185204
}

‎agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,17 @@
3636
import java.io.InputStream;
3737
import java.io.OutputStream;
3838
import java.io.IOException;
39+
import java.util.HashMap;
3940
import java.util.Map;
4041

4142
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
4243

43-
private HttpClient client;
44+
protected HttpClient client;
4445
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
45-
private static final int CHUNK_SIZE = 1024 * 1024; //1M
4646
protected HttpMethodRetryHandler myretryhandler;
4747
public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName());
4848
protected GetMethod request;
49+
protected Map<String, String> reqHeaders = new HashMap<>();
4950

5051
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
5152
super(url, destPoolPath, templateId, checksum);
@@ -69,6 +70,7 @@ protected GetMethod createRequest(String downloadUrl, Map<String, String> header
6970
if (MapUtils.isNotEmpty(headers)) {
7071
for (String key : headers.keySet()) {
7172
request.setRequestHeader(key, headers.get(key));
73+
reqHeaders.put(key, headers.get(key));
7274
}
7375
}
7476
return request;

‎agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import com.cloud.utils.exception.CloudRuntimeException;
2323
import com.cloud.utils.script.Script;
24+
import org.apache.commons.collections.MapUtils;
2425
import org.apache.http.client.methods.HttpGet;
2526
import org.apache.http.client.methods.HttpUriRequest;
2627
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
@@ -38,14 +39,15 @@
3839
import java.security.KeyStoreException;
3940
import java.security.NoSuchAlgorithmException;
4041
import java.security.cert.CertificateException;
42+
import java.util.Map;
4143

4244
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
4345

4446
private CloseableHttpClient httpsClient;
4547
private HttpUriRequest req;
4648

47-
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) {
48-
super(url, templateId, destPoolPath, checksum, null);
49+
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
50+
super(url, templateId, destPoolPath, checksum, headers);
4951
SSLContext sslcontext = null;
5052
try {
5153
sslcontext = getSSLContext();
@@ -54,11 +56,16 @@ public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoo
5456
}
5557
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
5658
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
57-
req = createUriRequest(url);
59+
createUriRequest(url, headers);
5860
}
5961

60-
protected HttpUriRequest createUriRequest(String downloadUrl) {
61-
return new HttpGet(downloadUrl);
62+
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
63+
req = new HttpGet(downloadUrl);
64+
if (MapUtils.isNotEmpty(headers)) {
65+
for (String headerKey: headers.keySet()) {
66+
req.setHeader(headerKey, headers.get(headerKey));
67+
}
68+
}
6269
}
6370

6471
private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
@@ -86,4 +93,4 @@ public boolean downloadTemplate() {
8693
}
8794
return performDownload();
8895
}
89-
}
96+
}

‎agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,50 @@
1818
//
1919
package com.cloud.agent.direct.download;
2020

21-
import com.cloud.utils.script.Script;
21+
import com.cloud.utils.UriUtils;
22+
import org.apache.commons.collections.CollectionUtils;
2223

2324
import java.io.File;
25+
import java.util.List;
26+
import java.util.Map;
2427

25-
public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
28+
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
2629

27-
private String downloadDir;
28-
29-
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) {
30-
super(url, destPoolPath, templateId, checksum);
31-
String relativeDir = getDirectDownloadTempPath(templateId);
32-
downloadDir = getDestPoolPath() + File.separator + relativeDir;
33-
createFolder(downloadDir);
30+
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers) {
31+
super(url, templateId, destPoolPath, checksum, headers);
3432
}
3533

3634
@Override
3735
public boolean downloadTemplate() {
38-
String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true";
39-
Script.runSimpleBashScript(downloadCommand);
40-
//Remove .metalink file
41-
Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl());
42-
String fileName = Script.runSimpleBashScript("ls " + downloadDir);
43-
if (fileName == null) {
44-
return false;
36+
s_logger.debug("Retrieving metalink file from: " + getUrl() + " to file: " + getDownloadedFilePath());
37+
List<String> metalinkUrls = UriUtils.getMetalinkUrls(getUrl());
38+
if (CollectionUtils.isNotEmpty(metalinkUrls)) {
39+
String downloadDir = getDirectDownloadTempPath(getTemplateId());
40+
boolean downloaded = false;
41+
int i = 0;
42+
while (!downloaded && i < metalinkUrls.size()) {
43+
try {
44+
setUrl(metalinkUrls.get(i));
45+
s_logger.debug("Trying to download template from metalink url: " + getUrl());
46+
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
47+
if (f.exists()) {
48+
f.delete();
49+
f.createNewFile();
50+
}
51+
setDownloadedFilePath(f.getAbsolutePath());
52+
request = createRequest(getUrl(), reqHeaders);
53+
downloaded = super.downloadTemplate();
54+
if (downloaded) {
55+
s_logger.debug("Successfully downloaded template from metalink url: " + getUrl());
56+
break;
57+
}
58+
} catch (Exception e) {
59+
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
60+
}
61+
i++;
62+
}
63+
return downloaded;
4564
}
46-
setDownloadedFilePath(downloadDir + File.separator + fileName);
4765
return true;
4866
}
4967
}

‎core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
2323
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
2424

25+
import java.util.Map;
26+
2527
public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
2628

2729
public enum DownloadProtocol {
@@ -32,12 +34,14 @@ public enum DownloadProtocol {
3234
private Long templateId;
3335
private PrimaryDataStoreTO destPool;
3436
private String checksum;
37+
private Map<String, String> headers;
3538

36-
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
39+
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
3740
this.url = url;
3841
this.templateId = templateId;
3942
this.destPool = destPool;
4043
this.checksum = checksum;
44+
this.headers = headers;
4145
}
4246

4347
public String getUrl() {
@@ -56,6 +60,10 @@ public String getChecksum() {
5660
return checksum;
5761
}
5862

63+
public Map<String, String> getHeaders() {
64+
return headers;
65+
}
66+
5967
@Override
6068
public void setExecuteInSequence(boolean inSeq) {
6169
}

‎core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,8 @@
2424

2525
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
2626

27-
private Map<String, String> headers;
28-
2927
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
30-
super(url, templateId, destPool, checksum);
31-
this.headers = headers;
32-
}
33-
34-
public Map<String, String> getHeaders() {
35-
return headers;
28+
super(url, templateId, destPool, checksum, headers);
3629
}
3730

3831
}

‎core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
2727

2828
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
29-
super(url, templateId, destPool, checksum);
29+
super(url, templateId, destPool, checksum, headers);
3030
}
3131
}

‎core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020

2121
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
2222

23+
import java.util.Map;
24+
2325
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
2426

25-
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) {
26-
super(url, templateId, destPool, checksum);
27+
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
28+
super(url, templateId, destPool, checksum, headers);
2729
}
2830

2931
}

‎core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020

2121
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
2222

23+
import java.util.Map;
24+
2325
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
2426

25-
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
26-
super(url, templateId, destPool, checksum);
27+
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
28+
super(url, templateId, destPool, checksum, headers);
2729
}
2830

2931
}

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1606,13 +1606,13 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
16061606
DirectTemplateDownloader downloader;
16071607

16081608
if (cmd instanceof HttpDirectDownloadCommand) {
1609-
downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders());
1609+
downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
16101610
} else if (cmd instanceof HttpsDirectDownloadCommand) {
1611-
downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum());
1611+
downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders());
16121612
} else if (cmd instanceof NfsDirectDownloadCommand) {
16131613
downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
16141614
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
1615-
downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
1615+
downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders());
16161616
} else {
16171617
return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink");
16181618
}

0 commit comments

Comments
 (0)