Skip to content

Commit 60f3db5

Browse files
syedsyed
committed
Update with cleanup test
1 parent 081fbb7 commit 60f3db5

5 files changed

Lines changed: 402 additions & 89 deletions

File tree

‎services/secondary-storage/server/conf/log4j.xml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
23+
24+
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="false">
25+
26+
<throwableRenderer class="com.cloud.utils.log.CglibThrowableRenderer"/>
27+
<!-- ================================= -->
28+
<!-- Preserve messages in a local file -->
29+
<!-- ================================= -->
30+
31+
<!-- A regular appender FIXME implement code that will close/reopen logs on SIGHUP by logrotate FIXME make the paths configurable using the build system -->
32+
<!-- <appender name="FILE" class="org.apache.log4j.rolling.RollingFileAppender">
33+
<param name="Append" value="true"/>
34+
<param name="Threshold" value="TRACE"/>
35+
<rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
36+
<param name="FileNamePattern" value="@MSLOG@.%d{yyyy-MM-dd}.gz"/>
37+
<param name="ActiveFileName" value="@MSLOG@"/>
38+
</rollingPolicy>
39+
<layout class="org.apache.log4j.EnhancedPatternLayout">
40+
<param name="ConversionPattern" value="%d{ISO8601} %-5p [%c{3}] (%t:%x) %m%n"/>
41+
</layout>
42+
</appender>
43+
-->
44+
<!--
45+
<appender name="APISERVER" class="org.apache.log4j.rolling.RollingFileAppender">
46+
<param name="Append" value="true"/>
47+
<param name="Threshold" value="TRACE"/>
48+
<rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
49+
<param name="FileNamePattern" value="@APISERVERLOG@.%d{yyyy-MM-dd}.gz"/>
50+
<param name="ActiveFileName" value="@APISERVERLOG@"/>
51+
</rollingPolicy>
52+
<layout class="org.apache.log4j.EnhancedPatternLayout">
53+
<param name="ConversionPattern" value="%d{ISO8601} %-5p [%c{3}] (%t:%x) %m%n"/>
54+
</layout>
55+
</appender>
56+
-->
57+
58+
59+
<!-- ============================== -->
60+
<!-- Append warnings+ to the syslog if it is listening on UDP port FIXME make sysloghost configurable! -->
61+
<!-- ============================== -->
62+
<!--
63+
<appender name="SYSLOG" class="org.apache.log4j.net.SyslogAppender">
64+
<param name="Threshold" value="WARN"/>
65+
<param name="SyslogHost" value="localhost"/>
66+
<param name="Facility" value="LOCAL6"/>
67+
<layout class="org.apache.log4j.PatternLayout">
68+
<param name="ConversionPattern" value="%-5p [%c{3}] (%t:%x) %m%n"/>
69+
</layout>
70+
</appender>
71+
-->
72+
<!-- ============================== -->
73+
<!-- Append messages to the console -->
74+
<!-- ============================== -->
75+
76+
<appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
77+
<param name="Target" value="System.out"/>
78+
<param name="Threshold" value="TRACE"/>
79+
<layout class="org.apache.log4j.PatternLayout">
80+
<param name="ConversionPattern" value="%d{ISO8601} %-5p [%c{3}] (%t:%x) %m%n"/>
81+
</layout>
82+
</appender>
83+
84+
<!-- ================ -->
85+
<!-- Limit categories -->
86+
<!-- ================ -->
87+
88+
<category name="com.cloud">
89+
<priority value="DEBUG"/>
90+
</category>
91+
92+
<!-- Limit the org.apache category to INFO as its DEBUG is verbose -->
93+
<category name="org.apache">
94+
<priority value="INFO"/>
95+
</category>
96+
97+
<category name="org">
98+
<priority value="INFO"/>
99+
</category>
100+
101+
<category name="net">
102+
<priority value="INFO"/>
103+
</category>
104+
105+
<category name="apiserver.com.cloud">
106+
<priority value="DEBUG"/>
107+
</category>
108+
109+
<!-- ======================= -->
110+
<!-- Setup the Root category -->
111+
<!-- ======================= -->
112+
113+
<root>
114+
<level value="INFO"/>
115+
<appender-ref ref="CONSOLE"/>
116+
</root>
117+
118+
</log4j:configuration>

‎services/secondary-storage/server/pom.xml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,6 @@
2626
<version>4.7.0-SNAPSHOT</version>
2727
<relativePath>../pom.xml</relativePath>
2828
</parent>
29-
<properties>
30-
<skipTests>true</skipTests>
31-
</properties>
3229
<dependencies>
3330
<dependency>
3431
<groupId>log4j</groupId>

‎services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java‎

Lines changed: 82 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -16,91 +16,7 @@
1616
// under the License.
1717
package org.apache.cloudstack.storage.resource;
1818

19-
import static com.cloud.utils.storage.S3.S3Utils.putFile;
20-
import static com.cloud.utils.StringUtils.join;
21-
import static java.lang.String.format;
22-
import static java.util.Arrays.asList;
23-
import static org.apache.commons.lang.StringUtils.substringAfterLast;
24-
25-
import java.io.BufferedReader;
26-
import java.io.BufferedWriter;
27-
import java.io.File;
28-
import java.io.FileInputStream;
29-
import java.io.FileOutputStream;
30-
import java.io.FileReader;
31-
import java.io.FileWriter;
32-
import java.io.IOException;
33-
import java.io.InputStream;
34-
import java.io.UnsupportedEncodingException;
35-
import java.math.BigInteger;
36-
import java.net.InetAddress;
37-
import java.net.URI;
38-
import java.net.UnknownHostException;
39-
import java.security.MessageDigest;
40-
import java.security.NoSuchAlgorithmException;
41-
import java.util.ArrayList;
42-
import java.util.HashMap;
43-
import java.util.List;
44-
import java.util.Map;
45-
import java.util.UUID;
46-
47-
import javax.naming.ConfigurationException;
48-
49-
import com.cloud.exception.InvalidParameterValueException;
50-
import com.cloud.storage.Storage;
51-
import com.cloud.storage.template.TemplateConstants;
52-
import com.cloud.utils.EncryptionUtil;
53-
import com.google.gson.Gson;
54-
import com.google.gson.GsonBuilder;
55-
import io.netty.bootstrap.ServerBootstrap;
56-
import io.netty.channel.Channel;
57-
import io.netty.channel.ChannelInitializer;
58-
import io.netty.channel.ChannelPipeline;
59-
import io.netty.channel.EventLoopGroup;
60-
import io.netty.channel.nio.NioEventLoopGroup;
61-
import io.netty.channel.socket.SocketChannel;
62-
import io.netty.channel.socket.nio.NioServerSocketChannel;
63-
import io.netty.handler.codec.http.HttpContentCompressor;
64-
import io.netty.handler.codec.http.HttpRequestDecoder;
65-
import io.netty.handler.codec.http.HttpResponseEncoder;
66-
import io.netty.handler.logging.LogLevel;
67-
import io.netty.handler.logging.LoggingHandler;
68-
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
69-
import org.apache.cloudstack.storage.template.UploadEntity;
70-
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
71-
import org.apache.commons.codec.digest.DigestUtils;
72-
import org.apache.commons.io.FileUtils;
73-
import org.apache.commons.io.FilenameUtils;
74-
import org.apache.commons.lang.StringUtils;
75-
import org.apache.http.HttpEntity;
76-
import org.apache.http.HttpResponse;
77-
import org.apache.http.NameValuePair;
78-
import org.apache.http.client.HttpClient;
79-
import org.apache.http.client.methods.HttpGet;
80-
import org.apache.http.client.utils.URLEncodedUtils;
81-
import org.apache.http.impl.client.DefaultHttpClient;
82-
import org.apache.log4j.Logger;
83-
8419
import com.amazonaws.services.s3.model.S3ObjectSummary;
85-
86-
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
87-
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
88-
import org.apache.cloudstack.storage.command.CopyCommand;
89-
import org.apache.cloudstack.storage.command.DeleteCommand;
90-
import org.apache.cloudstack.storage.command.DownloadCommand;
91-
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
92-
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
93-
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
94-
import org.apache.cloudstack.storage.command.UploadStatusCommand;
95-
import org.apache.cloudstack.storage.template.DownloadManager;
96-
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
97-
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
98-
import org.apache.cloudstack.storage.template.UploadManager;
99-
import org.apache.cloudstack.storage.template.UploadManagerImpl;
100-
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
101-
import org.apache.cloudstack.storage.to.TemplateObjectTO;
102-
import org.apache.cloudstack.storage.to.VolumeObjectTO;
103-
10420
import com.cloud.agent.api.Answer;
10521
import com.cloud.agent.api.CheckHealthAnswer;
10622
import com.cloud.agent.api.CheckHealthCommand;
@@ -135,11 +51,13 @@
13551
import com.cloud.agent.api.to.S3TO;
13652
import com.cloud.agent.api.to.SwiftTO;
13753
import com.cloud.exception.InternalErrorException;
54+
import com.cloud.exception.InvalidParameterValueException;
13855
import com.cloud.host.Host;
13956
import com.cloud.host.Host.Type;
14057
import com.cloud.hypervisor.Hypervisor.HypervisorType;
14158
import com.cloud.resource.ServerResourceBase;
14259
import com.cloud.storage.DataStoreRole;
60+
import com.cloud.storage.Storage;
14361
import com.cloud.storage.Storage.ImageFormat;
14462
import com.cloud.storage.StorageLayer;
14563
import com.cloud.storage.VMTemplateStorageResourceAssoc;
@@ -149,21 +67,99 @@
14967
import com.cloud.storage.template.QCOW2Processor;
15068
import com.cloud.storage.template.RawImageProcessor;
15169
import com.cloud.storage.template.TARProcessor;
70+
import com.cloud.storage.template.TemplateConstants;
15271
import com.cloud.storage.template.TemplateLocation;
15372
import com.cloud.storage.template.TemplateProp;
15473
import com.cloud.storage.template.VhdProcessor;
15574
import com.cloud.storage.template.VmdkProcessor;
75+
import com.cloud.utils.EncryptionUtil;
15676
import com.cloud.utils.NumbersUtil;
157-
import com.cloud.utils.storage.S3.S3Utils;
15877
import com.cloud.utils.SwiftUtil;
15978
import com.cloud.utils.exception.CloudRuntimeException;
16079
import com.cloud.utils.net.NetUtils;
16180
import com.cloud.utils.script.OutputInterpreter;
16281
import com.cloud.utils.script.Script;
82+
import com.cloud.utils.storage.S3.S3Utils;
16383
import com.cloud.vm.SecondaryStorageVm;
84+
import com.google.gson.Gson;
85+
import com.google.gson.GsonBuilder;
86+
import io.netty.bootstrap.ServerBootstrap;
87+
import io.netty.channel.Channel;
88+
import io.netty.channel.ChannelInitializer;
89+
import io.netty.channel.ChannelPipeline;
90+
import io.netty.channel.EventLoopGroup;
91+
import io.netty.channel.nio.NioEventLoopGroup;
92+
import io.netty.channel.socket.SocketChannel;
93+
import io.netty.channel.socket.nio.NioServerSocketChannel;
94+
import io.netty.handler.codec.http.HttpContentCompressor;
95+
import io.netty.handler.codec.http.HttpRequestDecoder;
96+
import io.netty.handler.codec.http.HttpResponseEncoder;
97+
import io.netty.handler.logging.LogLevel;
98+
import io.netty.handler.logging.LoggingHandler;
99+
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
100+
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
101+
import org.apache.cloudstack.storage.command.CopyCommand;
102+
import org.apache.cloudstack.storage.command.DeleteCommand;
103+
import org.apache.cloudstack.storage.command.DownloadCommand;
104+
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
105+
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
106+
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
107+
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
108+
import org.apache.cloudstack.storage.command.UploadStatusCommand;
109+
import org.apache.cloudstack.storage.template.DownloadManager;
110+
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
111+
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
112+
import org.apache.cloudstack.storage.template.UploadEntity;
113+
import org.apache.cloudstack.storage.template.UploadManager;
114+
import org.apache.cloudstack.storage.template.UploadManagerImpl;
115+
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
116+
import org.apache.cloudstack.storage.to.TemplateObjectTO;
117+
import org.apache.cloudstack.storage.to.VolumeObjectTO;
118+
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
119+
import org.apache.commons.codec.digest.DigestUtils;
120+
import org.apache.commons.io.FileUtils;
121+
import org.apache.commons.io.FilenameUtils;
122+
import org.apache.commons.lang.StringUtils;
123+
import org.apache.http.HttpEntity;
124+
import org.apache.http.HttpResponse;
125+
import org.apache.http.NameValuePair;
126+
import org.apache.http.client.HttpClient;
127+
import org.apache.http.client.methods.HttpGet;
128+
import org.apache.http.client.utils.URLEncodedUtils;
129+
import org.apache.http.impl.client.DefaultHttpClient;
130+
import org.apache.log4j.Logger;
164131
import org.joda.time.DateTime;
165132
import org.joda.time.format.ISODateTimeFormat;
166133

134+
import javax.naming.ConfigurationException;
135+
import java.io.BufferedReader;
136+
import java.io.BufferedWriter;
137+
import java.io.File;
138+
import java.io.FileInputStream;
139+
import java.io.FileOutputStream;
140+
import java.io.FileReader;
141+
import java.io.FileWriter;
142+
import java.io.IOException;
143+
import java.io.InputStream;
144+
import java.io.UnsupportedEncodingException;
145+
import java.math.BigInteger;
146+
import java.net.InetAddress;
147+
import java.net.URI;
148+
import java.net.UnknownHostException;
149+
import java.security.MessageDigest;
150+
import java.security.NoSuchAlgorithmException;
151+
import java.util.ArrayList;
152+
import java.util.HashMap;
153+
import java.util.List;
154+
import java.util.Map;
155+
import java.util.UUID;
156+
157+
import static com.cloud.utils.StringUtils.join;
158+
import static com.cloud.utils.storage.S3.S3Utils.putFile;
159+
import static java.lang.String.format;
160+
import static java.util.Arrays.asList;
161+
import static org.apache.commons.lang.StringUtils.substringAfterLast;
162+
167163
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
168164

169165
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
@@ -636,6 +632,7 @@ protected void cleanupStagingNfs(TemplateObjectTO newTemplate) {
636632
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
637633
execute(deleteCommand);
638634
} catch (Exception e) {
635+
System.out.println("LOG EVENT ################");
639636
s_logger.debug("Failed to clean up staging area:", e);
640637
}
641638
}

‎services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
*/
1919
package org.apache.cloudstack.storage.resource;
2020

21+
import org.apache.cloudstack.storage.command.DeleteCommand;
22+
import org.apache.cloudstack.storage.to.TemplateObjectTO;
23+
import org.apache.log4j.Level;
24+
import org.apache.log4j.Logger;
2125
import org.junit.Assert;
2226
import org.junit.Before;
2327
import org.junit.Test;
@@ -31,6 +35,10 @@
3135
import java.io.FileWriter;
3236
import java.io.StringWriter;
3337

38+
import static org.mockito.Matchers.any;
39+
import static org.mockito.Mockito.doThrow;
40+
import static org.mockito.Mockito.spy;
41+
3442
@RunWith(PowerMockRunner.class)
3543
public class NfsSecondaryStorageResourceTest {
3644

@@ -48,10 +56,30 @@ public void testSwiftWriteMetadataFile() throws Exception {
4856

4957
StringWriter stringWriter = new StringWriter();
5058
BufferedWriter bufferWriter = new BufferedWriter(stringWriter);
51-
PowerMockito.whenNew(BufferedWriter.class).withArguments(Mockito.any(FileWriter.class)).thenReturn(bufferWriter);
59+
PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter);
5260

5361
resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000);
5462

5563
Assert.assertEquals(expected, stringWriter.toString());
5664
}
65+
66+
@Test
67+
public void testCleanupStagingNfs() throws Exception{
68+
69+
NfsSecondaryStorageResource spyResource = spy(resource);
70+
RuntimeException exception = new RuntimeException();
71+
doThrow(exception).when(spyResource).execute(any(DeleteCommand.class));
72+
TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class);
73+
74+
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
75+
appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:");
76+
TestAppender testLogAppender = appenderBuilder.build();
77+
Logger testLogger = Logger.getLogger(NfsSecondaryStorageResource.class);
78+
TestAppender.safeAddAppender(testLogger, testLogAppender);
79+
80+
spyResource.cleanupStagingNfs(mockTemplate);
81+
82+
testLogAppender.assertMessagesLogged();
83+
84+
}
5785
}

0 commit comments

Comments
 (0)