issue #795 added data encoding detection and conversion to UTF-8

soapui/pom.xml

@@ -787,6 +787,18 @@
             <version>${io.swagger.swagger-parser.version}</version>
         </dependency>
 
+        <!--Tika-->
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>1.27</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-parsers</artifactId>
+            <version>1.27</version>
+        </dependency>
+
 
     </dependencies>
 

soapui/src/main/java/com/eviware/soapui/Util/ByteEncodingDetector.java

@@ -0,0 +1,24 @@
+package com.eviware.soapui.Util;
+
+import com.eviware.soapui.SoapUI;
+import org.apache.tika.parser.txt.CharsetDetector;
+import org.apache.tika.parser.txt.CharsetMatch;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+public class ByteEncodingDetector {
+    public String detectEncoding(byte[] data) {
+        try {
+            InputStream stream = new ByteArrayInputStream(data);
+            CharsetDetector detector = new CharsetDetector();
+            detector.setText(stream);
+            CharsetMatch match = detector.detect();
+
+            return match.getName();
+        } catch (Exception e) {
+            SoapUI.logError(e);
+            return null;
+        }
+    }
+}

soapui/src/main/java/com/eviware/soapui/impl/support/AbstractMockRequest.java

@@ -17,6 +17,7 @@
 package com.eviware.soapui.impl.support;
 
 import com.eviware.soapui.SoapUI;
+import com.eviware.soapui.Util.ByteEncodingDetector;
 import com.eviware.soapui.impl.rest.RestRequestInterface;
 import com.eviware.soapui.impl.wsdl.WsdlOperation;
 import com.eviware.soapui.impl.wsdl.mock.WsdlMockRunContext;
@@ -42,6 +43,7 @@
 import javax.servlet.http.HttpServletResponse;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Enumeration;
 
 public abstract class AbstractMockRequest implements MockRequest {
@@ -175,11 +177,19 @@ private String readRequestContent(HttpServletRequest request) throws Exception {
         int contentOffset = 0;
 
         String contentType = request.getContentType();
+        String dataContent;
+
         if (contentType != null && data.length > 0) {
             if (contentType.toLowerCase().endsWith("xml")) {
                 if (data.length > 3 && data[0] == (byte) 239 && data[1] == (byte) 187 && data[2] == (byte) 191) {
                     encoding = "UTF-8";
                     contentOffset = 3;
+                } else {
+                    //converting the original file encoding to UTF-8
+                    ByteEncodingDetector byteEncodingDetector = new ByteEncodingDetector();
+                    encoding = byteEncodingDetector.detectEncoding(data);
+                    dataContent = new String(data, encoding);
+                    data = dataContent.getBytes(StandardCharsets.UTF_8);
                 }
             }