make MR build

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,23 +1,24 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-
+labels: "check_for_bug"
 ---
 
-<!-- Please format any piece of code you are pasting to make it readable -->
-
 **Describe the bug**
 
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**Job Settings**
 
-Steps to reproduce the behavior:
+```yml
+FULL FSCrawler Job Settings HERE
+```
 
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
+**Logs**
+
+```
+FULL FSCrawler LOGS HERE
+```
 
 **Expected behavior**
 
@@ -27,6 +28,8 @@ A clear and concise description of what you expected to happen.
  - OS: [e.g. MacOS]
  - Version [e.g. 2.5]
 
-**Screenshots**
+**Attachment**
+
+If the bug is related to a given file, please share this file so we can reuse it in tests
+to reproduce the problem and may be use it in our integration tests.
 
-If applicable, add screenshots to help explain your problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question
+    url: https://discuss.elastic.co/c/elasticsearch
+    about: Ask (and answer) questions here. Use fscrawler anywhere in the text as it is monitored.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,7 +1,7 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
-
+labels: "feature_request"
 ---
 
 **Is your feature request related to a problem? Please describe.**

.gitignore

@@ -4,3 +4,5 @@ target
 /.project
 .idea
 *.iml
+/.run
+/logs/

.travis.yml

@@ -1,7 +1,7 @@
 language: java
 sudo: true
 jdk:
-  - openjdk8
+  - oraclejdk15
 install: true
 services:
   - docker

3rdparty/pom.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>fr.pilato.elasticsearch.crawler</groupId>
+        <artifactId>fscrawler-parent</artifactId>
+        <version>2.7-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>fscrawler-3rdparty</artifactId>
+    <name>FSCrawler 3rd Party libraries</name>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>workplacesearch-client</module>
+    </modules>
+
+</project>

3rdparty/workplacesearch-client/pom.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>fscrawler-3rdparty</artifactId>
+        <groupId>fr.pilato.elasticsearch.crawler</groupId>
+        <version>2.7-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>fscrawler-workplacesearch-client</artifactId>
+    <name>FSCrawler Workplace Search Client</name>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+    </build>
+
+    <dependencies>
+        <!-- Our framework -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-framework</artifactId>
+        </dependency>
+
+        <!-- Rest Http Client -->
+        <dependency>
+            <groupId>org.glassfish.jersey.core</groupId>
+            <artifactId>jersey-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.media</groupId>
+            <artifactId>jersey-media-json-jackson</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchAdminClient.java

@@ -0,0 +1,139 @@
+/*
+ * Licensed to David Pilato under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch;
+
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import jakarta.ws.rs.HttpMethod;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+
+import static fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch.WPSearchClient.DEFAULT_HOST;
+
+/**
+ * This class is useless at the moment as we don't have an Admin API yet
+ * TODO: implement when we have an Admin API for Workplace Search
+ */
+public class WPSearchAdminClient implements Closeable {
+
+    private static final Logger logger = LogManager.getLogger(WPSearchAdminClient.class);
+    private static final String DEFAULT_USERNAME = "enterprise_search";
+    private static final String DEFAULT_PASSWORD = "changeme";
+
+    private String host = DEFAULT_HOST;
+    private String username = DEFAULT_USERNAME;
+    private String password = DEFAULT_PASSWORD;
+
+    /**
+     * Define a specific host. Defaults to "http://localhost:3002"
+     * @param host  If we need to change the default host
+     * @return the current instance
+     */
+    public WPSearchAdminClient withHost(String host) {
+        this.host = host;
+        return this;
+    }
+
+    /**
+     * Define the username. Defaults to "enterprise_search"
+     * @param username  If we need to change the default username
+     * @return the current instance
+     */
+    public WPSearchAdminClient withUsername(String username) {
+        this.username = username;
+        return this;
+    }
+
+    /**
+     * Define the password. Defaults to "changeme"
+     * @param password  If we need to change the default password
+     * @return the current instance
+     */
+    public WPSearchAdminClient withPassword(String password) {
+        this.password = password;
+        return this;
+    }
+
+    public void start() throws Exception {
+        // Create the client
+        login(username, password);
+    }
+
+    @Override
+    public void close() {
+        // Close the client
+    }
+
+    private void checkStarted() {
+        // This method is empty as we are waiting for an Admin API for Workplace Search
+    }
+
+    public Map<String, Object> createCustomSource(String sourceName) throws Exception {
+        checkStarted();
+
+        String response = callApi(HttpMethod.POST, "/ws/org/sources/form_create",
+                "{service_type: \"custom\", name: \"" + sourceName + "\"}");
+        // TODO: remove when we have an Admin API for Workplace Search
+        response = "{" +
+                "\"id\":\"FAKE_ID\"," +
+                "\"acces_token\":\"FAKE_TOKEN\"," +
+                "\"key\":\"FAKE_KEY\"" +
+                "}";
+        JsonMapper mapper = JsonMapper.builder().build();
+        Map<String, Object> map = mapper.readValue(response, Map.class);
+
+        logger.debug("Source [{}] created. id={}, acces_token={}, key={}",
+                sourceName, map.get("id"), map.get("accessToken"), map.get("key"));
+
+        return map;
+    }
+
+    public void removeCustomSource(String id) throws Exception {
+        checkStarted();
+
+        // Delete the source
+        callApi(HttpMethod.DELETE, "/ws/org/sources/" + id, null);
+    }
+
+    public void login(String username, String password) {
+        logger.debug("login to Workplace Search as user {}", username);
+    }
+
+    private String callApi(String method, String url, String data)
+            throws IOException {
+        logger.debug("Calling {}", url);
+
+        logger.debug("Faking a {} call to {}", method, new URL(host + url));
+
+        // Create a web request with
+//        url: host + url
+//        httpMethod: method;
+//        requestBody: data
+//        additionalHeader: Content-Type: application/json;charset=UTF-8
+//        additionalHeader: Cookie: session.getName()=session.getValue()
+//        additionalHeader: x-csrf-token: csrfToken
+//        additionalHeader: Accept: application/json, text/plain, */*
+        return null;
+    }
+}

3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkRequest.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch;
+
+import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerBulkRequest;
+
+public class WPSearchBulkRequest extends FsCrawlerBulkRequest<WPSearchOperation> {
+}

3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkResponse.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch;
+
+import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerBulkResponse;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class WPSearchBulkResponse extends FsCrawlerBulkResponse<WPSearchOperation> {
+    private static final Logger logger = LogManager.getLogger(WPSearchBulkResponse.class);
+
+    public WPSearchBulkResponse(String response) {
+        logger.debug(response);
+    }
+}