diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index d96515d6a..a6798ed5d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,23 +1,24 @@ --- name: Bug report about: Create a report to help us improve - +labels: "check_for_bug" --- - - **Describe the bug** A clear and concise description of what the bug is. -**To Reproduce** +**Job Settings** -Steps to reproduce the behavior: +```yml +FULL FSCrawler Job Settings HERE +``` -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error +**Logs** + +``` +FULL FSCrawler LOGS HERE +``` **Expected behavior** @@ -27,6 +28,8 @@ A clear and concise description of what you expected to happen. - OS: [e.g. MacOS] - Version [e.g. 2.5] -**Screenshots** +**Attachment** + +If the bug is related to a given file, please share this file so we can reuse it in tests +to reproduce the problem and may be use it in our integration tests. -If applicable, add screenshots to help explain your problem. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..c8f447719 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Question + url: https://discuss.elastic.co/c/elasticsearch + about: Ask (and answer) questions here. Use fscrawler anywhere in the text as it is monitored. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 2578dc5d2..8746570d5 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea for this project - +labels: "feature_request" --- **Is your feature request related to a problem? Please describe.** diff --git a/.gitignore b/.gitignore index 4db33f382..c9b58b2ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ target /.project .idea *.iml +/.run +/logs/ diff --git a/.travis.yml b/.travis.yml index d9c76d94c..5d0fde3a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: java sudo: true jdk: - - openjdk8 + - oraclejdk15 install: true services: - docker diff --git a/3rdparty/pom.xml b/3rdparty/pom.xml new file mode 100644 index 000000000..52c205e3d --- /dev/null +++ b/3rdparty/pom.xml @@ -0,0 +1,20 @@ + + + + fr.pilato.elasticsearch.crawler + fscrawler-parent + 2.7-SNAPSHOT + + 4.0.0 + + fscrawler-3rdparty + FSCrawler 3rd Party libraries + pom + + + workplacesearch-client + + + diff --git a/3rdparty/workplacesearch-client/pom.xml b/3rdparty/workplacesearch-client/pom.xml new file mode 100644 index 000000000..14a7d56b9 --- /dev/null +++ b/3rdparty/workplacesearch-client/pom.xml @@ -0,0 +1,42 @@ + + + + fscrawler-3rdparty + fr.pilato.elasticsearch.crawler + 2.7-SNAPSHOT + + 4.0.0 + + fscrawler-workplacesearch-client + FSCrawler Workplace Search Client + + + + + src/main/resources + true + + + + + + + + fr.pilato.elasticsearch.crawler + fscrawler-framework + + + + + org.glassfish.jersey.core + jersey-client + + + org.glassfish.jersey.media + jersey-media-json-jackson + + + + diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchAdminClient.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchAdminClient.java new file mode 100644 index 000000000..2d81c5e69 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchAdminClient.java @@ -0,0 +1,139 @@ +/* + * Licensed to David Pilato under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import com.fasterxml.jackson.databind.json.JsonMapper; +import jakarta.ws.rs.HttpMethod; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URL; +import java.util.Map; + +import static fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch.WPSearchClient.DEFAULT_HOST; + +/** + * This class is useless at the moment as we don't have an Admin API yet + * TODO: implement when we have an Admin API for Workplace Search + */ +public class WPSearchAdminClient implements Closeable { + + private static final Logger logger = LogManager.getLogger(WPSearchAdminClient.class); + private static final String DEFAULT_USERNAME = "enterprise_search"; + private static final String DEFAULT_PASSWORD = "changeme"; + + private String host = DEFAULT_HOST; + private String username = DEFAULT_USERNAME; + private String password = DEFAULT_PASSWORD; + + /** + * Define a specific host. Defaults to "http://localhost:3002" + * @param host If we need to change the default host + * @return the current instance + */ + public WPSearchAdminClient withHost(String host) { + this.host = host; + return this; + } + + /** + * Define the username. Defaults to "enterprise_search" + * @param username If we need to change the default username + * @return the current instance + */ + public WPSearchAdminClient withUsername(String username) { + this.username = username; + return this; + } + + /** + * Define the password. Defaults to "changeme" + * @param password If we need to change the default password + * @return the current instance + */ + public WPSearchAdminClient withPassword(String password) { + this.password = password; + return this; + } + + public void start() throws Exception { + // Create the client + login(username, password); + } + + @Override + public void close() { + // Close the client + } + + private void checkStarted() { + // This method is empty as we are waiting for an Admin API for Workplace Search + } + + public Map createCustomSource(String sourceName) throws Exception { + checkStarted(); + + String response = callApi(HttpMethod.POST, "/ws/org/sources/form_create", + "{service_type: \"custom\", name: \"" + sourceName + "\"}"); + // TODO: remove when we have an Admin API for Workplace Search + response = "{" + + "\"id\":\"FAKE_ID\"," + + "\"acces_token\":\"FAKE_TOKEN\"," + + "\"key\":\"FAKE_KEY\"" + + "}"; + JsonMapper mapper = JsonMapper.builder().build(); + Map map = mapper.readValue(response, Map.class); + + logger.debug("Source [{}] created. id={}, acces_token={}, key={}", + sourceName, map.get("id"), map.get("accessToken"), map.get("key")); + + return map; + } + + public void removeCustomSource(String id) throws Exception { + checkStarted(); + + // Delete the source + callApi(HttpMethod.DELETE, "/ws/org/sources/" + id, null); + } + + public void login(String username, String password) { + logger.debug("login to Workplace Search as user {}", username); + } + + private String callApi(String method, String url, String data) + throws IOException { + logger.debug("Calling {}", url); + + logger.debug("Faking a {} call to {}", method, new URL(host + url)); + + // Create a web request with +// url: host + url +// httpMethod: method; +// requestBody: data +// additionalHeader: Content-Type: application/json;charset=UTF-8 +// additionalHeader: Cookie: session.getName()=session.getValue() +// additionalHeader: x-csrf-token: csrfToken +// additionalHeader: Accept: application/json, text/plain, */* + return null; + } +} diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkRequest.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkRequest.java new file mode 100644 index 000000000..c1a628fff --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkRequest.java @@ -0,0 +1,25 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerBulkRequest; + +public class WPSearchBulkRequest extends FsCrawlerBulkRequest { +} diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkResponse.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkResponse.java new file mode 100644 index 000000000..044d2f4e5 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchBulkResponse.java @@ -0,0 +1,32 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerBulkResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class WPSearchBulkResponse extends FsCrawlerBulkResponse { + private static final Logger logger = LogManager.getLogger(WPSearchBulkResponse.class); + + public WPSearchBulkResponse(String response) { + logger.debug(response); + } +} diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchClient.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchClient.java new file mode 100644 index 000000000..41d7f4a96 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchClient.java @@ -0,0 +1,272 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerBulkProcessor; +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerRetryBulkProcessorListener; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readPropertiesFromClassLoader; + +/** + * Workplace Search Java client + * We make it as dumb as possible as it should be replaced in the future by + * an official implementation. + */ +public class WPSearchClient implements Closeable { + + private static final Logger logger = LogManager.getLogger(WPSearchClient.class); + + private final static String WORKPLACESEARCH_PROPERTIES = "workplacesearch.properties"; + private static final Properties properties; + private final static String CLIENT_VERSION; + + static { + properties = readPropertiesFromClassLoader(WORKPLACESEARCH_PROPERTIES); + CLIENT_VERSION = properties.getProperty("workplacesearch.version"); + } + + private final static String CLIENT_NAME = "elastic-workplace-search-java"; + private final static String DEFAULT_ENDPOINT = "/api/ws/v1/"; + public final static String DEFAULT_HOST = "http://127.0.0.1:3002"; + + private Client client; + private String userAgent; + private String endpoint = DEFAULT_ENDPOINT; + private String host = DEFAULT_HOST; + private final String accessToken; + final String urlForBulkCreate; + final String urlForBulkDestroy; + private int bulkSize; + private TimeValue flushInterval; + String urlForApi; + final String urlForSearch; + + private FsCrawlerBulkProcessor bulkProcessor; + + /** + * Create a client + * @param accessToken Access Token to use (get it from the Custom API interface) + * @param key Key to use (get it from the Custom API interface) + */ + public WPSearchClient(String accessToken, String key) { + this.accessToken = accessToken; + this.urlForBulkCreate = "sources/" + key + "/documents/bulk_create"; + this.urlForBulkDestroy = "sources/" + key + "/documents/bulk_destroy"; + this.urlForSearch = "search"; + this.urlForApi = host + endpoint; + } + + /** + * If needed we can allow passing a specific user-agent + * @param userAgent User Agent + * @return the current instance + */ + public WPSearchClient withUserAgent(String userAgent) { + this.userAgent = userAgent; + return this; + } + + /** + * Define a specific endpoint. Defaults to "/api/ws/v1" + * @param endpoint If we need to change the default endpoint + * @return the current instance + */ + public WPSearchClient withEndpoint(String endpoint) { + this.endpoint = endpoint; + this.urlForApi = host + endpoint; + return this; + } + + /** + * Define a specific host. Defaults to "http://localhost:3002" + * @param host If we need to change the default host + * @return the current instance + */ + public WPSearchClient withHost(String host) { + this.host = host; + this.urlForApi = host + endpoint; + return this; + } + + /** + * Defines the bulk size, which is the number of expected operations added to the bulk + * processor before actually sending the bulk request over the network. + * @param bulkSize Number of documents + * @return the current instance + * @see #withFlushInterval(TimeValue) for setting the flush interval + */ + public WPSearchClient withBulkSize(int bulkSize) { + this.bulkSize = bulkSize; + return this; + } + + /** + * Interval to use to flush the existing operations within the bulk processor whatever + * the number of documents. Which means that event you did not reach {@link #withBulkSize(int)} + * number of elements, the content will be flushed after the flushInterval period. + * @param flushInterval A duration. + * @return the current instance + * @see #withBulkSize(int) to set the maximum number of items in a bulk + */ + public WPSearchClient withFlushInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + /** + * Start the client + */ + public void start() { + logger.debug("Starting the WPSearchClient"); + ClientConfig config = new ClientConfig(); + // We need to suppress this so we can do DELETE with body + config.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); + client = ClientBuilder.newClient(config); + + // Create the BulkProcessor instance + bulkProcessor = new FsCrawlerBulkProcessor.Builder<>( + new WPSearchEngine(this), + new FsCrawlerRetryBulkProcessorListener<>(), + WPSearchBulkRequest::new) + .setBulkActions(bulkSize) + .setFlushInterval(flushInterval) + .build(); + } + + /** + * Index one single document + * @param document Document to index + */ + public void indexDocument(Map document) { + logger.debug("Adding document {}", document); + bulkProcessor.add(new WPSearchOperation(document)); + } + + /** + * Delete existing documents + * @param ids List of document ids to delete + */ + public void destroyDocuments(List ids) { + logger.debug("Removing documents {}", ids); + try { + String response = delete(urlForBulkDestroy, ids, String.class); + logger.debug("Removing documents response: {}", response); + // TODO parse the response to check for errors + } catch (NotFoundException e) { + logger.warn("We did not find the resources: {}", ids); + } + } + + /** + * Delete one document + * @param id Document id to delete + */ + public void destroyDocument(String id) { + destroyDocuments(Collections.singletonList(id)); + } + + public String search(String query) { + logger.debug("Searching for {}", query); + Map request = new HashMap<>(); + request.put("query", query); + + // TODO Fix this. It needs a OAuth access apparently and we can't just use the existing credentials + // String response = post(urlForSearch, request, String.class); + String response = "Needs to be implemented..."; + logger.warn("Searching response: {}", response); + return response; + } + + @Override + public void close() { + logger.debug("Closing the WPSearchClient"); + if (bulkProcessor != null) { + try { + bulkProcessor.close(); + } catch (IOException e) { + logger.warn("Error caught while trying to close the Bulk Processor for Workplace Search", e); + } + } + if (client != null) { + client.close(); + } + } + + T post(String path, Object data, Class clazz) { + return prepareHttpCall(path).post(Entity.entity(data, MediaType.APPLICATION_JSON), clazz); + } + + private T delete(String path, Object data, Class clazz) { + // TODO This does not remove the entity. Something to fix in the future... + return prepareHttpCall(path).method("DELETE", Entity.entity(data, MediaType.APPLICATION_JSON), clazz); + } + + private Invocation.Builder prepareHttpCall(String path) { + WebTarget target = client + .target(urlForApi) + .path(path); + + Invocation.Builder builder = target + .request(MediaType.APPLICATION_JSON) + .header("Content-Type", "application/json") + .header("X-Swiftype-Client", CLIENT_NAME) + .header("X-Swiftype-Client-Version", CLIENT_VERSION) + .header("Authorization", "Bearer " + accessToken); + + if (userAgent != null) { + builder.header("User-Agent", userAgent); + } + + return builder; + } + + @Override + public String toString() { + final StringBuffer sb = new StringBuffer("WPSearchClient{"); + sb.append("endpoint='").append(endpoint).append('\''); + sb.append(", host='").append(host).append('\''); + sb.append(", urlForBulkCreate='").append(urlForBulkCreate).append('\''); + sb.append(", urlForBulkDestroy='").append(urlForBulkDestroy).append('\''); + sb.append(", urlForApi='").append(urlForApi).append('\''); + sb.append(", urlForSearch='").append(urlForSearch).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchEngine.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchEngine.java new file mode 100644 index 000000000..551317d61 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchEngine.java @@ -0,0 +1,53 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.Engine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class WPSearchEngine implements Engine { + private static final Logger logger = LogManager.getLogger(WPSearchEngine.class); + private final WPSearchClient wpSearchClient; + + public WPSearchEngine(WPSearchClient wpSearchClient) { + this.wpSearchClient = wpSearchClient; + } + + @Override + public WPSearchBulkResponse bulk(WPSearchBulkRequest request) { + List> documents = new ArrayList<>(); + for (WPSearchOperation operation : request.getOperations()) { + documents.add(operation.getDocument()); + } + + try { + String response = wpSearchClient.post(wpSearchClient.urlForBulkCreate, documents, String.class); + return new WPSearchBulkResponse(response); + } catch (Exception e) { + logger.error(e); + return new WPSearchBulkResponse(e.getMessage()); + } + } +} diff --git a/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchOperation.java b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchOperation.java new file mode 100644 index 000000000..4bef30ea0 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchOperation.java @@ -0,0 +1,44 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch; + +import fr.pilato.elasticsearch.crawler.fs.framework.bulk.FsCrawlerOperation; + +import java.util.Map; + +public class WPSearchOperation implements FsCrawlerOperation { + private final Map document; + + public WPSearchOperation(Map document) { + this.document = document; + } + + public Map getDocument() { + return document; + } + + @Override + public int compareTo(WPSearchOperation request) { + // We check on the id field + String id1 = (String) document.get("id"); + String id2 = (String) request.document.get("id"); + return id1.compareTo(id2); + } +} diff --git a/3rdparty/workplacesearch-client/src/main/resources/workplacesearch.properties b/3rdparty/workplacesearch-client/src/main/resources/workplacesearch.properties new file mode 100644 index 000000000..750b7bce1 --- /dev/null +++ b/3rdparty/workplacesearch-client/src/main/resources/workplacesearch.properties @@ -0,0 +1 @@ +workplacesearch.version=${elasticsearch.version} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d62c1858..fab89528d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,157 +5,12 @@ some tricks which might help. ## Building -The project is built using [Apache Maven](https://maven.apache.org/). -It needs Java >= 1.8. - -To build it, just run: - -```sh -git clone https://github.com/dadoonet/fscrawler.git -cd fscrawler -mvn clean install -``` - -You can also skip running tests with: - -```sh -mvn clean install -DskipTests -``` +Read the [developer guide](https://fscrawler.readthedocs.io/en/latest/dev/build.html) to get started. ## Testing -### Integration tests - -Unless a node is already running at this address, integration tests use by default a Docker configuration -which starts a local node running at [127.0.0.1:9200](http://127.0.0.1:9200). -The elasticsearch version used is defined in the `pom.xml` file. - -By default, it will run integration tests against elasticsearch 7.x, 6.x and 5.x series. - -You can also run test using security feature by using the `security` maven profile: - -```sh -mvn install -Psecurity -``` - -### Running tests against an external cluster - -By default, FS Crawler will run tests against a cluster running at `127.0.0.1` on port `9200` -which is started by the maven plugin. - -If you started manually an elasticsearch cluster locally, integration tests can use it if you -define the `tests.cluster.url` setting. -This could speed up dramatically running the integration while developing on FSCrawler. - -If you want to run the test suite against an external cluster, using other credentials, you can use the following -system parameters: - -* `tests.cluster.url`: http://127.0.0.1:9200 (if set, local Docker instance won't be started) -* `tests.cluster.user`: username (defaults to `elastic`) -* `tests.cluster.pass`: password (defaults to `changeme`) - -For example, if you have a cluster running on [Elastic Cloud](https://cloud.elastic.co/), you can use: - -```sh -mvn clean install -Dtests.cluster.url=https://CLUSTERID.eu-west-1.aws.found.io:9243 -Dtests.cluster.user=elastic -Dtests.cluster.pass=GENERATEDPASSWORD -``` - -### Running REST tests using another port - -By default, FS crawler will run the integration tests using port `8080` for the REST service. -If you want to change it because this port is already used, you can start your tests with `-Dtests.rest.port=8280` which -will start the REST service on port `8280` - - -### Randomized testing - -FS Crawler uses [Randomized testing framework](https://github.com/randomizedtesting/randomizedtesting). -In case of failure, it will print a line like: - -``` -REPRODUCE WITH: -mvn test -Dtests.seed=AC6992149EB4B547 -Dtests.class=fr.pilato.elasticsearch.crawler.fs.test.unit.tika.TikaDocParserTest -Dtests.method="testExtractFromRtf" -Dtests.locale=ga-IE -Dtests.timezone=Canada/Saskatchewan -``` - - -It also exposes some parameters you can use at build time: - -* `tests.output`: display test output. For example: - -```sh -mvn install -Dtests.output=always -mvn install -Dtests.output=onError -``` - -* `tests.locale`: run the tests using a given Locale. For example: - -```sh -mvn install -Dtests.locale=random -mvn install -Dtests.locale=fr-FR -``` - -* `tests.timezone`: run the tests using a given Timezone. For example: - -```sh -mvn install -Dtests.timezone=random -mvn install -Dtests.timezone=CEST -mvn install -Dtests.timezone=-0200 -``` - -* `tests.verbose`: adds running tests details while executing tests - -```sh -mvn install -Dtests.verbose -``` - -* `tests.parallelism`: number of JVMs to start to run tests - -```sh -mvn install -Dtests.parallelism=auto -mvn install -Dtests.parallelism=max -mvn install -Dtests.parallelism=1 -``` - -* `tests.seed`: specify the seed to use to run the test suite if you need to reproduce a failure -given a specific test context. - -```sh -mvn test -Dtests.seed=E776CE45185A6E7A -``` - -* `tests.leaveTemporary`: leave temporary files on disk - -```sh -mvn test -Dtests.leaveTemporary -``` - +Read the [developer guide](https://fscrawler.readthedocs.io/en/latest/dev/build.html#integration-tests) to get started. ## Releasing -To release a version, just run: - -```sh -./release.sh -``` - -The release script will: - -* Create a release branch -* Replace SNAPSHOT version by the final version number -* Commit the change -* Run tests against all supported elasticsearch series -* Build the final artifacts using release profile (signing artifacts and generating all needed files) -* Tag the version -* Prepare the announcement email -* Deploy to https://oss.sonatype.org/ -* Prepare the next SNAPSHOT version -* Commit the change -* Release the Sonatype staging repository -* Merge the release branch to the branch we started from -* Push the changes to origin -* Announce the version on https://discuss.elastic.co/c/annoucements/community-ecosystem - -You will be guided through all the steps. - -You can add some maven options while executing the release script such as `-DskipTests` if you want to skip -the tests while building the release. +Read the [release guide](https://fscrawler.readthedocs.io/en/latest/dev/release.html). diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Doc.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Doc.java index aac4c1ba8..9a80cc6ee 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Doc.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/Doc.java @@ -19,7 +19,6 @@ package fr.pilato.elasticsearch.crawler.fs.beans; -import java.util.HashMap; import java.util.Map; /** @@ -45,7 +44,7 @@ static public final class FIELD_NAMES { private File file; private Path path; private Attributes attributes; - private Map object; + private Map object; private Map external; public Doc() { @@ -55,11 +54,16 @@ public Doc() { external = null; } - public Map getObject() { + public Doc(String content) { + this(); + this.content = content; + } + + public Map getObject() { return object; } - public void setObject(Map newObject) { + public void setObject(Map newObject) { this.object = newObject; } diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocParser.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocParser.java index dc896ab90..9ebe05b4e 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocParser.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocParser.java @@ -19,8 +19,6 @@ package fr.pilato.elasticsearch.crawler.fs.beans; -import com.fasterxml.jackson.core.JsonProcessingException; - import java.io.IOException; import java.util.Map; @@ -28,15 +26,21 @@ public class DocParser { - public static String toJson(Doc doc) throws JsonProcessingException { - return prettyMapper.writeValueAsString(doc); + public static String toJson(Doc doc) { + try { + return prettyMapper.writeValueAsString(doc); + } catch (IOException e) { + // TODO Fix that code. We should log here and return null. + throw new RuntimeException(e); + } } public static Doc fromJson(String json) throws IOException { return prettyMapper.readValue(json, Doc.class); } - public static Map asMap(String json) throws IOException { - return prettyMapper.readValue(json, Map.class); + public static Map asMap(String json) throws IOException { + //noinspection unchecked + return (Map) prettyMapper.readValue(json, Map.class); } } diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/FsJob.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/FsJob.java index 557ea9591..d24a518e6 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/FsJob.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/FsJob.java @@ -20,6 +20,7 @@ package fr.pilato.elasticsearch.crawler.fs.beans; import java.time.LocalDateTime; +import java.util.Objects; /** * Define a FS Job meta data @@ -118,8 +119,8 @@ public boolean equals(Object o) { if (indexed != fsJob.indexed) return false; if (deleted != fsJob.deleted) return false; - if (name != null ? !name.equals(fsJob.name) : fsJob.name != null) return false; - return !(lastrun != null ? !lastrun.equals(fsJob.lastrun) : fsJob.lastrun != null); + if (!Objects.equals(name, fsJob.name)) return false; + return Objects.equals(lastrun, fsJob.lastrun); } diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/PathParser.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/PathParser.java index aa9389c3c..e280158b8 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/PathParser.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/PathParser.java @@ -19,16 +19,19 @@ package fr.pilato.elasticsearch.crawler.fs.beans; -import com.fasterxml.jackson.core.JsonProcessingException; - import java.io.IOException; import static fr.pilato.elasticsearch.crawler.fs.framework.MetaParser.prettyMapper; public class PathParser { - public static String toJson(Path path) throws JsonProcessingException { - return prettyMapper.writeValueAsString(path); + public static String toJson(Path path) { + try { + return prettyMapper.writeValueAsString(path); + } catch (IOException e) { + // TODO Fix that code. We should log here and return null. + throw new RuntimeException(e); + } } public static Path fromJson(String json) throws IOException { diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/ScanStatistic.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/ScanStatistic.java index 644d51b37..ece6f55c5 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/ScanStatistic.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/ScanStatistic.java @@ -25,8 +25,8 @@ * @author David Pilato (aka dadoonet) */ public class ScanStatistic { - private int nbDocScan = 0; - private int nbDocDeleted = 0; + private int nbDocScan; + private int nbDocDeleted; private String rootPath; private String rootPathId; diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/BootstrapChecks.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/BootstrapChecks.java index 54279ef93..818198d3c 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/BootstrapChecks.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/BootstrapChecks.java @@ -26,6 +26,7 @@ import org.apache.logging.log4j.Logger; import java.math.BigDecimal; +import java.math.RoundingMode; import java.nio.charset.StandardCharsets; /** @@ -59,7 +60,7 @@ static Percentage computePercentage(ByteSizeValue current, ByteSizeValue max) { if (max.getBytes() <= 0) { return new Percentage(); } - return new Percentage((new BigDecimal(((double) current.getBytes())/(double) max.getBytes()*100)).setScale(2, BigDecimal.ROUND_HALF_EVEN).doubleValue(), true); + return new Percentage((BigDecimal.valueOf(((double) current.getBytes()) / (double) max.getBytes() * 100)).setScale(2, RoundingMode.HALF_EVEN).doubleValue(), true); } private static void checkUTF8() { diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index e48ea940b..a95ff0642 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -23,8 +23,10 @@ import com.beust.jcommander.Parameter; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.beans.FsJobFileHandler; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.MetaFileHandler; +import fr.pilato.elasticsearch.crawler.fs.framework.Version; import fr.pilato.elasticsearch.crawler.fs.rest.RestServer; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; @@ -32,12 +34,17 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.filter.LevelMatchFilter; +import org.apache.logging.log4j.core.filter.LevelRangeFilter; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -46,9 +53,7 @@ import java.util.List; import java.util.Scanner; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readDefaultJsonVersionedFile; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; /** * Main entry point to launch FsCrawler @@ -97,7 +102,6 @@ public static class FsCrawlerCommand { } - @SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { // create a scanner so we can read the command-line input Scanner scanner = new Scanner(System.in); @@ -110,29 +114,33 @@ public static void main(String[] args) throws Exception { if (commands.debug || commands.trace || commands.silent) { LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(FsCrawlerCli.class.getPackage().getName()); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + ConsoleAppender console = config.getAppender("Console"); if (commands.silent) { - // Check if the user also asked for --debug or --trace which is contradictory - if (commands.debug || commands.trace) { - logger.warn("--debug or --trace can't be used when --silent is set. Only silent mode will be activated."); - } // If the user did not enter any job name, nothing will be displayed if (commands.jobName == null) { + banner(); logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting."); jCommander.usage(); return; } - // We change the full rootLogger level - LoggerConfig rootLogger = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(Level.OFF); - rootLogger.setLevel(Level.OFF); + // We don't write anything on the console anymore + console.addFilter(LevelMatchFilter.newBuilder().setLevel(Level.ALL).setOnMatch(Filter.Result.DENY).build()); } else { - loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); + console.addFilter(LevelRangeFilter.createFilter( + commands.debug ? Level.DEBUG : Level.TRACE, + Level.ALL, + Filter.Result.DENY, + Filter.Result.ACCEPT)); } + + loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); ctx.updateLoggers(); } + banner(); + if (commands.help) { jCommander.usage(); return; @@ -162,23 +170,23 @@ public static void main(String[] args) throws Exception { if (commands.jobName == null) { // The user did not enter a job name. // We can list available jobs for him - logger.info("No job specified. Here is the list of existing jobs:"); + FSCrawlerLogger.console("No job specified. Here is the list of existing jobs:"); List files = FsCrawlerJobsUtil.listExistingJobs(configDir); if (!files.isEmpty()) { for (int i = 0; i < files.size(); i++) { - logger.info("[{}] - {}", i+1, files.get(i)); + FSCrawlerLogger.console("[{}] - {}", i+1, files.get(i)); } int chosenFile = 0; while (chosenFile <= 0 || chosenFile > files.size()) { - logger.info("Choose your job [1-{}]...", files.size()); + FSCrawlerLogger.console("Choose your job [1-{}]...", files.size()); chosenFile = scanner.nextInt(); } jobName = files.get(chosenFile - 1); } else { - logger.info("No job exists in [{}].", configDir); - logger.info("To create your first job, run 'fscrawler job_name' with 'job_name' you want"); + FSCrawlerLogger.console("No job exists in [{}].", configDir); + FSCrawlerLogger.console("To create your first job, run 'fscrawler job_name' with 'job_name' you want"); return; } @@ -210,18 +218,24 @@ public static void main(String[] args) throws Exception { } if (username != null && fsSettings.getElasticsearch().getPassword() == null) { - logger.info("Password for " + username + ":"); + FSCrawlerLogger.console("Password for {}:", username); String password = scanner.next(); fsSettings.getElasticsearch().setUsername(username); fsSettings.getElasticsearch().setPassword(password); } } catch (NoSuchFileException e) { - logger.warn("job [{}] does not exist", jobName); + // We can only have a dialog with the end user if we are not silent + if (commands.silent) { + logger.error("job [{}] does not exist. Exiting as we are in silent mode.", jobName); + return; + } + + FSCrawlerLogger.console("job [{}] does not exist", jobName); String yesno = null; while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) { - logger.info("Do you want to create it (Y/N)?"); + FSCrawlerLogger.console("Do you want to create it (Y/N)?"); yesno = scanner.next(); } @@ -233,39 +247,48 @@ public static void main(String[] args) throws Exception { fsSettingsFileHandler.write(fsSettings); Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML); - logger.info("Settings have been created in [{}]. Please review and edit before relaunch", config); + FSCrawlerLogger.console("Settings have been created in [{}]. Please review and edit before relaunch", config); } return; } - logger.trace("settings used for this crawler: [{}]", FsSettingsParser.toYaml(fsSettings)); + if (logger.isTraceEnabled()) { + logger.trace("settings used for this crawler: [{}]", FsSettingsParser.toYaml(fsSettings)); + } if (FsCrawlerValidator.validateSettings(logger, fsSettings, commands.rest)) { // We don't go further as we have critical errors return; } - FsCrawlerImpl fsCrawler = new FsCrawlerImpl(configDir, fsSettings, commands.loop, commands.rest); - Runtime.getRuntime().addShutdownHook(new FSCrawlerShutdownHook(fsCrawler)); + // We add a special case here in case someone tries to use workplace search + if (fsSettings.getWorkplaceSearch() != null) { + logger.info("Workplace Search integration is an experimental feature. " + + "As is it is not fully implemented and settings might change in the future."); + if (commands.loop == -1 || commands.loop > 1) { + logger.warn("Workplace Search integration does not support yet watching a directory. " + + "It will be able to run only once and exit. We manually force from --loop {} to --loop 1. " + + "If you want to remove this message next time, please start FSCrawler with --loop 1", + commands.loop); + commands.loop = 1; + } + } - try { + try (FsCrawlerImpl fsCrawler = new FsCrawlerImpl(configDir, fsSettings, commands.loop, commands.rest)) { + Runtime.getRuntime().addShutdownHook(new FSCrawlerShutdownHook(fsCrawler)); // Let see if we want to upgrade an existing cluster to latest version if (commands.upgrade) { logger.info("Upgrading job [{}]. No rule implemented. Skipping.", jobName); } else { - try { - fsCrawler.getEsClient().start(); - } catch (Exception t) { - logger.fatal("We can not start Elasticsearch Client. Exiting.", t); + if (!startEsClient(fsCrawler)) { return; } - String elasticsearchVersion = fsCrawler.getEsClient().getVersion(); + String elasticsearchVersion = fsCrawler.getManagementService().getClient().getVersion(); checkForDeprecatedResources(configDir, elasticsearchVersion); - fsCrawler.start(); // Start the REST Server if needed if (commands.rest) { - RestServer.start(fsSettings, fsCrawler.getEsClient()); + RestServer.start(fsSettings, fsCrawler.getManagementService(), fsCrawler.getDocumentService()); } // We just have to wait until the process is stopped @@ -276,11 +299,77 @@ public static void main(String[] args) throws Exception { } catch (Exception e) { logger.fatal("Fatal error received while running the crawler: [{}]", e.getMessage()); logger.debug("error caught", e); - } finally { - fsCrawler.close(); } } + private static boolean startEsClient(FsCrawlerImpl fsCrawler) { + try { + fsCrawler.getManagementService().start(); + fsCrawler.getDocumentService().start(); + return true; + } catch (Exception t) { + logger.fatal("We can not start Elasticsearch Client. Exiting.", t); + return false; + } + } + + private static final int BANNER_LENGTH = 100; + + /** + * This is coming from: https://patorjk.com/software/taag/#p=display&f=3D%20Diagonal&t=FSCrawler + */ + private static final String ASCII_ART = "" + + " ,---,. .--.--. ,----.. ,--, \n" + + " ,' .' | / / '. / / \\ ,--.'| \n" + + ",---.' || : /`. / | : : __ ,-. .---.| | : __ ,-.\n" + + "| | .'; | |--` . | ;. /,' ,'/ /| /. ./|: : ' ,' ,'/ /|\n" + + ": : : | : ;_ . ; /--` ' | |' | ,--.--. .-'-. ' || ' | ,---. ' | |' |\n" + + ": | |-, \\ \\ `. ; | ; | | ,'/ \\ /___/ \\: |' | | / \\ | | ,'\n" + + "| : ;/| `----. \\| : | ' : / .--. .-. | .-'.. ' ' .| | : / / |' : / \n" + + "| | .' __ \\ \\ |. | '___ | | ' \\__\\/: . ./___/ \\: '' : |__ . ' / || | ' \n" + + "' : ' / /`--' /' ; : .'|; : | ,\" .--.; |. \\ ' .\\ | | '.'|' ; /|; : | \n" + + "| | | '--'. / ' | '/ :| , ; / / ,. | \\ \\ ' \\ |; : ;' | / || , ; \n" + + "| : \\ `--'---' | : / ---' ; : .' \\ \\ \\ |--\" | , / | : | ---' \n" + + "| | ,' \\ \\ .' | , .-./ \\ \\ | ---`-' \\ \\ / \n" + + "`----' `---` `--`---' '---\" `----' \n"; + + private static void banner() { + FSCrawlerLogger.console( + separatorLine(",", ".") + + centerAsciiArt() + + separatorLine("+", "+") + + bannerLine("You know, for Files!") + + bannerLine("Made from France with Love") + + bannerLine("Source: https://github.com/dadoonet/fscrawler/") + + bannerLine("Documentation: https://fscrawler.readthedocs.io/") + + separatorLine("`", "'")); + } + + private static String centerAsciiArt() { + String[] lines = StringUtils.split(ASCII_ART, '\n'); + + // Edit line 0 as we want to add the version + String version = Version.getVersion(); + String firstLine = StringUtils.stripEnd(StringUtils.center(lines[0], BANNER_LENGTH), null); + String pad = StringUtils.rightPad(firstLine, BANNER_LENGTH - version.length() - 1) + version; + lines[0] = pad; + + StringBuilder content = new StringBuilder(); + for (String line : lines) { + content.append(bannerLine(line)); + } + + return content.toString(); + } + + private static String bannerLine(String text) { + return "|" + StringUtils.center(text, BANNER_LENGTH) + "|\n"; + } + + private static String separatorLine(String first, String last) { + return first + StringUtils.center("", BANNER_LENGTH, "-") + last + "\n"; + } + private static void checkForDeprecatedResources(Path configDir, String elasticsearchVersion) throws IOException { try { // If we are able to read an old configuration file, we should tell the user to check the documentation @@ -288,14 +377,18 @@ private static void checkForDeprecatedResources(Path configDir, String elasticse logger.warn("We found old configuration index settings: [{}/_default/doc.json]. You should look at the documentation" + " about upgrades: https://fscrawler.readthedocs.io/en/latest/installation.html#upgrade-fscrawler.", configDir); - } catch (IllegalArgumentException ignored) { } + } catch (IllegalArgumentException ignored) { + // If we can't find the deprecated resource, it will throw an exception which needs to be ignored. + } try { // If we are able to read an old configuration file, we should tell the user to check the documentation readDefaultJsonVersionedFile(configDir, extractMajorVersion(elasticsearchVersion), "folder"); logger.warn("We found old configuration index settings: [{}/_default/folder.json]. You should look at the documentation" + " about upgrades: https://fscrawler.readthedocs.io/en/latest/installation.html#upgrade-fscrawler.", configDir); - } catch (IllegalArgumentException ignored) { } + } catch (IllegalArgumentException ignored) { + // If we can't find the deprecated resource, it will throw an exception which needs to be ignored. + } } private static void sleep() { diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerJobsUtil.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerJobsUtil.java index 8b9694031..d576ec155 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerJobsUtil.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerJobsUtil.java @@ -41,12 +41,12 @@ public static List listExistingJobs(Path configDir) { // This is a directory. Let's see if we have the _settings.json file in it if (Files.isDirectory(path)) { String jobName = path.getFileName().toString(); - Path jobSettingsFile = path.resolve(FsSettingsFileHandler.FILENAME_JSON); - if (Files.exists(jobSettingsFile)) { + if (Files.exists(path.resolve(FsSettingsFileHandler.SETTINGS_YAML)) + || Files.exists(path.resolve(FsSettingsFileHandler.FILENAME_JSON))) { files.add(jobName); - logger.debug("Adding job [{}]", jobName, FsSettingsFileHandler.FILENAME_JSON); + logger.debug("Adding job [{}]", jobName); } else { - logger.debug("Ignoring [{}] dir as no [{}] has been found", jobName, FsSettingsFileHandler.FILENAME_JSON); + logger.debug("Ignoring [{}] dir as no settings file has been found", jobName); } } } diff --git a/cli/src/main/resources/log4j2.xml b/cli/src/main/resources/log4j2.xml index 934aff085..8b128b975 100644 --- a/cli/src/main/resources/log4j2.xml +++ b/cli/src/main/resources/log4j2.xml @@ -1,25 +1,71 @@ - + + + + info + + info + + logs + + - - + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - + - + - + + + + - - + + + diff --git a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliJobsUtilTest.java b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliJobsUtilTest.java index 8b99fd705..16e4ae88a 100644 --- a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliJobsUtilTest.java +++ b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliJobsUtilTest.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import java.util.List; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; @@ -49,7 +50,13 @@ public void testListExistingJobs() throws IOException { String jobName = jobNamePrefix + "-" + i; Path jobDir = metadataDir.resolve(jobName); Files.createDirectories(jobDir); - fsSettingsFileHandler.write(FsSettings.builder(jobName).build()); + if (randomBoolean()) { + // Yaml settings file + fsSettingsFileHandler.write(FsSettings.builder(jobName).build()); + } else { + // Json settings (empty dummy) + Files.createFile(jobDir.resolve(FsSettingsFileHandler.FILENAME_JSON)); + } } // We test that we can actually see the jobs diff --git a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliTest.java b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliTest.java index d4259b9f5..b8246439d 100644 --- a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliTest.java +++ b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliTest.java @@ -29,12 +29,10 @@ import org.junit.Test; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyResourceFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; diff --git a/contrib/docker-compose-example/.env b/contrib/docker-compose-example/.env new file mode 100644 index 000000000..6a0e6c949 --- /dev/null +++ b/contrib/docker-compose-example/.env @@ -0,0 +1 @@ +ELASTIC_VERSION=7.10.0 diff --git a/contrib/docker-compose-example/Dockerfile-fscrawler b/contrib/docker-compose-example/Dockerfile-fscrawler new file mode 100644 index 000000000..ac7b682ad --- /dev/null +++ b/contrib/docker-compose-example/Dockerfile-fscrawler @@ -0,0 +1,27 @@ +FROM openjdk:14-alpine AS builder +ENV LANG C.UTF-8 +ENV MAVEN_OPTS "-Xmx256m -Xms256m" +RUN apk add --update --no-cache openssl wget maven +WORKDIR /usr/src/ +RUN wget https://github.com/dadoonet/fscrawler/archive/master.zip && unzip master.zip +WORKDIR /usr/src/fscrawler-master +RUN mvn clean package + +FROM openjdk:14-alpine + +ENV ES es7 +ENV VERSION 2.7-SNAPSHOT +ENV FSCRAWLER_FNAME fscrawler-$ES-$VERSION + +RUN apk add --update \ + && addgroup -S fscrawler && adduser -S -G fscrawler fscrawler +WORKDIR /usr/app +COPY --from=builder /usr/src/fscrawler-master/distribution/es7/target/$FSCRAWLER_FNAME.zip . +RUN unzip $FSCRAWLER_FNAME.zip && rm $FSCRAWLER_FNAME.zip \ + && mkdir -p data/idx \ + && mkdir config \ + && chown -R fscrawler:fscrawler . + +VOLUME [ "/usr/app/config", "/usr/app/data" ] + +ENTRYPOINT $FSCRAWLER_FNAME/bin/fscrawler --config_dir /usr/app/config idx --rest diff --git a/contrib/docker-compose-example/config/_default/6/_settings.json b/contrib/docker-compose-example/config/_default/6/_settings.json new file mode 100644 index 000000000..3fa999288 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/6/_settings.json @@ -0,0 +1,214 @@ +{ + "settings": { + "number_of_shards": 1, + "index.mapping.total_fields.limit": 2000, + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "dynamic_templates": [ + { + "raw_as_text": { + "path_match": "meta.raw.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "attachment": { + "type": "binary", + "doc_values": false + }, + "attributes": { + "properties": { + "group": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + } + } + }, + "content": { + "type": "text" + }, + "file": { + "properties": { + "content_type": { + "type": "keyword" + }, + "filename": { + "type": "keyword", + "store": true + }, + "extension": { + "type": "keyword" + }, + "filesize": { + "type": "long" + }, + "indexed_chars": { + "type": "long" + }, + "indexing_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_modified": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_accessed": { + "type": "date", + "format": "dateOptionalTime" + }, + "checksum": { + "type": "keyword" + }, + "url": { + "type": "keyword", + "index": false + } + } + }, + "meta": { + "properties": { + "author": { + "type": "text" + }, + "date": { + "type": "date", + "format": "dateOptionalTime" + }, + "keywords": { + "type": "text" + }, + "title": { + "type": "text" + }, + "language": { + "type": "keyword" + }, + "format": { + "type": "text" + }, + "identifier": { + "type": "text" + }, + "contributor": { + "type": "text" + }, + "coverage": { + "type": "text" + }, + "modifier": { + "type": "text" + }, + "creator_tool": { + "type": "keyword" + }, + "publisher": { + "type": "text" + }, + "relation": { + "type": "text" + }, + "rights": { + "type": "text" + }, + "source": { + "type": "text" + }, + "type": { + "type": "text" + }, + "description": { + "type": "text" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "print_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "metadata_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "latitude": { + "type": "text" + }, + "longitude": { + "type": "text" + }, + "altitude": { + "type": "text" + }, + "rating": { + "type": "byte" + }, + "comments": { + "type": "text" + } + } + }, + "path": { + "properties": { + "real": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + }, + "root": { + "type": "keyword" + }, + "virtual": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/6/_settings_folder.json b/contrib/docker-compose-example/config/_default/6/_settings_folder.json new file mode 100644 index 000000000..2b63ec7f2 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/6/_settings_folder.json @@ -0,0 +1,32 @@ +{ + "settings": { + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "properties" : { + "real" : { + "type" : "keyword", + "store" : true + }, + "root" : { + "type" : "keyword", + "store" : true + }, + "virtual" : { + "type" : "keyword", + "store" : true + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/7/_settings.json b/contrib/docker-compose-example/config/_default/7/_settings.json new file mode 100644 index 000000000..3fa999288 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/7/_settings.json @@ -0,0 +1,214 @@ +{ + "settings": { + "number_of_shards": 1, + "index.mapping.total_fields.limit": 2000, + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "dynamic_templates": [ + { + "raw_as_text": { + "path_match": "meta.raw.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "attachment": { + "type": "binary", + "doc_values": false + }, + "attributes": { + "properties": { + "group": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + } + } + }, + "content": { + "type": "text" + }, + "file": { + "properties": { + "content_type": { + "type": "keyword" + }, + "filename": { + "type": "keyword", + "store": true + }, + "extension": { + "type": "keyword" + }, + "filesize": { + "type": "long" + }, + "indexed_chars": { + "type": "long" + }, + "indexing_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_modified": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_accessed": { + "type": "date", + "format": "dateOptionalTime" + }, + "checksum": { + "type": "keyword" + }, + "url": { + "type": "keyword", + "index": false + } + } + }, + "meta": { + "properties": { + "author": { + "type": "text" + }, + "date": { + "type": "date", + "format": "dateOptionalTime" + }, + "keywords": { + "type": "text" + }, + "title": { + "type": "text" + }, + "language": { + "type": "keyword" + }, + "format": { + "type": "text" + }, + "identifier": { + "type": "text" + }, + "contributor": { + "type": "text" + }, + "coverage": { + "type": "text" + }, + "modifier": { + "type": "text" + }, + "creator_tool": { + "type": "keyword" + }, + "publisher": { + "type": "text" + }, + "relation": { + "type": "text" + }, + "rights": { + "type": "text" + }, + "source": { + "type": "text" + }, + "type": { + "type": "text" + }, + "description": { + "type": "text" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "print_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "metadata_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "latitude": { + "type": "text" + }, + "longitude": { + "type": "text" + }, + "altitude": { + "type": "text" + }, + "rating": { + "type": "byte" + }, + "comments": { + "type": "text" + } + } + }, + "path": { + "properties": { + "real": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + }, + "root": { + "type": "keyword" + }, + "virtual": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/7/_settings_folder.json b/contrib/docker-compose-example/config/_default/7/_settings_folder.json new file mode 100644 index 000000000..2b63ec7f2 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/7/_settings_folder.json @@ -0,0 +1,32 @@ +{ + "settings": { + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "properties" : { + "real" : { + "type" : "keyword", + "store" : true + }, + "root" : { + "type" : "keyword", + "store" : true + }, + "virtual" : { + "type" : "keyword", + "store" : true + } + } + } +} diff --git a/contrib/docker-compose-example/config/idx/_settings.yaml b/contrib/docker-compose-example/config/idx/_settings.yaml new file mode 100644 index 000000000..95b7e774c --- /dev/null +++ b/contrib/docker-compose-example/config/idx/_settings.yaml @@ -0,0 +1,30 @@ +--- +name: "idx" +fs: + url: "/usr/app/data" + update_rate: "15m" + indexed_chars: 100% + json_support: false + filename_as_id: false + add_filesize: true + remove_deleted: true + add_as_inner_object: false + store_source: false + index_content: true + attributes_support: false + raw_metadata: false + xml_support: false + index_folders: true + lang_detect: true + continue_on_error: false + ocr: + language: "eng" + enabled: true + pdf_strategy: "ocr_and_text" + follow_symlinks: false +elasticsearch: + nodes: + - url: "http://es01:9200" + bulk_size: 100 + flush_interval: "5s" + byte_size: "10mb" diff --git a/contrib/docker-compose-example/config/idx/_status.json b/contrib/docker-compose-example/config/idx/_status.json new file mode 100644 index 000000000..e697ddb83 --- /dev/null +++ b/contrib/docker-compose-example/config/idx/_status.json @@ -0,0 +1,6 @@ +{ + "name" : "idx", + "lastrun" : "2020-07-15T23:52:08.174", + "indexed" : 2048, + "deleted" : 0 +} \ No newline at end of file diff --git a/contrib/docker-compose-example/docker-compose.yml b/contrib/docker-compose-example/docker-compose.yml new file mode 100644 index 000000000..dae476a57 --- /dev/null +++ b/contrib/docker-compose-example/docker-compose.yml @@ -0,0 +1,84 @@ +version: '3' +services: + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION + container_name: es01 + environment: + - node.name=es01 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es02 + - cluster.initial_master_nodes=es01,es02 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms768m -Xmx768m" + restart: always + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - fscrawler_net + + es02: + image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION + container_name: es02 + environment: + - node.name=es02 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01 + - cluster.initial_master_nodes=es01,es02 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms768m -Xmx768m" + restart: always + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data02:/usr/share/elasticsearch/data + ports: + - 9201:9200 + networks: + - fscrawler_net + + kib01: + image: docker.elastic.co/kibana/kibana:$ELASTIC_VERSION + container_name: kib01 + ports: + - 5601:5601 + environment: + ELASTICSEARCH_URL: http://es01:9200 + ELASTICSEARCH_HOSTS: http://es01:9200 + restart: always + networks: + - fscrawler_net + + fscrawler: + build: + context: ${PWD} + dockerfile: Dockerfile-fscrawler + container_name: fscrawler + restart: always + volumes: + - path_to_files_to_scan:/usr/app/data/:ro + - ./config/:/usr/app/config/ + depends_on: + - es01 + - es02 + - kib01 + command: [ "./wait-for-it.sh", "-h es01", "-t 30", "-p 9200" ] + networks: + - fscrawler_net + +volumes: + data01: + driver: local + data02: + driver: local + +networks: + fscrawler_net: + driver: bridge diff --git a/contrib/docker-compose-example/wait-for-it.sh b/contrib/docker-compose-example/wait-for-it.sh new file mode 100755 index 000000000..5e8679e54 --- /dev/null +++ b/contrib/docker-compose-example/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/contrib/docker-compose-it/.env b/contrib/docker-compose-it/.env new file mode 100644 index 000000000..a10102699 --- /dev/null +++ b/contrib/docker-compose-it/.env @@ -0,0 +1,15 @@ +# Elastic Stack settings +ELASTIC_VERSION=7.10.1 +ELASTIC_PASSWORD=changeme + +# Elasticsearch settings +IMG_ELASTICSEARCH=docker.elastic.co/elasticsearch/elasticsearch +ELASTICSEARCH_PORT=9200 + +# Kibana settings +IMG_KIBANA=docker.elastic.co/kibana/kibana +KIBANA_PORT=5601 + +# Enterprise Search settings +IMG_ENTERPRISE_SEARCH=docker.elastic.co/enterprise-search/enterprise-search +ENTERPRISE_SEARCH_PORT=3002 diff --git a/contrib/docker-compose-it/README.md b/contrib/docker-compose-it/README.md new file mode 100644 index 000000000..3f00bede4 --- /dev/null +++ b/contrib/docker-compose-it/README.md @@ -0,0 +1,81 @@ +# Developper Guide + +This documentation shows how to manually test the Workplace Search integration until we can make all that automatic. + +## Launch Elastic Stack + +Go to the `contrib/docker-compose-it` dir and type: + +```sh +docker-compose \ + -f docker-compose-elasticsearch.yml \ + -f docker-compose-enterprise-search.yml \ + up +``` + +If you want to also start the stack with Kibana, run instead: + +```sh +docker-compose \ + -f docker-compose-elasticsearch.yml \ + -f docker-compose-kibana.yml \ + -f docker-compose-enterprise-search.yml \ + up +``` + +Wait for everything to start. It could take several minutes: + +``` +enterprisesearch_1 | 2020-07-20 13:55:08.199:INFO:oejs.ServerConnector:main: Started ServerConnector@252570b9{HTTP/1.1}{0.0.0.0:3002} +enterprisesearch_1 | 2020-07-20 13:55:08.199:INFO:oejs.Server:main: Started @342868ms +``` + +Then you can open http://localhost:3002 and log in with `enterprise_search` / `changeme` account. +Then launch [Workplace Search](http://localhost:3002/ws) and [add a custom source](http://localhost:3002/ws/org/sources#/add/custom). +Name it `fscrawler`. + +You will be able to retrieve your Access Token and the Key. + +* `0bbc4c1c20ad6088e719154d8ebef41b4302677fe93710f9b06295a139b10d1e` +* `5f15a47dd26b57eaa88fb196` + +## Build FSCrawler + +```sh +mvn clean install -DskipTests +cd distribution/es7/target/ +unzip fscrawler-es7-2.7-SNAPSHOT.zip +``` + + +## Configure FSCrawler + +```sh +fscrawler-es7-2.7-SNAPSHOT/bin/fscrawler workplace --config_dir ./config +``` + +Type `Y` to create the default config file `./config/workplace/_settings.yaml` and edit it as is: + +```yml +--- +name: "workplace" +fs: + url: "/tmp/es" + update_rate: "15m" +elasticsearch: + nodes: + - url: "http://127.0.0.1:9200" + username: "elastic" + password: "changeme" +workplace_search: + content_source_key: "0bbc4c1c20ad6088e719154d8ebef41b4302677fe93710f9b06295a139b10d1e" + access_token: "5f15a47dd26b57eaa88fb196" +``` + +## Launch FSCrawler + +```sh +fscrawler-es7-2.7-SNAPSHOT/bin/fscrawler workplace --config_dir ./config +``` + + diff --git a/contrib/docker-compose-it/docker-compose-elasticsearch.yml b/contrib/docker-compose-it/docker-compose-elasticsearch.yml new file mode 100644 index 000000000..70cefe533 --- /dev/null +++ b/contrib/docker-compose-it/docker-compose-elasticsearch.yml @@ -0,0 +1,24 @@ +--- +version: '3' +services: + + elasticsearch: + image: $IMG_ELASTICSEARCH:$ELASTIC_VERSION + environment: + - bootstrap.memory_lock=true + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms3g -Xmx3g" + - cluster.routing.allocation.disk.threshold_enabled=false + - ELASTIC_PASSWORD=$ELASTIC_PASSWORD + - xpack.security.enabled=true + - xpack.security.authc.api_key.enabled=true + - xpack.ml.enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + ports: ["$ELASTICSEARCH_PORT:$ELASTICSEARCH_PORT"] + networks: ['fscrawler_it'] + +networks: + fscrawler_it: {} diff --git a/contrib/docker-compose-it/docker-compose-enterprise-search.yml b/contrib/docker-compose-it/docker-compose-enterprise-search.yml new file mode 100644 index 000000000..ac12f9fc5 --- /dev/null +++ b/contrib/docker-compose-it/docker-compose-enterprise-search.yml @@ -0,0 +1,23 @@ +--- +version: '3' +services: + + enterprisesearch: + image: $IMG_ENTERPRISE_SEARCH:$ELASTIC_VERSION + environment: + - "elasticsearch.username=elastic" + - "elasticsearch.password=$ELASTIC_PASSWORD" + - "elasticsearch.host=http://elasticsearch:9200" + - "allow_es_settings_modification=true" + - "ent_search.auth.native1.source=elasticsearch-native" + - "secret_management.encryption_keys=[XYZ]" + - "ENT_SEARCH_DEFAULT_PASSWORD=$ELASTIC_PASSWORD" + - "JAVA_OPTS=-Xms2g -Xmx2g" + ports: ["$ENTERPRISE_SEARCH_PORT:$ENTERPRISE_SEARCH_PORT"] + networks: ['fscrawler_it'] + links: ['elasticsearch'] + depends_on: ['elasticsearch'] + restart: always + +networks: + fscrawler_it: {} diff --git a/contrib/docker-compose-it/docker-compose-kibana.yml b/contrib/docker-compose-it/docker-compose-kibana.yml new file mode 100644 index 000000000..8c6282220 --- /dev/null +++ b/contrib/docker-compose-it/docker-compose-kibana.yml @@ -0,0 +1,18 @@ +--- +version: '3' +services: + + kibana: + image: $IMG_KIBANA:$ELASTIC_VERSION + environment: + - "ELASTICSEARCH_USERNAME=elastic" + - "ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD" + - "ENTERPRISESEARCH_HOST=http://enterprisesearch:3002" + ports: ['5601:5601'] + networks: ['fscrawler_it'] + links: ['elasticsearch', 'enterprisesearch'] + restart: always + depends_on: ['elasticsearch'] + +networks: + fscrawler_it: {} diff --git a/core/pom.xml b/core/pom.xml index 9919582e1..5f1bdfe4e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -80,6 +80,12 @@ fscrawler-elasticsearch-client-base + + + fr.pilato.elasticsearch.crawler + fscrawler-workplacesearch-client + + org.apache.tika tika-parsers @@ -95,8 +101,8 @@ to use them have to provided them manually in the lib dir. --> - com.levigo.jbig2 - levigo-jbig2-imageio + org.apache.pdfbox + jbig2-imageio true diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java index ed66b8538..f8bdf4f12 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java @@ -19,10 +19,13 @@ package fr.pilato.elasticsearch.crawler.fs; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementServiceElasticsearchImpl; import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; @@ -36,7 +39,7 @@ /** * @author dadoonet (David Pilato) */ -public class FsCrawlerImpl { +public class FsCrawlerImpl implements AutoCloseable { @Deprecated public static final String INDEX_TYPE_FOLDER = "folder"; @@ -53,7 +56,9 @@ public class FsCrawlerImpl { private Thread fsCrawlerThread; - private final ElasticsearchClient esClient; + private final FsCrawlerDocumentService documentService; + private final FsCrawlerManagementService managementService; + private FsParser fsParser; public FsCrawlerImpl(Path config, FsSettings settings, Integer loop, boolean rest) { @@ -63,7 +68,16 @@ public FsCrawlerImpl(Path config, FsSettings settings, Integer loop, boolean res this.settings = settings; this.loop = loop; this.rest = rest; - this.esClient = ElasticsearchClientUtil.getInstance(config, settings); + + this.managementService = new FsCrawlerManagementServiceElasticsearchImpl(config, settings); + + if (settings.getWorkplaceSearch() == null) { + // The documentService is using the esSearch instance + this.documentService = new FsCrawlerDocumentServiceElasticsearchImpl(config, settings); + } else { + // The documentService is using the wpSearch instance + this.documentService = new FsCrawlerDocumentServiceWorkplaceSearchImpl(config, settings); + } // We don't go further as we have critical errors // It's just a double check as settings must be validated before creating the instance @@ -80,8 +94,12 @@ public FsCrawlerImpl(Path config, FsSettings settings, Integer loop, boolean res } } - public ElasticsearchClient getEsClient() { - return esClient; + public FsCrawlerDocumentService getDocumentService() { + return documentService; + } + + public FsCrawlerManagementService getManagementService() { + return managementService; } public void start() throws Exception { @@ -95,18 +113,19 @@ public void start() throws Exception { return; } - esClient.start(); - esClient.createIndices(); + managementService.start(); + documentService.start(); + documentService.createSchema(); // Start the crawler thread - but not if only in rest mode if (loop != 0) { // What is the protocol used? if (settings.getServer() == null || Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol())) { // Local FS - fsParser = new FsParserLocal(settings, config, esClient, loop); + fsParser = new FsParserLocal(settings, config, managementService, documentService, loop); } else if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { // Remote SSH FS - fsParser = new FsParserSsh(settings, config, esClient, loop); + fsParser = new FsParserSsh(settings, config, managementService, documentService, loop); } else { // Non supported protocol throw new RuntimeException(settings.getServer().getProtocol() + " is not supported yet. Please use " + @@ -144,7 +163,8 @@ public void close() throws InterruptedException, IOException { logger.debug("FS crawler thread is now stopped"); } - esClient.close(); + managementService.close(); + documentService.close(); logger.debug("ES Client Manager stopped"); logger.info("FS crawler [{}] stopped", settings.getName()); diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index ecd26e342..bc40e3b9b 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -26,15 +26,15 @@ import fr.pilato.elasticsearch.crawler.fs.beans.FsJobFileHandler; import fr.pilato.elasticsearch.crawler.fs.beans.PathParser; import fr.pilato.elasticsearch.crawler.fs.beans.ScanStatistic; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; -import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; -import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; -import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; -import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; +import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser; import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser; @@ -55,7 +55,6 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.stream.Collectors; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; @@ -66,25 +65,25 @@ public abstract class FsParserAbstract extends FsParser { private static final Logger logger = LogManager.getLogger(FsParserAbstract.class); - private static final String PATH_ROOT = Doc.FIELD_NAMES.PATH + "." + fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.ROOT; - private static final String FILE_FILENAME = Doc.FIELD_NAMES.FILE + "." + fr.pilato.elasticsearch.crawler.fs.beans.File.FIELD_NAMES.FILENAME; - private static final String FSCRAWLER_IGNORE_FILENAME = ".fscrawlerignore"; - private static final int REQUEST_SIZE = 10000; - final FsSettings fsSettings; private final FsJobFileHandler fsJobFileHandler; - private final ElasticsearchClient esClient; + + private final FsCrawlerManagementService managementService; + private final FsCrawlerDocumentService documentService; private final Integer loop; private final MessageDigest messageDigest; + private final String pathSeparator; private ScanStatistic stats; - FsParserAbstract(FsSettings fsSettings, Path config, ElasticsearchClient esClient, Integer loop) { + FsParserAbstract(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, FsCrawlerDocumentService documentService, Integer loop) { this.fsSettings = fsSettings; this.fsJobFileHandler = new FsJobFileHandler(config); - this.esClient = esClient; + this.managementService = managementService; + this.documentService = documentService; + this.loop = loop; logger.debug("creating fs crawler thread [{}] for [{}] every [{}]", fsSettings.getName(), fsSettings.getFs().getUrl(), @@ -100,9 +99,17 @@ public abstract class FsParserAbstract extends FsParser { } else { messageDigest = null; } + + // On Windows, when using SSH server, we need to force the "Linux" separator + if (OsValidator.WINDOWS && fsSettings.getServer() != null) { + logger.debug("We are running on Windows with SSH Server settings so we need to force the Linux separator."); + pathSeparator = "/"; + } else { + pathSeparator = File.separator; + } } - protected abstract FileAbstractor buildFileAbstractor(); + protected abstract FileAbstractor buildFileAbstractor(); @Override public void run() { @@ -117,7 +124,7 @@ public void run() { } int run = runNumber.incrementAndGet(); - FileAbstractor path = null; + FileAbstractor path = null; try { logger.debug("Fs crawler thread [{}] is now running. Run #{}...", fsSettings.getName(), run); @@ -150,7 +157,7 @@ public void run() { updateFsJob(fsSettings.getName(), scanDatenew); } catch (Exception e) { - logger.warn("Error while crawling {}: {}", fsSettings.getFs().getUrl(), e.getMessage()); + logger.warn("Error while crawling {}: {}", fsSettings.getFs().getUrl(), e.getMessage() == null ? e.getClass().getName() : e.getMessage()); if (logger.isDebugEnabled()) { logger.warn("Full stacktrace", e); } @@ -159,7 +166,10 @@ public void run() { try { path.close(); } catch (Exception e) { - logger.warn("Error while closing the connection: {}", e, e.getMessage()); + logger.warn("Error while closing the connection: {}", e.getMessage()); + if (logger.isDebugEnabled()) { + logger.warn("Full stacktrace", e); + } } } } @@ -315,8 +325,8 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString()); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { - logger.trace("Removing file [{}] in elasticsearch", esfile); - esDelete(fsSettings.getElasticsearch().getIndex(), generateIdFromFilename(esfile, filepath)); + logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); + esDelete(documentService, fsSettings.getElasticsearch().getIndex(), generateIdFromFilename(esfile, filepath)); stats.removeFile(); } } @@ -340,71 +350,21 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD } } - // TODO Optimize it. We can probably use a search for a big array of filenames instead of - // Searching fo 10000 files (which is somehow limited). private Collection getFileDirectory(String path) throws Exception { - // If the crawler is being closed, we return if (closed) { - return Collections.emptyList(); - } - - logger.trace("Querying elasticsearch for files in dir [{}:{}]", PATH_ROOT, SignTool.sign(path)); - Collection files = new ArrayList<>(); - ESSearchResponse response = esClient.search( - new ESSearchRequest() - .withIndex(fsSettings.getElasticsearch().getIndex()) - .withSize(REQUEST_SIZE) - .addField(FILE_FILENAME) - .withESQuery(new ESTermQuery(PATH_ROOT, SignTool.sign(path)))); - - logger.trace("Response [{}]", response.toString()); - if (response.getHits() != null) { - for (ESSearchHit hit : response.getHits()) { - String name; - if (hit.getFields() != null - && hit.getFields().get(FILE_FILENAME) != null) { - // In case someone disabled _source which is not recommended - name = hit.getFields().get(FILE_FILENAME).getValue(); - } else { - // Houston, we have a problem ! We can't get the old files from ES - logger.warn("Can't find stored field name to check existing filenames in path [{}]. " + - "Please set store: true on field [{}]", path, FILE_FILENAME); - throw new RuntimeException("Mapping is incorrect: please set stored: true on field [" + - FILE_FILENAME + "]."); - } - files.add(name); - } + return new ArrayList<>(); } - - logger.trace("We found: {}", files); - - return files; + return managementService.getFileDirectory(path); } private Collection getFolderDirectory(String path) throws Exception { - Collection files = new ArrayList<>(); - // If the crawler is being closed, we return if (closed) { - return files; + return new ArrayList<>(); } - - ESSearchResponse response = esClient.search( - new ESSearchRequest() - .withIndex(fsSettings.getElasticsearch().getIndexFolder()) - .withSize(REQUEST_SIZE) // TODO: WHAT? DID I REALLY WROTE THAT? :p - .withESQuery(new ESTermQuery(fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.ROOT, SignTool.sign(path)))); - - if (response.getHits() != null) { - for (ESSearchHit hit : response.getHits()) { - String name = hit.getSourceAsMap().get(fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.REAL).toString(); - files.add(name); - } - } - - return files; + return managementService.getFolderDirectory(path); } /** @@ -420,12 +380,13 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); + String fullFilename = new File(dirname, filename).toString(); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) + String id = generateIdFromFilename(filename, dirname); if (fsSettings.getFs().isAddAsInnerObject() || (!fsSettings.getFs().isJsonSupport() && !fsSettings.getFs().isXmlSupport())) { - String fullFilename = new File(dirname, filename).toString(); Doc doc = new Doc(); @@ -476,27 +437,54 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, // We index the data structure if (isIndexable(doc.getContent(), fsSettings.getFs().getFilters())) { - esIndex(fsSettings.getElasticsearch().getIndex(), - generateIdFromFilename(filename, dirname), - DocParser.toJson(doc), - fsSettings.getElasticsearch().getPipeline()); + if (!closed) { + FSCrawlerLogger.documentDebug(id, + computeVirtualPathName(stats.getRootPath(), fullFilename), + "Indexing content"); + documentService.index( + fsSettings.getElasticsearch().getIndex(), + id, + doc, + fsSettings.getElasticsearch().getPipeline()); + } else { + logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", + fsSettings.getElasticsearch().getIndex(), id); + } } else { logger.debug("We ignore file [{}] because it does not match all the patterns {}", filename, fsSettings.getFs().getFilters()); } } else { if (fsSettings.getFs().isJsonSupport()) { + FSCrawlerLogger.documentDebug(generateIdFromFilename(filename, dirname), + computeVirtualPathName(stats.getRootPath(), fullFilename), + "Indexing json content"); // We index the json content directly - esIndex(fsSettings.getElasticsearch().getIndex(), - generateIdFromFilename(filename, dirname), - read(inputStream), - fsSettings.getElasticsearch().getPipeline()); + if (!closed) { + documentService.indexRawJson( + fsSettings.getElasticsearch().getIndex(), + id, + read(inputStream), + fsSettings.getElasticsearch().getPipeline()); + } else { + logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", + fsSettings.getElasticsearch().getIndex(), id); + } } else if (fsSettings.getFs().isXmlSupport()) { - // We index the xml content directly - esIndex(fsSettings.getElasticsearch().getIndex(), - generateIdFromFilename(filename, dirname), - XmlDocParser.generate(inputStream), - fsSettings.getElasticsearch().getPipeline()); + FSCrawlerLogger.documentDebug(generateIdFromFilename(filename, dirname), + computeVirtualPathName(stats.getRootPath(), fullFilename), + "Indexing xml content"); + // We index the xml content directly (after transformation to json) + if (!closed) { + documentService.indexRawJson( + fsSettings.getElasticsearch().getIndex(), + id, + XmlDocParser.generate(inputStream), + fsSettings.getElasticsearch().getPipeline()); + } else { + logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", + fsSettings.getElasticsearch().getIndex(), id); + } } } } finally { @@ -521,13 +509,14 @@ private String read(InputStream input) throws IOException { * Index a Path object (AKA a folder) in elasticsearch * @param id id of the path * @param path path object - * @throws Exception in case of error */ - private void indexDirectory(String id, fr.pilato.elasticsearch.crawler.fs.beans.Path path) throws Exception { - esIndex(fsSettings.getElasticsearch().getIndexFolder(), - id, - PathParser.toJson(path), - null); + private void indexDirectory(String id, fr.pilato.elasticsearch.crawler.fs.beans.Path path) throws IOException { + if (!closed) { + managementService.storeVisitedDirectory(fsSettings.getElasticsearch().getIndexFolder(), id, path); + } else { + logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", + fsSettings.getElasticsearch().getIndexFolder(), id); + } } /** @@ -538,7 +527,7 @@ private void indexDirectory(String path) throws Exception { fr.pilato.elasticsearch.crawler.fs.beans.Path pathObject = new fr.pilato.elasticsearch.crawler.fs.beans.Path(); // The real and complete path pathObject.setReal(path); - String rootdir = path.substring(0, path.lastIndexOf(File.separator)); + String rootdir = path.substring(0, path.lastIndexOf(pathSeparator)); // Encoded version of the parent dir pathObject.setRoot(SignTool.sign(rootdir)); // The virtual URL (not including the initial root dir) @@ -555,7 +544,7 @@ private void removeEsDirectoryRecursively(final String path) throws Exception { Collection listFile = getFileDirectory(path); for (String esfile : listFile) { - esDelete(fsSettings.getElasticsearch().getIndex(), SignTool.sign(path.concat(File.separator).concat(esfile))); + esDelete(managementService, fsSettings.getElasticsearch().getIndex(), SignTool.sign(path.concat(pathSeparator).concat(esfile))); } Collection listFolder = getFolderDirectory(path); @@ -563,30 +552,16 @@ private void removeEsDirectoryRecursively(final String path) throws Exception { removeEsDirectoryRecursively(esfolder); } - esDelete(fsSettings.getElasticsearch().getIndexFolder(), SignTool.sign(path)); - } - - /** - * Add to bulk an IndexRequest in JSon format - */ - private void esIndex(String index, String id, String json, String pipeline) { - logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline); - logger.trace("JSon indexed : {}", json); - - if (!closed) { - esClient.index(index, id, json, pipeline); - } else { - logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", index, id); - } + esDelete(managementService, fsSettings.getElasticsearch().getIndexFolder(), SignTool.sign(path)); } /** * Add to bulk a DeleteRequest */ - private void esDelete(String index, String id) { + private void esDelete(FsCrawlerService service, String index, String id) { logger.debug("Deleting {}/{}", index, id); if (!closed) { - esClient.delete(index, id); + service.getClient().delete(index, id); } else { logger.warn("trying to remove a file while closing crawler. Document [{}]/[{}] has been ignored", index, id); } diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserLocal.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserLocal.java index d4bceaaf3..84f114e84 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserLocal.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserLocal.java @@ -19,20 +19,22 @@ package fr.pilato.elasticsearch.crawler.fs; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import java.nio.file.Path; public class FsParserLocal extends FsParserAbstract { - public FsParserLocal(FsSettings fsSettings, Path config, ElasticsearchClient esClient, Integer loop) { - super(fsSettings, config, esClient, loop); + public FsParserLocal(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop) { + super(fsSettings, config, managementService, documentService, loop); } - protected FileAbstractor buildFileAbstractor() { + protected FileAbstractor buildFileAbstractor() { return new FileAbstractorFile(fsSettings); } } diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSsh.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSsh.java index 5ebf9a4c5..b721f72b0 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSsh.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSsh.java @@ -19,20 +19,22 @@ package fr.pilato.elasticsearch.crawler.fs; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; import fr.pilato.elasticsearch.crawler.fs.crawler.ssh.FileAbstractorSSH; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import java.nio.file.Path; public class FsParserSsh extends FsParserAbstract { - public FsParserSsh(FsSettings fsSettings, Path config, ElasticsearchClient esClient, Integer loop) { - super(fsSettings, config, esClient, loop); + public FsParserSsh(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop) { + super(fsSettings, config, managementService, documentService, loop); } - protected FileAbstractor buildFileAbstractor() { + protected FileAbstractor buildFileAbstractor() { return new FileAbstractorSSH(fsSettings); } } diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentService.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentService.java new file mode 100644 index 000000000..79324d5fa --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentService.java @@ -0,0 +1,48 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; + +public interface FsCrawlerDocumentService extends FsCrawlerService { + /** + * Create a schema for the dataset. This is called when the service starts + * @throws Exception in case of error + */ + void createSchema() throws Exception; + + /** + * Send a document to the target service + * @param index Index name + * @param id Document id + * @param doc Document to index + * @param pipeline Pipeline (can be null) + */ + void index(String index, String id, Doc doc, String pipeline); + + /** + * Send a Raw Json to the target service + * @param index Index name + * @param id Document ID + * @param json Document to index + * @param pipeline Pipeline (can be null) + */ + void indexRawJson(String index, String id, String json, String pipeline); +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceElasticsearchImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceElasticsearchImpl.java new file mode 100644 index 000000000..9cf489ae4 --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceElasticsearchImpl.java @@ -0,0 +1,76 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.beans.DocParser; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; + +public class FsCrawlerDocumentServiceElasticsearchImpl implements FsCrawlerDocumentService { + + private static final Logger logger = LogManager.getLogger(FsCrawlerDocumentServiceElasticsearchImpl.class); + + private final ElasticsearchClient client; + + public FsCrawlerDocumentServiceElasticsearchImpl(Path config, FsSettings settings) { + this.client = ElasticsearchClientUtil.getInstance(config, settings); + } + + @Override + public void start() throws IOException { + client.start(); + logger.debug("Elasticsearch Document Service started"); + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public void close() throws IOException { + client.close(); + logger.debug("Elasticsearch Document Service stopped"); + } + + @Override + public void createSchema() throws Exception { + client.createIndices(); + } + + @Override + public void index(String index, String id, Doc doc, String pipeline) { + indexRawJson(index, id, DocParser.toJson(doc), pipeline); + } + + @Override + public void indexRawJson(String index, String id, String json, String pipeline) { + logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline); + client.indexRawJson(index, id, json, pipeline); + } + +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceWorkplaceSearchImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceWorkplaceSearchImpl.java new file mode 100644 index 000000000..71d33d79f --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerDocumentServiceWorkplaceSearchImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.WorkplaceSearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.WorkplaceSearchClientUtil; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; + +public class FsCrawlerDocumentServiceWorkplaceSearchImpl implements FsCrawlerDocumentService { + + private static final Logger logger = LogManager.getLogger(FsCrawlerDocumentServiceWorkplaceSearchImpl.class); + + private final WorkplaceSearchClient client; + + public FsCrawlerDocumentServiceWorkplaceSearchImpl(Path config, FsSettings settings) throws RuntimeException { + this.client = WorkplaceSearchClientUtil.getInstance(config, settings); + + if (client == null) { + throw new FsCrawlerIllegalConfigurationException("As we can not find an existing Workplace Search client for elastic stack before 7.8," + + " you can't define workplace settings in your configuration. FSCrawler will refuse to start."); + } + } + + @Override + public void start() throws IOException { + client.start(); + logger.debug("Workplace Search Document Service started"); + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public void close() throws IOException { + client.close(); + logger.debug("Workplace Search Document Service stopped"); + } + + @Override + public void createSchema() { + // There is no way yet to create a schema in workplace before hand. + } + + @Override + public void index(String index, String id, Doc doc, String pipeline) { + logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline); + client.index(index, id, doc, pipeline); + } + + @Override + public void indexRawJson(String index, String id, String json, String pipeline) { + throw new RuntimeException("We can't send Raw Json Documents to Workplace Search"); + } +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementService.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementService.java new file mode 100644 index 000000000..5137304c3 --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementService.java @@ -0,0 +1,53 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.beans.Path; + +import java.io.IOException; +import java.util.Collection; + +public interface FsCrawlerManagementService extends FsCrawlerService { + + /** + * Retrieve the list of files that are currently available within a dir + * @param path the virtual path + * @return a list of known files + * @throws Exception In case of problems + */ + Collection getFileDirectory(String path) throws Exception; + + /** + * Retrieve the list of sub folders that are currently available within a dir + * @param path the virtual path + * @return a list of known folders + * @throws Exception In case of problems + */ + Collection getFolderDirectory(String path) throws Exception; + + /** + * Store a visited directory. It will be used to compare old dirs vs + * current directories. So we will be able to remove data if needed. + * @param indexFolder index name to store this information + * @param id id of the directory + * @param path Path to store + */ + void storeVisitedDirectory(String indexFolder, String id, Path path) throws IOException; +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementServiceElasticsearchImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementServiceElasticsearchImpl.java new file mode 100644 index 000000000..2fb0ecdaa --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerManagementServiceElasticsearchImpl.java @@ -0,0 +1,138 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.beans.PathParser; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; +import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; +import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; + +public class FsCrawlerManagementServiceElasticsearchImpl implements FsCrawlerManagementService { + + private static final Logger logger = LogManager.getLogger(FsCrawlerManagementServiceElasticsearchImpl.class); + private static final String PATH_ROOT = Doc.FIELD_NAMES.PATH + "." + fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.ROOT; + private static final String FILE_FILENAME = Doc.FIELD_NAMES.FILE + "." + fr.pilato.elasticsearch.crawler.fs.beans.File.FIELD_NAMES.FILENAME; + + // TODO Optimize it. We can probably use a search for a big array of filenames instead of + // searching fo 10000 files (which is somehow limited). + private static final int REQUEST_SIZE = 10000; + + private final ElasticsearchClient client; + private final FsSettings settings; + + public FsCrawlerManagementServiceElasticsearchImpl(Path config, FsSettings settings) { + this.settings = settings; + this.client = ElasticsearchClientUtil.getInstance(config, settings); + } + + @Override + public void start() throws IOException { + client.start(); + logger.debug("Elasticsearch Management Service started"); + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public void close() throws IOException { + client.close(); + logger.debug("Elasticsearch Management Service stopped"); + } + + @Override + public Collection getFileDirectory(String path) + throws Exception { + + if (logger.isTraceEnabled()) { + logger.trace("Querying elasticsearch for files in dir [{}:{}]", PATH_ROOT, SignTool.sign(path)); + } + + Collection files = new ArrayList<>(); + ESSearchResponse response = client.search( + new ESSearchRequest() + .withIndex(settings.getElasticsearch().getIndex()) + .withSize(REQUEST_SIZE) + .addField(FILE_FILENAME) + .withESQuery(new ESTermQuery(PATH_ROOT, SignTool.sign(path)))); + + if (response.getHits() != null) { + for (ESSearchHit hit : response.getHits()) { + String name; + if (hit.getFields() != null + && hit.getFields().get(FILE_FILENAME) != null) { + // In case someone disabled _source which is not recommended + name = hit.getFields().get(FILE_FILENAME).getValue(); + } else { + // Houston, we have a problem ! We can't get the old files from ES + logger.warn("Can't find stored field name to check existing filenames in path [{}]. " + + "Please set store: true on field [{}]", path, FILE_FILENAME); + throw new RuntimeException("Mapping is incorrect: please set stored: true on field [" + + FILE_FILENAME + "]."); + } + files.add(name); + } + } + + logger.trace("We found: {}", files); + + return files; + } + + @Override + public Collection getFolderDirectory(String path) throws Exception { + Collection files = new ArrayList<>(); + + ESSearchResponse response = client.search( + new ESSearchRequest() + .withIndex(settings.getElasticsearch().getIndexFolder()) + .withSize(REQUEST_SIZE) // TODO: WHAT? DID I REALLY WROTE THAT? :p + .withESQuery(new ESTermQuery(fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.ROOT, SignTool.sign(path)))); + + if (response.getHits() != null) { + for (ESSearchHit hit : response.getHits()) { + String name = hit.getSourceAsMap().get(fr.pilato.elasticsearch.crawler.fs.beans.Path.FIELD_NAMES.REAL).toString(); + files.add(name); + } + } + + return files; + } + + @Override + public void storeVisitedDirectory(String indexFolder, String id, fr.pilato.elasticsearch.crawler.fs.beans.Path path) throws IOException { + client.indexRawJson(indexFolder, id, PathParser.toJson(path), null); + } +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerService.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerService.java new file mode 100644 index 000000000..4d80c95cd --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/service/FsCrawlerService.java @@ -0,0 +1,20 @@ +package fr.pilato.elasticsearch.crawler.fs.service; + +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; + +import java.io.Closeable; +import java.io.IOException; + +public interface FsCrawlerService extends Closeable { + + /** + * Start the service + */ + void start() throws IOException; + + /** + * Get the elasticsearch client + * @return elasticsearch client + */ + ElasticsearchClient getClient(); +} diff --git a/crawler/crawler-abstract/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/FileAbstractModel.java b/crawler/crawler-abstract/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/FileAbstractModel.java index 1df965d71..e894fe654 100644 --- a/crawler/crawler-abstract/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/FileAbstractModel.java +++ b/crawler/crawler-abstract/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/FileAbstractModel.java @@ -42,19 +42,19 @@ import java.time.LocalDateTime; public class FileAbstractModel { - private String name; - private boolean file; - private boolean directory; - private LocalDateTime lastModifiedDate; - private LocalDateTime creationDate; - private LocalDateTime accessDate; - private String path; - private String fullpath; - private long size; - private String owner; - private String group; - private int permissions; - private String extension; + private final String name; + private final boolean file; + private final boolean directory; + private final LocalDateTime lastModifiedDate; + private final LocalDateTime creationDate; + private final LocalDateTime accessDate; + private final String path; + private final String fullpath; + private final long size; + private final String owner; + private final String group; + private final int permissions; + private final String extension; public FileAbstractModel(String name, boolean file, LocalDateTime lastModifiedDate, LocalDateTime creationDate, LocalDateTime accessDate, String extension, String path, String fullpath, long size, String owner, String group, int permissions) { diff --git a/distribution/pom.xml b/distribution/pom.xml index 7353fc1ad..8106ac865 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,7 +19,8 @@ - 3.04.* + 4.0.0* + 1:4.00* @@ -45,16 +46,16 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.lang.version} - + ${project.artifactId}-fra ${docker.username}/fscrawler:${project.version}-${module.name}-fra ${docker.fra.name} ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.lang.version} ${project.artifactId}-jpn @@ -63,7 +64,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.lang.version} diff --git a/distribution/src/main/assembly/assembly.xml b/distribution/src/main/assembly/assembly.xml index dc272d4d2..6037b3a29 100644 --- a/distribution/src/main/assembly/assembly.xml +++ b/distribution/src/main/assembly/assembly.xml @@ -29,5 +29,12 @@ README.md + + ${project.parent.basedir}/../cli/src/main/resources + config + + log4j2.xml + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index ba2b2d784..c169bf084 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,12 +1,13 @@ -FROM openjdk:8u222-jdk +FROM openjdk:15.0.2-jdk-slim-buster ARG langsPkg + RUN set -ex \ - && apt update \ - && apt install -y --no-install-recommends \ - "gettext-base=0.19.*" \ - ${langsPkg} \ - && apt clean \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + "gettext-base=0.19.*" \ + ${langsPkg} \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler diff --git a/distribution/src/main/scripts/fscrawler b/distribution/src/main/scripts/fscrawler index 2af2fddbd..1af401cd1 100755 --- a/distribution/src/main/scripts/fscrawler +++ b/distribution/src/main/scripts/fscrawler @@ -43,6 +43,9 @@ JAVA_OPTS="$JAVA_OPTS -Dsun.jnu.encoding=UTF-8" # Use LOG4J2 instead of java.util.logging JAVA_OPTS="$JAVA_OPTS -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" +# Define LOG4J2 config file +JAVA_OPTS="$JAVA_OPTS -Dlog4j.configurationFile=config/log4j2.xml" + # If the user defined FS_JAVA_OPTS, we will use it to start the crawler JAVA_OPTS="$JAVA_OPTS $FS_JAVA_OPTS" diff --git a/distribution/src/main/scripts/fscrawler.bat b/distribution/src/main/scripts/fscrawler.bat index 899dff832..c0c993bae 100644 --- a/distribution/src/main/scripts/fscrawler.bat +++ b/distribution/src/main/scripts/fscrawler.bat @@ -26,6 +26,9 @@ set JAVA_OPTS=%JAVA_OPTS% -Dsun.jnu.encoding=UTF-8 REM Use LOG4J2 instead of java.util.logging set JAVA_OPTS=%JAVA_OPTS% -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +REM Define LOG4J2 config file +set JAVA_OPTS=%JAVA_OPTS% -Dlog4j.configurationFile=config/log4j2.xml + REM If the user defined FS_JAVA_OPTS, we will use it to start the crawler set JAVA_OPTS=%JAVA_OPTS% %FS_JAVA_OPTS% diff --git a/docs/source/_static/wpsearch/fscrawler-custom-source.png b/docs/source/_static/wpsearch/fscrawler-custom-source.png new file mode 100644 index 000000000..13cc70338 Binary files /dev/null and b/docs/source/_static/wpsearch/fscrawler-custom-source.png differ diff --git a/docs/source/_static/wpsearch/fscrawler-display-settings-1.png b/docs/source/_static/wpsearch/fscrawler-display-settings-1.png new file mode 100644 index 000000000..bc6b596a3 Binary files /dev/null and b/docs/source/_static/wpsearch/fscrawler-display-settings-1.png differ diff --git a/docs/source/_static/wpsearch/fscrawler-display-settings-2.png b/docs/source/_static/wpsearch/fscrawler-display-settings-2.png new file mode 100644 index 000000000..8f4c20478 Binary files /dev/null and b/docs/source/_static/wpsearch/fscrawler-display-settings-2.png differ diff --git a/docs/source/admin/cli-options.rst b/docs/source/admin/cli-options.rst index 540dc7d71..0bd67ba06 100644 --- a/docs/source/admin/cli-options.rst +++ b/docs/source/admin/cli-options.rst @@ -4,9 +4,9 @@ CLI options =========== - ``--help`` displays help -- ``--silent`` runs in silent mode. No output is generated. -- ``--debug`` runs in debug mode. -- ``--trace`` runs in trace mode (more verbose than debug). +- ``--silent`` runs in silent mode. No output is generated on the console. +- ``--debug`` runs in debug mode. This applies to log files only. See also :ref:`logger`. +- ``--trace`` runs in trace mode (more verbose than debug). This applies to log files only. See also :ref:`logger`. - ``--config_dir`` defines directory where jobs are stored instead of default ``~/.fscrawler``. - ``--username`` defines the username to use when using an secured diff --git a/docs/source/admin/fs/elasticsearch.rst b/docs/source/admin/fs/elasticsearch.rst index b6cfbc513..8758fcca2 100644 --- a/docs/source/admin/fs/elasticsearch.rst +++ b/docs/source/admin/fs/elasticsearch.rst @@ -5,29 +5,31 @@ Elasticsearch settings Here is a list of Elasticsearch settings (under ``elasticsearch.`` prefix)`: -+----------------------------------+---------------------------+---------------------------------+ -| Name | Default value | Documentation | -+==================================+===========================+=================================+ -| ``elasticsearch.index`` | job name | `Index settings for documents`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.index_folder`` | job name + ``_folder`` | `Index settings for folders`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.bulk_size`` | ``100`` | `Bulk settings`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.flush_interval`` | ``"5s"`` | `Bulk settings`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.byte_size`` | ``"10mb"`` | `Bulk settings`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.pipeline`` | ``null`` | :ref:`ingest_node` | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.nodes`` | ``http://127.0.0.1:9200`` | `Node settings`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.path_prefix`` | ``null`` | `Path prefix`_ | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.username`` | ``null`` | :ref:`credentials` | -+----------------------------------+---------------------------+---------------------------------+ -| ``elasticsearch.password`` | ``null`` | :ref:`credentials` | -+----------------------------------+---------------------------+---------------------------------+ ++-----------------------------------+---------------------------+---------------------------------+ +| Name | Default value | Documentation | ++===================================+===========================+=================================+ +| ``elasticsearch.index`` | job name | `Index settings for documents`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.index_folder`` | job name + ``_folder`` | `Index settings for folders`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.bulk_size`` | ``100`` | `Bulk settings`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.flush_interval`` | ``"5s"`` | `Bulk settings`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.byte_size`` | ``"10mb"`` | `Bulk settings`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.pipeline`` | ``null`` | :ref:`ingest_node` | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.nodes`` | ``http://127.0.0.1:9200`` | `Node settings`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.path_prefix`` | ``null`` | `Path prefix`_ | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.username`` | ``null`` | :ref:`credentials` | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.password`` | ``null`` | :ref:`credentials` | ++-----------------------------------+---------------------------+---------------------------------+ +| ``elasticsearch.ssl_verification``| ``true`` | :ref:`credentials` | ++-----------------------------------+---------------------------+---------------------------------+ Index settings ^^^^^^^^^^^^^^ @@ -575,7 +577,7 @@ steps: .. important:: Prerequisite: you need to have root CA chain certificate or Elasticsearch server certificate - in DER format. DER format files have a ``.cer`` extension. + in DER format. DER format files have a ``.cer`` extension. Certificate verification can be disabled by option ``ssl_verification: false`` 1. Logon to server (or client machine) where FSCrawler is running 2. Run: diff --git a/docs/source/admin/fs/wpsearch.rst b/docs/source/admin/fs/wpsearch.rst new file mode 100644 index 000000000..b0623a4ed --- /dev/null +++ b/docs/source/admin/fs/wpsearch.rst @@ -0,0 +1,155 @@ +.. _wpsearch-settings: + +Workplace Search settings +------------------------- + +.. versionadded:: 2.7 + +FSCrawler can now send documents to `Workplace Search `_. + +.. note:: + + Although this won't be needed in the future, it is still mandatory to have access to the elasticsearch + instance running behind Workplace Search. In this section of the documentation, we will only cover the + specifics for workplace search. Please refer to :ref:`elasticsearch-settings` chapter. + +.. hint:: + + To easily start locally with Workplace Search, follow the steps: + + * Check-out the source code on `GitHub `_:: + + git clone git@github.com:dadoonet/fscrawler.git + cd fscrawler + cd contrib/docker-compose-workplacesearch + docker-compose up + + This will start Elasticsearch, Kibana (not used) and Workplace Search. + + * Wait for it to start and open http://127.0.0.1:3002/ws. + * Enter ``enterprise_search`` as the login and ``changeme`` as the password. + * Click on "Add sources" button and choose `Custom API `_. + * Name it ``fscrawler`` and click on "Create Custom API Source" button. + * Copy the "Access Token" value. We will mention it as ``ACCESS_TOKEN`` for the rest of this documentation. + * Copy the "Key" value. We will mention it as ``KEY`` for the rest of this documentation. + + .. image:: /_static/wpsearch/fscrawler-custom-source.png + +Here is a list of Workplace Search settings (under ``workplace_search.`` prefix)`: + ++-------------------------------------+---------------------------+---------------------------------+ +| Name | Default value | Documentation | ++=====================================+===========================+=================================+ +| ``workplace_search.access_token`` | None (Must be set) | `Keys`_ | ++-------------------------------------+---------------------------+---------------------------------+ +| ``workplace_search.key`` | None (Must be set) | `Keys`_ | ++-------------------------------------+---------------------------+---------------------------------+ +| ``workplace_search.server`` | ``http://127.0.0.1:3002`` | `Server`_ | ++-------------------------------------+---------------------------+---------------------------------+ +| ``workplace_search.bulk_size`` | ``100`` | `Bulk settings`_ | ++-------------------------------------+---------------------------+---------------------------------+ +| ``workplace_search.flush_interval`` | ``"5s"`` | `Bulk settings`_ | ++-------------------------------------+---------------------------+---------------------------------+ +| ``workplace_search.url_prefix`` | ``http://127.0.0.1`` | `Documents Repository URL`_ | ++-------------------------------------+---------------------------+---------------------------------+ + + +Keys +^^^^ + +Once you have created your Custom API and have the ``ACCESS_TOKEN`` and ``KEY``, you can add to your existing +FSCrawler configuration file: + +.. code:: yaml + + name: "test" + workplace_search: + access_token: "ACCESS_TOKEN" + key: "KEY" + +Server +^^^^^^ + +When using Workplace Search, FSCrawler will by default connect to ``http://127.0.0.1:3002`` +which is the default when running a local node on your machine. + +Of course, in production, you would probably change this and connect to +a production cluster: + +.. code:: yaml + + name: "test" + workplace_search: + access_token: "ACCESS_TOKEN" + key: "KEY" + server: "http://wpsearch.mycompany.com:3002" + +Running on Cloud +^^^^^^^^^^^^^^^^ + +The easiest way to get started is to deploy Enterprise Search on +`Elastic Cloud Service `_. + +Then you can define the following: + +.. code:: yaml + + name: "test" + elasticsearch: + username: "elastic" + password: "PASSWORD" + nodes: + - cloud_id: "CLOUD_ID" + workplace_search: + access_token: "ACCESS_TOKEN" + key: "KEY" + server: "https://XYZ.ent-search.ZONE.CLOUD_PROVIDER.elastic-cloud.com" + +.. note:: + + Change the ``PASSWORD``, ``CLOUD_ID`` by values coming from the `Elastic Console `_. + And get the ``ACCESS_TOKEN`` and ``KEY`` from your Enterprise Search deployment once you have created the + Custom API source as seen previously. + +Bulk settings +^^^^^^^^^^^^^ + +FSCrawler is using bulks to send data to Workplace Search. By default the +bulk is executed every 100 operations or every 5 seconds. You can change +default settings using ``workplace_search.bulk_size`` and ``workplace_search.flush_interval``: + +.. code:: yaml + + name: "test" + workplace_search: + bulk_size: 1000 + flush_interval: "2s" + + +Documents Repository URL +^^^^^^^^^^^^^^^^^^^^^^^^ + +The URL that will be used to give access to your users to the source document is +prefixed by default with ``http://127.0.0.1``. That means that if you are able to run +a Web Server locally which can serve the directory you defined in ``fs.url`` setting +(see :ref:`root-directory`), your users will be able to click in the Workplace Search interface +to have access to the documents. + +Of course, in production, you would probably change this and connect to +another url. This can be done by changing the ``workplace_search.url_prefix`` setting: + +.. code:: yaml + + name: "test" + workplace_search: + access_token: "ACCESS_TOKEN" + key: "KEY" + url_prefix: "https://repository.mycompany.com/docs" + +.. note:: + + If ``fs.url`` is set to ``/tmp/es`` and you have indexed a document named + ``/tmp/es/path/to/foobar.txt``, the default url will be ``http://127.0.0.1/path/to/foobar.txt``. + + If you change ``workplace_search.url_prefix`` to ``https://repository.mycompany.com/docs``, the + same document will be served as ``https://repository.mycompany.com/docs/path/to/foobar.txt``. diff --git a/docs/source/admin/logger.rst b/docs/source/admin/logger.rst index 9b8326f8b..a5883df0f 100644 --- a/docs/source/admin/logger.rst +++ b/docs/source/admin/logger.rst @@ -1,15 +1,38 @@ -Configuring an external logger configuration file -================================================= +.. _logger: -If you want to define an external ``log4j2.xml`` file, you can use the -``log4j.configurationFile`` JVM parameter which you can define in -``FS_JAVA_OPTS`` variable if you wish: +Configuring the logger +====================== + +In addition to the :ref:`cli-options`, FSCrawler comes with a default logger configuration which can be found in the +FSCrawler installation dir as ``config/log4j2.xml`` file. + +You can modify it to suit your needs. It will be automatically reloaded every 30 seconds. + +There are some properties to make your life easier to change the log levels or the log dir: + +.. code:: xml + + + info + info + logs + + +You can control where FSCrawler will store the logs and the log levels by setting +``LOG_DIR``, ``LOG_LEVEL`` and ``DOC_LEVEL`` Java properties. .. code:: sh - FS_JAVA_OPTS="-Dlog4j.configurationFile=path/to/log4j2.xml" bin/fscrawler + FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir -DLOG_LEVEL=trace -DDOC_LEVEL=debug" bin/fscrawler + +By default, it will log everything in the ``logs`` directory inside the installation folder. + +Two log files are generated: -You can use `the default log4j2.xml -file `__ -as an example to start with. +* One is used to log FSCrawler code execution, named ``fscrawler.log``. It's automatically +rotated every day or after 20mb of logs and gzipped. Logs are removed after 7 days. +* One is used to trace all information about documents, named ``documents.log``. It's automatically +rotated every day or after 20mb of logs and gzipped. Logs are removed after 7 days. +You can change this strategy by modifying the ``config/log4j2.xml`` file. +Please read `Log4J2 documentation `_ on how to configure Log4J. diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index e9141796f..8ffa34f64 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -1,7 +1,7 @@ Building the project -------------------- -This project is built with `Maven `_. +This project is built with `Maven `_. It needs Java >= 1.14. Source code is available on `GitHub `_. Thanks to `JetBrains `_ for the IntelliJ IDEA License! @@ -72,7 +72,7 @@ To run the test suite against an elasticsearch instance running locally, just ru mvn verify -pl fr.pilato.elasticsearch.crawler:fscrawler-it-v7 \ -Dtests.cluster.user=elastic \ -Dtests.cluster.pass=changeme \ - -Dtests.cluster.url=https://127.0.0.1:9200 \ + -Dtests.cluster.url=http://127.0.0.1:9200 \ .. hint:: @@ -95,20 +95,95 @@ To run the test suite against an elasticsearch instance running locally, just ru Using security feature """""""""""""""""""""" -Integration tests are run by default against a standard Elasticsearch cluster, which means -with no security feature activated. +Integration tests are run by default against a secured Elasticsearch cluster. .. versionadded:: 2.7 -You can run all the integration tests against a secured cluster by using the ``security`` profile:: +Secured tests are using by default ``changeme`` as the password. +You can change this by using ``tests.cluster.pass`` option:: - mvn verify -Psecurity + mvn verify -Dtests.cluster.pass=mystrongpassword -Note that secured tests are using by default ``changeme`` as the password. -You can change this by using ``tests.cluster.pass`` option:: - mvn verify -Psecurity -Dtests.cluster.pass=mystrongpassword +Testing Workplace Search connector +"""""""""""""""""""""""""""""""""" + +To test the Workplace Search connector, some manual steps needs to be performed as you need to start +Enterprise Search and create manually the custom source as there is no API yet to do that. + + .. warning:: + + Running the integration tests **will remove everything** you have indexed so far in the workplace local instance. + +.. versionadded:: 2.7 + +* Run the following steps:: + + mvn docker-compose:up waitfor:waitfor -pl fr.pilato.elasticsearch.crawler:fscrawler-it-v7 + +* Wait for it to end and open http://localhost:3002/ws. +* Enter ``enterprise_search`` as the login and ``changeme`` as the password. +* Click on "Add sources" button and choose `Custom API `_. +* Name it ``fscrawler`` and click on "Create Custom API Source" button. +* Copy the "Access Token" value. We will mention it as ``ACCESS_TOKEN`` for the rest of this documentation. +* Copy the "Key" value. We will mention it as ``KEY`` for the rest of this documentation. + +.. image:: /_static/wpsearch/fscrawler-custom-source.png + +* You can now run in another terminal:: + + mvn verify -pl fr.pilato.elasticsearch.crawler:fscrawler-it-v7 \ + -Dtests.cluster.url=http://127.0.0.1:9200 \ + -Dtests.workplace.access_token=ACCESS_TOKEN \ + -Dtests.workplace.key=KEY + +* Then you should be able to see the documents in http://localhost:3002/ws/search + +* Once you're done and want to switch off the stack, run:: + + mvn docker-compose:down -pl fr.pilato.elasticsearch.crawler:fscrawler-it-v7 + +.. hint:: + + If you want to modify the look, go to the source and choose "Display Settings". + Adapt the settings accordingly. + + .. image:: /_static/wpsearch/fscrawler-display-settings-1.png + + In "Result Detail" tab, add the missing fields. And click on "Save". + + .. image:: /_static/wpsearch/fscrawler-display-settings-2.png + +To run Workplace Search tests against another instance (ie. running on +`Enterprise Search service by Elastic `_, +you can also use ``tests.workplace.url`` to set where Enterprise Search is running:: + + mvn verify -pl fr.pilato.elasticsearch.crawler:fscrawler-it-v7 \ + -Dtests.cluster.user=elastic \ + -Dtests.cluster.pass=changeme \ + -Dtests.cluster.cloud_id=CLOUD_ID + -Dtests.workplace.url=https://XYZ.ent-search.ZONE.CLOUD_PROVIDER.elastic-cloud.com \ + -Dtests.workplace.access_token=ACCESS_TOKEN \ + -Dtests.workplace.key=KEY + +Changing the REST port +"""""""""""""""""""""" + +By default, FS crawler will run the integration tests using port ``8080`` for the REST service. +You can change this by using ``tests.rest.port`` option:: + + mvn verify -Dtests.rest.port=8280 + +Randomized testing +"""""""""""""""""" + +FS Crawler uses the `randomized testing framework `_. +In case of failure, it will print a line like:: + + REPRODUCE WITH: + mvn test -Dtests.seed=AC6992149EB4B547 -Dtests.class=fr.pilato.elasticsearch.crawler.fs.test.unit.tika.TikaDocParserTest -Dtests.method="testExtractFromRtf" -Dtests.locale=ga-IE -Dtests.timezone=Canada/Saskatchewan +You can just run the test again using the same seed to make sure you always run the test in the same context as before. Tests options """"""""""""" @@ -116,19 +191,23 @@ Tests options Some options are available from the command line when running the tests: * ``tests.leaveTemporary`` leaves temporary files after tests. ``false`` by default. -* ``tests.parallelism`` how many JVM to launch in parallel for tests. Set to ``auto`` by default - which means that it depends on the number of processors you have. -* ``tests.output`` what should be displayed to the console while running tests. By default it is set to - ``onError`` but can be set to ``always`` +* ``tests.parallelism`` how many JVM to launch in parallel for tests. ``auto`` by default which means that it depends on the number of processors you have. It can be set to ``max`` if you want to use all the available processors, or a given value like ``1`` to use that exact number of JVMs. +* ``tests.output`` what should be displayed to the console while running tests. By default it is set to ``onError`` but can be set to ``always`` * ``tests.verbose`` ``false`` by default * ``tests.seed`` if you need to reproduce a specific failure using the exact same random seed * ``tests.timeoutSuite`` how long a single can run. It's set by default to ``600000`` which means 5 minutes. * ``tests.locale`` by default it's set to ``random`` but you can force the locale to use. -* ``tests.timezone`` by default it's set to ``random`` but you can force the timezone to use. +* ``tests.timezone`` by default it's set to ``random`` but you can force the timezone to use, like ``CEST`` or ``-0200``. For example:: - mvn install -rf :fscrawler-it -Dtests.output=always + mvn install -rf :fscrawler-it \ + -Dtests.output=always \ + -Dtests.locale=fr-FR \ + -Dtests.timezone=CEST \ + -Dtests.verbose \ + -Dtests.leaveTemporary \ + -Dtests.seed=E776CE45185A6E7A Check for vulnerabilities (CVE) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/dev/release.rst b/docs/source/dev/release.rst index 7fc012f17..d3667c4c4 100644 --- a/docs/source/dev/release.rst +++ b/docs/source/dev/release.rst @@ -5,7 +5,27 @@ To release the project, run:: $ release.sh -And follow the instructions. +The release script will: + +* Create a release branch +* Replace SNAPSHOT version by the final version number +* Commit the change +* Run tests against all supported elasticsearch series +* Build the final artifacts using release profile (signing artifacts and generating all needed files) +* Tag the version +* Prepare the announcement email +* Deploy to https://oss.sonatype.org/ +* Prepare the next SNAPSHOT version +* Commit the change +* Release the Sonatype staging repository +* Merge the release branch to the branch we started from +* Push the changes to origin +* Announce the version on https://discuss.elastic.co/c/annoucements/community-ecosystem + +You will be guided through all the steps. + +You can add some maven options while executing the release script such as ``-DskipTests`` if you want to skip +the tests while building the release. .. note:: diff --git a/docs/source/fscrawler.ini b/docs/source/fscrawler.ini index 1f786fa0f..de46b7589 100644 --- a/docs/source/fscrawler.ini +++ b/docs/source/fscrawler.ini @@ -2,9 +2,9 @@ Version=2.7-SNAPSHOT [3rdParty] -TikaVersion=1.24 -ElasticsearchVersion6=6.8.8 -ElasticsearchVersion7=7.6.2 +TikaVersion=1.25 +ElasticsearchVersion6=6.8.13 +ElasticsearchVersion7=7.10.1 LevigoVersion=2.0 TiffVersion=1.4.0 -JpegVersion=1.3.0 +JpegVersion=1.4.0 diff --git a/docs/source/index.rst b/docs/source/index.rst index 0e399a65f..ee0d19b57 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,6 +57,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office admin/fs/local-fs admin/fs/ssh admin/fs/elasticsearch + admin/fs/wpsearch admin/fs/rest diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 597d4aa87..ae9e147b1 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -54,6 +54,8 @@ The distribution contains: ├── bin │   ├── fscrawler │   └── fscrawler.bat + ├── config + │   └── log4j2.xml └── lib ├── ... All needed jars @@ -123,7 +125,7 @@ And, prepare the following ``docker-compose.yml``. services: # FSCrawler fscrawler: - image: toto1310/fscrawler + image: dadoonet/fscrawler container_name: fscrawler volumes: - ${PWD}/config:/root/.fscrawler @@ -209,7 +211,7 @@ Create a ``fscrawlerRunner.bat`` as: .. code:: sh - set JAVA_HOME=c:\Program Files\Java\jdk1.8.0_144 + set JAVA_HOME=c:\Program Files\Java\jdk15.0.1 set FS_JAVA_OPTS=-Xmx2g -Xms2g /Elastic/fscrawler/bin/fscrawler.bat --config_dir /Elastic/fscrawler data >> /Elastic/logs/fscrawler.log 2>&1 @@ -486,3 +488,4 @@ Upgrade to 2.7 to ``true`` if you wish revert to the previous behavior. - The mapping for elasticsearch 6.x can not contain anymore the type name. - We removed the Elasticsearch V5 compatibility as it's not maintained anymore by elastic. +- You need to use a recent JVM to run FSCrawler (Java 15+ recommended) diff --git a/docs/source/user/ocr.rst b/docs/source/user/ocr.rst index 19cb24080..a329935b8 100644 --- a/docs/source/user/ocr.rst +++ b/docs/source/user/ocr.rst @@ -139,21 +139,24 @@ OCR PDF Strategy By default, FSCrawler will also try to extract also images from your PDF documents and run OCR on them. This can be a CPU intensive operation. If you don’t mean to run OCR on PDF but only on images, you can set -``fs.ocr.pdf_strategy`` to ``"no_ocr"``: +``fs.ocr.pdf_strategy`` to ``"no_ocr"`` or to ``"auto"``: .. code:: yaml name: "test" fs: ocr: - pdf_strategy: "no_ocr" + pdf_strategy: "auto" Supported strategies are: +* ``auto``: No OCR is performed on PDF documents if there is more than 10 characters extracted. See `PDFParser OCR Options `__. + * ``no_ocr``: No OCR is performed on PDF documents. OCR might be performed on images though if OCR is not disabled. See `Disable/Enable OCR`_. * ``ocr_only``: Only OCR is performed. * ``ocr_and_text``: OCR and text extraction is performed. -.. note:: When omitted, ``ocr_and_text`` value is used. +.. note:: When omitted, ``ocr_and_text`` value is used. If you have performance issues, it's worth using the ``auto`` option +instead as only documents with barely no text will go through the OCR process. diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index 5022e71a9..740eb851d 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -32,3 +32,32 @@ crawler on this mount point. You can also read details about `HDFS NFS Gateway `__. +Using docker-compose +-------------------- +To standup a full environment you can use docker-compose from the contrib directory. +This environment will setup a node ElasticSearch cluster, a copy of Kibana +for searching and FSCrawler as containers. No other installs are neeeded, aside form Docker and docker-compose. + +Steps: + + 1. Download and install `docker `__. + 2. Download and install `docker-compose `__. + 3. Copy the contrib directory into your home directory. + 4. Edit the docker-compose.yaml + 1. Edit the line (somewhere around 66) that points to the "files to be scanned". + This is the path on the host machine prior to the colon. (ex: /fs/resume) + 2. In the ./config/ directory exists the name of the index name that FSCrawler will use. + By default, it's set to 'idx'. You can change it by renaming this directory, and changing the _settings.yaml file. + Check the ./config/idx/_settings.yaml to update any changes you like. + If you have multiple directories that you like to scan, I would suggest linking them under a single directory and + changing the "follow_links" option. + 5. Check the Dockerfile-fscrawler file. This is where the version of the package is determined. By default I have set to + download the 'master' branch which is currently producing a es7-2.7-SNAPSHOT version but you can lock this into a + specific version to make it more reliable. Update (DO NOT MOVE) the ENV variables to match what you want the build to be. + 6. Issue `docker-compose up -d` in that directory and it'll download and create the containers. It'll also compile and build a + custom container for fscrawler. + 7. After the containers are up and running, wait about 30 seconds for everything to start syncing. You can now access Kibana and + build your index (just need to do it once). After that the search will be available via Kibana. + +TODO: Build a more robust link to a specific version in the Dockerfile so it's a little more specific about what it downloads and builds.0:w + diff --git a/elasticsearch-client/elasticsearch-client-base/pom.xml b/elasticsearch-client/elasticsearch-client-base/pom.xml index 9fb831a1b..c1f361053 100644 --- a/elasticsearch-client/elasticsearch-client-base/pom.xml +++ b/elasticsearch-client/elasticsearch-client-base/pom.xml @@ -12,4 +12,11 @@ fscrawler-elasticsearch-client-base FSCrawler Elasticsearch Client Base + + + fr.pilato.elasticsearch.crawler + fscrawler-beans + + + diff --git a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESDocumentField.java b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESDocumentField.java index 5940bac1c..3578ad9f7 100644 --- a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESDocumentField.java +++ b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESDocumentField.java @@ -29,6 +29,7 @@ * * @see ESSearchHit */ +@SuppressWarnings("unchecked") public class ESDocumentField implements Iterable { private String name; @@ -56,7 +57,7 @@ public V getValue() { if (values == null || values.isEmpty()) { return null; } - return (V)values.get(0); + return (V) values.get(0); } /** diff --git a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java index 767c85923..49044d6ad 100644 --- a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java +++ b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java @@ -20,6 +20,8 @@ package fr.pilato.elasticsearch.crawler.fs.client; +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; + import java.io.Closeable; import java.io.IOException; @@ -97,19 +99,28 @@ public interface ElasticsearchClient extends Closeable { String getDefaultTypeName(); /** - * Index a document using a BulkProcessor behind the scenes + * Index a document (might use a BulkProcessor behind the scenes) + * @param index Index name + * @param id Document ID + * @param doc Document to index + * @param pipeline Pipeline (can be null) + */ + void index(String index, String id, Doc doc, String pipeline); + + /** + * Index a Raw Json in Elasticsearch * @param index Index name * @param id Document ID - * @param json JSON + * @param json Document to index * @param pipeline Pipeline (can be null) */ - void index(String index, String id, String json, String pipeline); + void indexRawJson(String index, String id, String json, String pipeline); /** * Index a document (for test purposes only) * @param index Index name * @param id Document ID - * @param json JSON + * @param json Document to index */ void indexSingle(String index, String id, String json) throws IOException; diff --git a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientUtil.java b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientUtil.java index f4025f341..84b4dd2cf 100644 --- a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientUtil.java +++ b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientUtil.java @@ -97,6 +97,7 @@ public static ElasticsearchClient getInstance(Path config, FsSettings settings, * @param version Version number to use. It's only the major part of the version. Like 5 or 6. * @return The Client class */ + @SuppressWarnings("unchecked") private static Class findClass(int version) throws ClassNotFoundException { String className = "fr.pilato.elasticsearch.crawler.fs.client.v" + version + ".ElasticsearchClientV" + version; logger.trace("Trying to find a class named [{}]", className); diff --git a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClient.java b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClient.java new file mode 100644 index 000000000..5769fdc70 --- /dev/null +++ b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClient.java @@ -0,0 +1,4 @@ +package fr.pilato.elasticsearch.crawler.fs.client; + +public interface WorkplaceSearchClient extends ElasticsearchClient { +} diff --git a/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClientUtil.java b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClientUtil.java new file mode 100644 index 000000000..93908712d --- /dev/null +++ b/elasticsearch-client/elasticsearch-client-base/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClientUtil.java @@ -0,0 +1,114 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.client; + +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.util.Objects; + +public abstract class WorkplaceSearchClientUtil { + + protected static final Logger logger = LogManager.getLogger(WorkplaceSearchClientUtil.class); + + private WorkplaceSearchClientUtil() { + + } + + /** + * Try to find a client version in the classpath + * @param config Path to FSCrawler configuration files (elasticsearch templates) + * @param settings FSCrawler settings. Can not be null. + * @return A Client instance + */ + public static WorkplaceSearchClient getInstance(Path config, FsSettings settings) { + + Objects.requireNonNull(settings, "settings can not be null"); + + for (int i = 7; i >= 1; i--) { + logger.debug("Trying to find a client version {}", i); + + try { + return getInstance(config, settings, i); + } catch (ClassNotFoundException ignored) { + } + } + + return null; + } + + /** + * Try to find a client version in the classpath + * @param config Path to FSCrawler configuration files (elasticsearch templates) + * @param settings FSCrawler settings. Can not be null. + * @param version Version to load + * @return A Client instance + */ + public static WorkplaceSearchClient getInstance(Path config, FsSettings settings, int version) throws ClassNotFoundException { + Objects.requireNonNull(settings, "settings can not be null"); + + if (settings.getWorkplaceSearch() == null) { + return null; + } + + Class clazz = null; + try { + clazz = findClass(version); + } catch (ClassNotFoundException e) { + logger.trace("WorkplaceSearchClient class not found for version {} in the classpath. Skipping...", version); + } + + if (clazz == null) { + throw new ClassNotFoundException("Can not find any WorkplaceSearchClient in the classpath. " + + "Did you forget to add the elasticsearch client library?"); + } + + logger.trace("Found [{}] class as the elasticsearch client implementation.", clazz.getName()); + try { + Constructor constructor = clazz.getConstructor(Path.class, FsSettings.class); + return constructor.newInstance(config, settings); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Class " + clazz.getName() + " does not have the expected ctor (Path, FsSettings).", e); + } catch (IllegalAccessException|InstantiationException| InvocationTargetException e) { + throw new IllegalArgumentException("Can not create an instance of " + clazz.getName(), e); + } + } + + /** + * Try to load an WorkplaceSearchClient class + * @param version Version number to use. It's only the major part of the version. Like 5 or 6. + * @return The Client class + */ + private static Class findClass(int version) throws ClassNotFoundException { + String className = "fr.pilato.elasticsearch.crawler.fs.client.v" + version + ".WorkplaceSearchClientV" + version; + logger.trace("Trying to find a class named [{}]", className); + Class aClass = Class.forName(className); + boolean isImplementingInterface = WorkplaceSearchClient.class.isAssignableFrom(aClass); + if (!isImplementingInterface) { + throw new IllegalArgumentException("Class " + className + " does not implement " + WorkplaceSearchClient.class.getName() + " interface"); + } + + return (Class) aClass; + } +} diff --git a/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientDummyBase.java b/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientDummyBase.java index dee7c80bf..071594273 100644 --- a/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientDummyBase.java +++ b/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientDummyBase.java @@ -19,7 +19,10 @@ package fr.pilato.elasticsearch.crawler.fs.client; +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; + import java.io.IOException; +import java.util.function.Supplier; public abstract class ElasticsearchClientDummyBase implements ElasticsearchClient { @@ -74,12 +77,17 @@ public String getDefaultTypeName() { } @Override - public void index(String index, String id, String json, String pipeline) { + public void index(String index, String id, Doc doc, String pipeline) { + // Testing purpose only + } + + @Override + public void indexRawJson(String index, String id, String json, String pipeline) { // Testing purpose only } @Override - public void indexSingle(String index, String id, String json) throws IOException { + public void indexSingle(String index, String id, String json) { // Testing purpose only } @@ -114,12 +122,12 @@ public void performLowLevelRequest(String method, String endpoint, String jsonEn } @Override - public ESSearchHit get(String index, String id) throws IOException { + public ESSearchHit get(String index, String id) { return null; } @Override - public boolean exists(String index, String id) throws IOException { + public boolean exists(String index, String id) { return false; } diff --git a/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientTest.java b/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientTest.java index 83d3e0189..d67b16977 100644 --- a/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientTest.java +++ b/elasticsearch-client/elasticsearch-client-base/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClientTest.java @@ -31,6 +31,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +@SuppressWarnings("ConstantConditions") public class ElasticsearchClientTest extends AbstractFSCrawlerTestCase { @Test diff --git a/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/fscrawler-client.properties b/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/fscrawler-client.properties deleted file mode 100644 index 98a002a2c..000000000 --- a/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/fscrawler-client.properties +++ /dev/null @@ -1 +0,0 @@ -es-client.class=fr.pilato.elasticsearch.crawler.fs.client.v0.ElasticsearchClientV0 diff --git a/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/v0/fscrawler-client-wrong-version.properties b/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/v0/fscrawler-client-wrong-version.properties deleted file mode 100644 index 931d5bbfa..000000000 --- a/elasticsearch-client/elasticsearch-client-base/src/test/resources/fr/pilato/elasticsearch/crawler/fs/client/v0/fscrawler-client-wrong-version.properties +++ /dev/null @@ -1 +0,0 @@ -es-client.class=fr.pilato.elasticsearch.crawler.fs.client.v1.ElasticsearchClientV1 diff --git a/elasticsearch-client/elasticsearch-client-v6/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v6/ElasticsearchClientV6.java b/elasticsearch-client/elasticsearch-client-v6/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v6/ElasticsearchClientV6.java index f6ec5a25d..4ad399e4a 100644 --- a/elasticsearch-client/elasticsearch-client-v6/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v6/ElasticsearchClientV6.java +++ b/elasticsearch-client/elasticsearch-client-v6/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v6/ElasticsearchClientV6.java @@ -20,6 +20,8 @@ package fr.pilato.elasticsearch.crawler.fs.client.v6; +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.beans.DocParser; import fr.pilato.elasticsearch.crawler.fs.client.ESBoolQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESDocumentField; import fr.pilato.elasticsearch.crawler.fs.client.ESHighlightField; @@ -33,6 +35,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESTermsAggregation; import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; @@ -41,6 +44,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; @@ -84,21 +88,26 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.nio.file.Path; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SETTINGS_FILE; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SETTINGS_FOLDER_FILE; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMinorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isNullOrEmpty; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readJsonFile; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN; /** @@ -143,7 +152,7 @@ public void start() throws IOException { checkVersion(); logger.info("Elasticsearch Client for version {}.x connected to a node running version {}", compatibleVersion(), getVersion()); } catch (Exception e) { - logger.warn("failed to create elasticsearch client, disabling crawler..."); + logger.warn("failed to create elasticsearch client on {}, disabling crawler...", settings.getElasticsearch().toString()); throw e; } @@ -158,7 +167,7 @@ public void start() throws IOException { BiConsumer> bulkConsumer = (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); - bulkProcessor = BulkProcessor.builder(bulkConsumer, new DebugListener(logger)) + bulkProcessor = BulkProcessor.builder(bulkConsumer, new DebugListener()) .setBulkActions(settings.getElasticsearch().getBulkSize()) .setFlushInterval(TimeValue.timeValueMillis(settings.getElasticsearch().getFlushInterval().millis())) .setBulkSize(new ByteSizeValue(settings.getElasticsearch().getByteSize().getBytes())) @@ -186,17 +195,11 @@ public void checkVersion() throws IOException { if (Integer.parseInt(extractMinorVersion(esVersion)) < 4) { throw new RuntimeException("This version of FSCrawler is not compatible with " + "Elasticsearch version [" + - esVersion.toString() + "]. Please upgrade Elasticsearch to at least a 6.4.x version."); + esVersion + "]. Please upgrade Elasticsearch to at least a 6.4.x version."); } } - class DebugListener implements BulkProcessor.Listener { - private final Logger logger; - - DebugListener(Logger logger) { - this.logger = logger; - } - + static class DebugListener implements BulkProcessor.Listener { @Override public void beforeBulk(long executionId, BulkRequest request) { logger.trace("Sending a bulk request of [{}] requests", request.numberOfActions()); } @@ -208,6 +211,10 @@ class DebugListener implements BulkProcessor.Listener { response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; + FSCrawlerLogger.documentError( + bir.getId(), + null, + bir.getFailureMessage()); logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } @@ -334,7 +341,7 @@ public int reindex(String sourceIndex, String sourceType, String targetIndex) th Map response = asMap(restResponse); logger.debug("reindex response: {}", response); - return (int) response.get("total"); + return (int) Objects.requireNonNull(response).get("total"); } /** @@ -369,13 +376,40 @@ public String getDefaultTypeName() { return INDEX_TYPE_DOC; } + private static final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) {} + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) {} + + @Override + public X509Certificate[] getAcceptedIssuers() { return null; } + }}; + + public static class NullHostNameVerifier implements HostnameVerifier { + + @Override + public boolean verify(String arg0, SSLSession arg1) { return true; } + + } + + @Override + public void index(String index, String id, Doc doc, String pipeline) { + String json = DocParser.toJson(doc); + indexRawJson(index, id, json, pipeline); + } + @Override - public void index(String index, String id, String json, String pipeline) { + public void indexRawJson(String index, String id, String json, String pipeline) { + logger.trace("JSon indexed : {}", json); bulkProcessor.add(new IndexRequest(index, getDefaultTypeName(), id).setPipeline(pipeline).source(json, XContentType.JSON)); } @Override public void indexSingle(String index, String id, String json) throws IOException { + logger.trace("JSon indexed : {}", json); IndexRequest request = new IndexRequest(index, getDefaultTypeName(), id); request.source(json, XContentType.JSON); client.index(request, RequestOptions.DEFAULT); @@ -406,7 +440,7 @@ private static RestClientBuilder buildRestClient(Elasticsearch settings) { List hosts = new ArrayList<>(settings.getNodes().size()); settings.getNodes().forEach(node -> hosts.add(HttpHost.create(node.decodedUrl()))); - RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()])); + RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0])); if (settings.getPathPrefix() != null) { builder.setPathPrefix(settings.getPathPrefix()); @@ -415,10 +449,24 @@ private static RestClientBuilder buildRestClient(Elasticsearch settings) { if (settings.getUsername() != null) { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); - builder.setHttpClientConfigCallback(httpClientBuilder -> - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + if (settings.getSslVerification()) { + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } else { + builder.setHttpClientConfigCallback(httpClientBuilder -> { + SSLContext sc; + try { + sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new SecureRandom()); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + logger.warn("Failed to get SSL Context", e); + throw new RuntimeException(e); + } + httpClientBuilder.setSSLStrategy(new SSLIOSessionStrategy(sc, new NullHostNameVerifier())); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + }); + } } - return builder; } diff --git a/elasticsearch-client/elasticsearch-client-v7/pom.xml b/elasticsearch-client/elasticsearch-client-v7/pom.xml index 50884713a..673745779 100644 --- a/elasticsearch-client/elasticsearch-client-v7/pom.xml +++ b/elasticsearch-client/elasticsearch-client-v7/pom.xml @@ -32,6 +32,10 @@ elasticsearch-rest-high-level-client ${elasticsearch7.version} + + fr.pilato.elasticsearch.crawler + fscrawler-workplacesearch-client + diff --git a/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7.java b/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7.java index ed9922c31..66ebfaa6f 100644 --- a/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7.java +++ b/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7.java @@ -20,6 +20,8 @@ package fr.pilato.elasticsearch.crawler.fs.client.v7; +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.beans.DocParser; import fr.pilato.elasticsearch.crawler.fs.client.ESBoolQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESDocumentField; import fr.pilato.elasticsearch.crawler.fs.client.ESHighlightField; @@ -33,6 +35,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESTermsAggregation; import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; @@ -41,6 +44,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; @@ -84,20 +88,26 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.nio.file.Path; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SETTINGS_FILE; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SETTINGS_FOLDER_FILE; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isNullOrEmpty; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readJsonFile; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN; /** @@ -142,7 +152,7 @@ public void start() throws IOException { checkVersion(); logger.info("Elasticsearch Client for version {}.x connected to a node running version {}", compatibleVersion(), getVersion()); } catch (Exception e) { - logger.warn("failed to create elasticsearch client, disabling crawler..."); + logger.warn("failed to create elasticsearch client on {}, disabling crawler...", settings.getElasticsearch().toString()); throw e; } @@ -157,7 +167,7 @@ public void start() throws IOException { BiConsumer> bulkConsumer = (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); - bulkProcessor = BulkProcessor.builder(bulkConsumer, new DebugListener(logger)) + bulkProcessor = BulkProcessor.builder(bulkConsumer, new DebugListener()) .setBulkActions(settings.getElasticsearch().getBulkSize()) .setFlushInterval(TimeValue.timeValueMillis(settings.getElasticsearch().getFlushInterval().millis())) .setBulkSize(new ByteSizeValue(settings.getElasticsearch().getByteSize().getBytes())) @@ -170,13 +180,7 @@ public String getVersion() throws IOException { return version.getNumber(); } - class DebugListener implements BulkProcessor.Listener { - private final Logger logger; - - DebugListener(Logger logger) { - this.logger = logger; - } - + static class DebugListener implements BulkProcessor.Listener { @Override public void beforeBulk(long executionId, BulkRequest request) { logger.trace("Sending a bulk request of [{}] requests", request.numberOfActions()); } @@ -188,6 +192,10 @@ class DebugListener implements BulkProcessor.Listener { response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; + FSCrawlerLogger.documentError( + bir.getId(), + null, + bir.getFailureMessage()); logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } @@ -208,6 +216,7 @@ class DebugListener implements BulkProcessor.Listener { * @param indexSettings index settings if any * @throws IOException In case of error */ + @Override public void createIndex(String index, boolean ignoreErrors, String indexSettings) throws IOException { logger.debug("create index [{}]", index); logger.trace("index settings: [{}]", indexSettings); @@ -234,6 +243,7 @@ public void createIndex(String index, boolean ignoreErrors, String indexSettings * @return true if the index exists, false otherwise * @throws IOException In case of error */ + @Override public boolean isExistingIndex(String index) throws IOException { logger.debug("is existing index [{}]", index); return client.indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT); @@ -245,6 +255,7 @@ public boolean isExistingIndex(String index) throws IOException { * @return true if the pipeline exists, false otherwise * @throws IOException In case of error */ + @Override public boolean isExistingPipeline(String pipelineName) throws IOException { logger.debug("is existing pipeline [{}]", pipelineName); try { @@ -262,6 +273,7 @@ public boolean isExistingPipeline(String pipelineName) throws IOException { * @param index index name * @throws IOException In case of error */ + @Override public void refresh(String index) throws IOException { logger.debug("refresh index [{}]", index); RefreshRequest request = new RefreshRequest(); @@ -277,6 +289,7 @@ public void refresh(String index) throws IOException { * @param index index name * @throws IOException In case of error */ + @Override public void waitForHealthyIndex(String index) throws IOException { logger.debug("wait for yellow health on index [{}]", index); ClusterHealthResponse health = client.cluster().health(new ClusterHealthRequest(index).waitForYellowStatus(), @@ -314,7 +327,7 @@ public int reindex(String sourceIndex, String sourceType, String targetIndex) th Map response = asMap(restResponse); logger.debug("reindex response: {}", response); - return (int) response.get("total"); + return (int) Objects.requireNonNull(response).get("total"); } /** @@ -341,21 +354,50 @@ public void deleteByQuery(String index, String type) throws IOException { // Utility methods + @Override public boolean isIngestSupported() { return true; } + @Override public String getDefaultTypeName() { return INDEX_TYPE_DOC; } + private static final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) {} + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) {} + + @Override + public X509Certificate[] getAcceptedIssuers() { return null; } + }}; + + public static class NullHostNameVerifier implements HostnameVerifier { + + @Override + public boolean verify(String arg0, SSLSession arg1) { return true; } + + } + + @Override + public void index(String index, String id, Doc doc, String pipeline) { + String json = DocParser.toJson(doc); + indexRawJson(index, id, json, pipeline); + } + @Override - public void index(String index, String id, String json, String pipeline) { + public void indexRawJson(String index, String id, String json, String pipeline) { + logger.trace("JSon indexed : {}", json); bulkProcessor.add(new IndexRequest(index).id(id).setPipeline(pipeline).source(json, XContentType.JSON)); } @Override public void indexSingle(String index, String id, String json) throws IOException { + logger.trace("JSon indexed : {}", json); client.index(new IndexRequest(index).id(id).source(json, XContentType.JSON), RequestOptions.DEFAULT); } @@ -384,7 +426,7 @@ private static RestClientBuilder buildRestClient(Elasticsearch settings) { List hosts = new ArrayList<>(settings.getNodes().size()); settings.getNodes().forEach(node -> hosts.add(HttpHost.create(node.decodedUrl()))); - RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()])); + RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0])); if (settings.getPathPrefix() != null) { builder.setPathPrefix(settings.getPathPrefix()); @@ -393,13 +435,29 @@ private static RestClientBuilder buildRestClient(Elasticsearch settings) { if (settings.getUsername() != null) { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); - builder.setHttpClientConfigCallback(httpClientBuilder -> - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + if (settings.getSslVerification()) { + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } else { + builder.setHttpClientConfigCallback(httpClientBuilder -> { + SSLContext sc; + try { + sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new SecureRandom()); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + logger.warn("Failed to get SSL Context", e); + throw new RuntimeException(e); + } + httpClientBuilder.setSSLStrategy(new SSLIOSessionStrategy(sc, new NullHostNameVerifier())); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + }); + } } return builder; } + @Override public void createIndices() throws Exception { String elasticsearchVersion; Path jobMappingDir = config.resolve(settings.getName()).resolve("_mappings"); diff --git a/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/WorkplaceSearchClientV7.java b/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/WorkplaceSearchClientV7.java new file mode 100644 index 000000000..26610e5ab --- /dev/null +++ b/elasticsearch-client/elasticsearch-client-v7/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/v7/WorkplaceSearchClientV7.java @@ -0,0 +1,248 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.client.v7; + + +import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; +import fr.pilato.elasticsearch.crawler.fs.client.WorkplaceSearchClient; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch.WPSearchClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * Workplace Search Client for Clusters running v7. + * It also starts an embedded Elasticsearch Client. + */ +public class WorkplaceSearchClientV7 implements WorkplaceSearchClient { + + private static final Logger logger = LogManager.getLogger(WorkplaceSearchClientV7.class); + private final Path config; + private final FsSettings settings; + private final SimpleDateFormat rfc3339 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ssZZZZZ"); + + private ElasticsearchClient esClient = null; + private WPSearchClient wpSearchClient = null; + + public WorkplaceSearchClientV7(Path config, FsSettings settings) { + this.config = config; + this.settings = settings; + } + + @Override + public String compatibleVersion() { + return "7"; + } + + @Override + public void start() throws IOException { + logger.debug("Starting Workplace Search V7 client"); + wpSearchClient = new WPSearchClient( + settings.getWorkplaceSearch().getAccessToken(), + settings.getWorkplaceSearch().getKey()) + .withHost(settings.getWorkplaceSearch().getServer().decodedUrl()) + .withBulkSize(settings.getWorkplaceSearch().getBulkSize()) + .withFlushInterval(settings.getWorkplaceSearch().getFlushInterval()); + wpSearchClient.start(); + esClient = ElasticsearchClientUtil.getInstance(config, settings); + esClient.start(); + logger.debug("Workplace Search V7 client started"); + } + + @Override + public String getVersion() throws IOException { + return esClient.getVersion(); + } + + /** + * Create an index + * @param index index name + * @param ignoreErrors don't fail if the index already exists + * @param indexSettings index settings if any + * @throws IOException In case of error + */ + public void createIndex(String index, boolean ignoreErrors, String indexSettings) throws IOException { + esClient.createIndex(index, ignoreErrors, indexSettings); + } + + /** + * Check if an index exists + * @param index index name + * @return true if the index exists, false otherwise + * @throws IOException In case of error + */ + public boolean isExistingIndex(String index) throws IOException { + return esClient.isExistingIndex(index); + } + + /** + * Check if a pipeline exists + * @param pipelineName pipeline name + * @return true if the pipeline exists, false otherwise + * @throws IOException In case of error + */ + public boolean isExistingPipeline(String pipelineName) throws IOException { + return esClient.isExistingPipeline(pipelineName); + } + + /** + * Refresh an index + * @param index index name + * @throws IOException In case of error + */ + public void refresh(String index) throws IOException { + esClient.refresh(index); + } + + /** + * Wait for an index to become at least yellow (all primaries assigned) + * @param index index name + * @throws IOException In case of error + */ + public void waitForHealthyIndex(String index) throws IOException { + esClient.waitForHealthyIndex(index); + } + + // Utility methods + + public boolean isIngestSupported() { + return true; + } + + public String getDefaultTypeName() { + return esClient.getDefaultTypeName(); + } + + @Override + public void index(String index, String id, Doc doc, String pipeline) { + Map document = new HashMap<>(); + // Id + document.put("id", id); + + // Index content + document.put("body", doc.getContent()); + + // Index main metadata + // We use the name of the file if no title has been found in the document metadata + document.put("title", FsCrawlerUtil.isNullOrEmpty(doc.getMeta().getTitle()) ? doc.getFile().getFilename() : doc.getMeta().getTitle()); + document.put("author", doc.getMeta().getAuthor()); + document.put("keywords", doc.getMeta().getKeywords()); + document.put("language", doc.getMeta().getLanguage()); + document.put("comments", doc.getMeta().getComments()); + + // Index main file attributes + document.put("name", doc.getFile().getFilename()); + document.put("mime_type", doc.getFile().getContentType()); + document.put("extension", doc.getFile().getExtension()); + document.put("size", doc.getFile().getFilesize()); + document.put("text_size", doc.getFile().getIndexedChars()); + document.put("last_modified", toRFC3339(doc.getFile().getLastModified())); + document.put("created_at", toRFC3339(doc.getFile().getCreated())); + + // Index main path attributes + document.put("url", settings.getWorkplaceSearch().getUrlPrefix() + doc.getPath().getVirtual()); + document.put("path", doc.getPath().getReal()); + + wpSearchClient.indexDocument(document); + } + + @Override + public void indexRawJson(String index, String id, String json, String pipeline) { + throw new RuntimeException("Not supported by the workplace search client. Should not be called."); + } + + @Override + public void indexSingle(String index, String id, String json) { + throw new RuntimeException("Not supported by the workplace search client. Should not be called."); + } + + @Override + public void delete(String index, String id) { + wpSearchClient.destroyDocument(id); + } + + @Override + public void close() throws IOException { + logger.debug("Closing Workplace Search V7 client"); + if (esClient != null) { + esClient.close(); + } + if (wpSearchClient != null) { + wpSearchClient.close(); + } + logger.debug("Workplace Search V7 client closed"); + } + + public void createIndices() throws Exception { + esClient.createIndices(); + } + + @Override + public ESSearchResponse search(ESSearchRequest request) throws IOException { + // For now we are going to run a dummy search in elasticsearch directly + // and ignore the request in most times + request.withIndex(".ent-search-engine-*"); + return esClient.search(request); + } + + @Override + public void deleteIndex(String index) throws IOException { + esClient.deleteIndex(index); + } + + @Override + public void flush() { + esClient.flush(); + } + + @Override + public void performLowLevelRequest(String method, String endpoint, String jsonEntity) throws IOException { + esClient.performLowLevelRequest(method, endpoint, jsonEntity); + } + + @Override + public ESSearchHit get(String index, String id) { + throw new RuntimeException("Not implemented yet"); + } + + @Override + public boolean exists(String index, String id) { + throw new RuntimeException("Not implemented yet"); + // return esClient.exists(index, id); + } + + private String toRFC3339(Date d) + { + return rfc3339.format(d).replaceAll("(\\d\\d)(\\d\\d)$", "$1:$2"); + } +} diff --git a/elasticsearch-client/elasticsearch-client-v7/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7Test.java b/elasticsearch-client/elasticsearch-client-v7/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7Test.java index 77e57d82f..da9bb55a1 100644 --- a/elasticsearch-client/elasticsearch-client-v7/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7Test.java +++ b/elasticsearch-client/elasticsearch-client-v7/src/test/java/fr/pilato/elasticsearch/crawler/fs/client/v7/ElasticsearchClientV7Test.java @@ -21,7 +21,10 @@ import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; +import fr.pilato.elasticsearch.crawler.fs.client.WorkplaceSearchClient; +import fr.pilato.elasticsearch.crawler.fs.client.WorkplaceSearchClientUtil; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.WorkplaceSearch; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; import org.junit.Test; @@ -40,4 +43,13 @@ public void testGetInstance() { ElasticsearchClient instance = ElasticsearchClientUtil.getInstance(null, FsSettings.builder("foo").build()); assertThat(instance, instanceOf(ElasticsearchClientV7.class)); } + + @Test + public void testGetWorkplaceSearchInstance() { + WorkplaceSearchClient instance = WorkplaceSearchClientUtil.getInstance(null, + FsSettings.builder("foo") + .setWorkplaceSearch(WorkplaceSearch.builder().build()) + .build()); + assertThat(instance, instanceOf(WorkplaceSearchClientV7.class)); + } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java new file mode 100644 index 000000000..035a00c82 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java @@ -0,0 +1,60 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FSCrawlerLogger { + + /** + * This logger is for the console + */ + private final static Logger consoleLogger = LogManager.getLogger("fscrawler.console"); + + /** + * This logger is used to log information related to documents + */ + private final static Logger documentLogger = LogManager.getLogger("fscrawler.document"); + + public static void console(String message, Object... params) { + consoleLogger.info(message, params); + } + + /** + * Log information in Debug Level about documents + * @param id Document ID + * @param path Virtual path to the document + * @param message Message to display + */ + public static void documentDebug(String id, String path, String message) { + documentLogger.debug("[{}][{}] {}", id, path, message); + } + + /** + * Log information in Error Level about documents + * @param id Document ID + * @param path Virtual path to the document + * @param error Error to display + */ + public static void documentError(String id, String path, String error) { + documentLogger.error("[{}][{}] {}", id, path, error); + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerIllegalConfigurationException.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerIllegalConfigurationException.java new file mode 100644 index 000000000..dd164d1c1 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerIllegalConfigurationException.java @@ -0,0 +1,29 @@ +/* + * Licensed to David Pilato under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework; + +/** + * In case we need to throw a Runtime Exception when we detect something wrong + */ +public class FsCrawlerIllegalConfigurationException extends RuntimeException { + public FsCrawlerIllegalConfigurationException(String s) { + super(s); + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index c1163a280..e6e39a4f0 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -99,7 +99,7 @@ public static String readDefaultJsonVersionedFile(Path config, String version, S */ private static String readJsonVersionedFile(Path dir, String version, String type) throws IOException { Path file = dir.resolve(version).resolve(type + ".json"); - return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + return Files.readString(file); } /** @@ -547,7 +547,7 @@ public static void unzip(String jarFile, Path destination) throws IOException { try (FileSystem zipfs = FileSystems.newFileSystem(zipFile, zipProperties)) { Path rootPath = zipfs.getPath("/"); - Files.walkFileTree(rootPath, new SimpleFileVisitor() { + Files.walkFileTree(rootPath, new SimpleFileVisitor<>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { Path targetPath = destination.resolve(rootPath.relativize(dir).toString()); diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java index 66d132554..a5556a322 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java @@ -48,7 +48,7 @@ public static T deserialize(InputStream stream, Class clazz) { public static Map asMap(InputStream stream) { try { - return mapper.readValue(stream, new TypeReference>(){}); + return mapper.readValue(stream, new TypeReference<>() {}); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/MetaFileHandler.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/MetaFileHandler.java index 2f3c0574a..ae472a925 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/MetaFileHandler.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/MetaFileHandler.java @@ -21,7 +21,6 @@ import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -51,7 +50,7 @@ protected String readFile(String subdir, String filename) throws IOException { if (subdir != null) { dir = dir.resolve(subdir); } - return new String(Files.readAllBytes(dir.resolve(filename)), StandardCharsets.UTF_8); + return Files.readString(dir.resolve(filename)); } /** @@ -71,7 +70,7 @@ protected void writeFile(String subdir, String filename, String content) throws Files.createDirectory(dir); } } - Files.write(dir.resolve(filename), content.getBytes(StandardCharsets.UTF_8)); + Files.writeString(dir.resolve(filename), content); } /** diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/OsValidator.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/OsValidator.java index f94f94565..fb183dfbc 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/OsValidator.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/OsValidator.java @@ -31,7 +31,7 @@ public class OsValidator { OS = System.getProperty("os.name").toLowerCase(); WINDOWS = (OS.contains("win")); MAC = (OS.contains("mac")); - UNIX = (OS.contains("nix") || OS.contains("nux") || OS.indexOf("aix") >= 0 ); + UNIX = (OS.contains("nix") || OS.contains("nux") || OS.contains("aix")); SOLARIS = (OS.contains("sunos")); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/SignTool.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/SignTool.java index 77f742264..bfa0a4cfb 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/SignTool.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/SignTool.java @@ -35,7 +35,7 @@ public static String sign(String toSign) throws NoSuchAlgorithmException { md.update(toSign.getBytes()); StringBuilder key = new StringBuilder(); - byte b[] = md.digest(); + byte[] b = md.digest(); for (byte aB : b) { long t = aB < 0 ? 256 + aB : aB; key.append(Long.toHexString(t)); diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtil.java deleted file mode 100644 index 6fd90053c..000000000 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package fr.pilato.elasticsearch.crawler.fs.framework; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class StreamsUtil { - private static final int BUF_SIZE = 0x1000; // 4K - - /** - * Copies all bytes from the input stream to the output stream. - * Does not close or flush either stream. - * - * @param from the input stream to read from - * @param to the output stream to write to - * @return the number of bytes copied - * @throws IOException if an I/O error occurs - */ - public static long copy(InputStream from, OutputStream to) - throws IOException { - assert from != null; - assert to != null; - byte[] buf = new byte[BUF_SIZE]; - long total = 0; - while (true) { - int r = from.read(buf); - if (r == -1) { - break; - } - to.write(buf, 0, r); - total += r; - } - return total; - } -} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Version.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Version.java index 6847897ec..e9ae16984 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Version.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Version.java @@ -25,7 +25,7 @@ public class Version { private final static String FSCRAWLER_PROPERTIES = "fscrawler.properties"; - public static final Properties properties; + private static final Properties properties; static { properties = readPropertiesFromClassLoader(FSCRAWLER_PROPERTIES); diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Waiter.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Waiter.java new file mode 100644 index 000000000..38c5f4181 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/Waiter.java @@ -0,0 +1,73 @@ +/* + * Licensed to David Pilato under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework; + +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * Helper to wait for a given resource to be available + * @param Type of object that the supplier must provide (the resource) + */ +public class Waiter { + // After 1s, we stop growing the sleep interval exponentially and just sleep 1s until maxWaitTime + private static final long AWAIT_BUSY_THRESHOLD = 1000L; + + private final long maxTimeInMillis; + + /** + * Wait by default for 10s + */ + public Waiter() { + maxTimeInMillis = 10000L; + } + + /** + * Build a waiter for a given timeout + * @param maxWaitTime Max time + * @param unit Unit for maxWaitTime + */ + public Waiter(long maxWaitTime, TimeUnit unit) { + maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit); + } + + /** + * Wait for a given amount of time for a resource to be available + * @param breakSupplier the resource supplier. While the resource is null, the waiter will continue to wait. + * @return the resource found + * @throws InterruptedException in case the thread was interrupted while waiting + */ + public T awaitBusy(Supplier breakSupplier) throws InterruptedException { + long timeInMillis = 1; + long sum = 0; + while (sum + timeInMillis < maxTimeInMillis) { + T resource = breakSupplier.get(); + if (resource != null) { + return resource; + } + Thread.sleep(timeInMillis); + sum += timeInMillis; + timeInMillis = Math.min(AWAIT_BUSY_THRESHOLD, timeInMillis * 2); + } + timeInMillis = maxTimeInMillis - sum; + Thread.sleep(Math.max(timeInMillis, 0)); + return null; + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/Engine.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/Engine.java new file mode 100644 index 000000000..eb4d3959c --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/Engine.java @@ -0,0 +1,34 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +/** + * Defines an Engine which is responsible to execute a bulk operation + */ +public interface Engine, + REQ extends FsCrawlerBulkRequest, + RES extends FsCrawlerBulkResponse> { + /** + * Actually execute the Bulk Request behind the scene + * @param request The request to execute + * @return The outcome + */ + RES bulk(REQ request); +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerAdvancedBulkProcessorListener.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerAdvancedBulkProcessorListener.java new file mode 100644 index 000000000..eddf889df --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerAdvancedBulkProcessorListener.java @@ -0,0 +1,60 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This Listener exposes the number of successive errors that might have seen previously. So the caller can if needed slow down + * a bit the injection. + * A retry mechanism is implemented in RetryBulkProcessorListener + * @see FsCrawlerRetryBulkProcessorListener + */ +public class FsCrawlerAdvancedBulkProcessorListener< + O extends FsCrawlerOperation, + REQ extends FsCrawlerBulkRequest, + RES extends FsCrawlerBulkResponse + > extends FsCrawlerSimpleBulkProcessorListener { + private static final Logger logger = LogManager.getLogger(FsCrawlerAdvancedBulkProcessorListener.class); + + private final AtomicInteger successiveErrors = new AtomicInteger(0); + + public int getErrors() { + return successiveErrors.get(); + } + + @Override + public void afterBulk(long executionId, REQ request, RES response) { + super.afterBulk(executionId, request, response); + if (response.hasFailures()) { + int previousErrors = successiveErrors.getAndIncrement(); + logger.warn("Throttling is activated. Got [{}] successive errors so far.", previousErrors); + } else { + int previousErrors = successiveErrors.get(); + if (previousErrors > 0) { + successiveErrors.set(0); + logger.debug("We are back to normal behavior after [{}] errors. \\o/", previousErrors); + } + } + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkProcessor.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkProcessor.java new file mode 100644 index 000000000..1f155fa33 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkProcessor.java @@ -0,0 +1,204 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +/** + * Bulk processor + */ +public class FsCrawlerBulkProcessor< + O extends FsCrawlerOperation, + Req extends FsCrawlerBulkRequest, + Res extends FsCrawlerBulkResponse + > implements Closeable { + + private static final Logger logger = LogManager.getLogger(FsCrawlerBulkProcessor.class); + + private final int bulkActions; + private final Listener listener; + private final Engine engine; + private Req bulkRequest; + private final Supplier requestSupplier; + private final ScheduledExecutorService executor; + private volatile boolean closed = false; + private final AtomicLong executionIdGen = new AtomicLong(); + + public FsCrawlerBulkProcessor(Engine engine, + Listener listener, + int bulkActions, + TimeValue flushInterval, + Supplier requestSupplier) { + this.engine = engine; + this.listener = listener; + this.bulkActions = bulkActions; + this.requestSupplier = requestSupplier; + this.bulkRequest = requestSupplier.get(); + this.listener.setBulkProcessor(this); + + if (flushInterval != null) { + executor = Executors.newScheduledThreadPool(1); + executor.scheduleWithFixedDelay(this::executeWhenNeeded, 0, flushInterval.millis(), TimeUnit.MILLISECONDS); + } else { + executor = null; + } + } + + @Override + public void close() throws IOException { + try { + internalClose(); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + + private void internalClose() throws InterruptedException { + if (closed) { + return; + } + closed = true; + + if (executor != null) { + logger.debug("Closing BulkProcessor"); + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + logger.debug("BulkProcessor is now closed"); + } + + if (bulkRequest.numberOfActions() > 0) { + logger.debug("Executing [{}] remaining actions", bulkRequest.numberOfActions()); + execute(); + } + } + + /** + * Add a request to the processor + * @param request request to add + * @return this so we can link methods. + */ + public synchronized FsCrawlerBulkProcessor add(O request) { + ensureOpen(); + bulkRequest.add(request); + executeIfNeeded(); + return this; + } + + private void ensureOpen() { + if (closed) { + throw new IllegalStateException("bulk process already closed"); + } + } + + private void executeIfNeeded() { + ensureOpen(); + if (isOverTheLimit()) { + execute(); + } + } + + private void executeWhenNeeded() { + ensureOpen(); + if (bulkRequest.numberOfActions() > 0) { + execute(); + } + } + + private void execute() { + final Req bulkRequest = this.bulkRequest; + this.bulkRequest = requestSupplier.get(); + final long executionId = executionIdGen.incrementAndGet(); + + // execute in a blocking fashion... + boolean afterCalled = false; + try { + listener.beforeBulk(executionId, bulkRequest); + Res bulkItemResponses = engine.bulk(bulkRequest); + afterCalled = true; + listener.afterBulk(executionId, bulkRequest, bulkItemResponses); + } catch (Exception e) { + if (!afterCalled) { + listener.afterBulk(executionId, bulkRequest, e); + } + } + } + + private boolean isOverTheLimit() { + return (bulkActions != -1) && (bulkRequest.numberOfActions() >= bulkActions); + } + + public Listener getListener() { + return listener; + } + + public static class Builder, + Req extends FsCrawlerBulkRequest, + Res extends FsCrawlerBulkResponse> { + + private int bulkActions; + private TimeValue flushInterval; + private final Engine engine; + private final Listener listener; + private final Supplier requestSupplier; + + public Builder(Engine engine, Listener listener, Supplier requestSupplier) { + this.engine = engine; + this.listener = listener; + this.requestSupplier = requestSupplier; + } + + public Builder setBulkActions(int bulkActions) { + this.bulkActions = bulkActions; + return this; + } + + public Builder setFlushInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + public FsCrawlerBulkProcessor build() { + return new FsCrawlerBulkProcessor<>(engine, listener, bulkActions, flushInterval, requestSupplier); + } + } + + public interface Listener, + Req extends FsCrawlerBulkRequest, + Res extends FsCrawlerBulkResponse> { + + void beforeBulk(long executionId, Req request); + + void afterBulk(long executionId, Req request, Res response); + + void afterBulk(long executionId, Req request, Throwable failure); + + void setBulkProcessor(FsCrawlerBulkProcessor bulkProcessor); + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkRequest.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkRequest.java new file mode 100644 index 000000000..17247e547 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkRequest.java @@ -0,0 +1,40 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import java.util.ArrayList; +import java.util.List; + +public abstract class FsCrawlerBulkRequest> { + + private final List operations = new ArrayList<>(); + + public int numberOfActions() { + return operations.size(); + } + + public void add(T request) { + operations.add(request); + } + + public List getOperations() { + return operations; + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkResponse.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkResponse.java new file mode 100644 index 000000000..d3a030ac3 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerBulkResponse.java @@ -0,0 +1,139 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Arrays; + +@SuppressWarnings("CanBeFinal") +public abstract class FsCrawlerBulkResponse> { + + private static final Logger logger = LogManager.getLogger(FsCrawlerBulkResponse.class); + + private boolean errors; + private BulkItemResponse[] items; + + @SuppressWarnings("ConstantConditions") + public boolean hasFailures() { + if (errors) return errors; + for (BulkItemResponse item : items) { + if (item.failed) { + return true; + } + } + return false; + } + + public BulkItemResponse[] getItems() { + return items; + } + + public boolean isErrors() { + return errors; + } + + public Throwable buildFailureMessage() { + StringBuilder sbf = new StringBuilder(); + int failures = 0; + for (BulkItemResponse item : items) { + if (item.failed) { + if (logger.isTraceEnabled()) { + sbf.append(item.getOperation()); + sbf.append(":").append(item.getOpType()).append(":").append(item.getFailureMessage()); + } + failures++; + } + } + if (logger.isTraceEnabled()) { + sbf.append("\n"); + } + sbf.append(failures).append(" failures"); + return new RuntimeException(sbf.toString()); + } + + @SuppressWarnings("CanBeFinal") + public static class BulkItemResponse> { + private boolean failed; + private O operation; + private String opType; + private String failureMessage; + + // We use Object here as in 1.7 it will be a String and from 2.0 it will be a Map + private Object error; + + public boolean isFailed() { + return failed; + } + + public O getOperation() { + return operation; + } + + public String getOpType() { + return opType; + } + + public String getFailureMessage() { + return failureMessage; + } + + public void setFailed(boolean failed) { + this.failed = failed; + } + + public void setOperation(O operation) { + this.operation = operation; + } + + public void setOpType(String opType) { + this.opType = opType; + } + + public void setFailureMessage(String failureMessage) { + this.failureMessage = failureMessage; + } + + public Object getError() { + return error; + } + + public void setError(Object error) { + this.error = error; + this.failed = true; + this.failureMessage = error.toString(); + } + + @Override + public String toString() { + return "BulkItemResponse{" + "failed=" + failed + + ", operation='" + operation + '\'' + + ", opType=" + opType + + ", failureMessage='" + failureMessage + '\'' + + '}'; + } + } + + @Override + public String toString() { + return "BulkResponse{" + "items=" + (items == null ? "null" : Arrays.asList(items).toString()) + '}'; + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerOperation.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerOperation.java new file mode 100644 index 000000000..d5c43b222 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerOperation.java @@ -0,0 +1,23 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +public interface FsCrawlerOperation> extends Comparable { +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerRetryBulkProcessorListener.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerRetryBulkProcessorListener.java new file mode 100644 index 000000000..980fa3258 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerRetryBulkProcessorListener.java @@ -0,0 +1,63 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This Listener implements a simple and naive retry mechanism. When a document is rejected because of a es_rejected_execution_exception + * the same document is sent again to the bulk processor. + */ +public class FsCrawlerRetryBulkProcessorListener< + O extends FsCrawlerOperation, + REQ extends FsCrawlerBulkRequest, + RES extends FsCrawlerBulkResponse + > extends FsCrawlerAdvancedBulkProcessorListener { + + private static final Logger logger = LogManager.getLogger(FsCrawlerRetryBulkProcessorListener.class); + + @Override + public void afterBulk(long executionId, REQ request, RES response) { + super.afterBulk(executionId, request, response); + if (response.hasFailures()) { + for (RES.BulkItemResponse item : response.getItems()) { + if (item.isFailed() && item.getFailureMessage().contains("es_rejected_execution_exception")) { + logger.debug("We are going to retry document [{}] because of [{}]", + item.getOperation(), item.getFailureMessage()); + // Find request + boolean requestFound = false; + for (O operation : request.getOperations()) { + if (operation.compareTo(item.getOperation()) == 0) { + this.bulkProcessor.add(operation); + requestFound = true; + logger.debug("Document [{}] found. Can be retried.", item.getOperation()); + break; + } + } + if (!requestFound) { + logger.warn("Can not retry document [{}] because we can't find it anymore.", + item.getOperation()); + } + } + } + } + } +} diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerSimpleBulkProcessorListener.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerSimpleBulkProcessorListener.java new file mode 100644 index 000000000..284628863 --- /dev/null +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/bulk/FsCrawlerSimpleBulkProcessorListener.java @@ -0,0 +1,64 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.framework.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FsCrawlerSimpleBulkProcessorListener< + O extends FsCrawlerOperation, + REQ extends FsCrawlerBulkRequest, + RES extends FsCrawlerBulkResponse + > implements FsCrawlerBulkProcessor.Listener { + private static final Logger logger = LogManager.getLogger(FsCrawlerSimpleBulkProcessorListener.class); + + protected FsCrawlerBulkProcessor bulkProcessor; + + @Override + public void beforeBulk(long executionId, REQ request) { + logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions()); + } + + @Override + public void afterBulk(long executionId, REQ request, RES response) { + logger.debug("Executed bulk composed of {} actions", request.numberOfActions()); + if (response.hasFailures()) { + logger.warn("There was failures while executing bulk", response.buildFailureMessage()); + if (logger.isDebugEnabled()) { + for (FsCrawlerBulkResponse.BulkItemResponse item : response.getItems()) { + if (item.isFailed()) { + logger.debug("Error for {} for {} operation: {}", item.getOperation(), + item.getOpType(), item.getFailureMessage()); + } + } + } + } + } + + @Override + public void afterBulk(long executionId, REQ request, Throwable failure) { + logger.warn("Error executing bulk", failure); + } + + @Override + public void setBulkProcessor(FsCrawlerBulkProcessor bulkProcessor) { + this.bulkProcessor = bulkProcessor; + } +} diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueTest.java index ecd5caa64..82125b365 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueTest.java @@ -39,14 +39,14 @@ public void testByteSizeConversion() { // Test 500 random values for (int i = 0; i < 500; i++) { int unitNumber = randomIntBetween(0, 5); - ByteSizeUnit unit = ByteSizeUnit.BYTES; - switch (unitNumber) { - case 1: unit = ByteSizeUnit.KB; break; - case 2: unit = ByteSizeUnit.MB; break; - case 3: unit = ByteSizeUnit.GB; break; - case 4: unit = ByteSizeUnit.TB; break; - case 5: unit = ByteSizeUnit.PB; break; - } + ByteSizeUnit unit = switch (unitNumber) { + case 1 -> ByteSizeUnit.KB; + case 2 -> ByteSizeUnit.MB; + case 3 -> ByteSizeUnit.GB; + case 4 -> ByteSizeUnit.TB; + case 5 -> ByteSizeUnit.PB; + default -> ByteSizeUnit.BYTES; + }; long value = randomLongBetween(1, 999); String randomByteSize = value + unit.getSuffix(); diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtilTest.java deleted file mode 100644 index 2a1e1d66b..000000000 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/StreamsUtilTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package fr.pilato.elasticsearch.crawler.fs.framework; - -import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLengthBetween; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -public class StreamsUtilTest extends AbstractFSCrawlerTestCase { - - @Test - public void copyStream() throws IOException { - String text = randomAsciiLettersOfLengthBetween(10, 1000); - ByteArrayInputStream bis = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - StreamsUtil.copy(bis, bos); - - String copiedText = bos.toString(StandardCharsets.UTF_8.name()); - assertThat(copiedText, is(text)); - } -} diff --git a/integration-tests/it-common/pom.xml b/integration-tests/it-common/pom.xml index 6900726de..61c9585f6 100644 --- a/integration-tests/it-common/pom.xml +++ b/integration-tests/it-common/pom.xml @@ -37,8 +37,8 @@ to use them have to provided them manually in the lib dir. --> - com.levigo.jbig2 - levigo-jbig2-imageio + org.apache.pdfbox + jbig2-imageio com.github.jai-imageio diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractFsCrawlerITCase.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractFsCrawlerITCase.java index 7e22ff7b9..8926a3e73 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractFsCrawlerITCase.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractFsCrawlerITCase.java @@ -39,12 +39,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -public abstract class AbstractFsCrawlerITCase extends AbstractITCase { +public abstract class AbstractFsCrawlerITCase extends AbstractRestITCase { @Before public void cleanExistingIndex() throws IOException { logger.info(" -> Removing existing index [{}*]", getCrawlerName()); - esClient.deleteIndex(getCrawlerName() + "*"); + documentService.getClient().deleteIndex(getCrawlerName() + "*"); } @After @@ -52,15 +52,15 @@ public void shutdownCrawler() throws InterruptedException, IOException { stopCrawler(); } - Fs.Builder startCrawlerDefinition() { + protected Fs.Builder startCrawlerDefinition() { return startCrawlerDefinition(currentTestResourceDir.toString(), TimeValue.timeValueSeconds(5)); } - Fs.Builder startCrawlerDefinition(TimeValue updateRate) { + protected Fs.Builder startCrawlerDefinition(TimeValue updateRate) { return startCrawlerDefinition(currentTestResourceDir.toString(), updateRate); } - Fs.Builder startCrawlerDefinition(String dir) { + protected Fs.Builder startCrawlerDefinition(String dir) { return startCrawlerDefinition(dir, TimeValue.timeValueSeconds(5)); } @@ -72,7 +72,7 @@ private Fs.Builder startCrawlerDefinition(String dir, TimeValue updateRate) { .setUpdateRate(updateRate); } - Elasticsearch endCrawlerDefinition(String indexName) { + protected Elasticsearch endCrawlerDefinition(String indexName) { return endCrawlerDefinition(indexName, indexName + INDEX_SUFFIX_FOLDER); } @@ -80,7 +80,7 @@ private Elasticsearch endCrawlerDefinition(String indexDocName, String indexFold return generateElasticsearchConfig(indexDocName, indexFolderName, 1, null, null); } - void startCrawler() throws Exception { + protected void startCrawler() throws Exception { startCrawler(getCrawlerName()); } @@ -88,19 +88,26 @@ private void startCrawler(final String jobName) throws Exception { startCrawler(jobName, startCrawlerDefinition().build(), endCrawlerDefinition(jobName), null); } - FsCrawlerImpl startCrawler(final String jobName, Fs fs, Elasticsearch elasticsearch, Server server) throws Exception { + protected FsCrawlerImpl startCrawler(final String jobName, Fs fs, Elasticsearch elasticsearch, Server server) throws Exception { return startCrawler(jobName, fs, elasticsearch, server, null, TimeValue.timeValueSeconds(10)); } - FsCrawlerImpl startCrawler(final String jobName, Fs fs, Elasticsearch elasticsearch, Server server, Rest rest, TimeValue duration) + protected FsCrawlerImpl startCrawler(final String jobName, Fs fs, Elasticsearch elasticsearch, Server server, Rest rest, + TimeValue duration) + throws Exception { + startCrawler(jobName, FsSettings.builder(jobName).setElasticsearch(elasticsearch).setFs(fs).setServer(server).setRest(rest).build(), duration); + return crawler; + } + + protected FsCrawlerImpl startCrawler(final String jobName, FsSettings fsSettings, TimeValue duration) throws Exception { logger.info(" --> starting crawler [{}]", jobName); crawler = new FsCrawlerImpl( metadataDir, - FsSettings.builder(jobName).setElasticsearch(elasticsearch).setFs(fs).setServer(server).setRest(rest).build(), + fsSettings, LOOP_INFINITE, - rest != null); + fsSettings.getRest() != null); crawler.start(); // We wait up to X seconds before considering a failing test diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java index a00e797fd..33f911d93 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java @@ -25,30 +25,27 @@ import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; -import fr.pilato.elasticsearch.crawler.fs.rest.RestJsonProvider; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.ServerUrl; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; import fr.pilato.elasticsearch.crawler.fs.test.framework.TestContainerThreadFilter; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; import org.apache.logging.log4j.Level; -import org.glassfish.jersey.jackson.JacksonFeature; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; -import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.hamcrest.Matcher; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; import java.io.File; import java.io.IOException; import java.net.ConnectException; @@ -59,7 +56,6 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -74,6 +70,7 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.fail; import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; /** * Integration tests expect to have an elasticsearch instance running on http://127.0.0.1:9200. @@ -97,10 +94,10 @@ @ThreadLeakLingering(linger = 5000) // 5 sec lingering public abstract class AbstractITCase extends AbstractFSCrawlerTestCase { - static Path metadataDir = null; + protected static Path metadataDir = null; - FsCrawlerImpl crawler = null; - Path currentTestResourceDir; + protected FsCrawlerImpl crawler = null; + protected Path currentTestResourceDir; private static final Path DEFAULT_RESOURCES = Paths.get(getUrl("samples", "common")); private final static String DEFAULT_TEST_CLUSTER_URL = "http://127.0.0.1:9200"; @@ -108,16 +105,14 @@ public abstract class AbstractITCase extends AbstractFSCrawlerTestCase { private final static String DEFAULT_PASSWORD = "changeme"; private final static Integer DEFAULT_TEST_REST_PORT = 8080; - static ElasticsearchClient esClient; + protected static String testClusterUrl; + protected final static String testClusterUser = System.getProperty("tests.cluster.user", DEFAULT_USERNAME); + protected final static String testClusterPass = System.getProperty("tests.cluster.pass", DEFAULT_PASSWORD); + protected final static int testRestPort = Integer.parseInt(System.getProperty("tests.rest.port", DEFAULT_TEST_REST_PORT.toString())); - private static String testClusterUrl; - private final static String testClusterUser = System.getProperty("tests.cluster.user", DEFAULT_USERNAME); - private final static String testClusterPass = System.getProperty("tests.cluster.pass", DEFAULT_PASSWORD); - final static int testRestPort = Integer.parseInt(System.getProperty("tests.rest.port", DEFAULT_TEST_REST_PORT.toString())); - - static Elasticsearch elasticsearchWithSecurity; - static WebTarget target; - static Client client; + protected static Elasticsearch elasticsearchWithSecurity; + protected static FsCrawlerManagementService managementService; + protected static FsCrawlerDocumentService documentService; /** * We suppose that each test has its own set of files. Even if we duplicate them, that will make the code @@ -197,7 +192,7 @@ private static void copyTestDocumentsToTargetDir(Path target, String sourceDirNa URL resource = AbstractFSCrawlerTestCase.class.getResource(marker); switch (resource.getProtocol()) { - case "file": { + case "file" -> { Path finalTarget = target.resolve(sourceDirName); if (Files.notExists(finalTarget)) { staticLogger.debug(" --> Creating test dir named [{}]", finalTarget); @@ -212,9 +207,8 @@ private static void copyTestDocumentsToTargetDir(Path target, String sourceDirNa staticLogger.info("-> Copying test documents from [{}] to [{}]", source, finalTarget); copyDirs(source, finalTarget); - break; } - case "jar": { + case "jar" -> { if (Files.notExists(target)) { staticLogger.debug(" --> Creating test dir named [{}]", target); Files.createDirectory(target); @@ -226,16 +220,13 @@ private static void copyTestDocumentsToTargetDir(Path target, String sourceDirNa staticLogger.info("-> Unzipping test documents from [{}] to [{}]", jarFile, target); unzip(jarFile, target); - break; } - default: - fail("Unknown protocol for IT document sources: " + resource.getProtocol()); - break; + default -> fail("Unknown protocol for IT document sources: " + resource.getProtocol()); } } @BeforeClass - public static void startElasticsearchRestClient() throws IOException { + public static void startDocumentService() throws IOException { String testClusterCloudId = System.getProperty("tests.cluster.cloud_id"); if (testClusterCloudId != null && !testClusterCloudId.isEmpty()) { testClusterUrl = decodeCloudId(testClusterCloudId); @@ -256,41 +247,13 @@ public static void startElasticsearchRestClient() throws IOException { .setPassword(testClusterPass) .build(); FsSettings fsSettings = FsSettings.builder("esClient").setElasticsearch(elasticsearchWithSecurity).build(); - esClient = ElasticsearchClientUtil.getInstance(null, fsSettings); - esClient.start(); - - // We make sure the cluster is running - testClusterRunning(); - } - - @BeforeClass - public static void startRestClient() { - // create the client - client = ClientBuilder.newBuilder() - .register(MultiPartFeature.class) - .register(RestJsonProvider.class) - .register(JacksonFeature.class) - .build(); - - target = client.target("http://127.0.0.1:" + testRestPort + "/fscrawler"); - } - - @AfterClass - public static void stopRestClient() throws IOException { - if (client != null) { - client.close(); - client = null; - } - if (esClient != null) { - esClient.close(); - esClient = null; - } - } + documentService = new FsCrawlerDocumentServiceElasticsearchImpl(metadataDir, fsSettings); + documentService.start(); - private static void testClusterRunning() throws IOException { + // We make sure the cluster is running try { - String version = esClient.getVersion(); + String version = documentService.getClient().getVersion(); staticLogger.info("Starting integration tests against an external cluster running elasticsearch [{}]", version); } catch (ConnectException e) { // If we have an exception here, let's ignore the test @@ -300,19 +263,19 @@ private static void testClusterRunning() throws IOException { } @AfterClass - public static void stopElasticsearchClient() throws IOException { + public static void stopDocumentService() throws IOException { staticLogger.info("Stopping integration tests against an external cluster"); - if (esClient != null) { - esClient.close(); - esClient = null; + if (documentService != null) { + documentService.close(); + documentService = null; staticLogger.info("Elasticsearch client stopped"); } } private static final String testCrawlerPrefix = "fscrawler_"; - static Elasticsearch generateElasticsearchConfig(String indexName, String indexFolderName, int bulkSize, - TimeValue timeValue, ByteSizeValue byteSize) { + protected static Elasticsearch generateElasticsearchConfig(String indexName, String indexFolderName, int bulkSize, + TimeValue timeValue, ByteSizeValue byteSize) { Elasticsearch.Builder builder = Elasticsearch.builder() .addNode(new ServerUrl(testClusterUrl)) .setBulkSize(bulkSize); @@ -337,8 +300,8 @@ static Elasticsearch generateElasticsearchConfig(String indexName, String indexF return builder.build(); } - static void refresh() throws IOException { - esClient.refresh(null); + protected static void refresh() throws IOException { + documentService.getClient().refresh(null); } /** @@ -376,12 +339,13 @@ public static ESSearchResponse countTestHelper(final ESSearchRequest request, fi // Let's search for entries try { - response[0] = esClient.search(request); - } catch (IOException e) { + // Make sure we refresh indexed docs before counting + refresh(); + response[0] = documentService.getClient().search(request); + } catch (RuntimeException|IOException e) { staticLogger.warn("error caught", e); return -1; } - staticLogger.trace("result {}", response[0].toString()); totalHits = response[0].getTotalHits(); staticLogger.debug("got so far [{}] hits on expected [{}]", totalHits, expected); @@ -408,7 +372,7 @@ public static ESSearchResponse countTestHelper(final ESSearchRequest request, fi return response[0]; } - static void logContentOfDir(Path path, Level level) { + protected static void logContentOfDir(Path path, Level level) { if (path != null) { try (Stream stream = Files.walk(path)) { stream.forEach(file -> { @@ -442,7 +406,7 @@ public static String getUrl(String... subdirs) { return dir.getAbsoluteFile().getAbsolutePath(); } - String getCrawlerName() { + protected String getCrawlerName() { String testName = testCrawlerPrefix.concat(getCurrentClassName()).concat("_").concat(getCurrentTestName()); return testName.contains(" ") ? split(testName, " ")[0] : testName; } @@ -464,29 +428,8 @@ public static String[] split(String toSplit, String delimiter) { return new String[]{beforeDelimiter, afterDelimiter}; } - public static T restCall(String path, Class clazz) { - if (staticLogger.isDebugEnabled()) { - String response = target.path(path).request().get(String.class); - staticLogger.debug("Rest response: {}", response); - } - return target.path(path).request().get(clazz); - } - - public static T restCall(String path, FormDataMultiPart mp, Class clazz, Map params) { - return restCall(target, path, mp, clazz, params); - } - - public static T restCall(WebTarget target, String path, FormDataMultiPart mp, Class clazz, Map params) { - WebTarget targetPath = target.path(path); - params.forEach(targetPath::queryParam); - - return targetPath.request(MediaType.MULTIPART_FORM_DATA) - .accept(MediaType.APPLICATION_JSON) - .post(Entity.entity(mp, mp.getMediaType()), clazz); - } - public static void deleteRecursively(Path root) throws IOException { - Files.walkFileTree(root, new SimpleFileVisitor() { + Files.walkFileTree(root, new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); @@ -500,4 +443,5 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx } }); } + } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java index 14356b669..b166eac5b 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java @@ -19,45 +19,59 @@ package fr.pilato.elasticsearch.crawler.fs.test.integration; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientUtil; -import fr.pilato.elasticsearch.crawler.fs.rest.RestServer; -import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator; -import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; -import fr.pilato.elasticsearch.crawler.fs.settings.Rest; -import org.junit.After; -import org.junit.Before; +import fr.pilato.elasticsearch.crawler.fs.rest.RestJsonProvider; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import org.glassfish.jersey.jackson.JacksonFeature; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.junit.AfterClass; +import org.junit.BeforeClass; -import java.io.IOException; +import java.util.Map; public abstract class AbstractRestITCase extends AbstractITCase { - private ElasticsearchClient esTmpClient; + protected static WebTarget target; + protected static Client client; - @Before - public void startRestServer() throws Exception { - FsSettings fsSettings = FsSettings.builder(getCrawlerName()) - .setRest(new Rest("http://127.0.0.1:" + testRestPort + "/fscrawler")) - .setElasticsearch(elasticsearchWithSecurity) - .build(); - fsSettings.getElasticsearch().setIndex(getCrawlerName()); - FsCrawlerValidator.validateSettings(logger, fsSettings, true); - esTmpClient = ElasticsearchClientUtil.getInstance(metadataDir, fsSettings); - esTmpClient.start(); - RestServer.start(fsSettings, esTmpClient); + public static T restCall(String path, Class clazz) { + if (staticLogger.isDebugEnabled()) { + String response = target.path(path).request().get(String.class); + staticLogger.debug("Rest response: {}", response); + } + return target.path(path).request().get(clazz); + } + + public static T restCall(WebTarget target, String path, FormDataMultiPart mp, Class clazz, Map params) { + WebTarget targetPath = target.path(path); + params.forEach(targetPath::queryParam); - logger.info(" -> Removing existing index [{}]", getCrawlerName() + "*"); - esTmpClient.deleteIndex(getCrawlerName() + "*"); + return targetPath.request(MediaType.MULTIPART_FORM_DATA) + .accept(MediaType.APPLICATION_JSON) + .post(Entity.entity(mp, mp.getMediaType()), clazz); + } + + @BeforeClass + public static void startRestClient() { + // create the client + client = ClientBuilder.newBuilder() + .register(MultiPartFeature.class) + .register(RestJsonProvider.class) + .register(JacksonFeature.class) + .build(); - logger.info(" -> Creating index [{}]", fsSettings.getElasticsearch().getIndex()); + target = client.target("http://127.0.0.1:" + testRestPort + "/fscrawler"); } - @After - public void stopRestServer() throws IOException { - RestServer.close(); - if (esTmpClient != null) { - esTmpClient.close(); - esTmpClient = null; + @AfterClass + public static void stopRestClient() { + if (client != null) { + client.close(); + client = null; } } } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/ElasticsearchClientIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/ElasticsearchClientIT.java similarity index 95% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/ElasticsearchClientIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/ElasticsearchClientIT.java index 1ae90b2cf..6626343f0 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/ElasticsearchClientIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/ElasticsearchClientIT.java @@ -17,11 +17,13 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; +import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractITCase; import org.junit.Before; import org.junit.Test; @@ -40,6 +42,8 @@ */ public class ElasticsearchClientIT extends AbstractITCase { + private ElasticsearchClient esClient = documentService.getClient(); + @Before public void cleanExistingIndex() throws IOException { logger.info(" -> Removing existing index [{}*]", getCrawlerName()); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerImplAllDocumentsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java similarity index 89% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerImplAllDocumentsIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java index 7cc929d1d..1a94bf9c6 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerImplAllDocumentsIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; @@ -30,8 +30,11 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractITCase; import org.apache.tika.parser.external.ExternalParser; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -52,7 +55,7 @@ /** * Test all type of documents we have */ -public class FsCrawlerImplAllDocumentsIT extends AbstractITCase { +public class FsCrawlerImplAllDocumentsIT extends AbstractFsCrawlerITCase { private static FsCrawlerImpl crawler = null; @@ -63,11 +66,11 @@ public static void startCrawling() throws Exception { copyResourcesToTestDir(); } - Long numFiles = 0L; + long numFiles; try { Files.walk(testResourceTarget) - .filter(path -> Files.isRegularFile(path)) + .filter(Files::isRegularFile) .forEach(path -> staticLogger.debug(" - [{}]", path)); numFiles = Files.list(testResourceTarget).count(); } catch (NoSuchFileException e) { @@ -76,7 +79,7 @@ public static void startCrawling() throws Exception { } staticLogger.info(" -> Removing existing index [fscrawler_test_all_documents*]"); - esClient.deleteIndex("fscrawler_test_all_documents*"); + documentService.getClient().deleteIndex("fscrawler_test_all_documents*"); staticLogger.info(" --> starting crawler in [{}] which contains [{}] files", testResourceTarget, numFiles); @@ -141,6 +144,15 @@ public void testExtractFromDocx() throws IOException { } } + @Test + public void testExtractFromEml() throws IOException { + ESSearchResponse response = runSearch("test.eml", "test"); + for (ESSearchHit hit : response.getHits()) { + assertThat(extractFromPath(hit.getSourceAsMap(), Doc.FIELD_NAMES.META).get(Meta.FIELD_NAMES.TITLE), is("Test")); + assertThat(extractFromPath(hit.getSourceAsMap(), Doc.FIELD_NAMES.META).get(Meta.FIELD_NAMES.AUTHOR), is("鲨掉 <2428617664@qq.com>")); + } + } + @Test public void testExtractFromHtml() throws IOException { runSearch("test.html", "sample"); @@ -242,7 +254,7 @@ private ESSearchResponse runSearch(String filename, String content) throws IOExc if (content != null) { query.addMust(new ESMatchQuery("content", content)); } - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex("fscrawler_test_all_documents") .withESQuery(query)); assertThat(response.getTotalHits(), is(1L)); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerRestIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java similarity index 82% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerRestIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java index c246e5d11..afd15aebb 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerRestIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; @@ -25,15 +25,26 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; import fr.pilato.elasticsearch.crawler.fs.framework.Version; +import fr.pilato.elasticsearch.crawler.fs.rest.RestServer; import fr.pilato.elasticsearch.crawler.fs.rest.ServerStatusResponse; import fr.pilato.elasticsearch.crawler.fs.rest.UploadResponse; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Rest; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractRestITCase; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; +import org.junit.After; import org.junit.Before; import org.junit.Test; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -52,9 +63,12 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +@SuppressWarnings("ALL") public class FsCrawlerRestIT extends AbstractRestITCase { private Path currentTestTagDir; + private FsCrawlerManagementService managementService; + private FsCrawlerDocumentService documentService; @Before public void copyTags() throws IOException { @@ -76,6 +90,49 @@ public void copyTags() throws IOException { } } + @Before + public void startRestServer() throws Exception { + FsSettings fsSettings = FsSettings.builder(getCrawlerName()) + .setRest(new Rest("http://127.0.0.1:" + testRestPort + "/fscrawler")) + .setElasticsearch(elasticsearchWithSecurity) + .build(); + fsSettings.getElasticsearch().setIndex(getCrawlerName()); + FsCrawlerValidator.validateSettings(logger, fsSettings, true); + + this.managementService = new FsCrawlerManagementServiceElasticsearchImpl(metadataDir, fsSettings); + + if (fsSettings.getWorkplaceSearch() == null) { + // The documentService is using the esSearch instance + this.documentService = new FsCrawlerDocumentServiceElasticsearchImpl(metadataDir, fsSettings); + } else { + // The documentService is using the wpSearch instance + this.documentService = new FsCrawlerDocumentServiceWorkplaceSearchImpl(metadataDir, fsSettings); + } + + managementService.start(); + documentService.start(); + + RestServer.start(fsSettings, managementService, documentService); + + logger.info(" -> Removing existing index [{}]", getCrawlerName() + "*"); + managementService.getClient().deleteIndex(getCrawlerName() + "*"); + + logger.info(" -> Creating index [{}]", fsSettings.getElasticsearch().getIndex()); + } + + @After + public void stopRestServer() throws IOException { + RestServer.close(); + if (managementService != null) { + managementService.close(); + managementService = null; + } + if (documentService != null) { + documentService.close(); + documentService = null; + } + } + @Test public void testCallRoot() { ServerStatusResponse status = restCall("/", ServerStatusResponse.class); @@ -91,7 +148,7 @@ public void testAllDocumentsWithRest() throws Exception { throw new RuntimeException(from + " doesn't seem to exist. Check your JUnit tests."); } Files.walk(from) - .filter(path -> Files.isRegularFile(path)) + .filter(Files::isRegularFile) .forEach(path -> { UploadResponse response = uploadFile(target, path); assertThat(response.getFilename(), is(path.getFileName().toString())); @@ -115,7 +172,7 @@ public void testAllDocumentsWithRestExternalIndex() throws Exception { } String index = "fscrawler_fs_custom"; Files.walk(from) - .filter(path -> Files.isRegularFile(path)) + .filter(Files::isRegularFile) .forEach(path -> { UploadResponse response = uploadFileOnIndex(target, path, index); assertThat(response.getFilename(), is(path.getFileName().toString())); @@ -136,7 +193,7 @@ public void testDocumentWithExternalTags() throws Exception { // which can overwrite the data we extracted AtomicInteger numFiles = new AtomicInteger(); Files.walk(currentTestResourceDir) - .filter(path -> Files.isRegularFile(path)) + .filter(Files::isRegularFile) .forEach(path -> { Path tagsFilePath = currentTestTagDir.resolve(path.getFileName().toString() + ".json"); logger.debug("Upload file #[{}]: [{}] with tags [{}]", numFiles.incrementAndGet(), path.getFileName(), tagsFilePath.getFileName()); @@ -255,7 +312,7 @@ public void testDocumentWithExternalTags() throws Exception { } private void checkDocument(String filename, HitChecker checker) throws IOException { - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESTermQuery("file.filename", filename))); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestAttributesIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestAttributesIT.java similarity index 95% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestAttributesIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestAttributesIT.java index 8c50f4862..b3f4556fb 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestAttributesIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestAttributesIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Attributes; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; @@ -26,6 +26,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil.extractFromPath; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestChecksumIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestChecksumIT.java similarity index 95% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestChecksumIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestChecksumIT.java index 1e9a7a410..a66f240c1 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestChecksumIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestChecksumIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.beans.File; @@ -25,6 +25,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.security.MessageDigest; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDatesIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDatesIT.java similarity index 97% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDatesIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDatesIT.java index 456a5ac5a..4952728da 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDatesIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDatesIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.beans.File; @@ -25,6 +25,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.nio.file.Files; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDefaultsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDefaultsIT.java similarity index 95% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDefaultsIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDefaultsIT.java index c5a1ae21a..65a7164da 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestDefaultsIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestDefaultsIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.beans.File; @@ -27,6 +27,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil.extractFromPath; @@ -98,11 +99,10 @@ public void test_highlight_documents() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); // Let's test highlighting - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESMatchQuery("content", "exemplo")) .addHighlighter("content")); - staticLogger.trace("result {}", response.toString()); assertThat(response.getTotalHits(), is(1L)); ESSearchHit hit = response.getHits().get(0); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilenameAsIdIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilenameAsIdIT.java similarity index 87% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilenameAsIdIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilenameAsIdIT.java index 483c7d8cb..6825fc3fe 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilenameAsIdIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilenameAsIdIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.io.IOException; @@ -46,7 +47,7 @@ public void test_filename_as_id() throws Exception { assertThat("Document should exists with [roottxtfile.txt] id...", awaitBusy(() -> { try { - return esClient.exists(getCrawlerName(), "roottxtfile.txt"); + return documentService.getClient().exists(getCrawlerName(), "roottxtfile.txt"); } catch (IOException e) { return false; } @@ -69,14 +70,14 @@ public void test_remove_deleted_with_filename_as_id() throws Exception { assertThat("Document should exists with [id1.txt] id...", awaitBusy(() -> { try { - return esClient.exists(getCrawlerName(), "id1.txt"); + return documentService.getClient().exists(getCrawlerName(), "id1.txt"); } catch (IOException e) { return false; } }), equalTo(true)); assertThat("Document should exists with [id2.txt] id...", awaitBusy(() -> { try { - return esClient.exists(getCrawlerName(), "id2.txt"); + return documentService.getClient().exists(getCrawlerName(), "id2.txt"); } catch (IOException e) { return false; } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilesizeIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilesizeIT.java similarity index 93% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilesizeIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilesizeIT.java index c40b4976b..ed02651f7 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFilesizeIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFilesizeIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.beans.File; @@ -27,6 +27,7 @@ import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.Percentage; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.nio.file.Files; @@ -109,7 +110,7 @@ public void test_filesize() throws Exception { ESSearchResponse searchResponse = countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); for (ESSearchHit hit : searchResponse.getHits()) { - Map file = (Map) hit.getSourceAsMap().get(Doc.FIELD_NAMES.FILE); + @SuppressWarnings("unchecked") Map file = (Map) hit.getSourceAsMap().get(Doc.FIELD_NAMES.FILE); assertThat(file, notNullValue()); assertThat(file.get(File.FIELD_NAMES.FILESIZE), is(12230)); } @@ -124,7 +125,7 @@ public void test_filesize_disabled() throws Exception { ESSearchResponse searchResponse = countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); for (ESSearchHit hit : searchResponse.getHits()) { - Map file = (Map) hit.getSourceAsMap().get(Doc.FIELD_NAMES.FILE); + @SuppressWarnings("unchecked") Map file = (Map) hit.getSourceAsMap().get(Doc.FIELD_NAMES.FILE); assertThat(file, notNullValue()); assertThat(file.get(File.FIELD_NAMES.FILESIZE), nullValue()); } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFiltersIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFiltersIT.java similarity index 93% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFiltersIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFiltersIT.java index d5f41ecc6..d86fb262f 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFiltersIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFiltersIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFollowSymlinksIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFollowSymlinksIT.java similarity index 93% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFollowSymlinksIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFollowSymlinksIT.java index aa2574f72..84abeac30 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFollowSymlinksIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFollowSymlinksIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.nio.file.Files; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIgnoreFoldersIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIgnoreFoldersIT.java similarity index 86% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIgnoreFoldersIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIgnoreFoldersIT.java index acf1c3e1c..7a697be47 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIgnoreFoldersIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIgnoreFoldersIT.java @@ -17,11 +17,12 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; @@ -47,8 +48,7 @@ public void test_ignore_folders() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 2L, null); // We expect having no folders - ESSearchResponse response = esClient.search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER)); - staticLogger.trace("result {}", response.toString()); + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER)); assertThat(response.getTotalHits(), is(0L)); } } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIncludesIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIncludesIT.java similarity index 94% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIncludesIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIncludesIT.java index c1adf4e45..c411f9418 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIncludesIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIncludesIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIndexContentIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIndexContentIT.java similarity index 92% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIndexContentIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIndexContentIT.java index 8678dec87..1c67a643e 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIndexContentIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIndexContentIT.java @@ -17,12 +17,13 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESPrefixQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIngestPipelineIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIngestPipelineIT.java similarity index 86% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIngestPipelineIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIngestPipelineIT.java index a2a5df214..93dfa1ce5 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestIngestPipelineIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestIngestPipelineIT.java @@ -17,13 +17,14 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESMatchQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; @@ -46,7 +47,7 @@ public void test_ingest_pipeline() throws Exception { String crawlerName = getCrawlerName(); // We can only run this test against a 5.0 cluster or > - assumeThat("We skip the test as we are not running it with a 5.0 cluster or >", esClient.isIngestSupported(), is(true)); + assumeThat("We skip the test as we are not running it with a 5.0 cluster or >", documentService.getClient().isIngestSupported(), is(true)); // Create an empty ingest pipeline String pipeline = "{\n" + @@ -60,7 +61,7 @@ public void test_ingest_pipeline() throws Exception { " }\n" + " ]\n" + "}"; - esClient.performLowLevelRequest("PUT", "/_ingest/pipeline/" + crawlerName, pipeline); + documentService.getClient().performLowLevelRequest("PUT", "/_ingest/pipeline/" + crawlerName, pipeline); Elasticsearch elasticsearch = endCrawlerDefinition(crawlerName); elasticsearch.setPipeline(crawlerName); @@ -73,7 +74,7 @@ public void test_ingest_pipeline() throws Exception { .withESQuery(new ESMatchQuery("my_content_field", "perniciosoque")), 1L, currentTestResourceDir); // We expect to have one folder - ESSearchResponse response = esClient.search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER)); + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER)); assertThat(response.getTotalHits(), is(1L)); } @@ -85,7 +86,7 @@ public void test_ingest_pipeline_392() throws Exception { String crawlerName = getCrawlerName(); // We can only run this test against a 5.0 cluster or > - assumeThat("We skip the test as we are not running it with a 5.0 cluster or >", esClient.isIngestSupported(), is(true)); + assumeThat("We skip the test as we are not running it with a 5.0 cluster or >", documentService.getClient().isIngestSupported(), is(true)); // Create an empty ingest pipeline String pipeline = "{\n" + @@ -108,7 +109,7 @@ public void test_ingest_pipeline_392() throws Exception { " }\n" + " ]\n" + "}"; - esClient.performLowLevelRequest("PUT", "/_ingest/pipeline/" + crawlerName, pipeline); + documentService.getClient().performLowLevelRequest("PUT", "/_ingest/pipeline/" + crawlerName, pipeline); Elasticsearch elasticsearch = endCrawlerDefinition(crawlerName); elasticsearch.setPipeline(crawlerName); @@ -117,7 +118,7 @@ public void test_ingest_pipeline_392() throws Exception { // We expect to have one file countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()) - .withESQuery(new ESTermQuery("ip_addr", "10.21.23.123")), 1L, currentTestResourceDir); + .withESQuery(new ESTermQuery("ip_addr", "127.0.0.1")), 1L, currentTestResourceDir); } /** @@ -129,7 +130,7 @@ public void test_ingest_missing_pipeline_490() throws Exception { // We can only run this test against a 5.0 cluster or > assumeThat("We skip the test as we are not running it with a 5.0 cluster or >", - esClient.isIngestSupported(), is(true)); + documentService.getClient().isIngestSupported(), is(true)); Elasticsearch elasticsearch = endCrawlerDefinition(crawlerName); elasticsearch.setPipeline(crawlerName); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestJsonSupportIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestJsonSupportIT.java similarity index 88% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestJsonSupportIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestJsonSupportIT.java index fdc515dca..7d7b32d10 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestJsonSupportIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestJsonSupportIT.java @@ -17,12 +17,13 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESMatchQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.io.IOException; @@ -47,7 +48,7 @@ public void test_json_support() throws Exception { assertThat("We should have 2 doc for tweet in text field...", awaitBusy(() -> { try { - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESMatchQuery("text", "tweet"))); return response.getTotalHits() == 2; @@ -70,7 +71,7 @@ public void test_json_disabled() throws Exception { assertThat("We should have 0 doc for tweet in text field...", awaitBusy(() -> { try { - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESMatchQuery("text", "tweet"))); return response.getTotalHits() == 0; @@ -82,7 +83,7 @@ public void test_json_disabled() throws Exception { assertThat("We should have 2 docs for tweet in content field...", awaitBusy(() -> { try { - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESMatchQuery("content", "tweet"))); return response.getTotalHits() == 2; @@ -106,7 +107,7 @@ public void test_add_as_inner_object() throws Exception { assertThat("We should have 2 doc for tweet in object.text field...", awaitBusy(() -> { try { - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withESQuery(new ESMatchQuery("object.text", "tweet"))); return response.getTotalHits() == 2; @@ -129,7 +130,7 @@ public void test_json_support_and_other_files() throws Exception { assertThat("We should have 2 docs only...", awaitBusy(() -> { try { - ESSearchResponse response = esClient.search(new ESSearchRequest().withIndex(getCrawlerName())); + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName())); return response.getTotalHits() == 2; } catch (IOException e) { logger.warn("Caught exception while running the test", e); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestLoopsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestLoopsIT.java similarity index 94% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestLoopsIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestLoopsIT.java index 039efe7ee..b6c7966b4 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestLoopsIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestLoopsIT.java @@ -17,12 +17,13 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestMultipleCrawlersIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestMultipleCrawlersIT.java similarity index 92% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestMultipleCrawlersIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestMultipleCrawlersIT.java index 57c868b93..d2ac593fd 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestMultipleCrawlersIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestMultipleCrawlersIT.java @@ -17,11 +17,12 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRawIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRawIT.java similarity index 91% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRawIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRawIT.java index 45ba2fc24..630eb0769 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRawIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRawIT.java @@ -17,13 +17,14 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static com.carrotsearch.randomizedtesting.RandomizedTest.rarely; @@ -67,9 +68,9 @@ public void test_mapping() throws Exception { // This will cause an Elasticsearch Exception as the String is not a Date // If the mapping is incorrect - esClient.index(getCrawlerName(), "1", json1, null); - esClient.index(getCrawlerName(), "2", json2, null); - esClient.flush(); + documentService.getClient().indexRawJson(getCrawlerName(), "1", json1, null); + documentService.getClient().indexRawJson(getCrawlerName(), "2", json2, null); + documentService.getClient().flush(); } @Test diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRemoveDeletedIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRemoveDeletedIT.java similarity index 96% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRemoveDeletedIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRemoveDeletedIT.java index a51f76dd1..f493139c5 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRemoveDeletedIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRemoveDeletedIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import com.google.common.base.Charsets; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; @@ -25,6 +25,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.apache.logging.log4j.Level; import org.junit.Test; @@ -175,7 +176,7 @@ public void test_moving_files() throws Exception { } Path file = Files.createFile(tmpDir.resolve(filename)); - Files.write(file, "Hello world".getBytes(Charsets.UTF_8)); + Files.writeString(file, "Hello world", Charsets.UTF_8); // We should have 1 doc first countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); @@ -232,7 +233,7 @@ private void checkDocVersions(ESSearchResponse response, long maxVersion) { for (ESSearchHit hit : response.getHits()) { // Read the document. This is needed since 5.0 as search does not return the _version field try { - ESSearchHit getHit = esClient.get(hit.getIndex(), hit.getId()); + ESSearchHit getHit = documentService.getClient().get(hit.getIndex(), hit.getId()); assertThat(getHit.getVersion(), lessThanOrEqualTo(maxVersion)); } catch (IOException e) { fail("We got an IOException: " + e.getMessage()); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRestOnlyIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRestOnlyIT.java similarity index 89% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRestOnlyIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRestOnlyIT.java index 331100f89..540e28d90 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestRestOnlyIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestRestOnlyIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; @@ -26,13 +26,14 @@ import fr.pilato.elasticsearch.crawler.fs.rest.UploadResponse; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Rest; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; +import jakarta.ws.rs.client.WebTarget; import org.junit.Test; -import javax.ws.rs.client.WebTarget; import java.nio.file.Files; import java.nio.file.Path; -import static fr.pilato.elasticsearch.crawler.fs.test.integration.FsCrawlerRestIT.uploadFile; +import static fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch.FsCrawlerRestIT.uploadFile; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -61,7 +62,7 @@ public void test_with_rest_only() throws Exception { 0, true); crawler.start(); - RestServer.start(fsSettings, crawler.getEsClient()); + RestServer.start(fsSettings, crawler.getManagementService(), crawler.getDocumentService()); Path from = rootTmpDir.resolve("resources").resolve("documents"); if (Files.notExists(from)) { @@ -71,7 +72,7 @@ public void test_with_rest_only() throws Exception { WebTarget target = client.target("http://127.0.0.1:" + (testRestPort+1) + "/fscrawler"); Files.walk(from) - .filter(path -> Files.isRegularFile(path)) + .filter(Files::isRegularFile) .forEach(path -> { UploadResponse response = uploadFile(target, path); assertThat(response.getFilename(), is(path.getFileName().toString())); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSettingsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSettingsIT.java similarity index 93% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSettingsIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSettingsIT.java index 2e4eec1ba..d74ff625f 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSettingsIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSettingsIT.java @@ -17,12 +17,13 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSshIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java similarity index 94% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSshIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java index 9955bf3e2..5aba9e21c 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSshIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java @@ -17,11 +17,12 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Ignore; import org.junit.Test; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestStoreSourceIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestStoreSourceIT.java similarity index 91% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestStoreSourceIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestStoreSourceIT.java index a8d4637d2..0a53712f4 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestStoreSourceIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestStoreSourceIT.java @@ -17,13 +17,14 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; @@ -55,7 +56,7 @@ public void test_do_not_store_source() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); - ESSearchResponse searchResponse = esClient.search(new ESSearchRequest().withIndex(getCrawlerName())); + ESSearchResponse searchResponse = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName())); for (ESSearchHit hit : searchResponse.getHits()) { // We check that the field is not part of _source assertThat(hit.getSourceAsMap().get(Doc.FIELD_NAMES.ATTACHMENT), nullValue()); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSubDirsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSubDirsIT.java similarity index 91% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSubDirsIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSubDirsIT.java index cc8aa763c..5e0ab18f9 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestSubDirsIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSubDirsIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; @@ -25,6 +25,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESTermsAggregation; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.hamcrest.Matcher; import org.junit.Test; @@ -69,6 +70,7 @@ public void test_subdirs() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()).withESQuery(new ESTermQuery("path.real.fulltext", "subdir")), 1L, null); } + @SuppressWarnings("unchecked") @Test public void test_subdirs_deep_tree() throws Exception { startCrawler(); @@ -77,7 +79,7 @@ public void test_subdirs_deep_tree() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 7L, null); // Run aggs - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withSize(0) .withAggregation(new ESTermsAggregation("folders", "path.virtual.tree"))); @@ -91,7 +93,7 @@ public void test_subdirs_deep_tree() throws Exception { assertThat(buckets, iterableWithSize(10)); // Check files - response = esClient.search(new ESSearchRequest().withIndex(getCrawlerName()).withSort("path.virtual")); + response = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName()).withSort("path.virtual")); assertThat(response.getTotalHits(), is(7L)); int i = 0; @@ -105,7 +107,7 @@ public void test_subdirs_deep_tree() throws Exception { // Check folders - response = esClient.search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER).withSort("virtual")); + response = documentService.getClient().search(new ESSearchRequest().withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER).withSort("virtual")); assertThat(response.getTotalHits(), is(7L)); i = 0; @@ -143,7 +145,7 @@ public void test_subdirs_very_deep_tree() throws Exception { countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), subdirs+1, null); // Run aggs - ESSearchResponse response = esClient.search(new ESSearchRequest() + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withSize(0) .withAggregation(new ESTermsAggregation("folders", "path.virtual.tree"))); @@ -157,18 +159,19 @@ public void test_subdirs_very_deep_tree() throws Exception { assertThat(buckets, iterableWithSize(10)); // Check files - response = esClient.search(new ESSearchRequest() + response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName()) .withSize(1000) .withSort("path.virtual")); assertThat(response.getTotalHits(), is(subdirs+1)); for (int i = 0; i < subdirs; i++) { + //noinspection unchecked pathHitTester(response, i, hit -> (Map) hit.getSourceAsMap().get("path"), "sample.txt", endsWith("/" + "sample.txt")); } // Check folders - response = esClient.search(new ESSearchRequest() + response = documentService.getClient().search(new ESSearchRequest() .withIndex(getCrawlerName() + INDEX_SUFFIX_FOLDER) .withSize(1000) .withSort("virtual")); diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestUnparsableIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestUnparsableIT.java similarity index 94% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestUnparsableIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestUnparsableIT.java index 9246e28f0..99e004512 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestUnparsableIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestUnparsableIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; import java.nio.file.FileSystems; diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestXmlSupportIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestXmlSupportIT.java similarity index 93% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestXmlSupportIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestXmlSupportIT.java index acc7a3c14..613dc8cd0 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestXmlSupportIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestXmlSupportIT.java @@ -17,7 +17,7 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESMatchQuery; import fr.pilato.elasticsearch.crawler.fs.client.ESRangeQuery; @@ -25,6 +25,7 @@ import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestZipFilesIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestZipFilesIT.java similarity index 91% rename from integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestZipFilesIT.java rename to integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestZipFilesIT.java index 4b2061e51..f0e41f612 100644 --- a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestZipFilesIT.java +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestZipFilesIT.java @@ -17,10 +17,11 @@ * under the License. */ -package fr.pilato.elasticsearch.crawler.fs.test.integration; +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; import org.junit.Test; /** diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/AbstractWorkplaceSearchITCase.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/AbstractWorkplaceSearchITCase.java new file mode 100644 index 000000000..de6524547 --- /dev/null +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/AbstractWorkplaceSearchITCase.java @@ -0,0 +1,80 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.test.integration.workplacesearch; + +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; +import fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch.WPSearchAdminClient; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.Assume.assumeNoException; + +public class AbstractWorkplaceSearchITCase extends AbstractFsCrawlerITCase { + + private final static String DEFAULT_TEST_WPSEARCH_URL = "http://127.0.0.1:3002"; + protected static String testWorkplaceUrl = System.getProperty("tests.workplace.url", DEFAULT_TEST_WPSEARCH_URL); + protected static String testWorkplaceAccessToken; + protected static String testWorkplaceKey; + + protected static WPSearchAdminClient adminClient; + private static String customSourceId; + + @BeforeClass + public static void createCustomSource() throws Exception { + adminClient = new WPSearchAdminClient() + .withHost(testWorkplaceUrl) + .withPassword(testClusterPass); + try { + adminClient.start(); + Map customSource = adminClient.createCustomSource("fscrawler_integration_tests"); + customSourceId = (String) customSource.get("id"); + testWorkplaceAccessToken = (String) customSource.get("accessToken"); + testWorkplaceKey = (String) customSource.get("key"); + + // Because we don't have an admin client yet for Workplace Search, we will be getting manually + // the custom source information from the command line as options. + testWorkplaceAccessToken = System.getProperty("tests.workplace.access_token"); + testWorkplaceKey = System.getProperty("tests.workplace.key"); + + assertThat(customSourceId, notNullValue()); + assertThat(testWorkplaceAccessToken, not(isEmptyOrNullString())); + assertThat(testWorkplaceKey, not(isEmptyOrNullString())); + } catch (AssertionError e) { + assumeNoException("We are skipping the test as we were not able to create a Workplace Search client", e); + } + } + + @AfterClass + public static void removeCustomSource() { + if (adminClient != null) { + try { + adminClient.removeCustomSource(customSourceId); + } catch (Exception e) { + staticLogger.warn("We have not been able to remove {}: {}", customSourceId, e.getMessage()); + } + adminClient.close(); + adminClient = null; + } + } +} diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchAllDocumentsIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchAllDocumentsIT.java new file mode 100644 index 000000000..083105804 --- /dev/null +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchAllDocumentsIT.java @@ -0,0 +1,294 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.test.integration.workplacesearch; + +import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; +import fr.pilato.elasticsearch.crawler.fs.client.ESBoolQuery; +import fr.pilato.elasticsearch.crawler.fs.client.ESMatchQuery; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchHit; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; +import fr.pilato.elasticsearch.crawler.fs.client.ESTermQuery; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; +import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; +import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.ServerUrl; +import fr.pilato.elasticsearch.crawler.fs.settings.WorkplaceSearch; +import org.apache.tika.parser.external.ExternalParser; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Map; + +import static fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl.LOOP_INFINITE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assume.assumeTrue; + +/** + * Test all type of documents we have with workplace search + */ +public class FsCrawlerTestWorkplaceSearchAllDocumentsIT extends AbstractWorkplaceSearchITCase { + + private static FsCrawlerImpl crawler = null; + private static FsCrawlerDocumentService oldDocumentService; + + @BeforeClass + public static void startCrawling() throws Exception { + Path testResourceTarget = rootTmpDir.resolve("resources").resolve("documents"); + if (Files.notExists(testResourceTarget)) { + copyResourcesToTestDir(); + } + + Long numFiles = 0L; + + try { + Files.walk(testResourceTarget) + .filter(path -> Files.isRegularFile(path)) + .forEach(path -> staticLogger.debug(" - [{}]", path)); + numFiles = Files.list(testResourceTarget).count(); + } catch (NoSuchFileException e) { + staticLogger.error("directory [{}] should exist before we can start tests.", testResourceTarget); + throw new RuntimeException(testResourceTarget + " doesn't seem to exist. Check your JUnit tests."); + } + + oldDocumentService = documentService; + + FsSettings fsSettings = FsSettings.builder("fscrawler_workplacesearch_test_all_documents") + .setElasticsearch(generateElasticsearchConfig("fscrawler_workplacesearch_test_all_documents", + "fscrawler_workplacesearch_test_all_documents_folder", + 5, TimeValue.timeValueSeconds(1), null)) + .setFs(Fs.builder() + .setUrl(testResourceTarget.toString()) + .setLangDetect(true) + .build()) + .setWorkplaceSearch(WorkplaceSearch.builder() + .setServer(new ServerUrl(testWorkplaceUrl)) + .setAccessToken(testWorkplaceAccessToken) + .setKey(testWorkplaceKey) + .setBulkSize(5) + .setFlushInterval(TimeValue.timeValueSeconds(1)) + .build()) + .build(); + + try { + documentService = new FsCrawlerDocumentServiceWorkplaceSearchImpl(metadataDir, fsSettings); + documentService.start(); + + staticLogger.info(" -> Removing existing index [.ent-search-engine-*]"); + documentService.getClient().deleteIndex(".ent-search-engine-*"); + + staticLogger.info(" --> starting crawler in [{}] which contains [{}] files", testResourceTarget, numFiles); + + crawler = new FsCrawlerImpl(metadataDir, fsSettings, LOOP_INFINITE, false); + crawler.start(); + + // We wait until we have all docs + countTestHelper(new ESSearchRequest().withIndex(".ent-search-engine-*"), numFiles, null, TimeValue.timeValueMinutes(5)); + } catch (FsCrawlerIllegalConfigurationException e) { + documentService = oldDocumentService; + Assume.assumeNoException("We don't have a compatible client for this version of the stack.", e); + } + } + + @AfterClass + public static void stopCrawling() throws Exception { + if (crawler != null) { + staticLogger.info(" --> Stopping crawler"); + crawler.close(); + crawler = null; + } + if (oldDocumentService != documentService) { + documentService.close(); + documentService = oldDocumentService; + } + } + + /** + * Test case for https://github.com/dadoonet/fscrawler/issues/163 + */ + @Test + public void testXmlIssue163() throws IOException { + runSearch("issue-163.xml"); + } + + @Test + public void testJson() throws IOException { + runSearch("test.json", "json"); + } + + @Test + public void testExtractFromDoc() throws IOException { + runSearch("test.doc", "sample"); + } + + @Test + public void testExtractFromDocx() throws IOException { + ESSearchResponse response = runSearch("test.docx", "sample"); + for (ESSearchHit hit : response.getHits()) { + Map source = hit.getSourceAsMap(); + assertThat(source, hasEntry(is("name$string"), notNullValue())); + assertThat(source, hasEntry(is("mime_type$string"), notNullValue())); + assertThat(source, hasEntry(is("url$string"), notNullValue())); + assertThat(source, hasKey(startsWith("size"))); + assertThat(source, hasEntry(is("last_modified$string"), notNullValue())); + assertThat(source, hasEntry(is("path$string"), notNullValue())); + assertThat(source, hasEntry(is("created_at$string"), notNullValue())); + assertThat(source, hasEntry(is("title$string"), notNullValue())); + assertThat(source, hasEntry(is("keywords$string"), notNullValue())); + assertThat(source, hasEntry(is("body$string"), notNullValue())); + } + } + + @Test + public void testExtractFromEml() throws IOException { + ESSearchResponse response = runSearch("test.eml", "test"); + for (ESSearchHit hit : response.getHits()) { + Map source = hit.getSourceAsMap(); + assertThat(source, hasEntry(is("title$string"), is("Test"))); + assertThat(source, hasEntry(is("author$string"), is("鲨掉 <2428617664@qq.com>"))); + } + } + + @Test + public void testExtractFromHtml() throws IOException { + runSearch("test.html", "sample"); + } + + @Test + public void testExtractFromMp3() throws IOException { + runSearch("test.mp3", "tika"); + } + + @Test + public void testExtractFromOdt() throws IOException { + runSearch("test.odt", "sample"); + } + + @Test + public void testExtractFromPdf() throws IOException { + runSearch("test.pdf", "sample"); + } + + @Test + public void testExtractFromRtf() throws IOException { + runSearch("test.rtf", "sample"); + } + + @Test + public void testExtractFromTxt() throws IOException { + runSearch("test.txt", "contains"); + } + + @Test + public void testExtractFromWav() throws IOException { + runSearch("test.wav"); + } + + /** + * Test case for https://github.com/dadoonet/fscrawler/issues/229 + */ + @Test + public void testProtectedDocument229() throws IOException { + runSearch("test-protected.docx"); + } + + /** + * Test case for https://github.com/dadoonet/fscrawler/issues/221 + */ + @Test + public void testProtectedDocument221() throws IOException { + runSearch("issue-221-doc1.pdf", "Formations"); + runSearch("issue-221-doc2.pdf", "FORMATIONS"); + } + + @Test + public void testLanguageDetection() throws IOException { + // TODO fix this hack. We can't make sure we are returning one single file + ESSearchResponse response = runSearch("test-fr.txt", "fichier"); + for (ESSearchHit hit : response.getHits()) { + if (hit.getSourceAsMap().get("name$string").equals("test-fr.txt")) { + assertThat(hit.getSourceAsMap(), hasEntry("language$string", "fr")); + } + } + response = runSearch("test-de.txt", "Datei"); + for (ESSearchHit hit : response.getHits()) { + if (hit.getSourceAsMap().get("name$string").equals("test-de.txt")) { + assertThat(hit.getSourceAsMap(), hasEntry("language$string", "de")); + } + } + response = runSearch("test.txt", "contains"); + for (ESSearchHit hit : response.getHits()) { + if (hit.getSourceAsMap().get("name$string").equals("test.txt")) { + assertThat(hit.getSourceAsMap(), hasEntry("language$string", "en")); + } + } + } + + @Test + public void testChineseContent369() throws IOException { + runSearch("issue-369.txt", "今天天气晴好"); + } + + @Test + public void testOcr() throws IOException { + assumeTrue("Tesseract is not installed so we are skipping this test", ExternalParser.check("tesseract")); + runSearch("test-ocr.png", "words"); + runSearch("test-ocr.pdf", "words"); + } + + @Test + public void testShiftJisEncoding() throws IOException { + runSearch("issue-400-shiftjis.txt", "elasticsearch"); + } + + @Test + public void testNonUtf8Filename418() throws IOException { + runSearch("issue-418-中文名称.txt"); + } + + private ESSearchResponse runSearch(String filename) throws IOException { + return runSearch(filename, null); + } + + private ESSearchResponse runSearch(String filename, String content) throws IOException { + logger.info(" -> Testing if file [{}] has been indexed correctly{}.", filename, + content == null ? "" : " and contains [" + content + "]"); + ESBoolQuery query = new ESBoolQuery().addMust(new ESTermQuery("name$string.enum", filename)); + if (content != null) { + query.addMust(new ESMatchQuery("body$string", content)); + } + ESSearchResponse response = documentService.getClient().search(new ESSearchRequest() + .withIndex(".ent-search-engine-*") + .withESQuery(query)); + assertThat(response.getTotalHits(), is(1L)); + return response; + } +} diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchIT.java new file mode 100644 index 000000000..4f81ebd1c --- /dev/null +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/FsCrawlerTestWorkplaceSearchIT.java @@ -0,0 +1,108 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.test.integration.workplacesearch; + +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchResponse; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; +import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; +import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; +import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.ServerUrl; +import fr.pilato.elasticsearch.crawler.fs.settings.WorkplaceSearch; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * Test workplace search + */ +public class FsCrawlerTestWorkplaceSearchIT extends AbstractWorkplaceSearchITCase { + + private FsCrawlerDocumentService oldDocumentService; + private FsSettings fsSettings; + + @Before + public void overrideDocumentService() throws IOException { + oldDocumentService = documentService; + Fs fs = startCrawlerDefinition().build(); + fsSettings = FsSettings.builder(getCrawlerName()) + .setFs(fs) + .setElasticsearch(Elasticsearch.builder() + .addNode(new ServerUrl(testClusterUrl)) + .setUsername(testClusterUser) + .setPassword(testClusterPass) + .build()) + .setWorkplaceSearch(WorkplaceSearch.builder() + .setServer(new ServerUrl(testWorkplaceUrl)) + .setAccessToken(testWorkplaceAccessToken) + .setKey(testWorkplaceKey) + .setBulkSize(1) + .setFlushInterval(TimeValue.timeValueSeconds(1)) + .build()) + .build(); + try { + documentService = new FsCrawlerDocumentServiceWorkplaceSearchImpl(metadataDir, fsSettings); + documentService.start(); + + logger.info(" -> Removing existing index [.ent-search-engine-*]"); + documentService.getClient().deleteIndex(".ent-search-engine-*"); + } catch (FsCrawlerIllegalConfigurationException e) { + documentService = oldDocumentService; + Assume.assumeNoException("We don't have a compatible client for this version of the stack.", e); + } + } + + @After + public void resetDocumentService() throws IOException { + if (oldDocumentService != documentService) { + documentService.close(); + documentService = oldDocumentService; + } + } + + @Test + public void testWorkplaceSearch() throws Exception { + startCrawler(getCrawlerName(), fsSettings, TimeValue.timeValueSeconds(10)); + ESSearchResponse searchResponse = countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); + + Map source = searchResponse.getHits().get(0).getSourceAsMap(); + assertThat(source, hasEntry(is("path$string"), notNullValue())); + assertThat(source, hasEntry("extension$string", "txt")); + assertThat(source, hasKey(startsWith("size"))); + assertThat(source, hasKey(startsWith("text_size"))); + assertThat(source, hasEntry(is("mime_type$string"), notNullValue())); + assertThat(source, hasEntry("name$string", "roottxtfile.txt")); + assertThat(source, hasEntry(is("created_at$string"), notNullValue())); + assertThat(source, hasEntry(is("body$string"), notNullValue())); + assertThat(source, hasEntry(is("last_modified$string"), notNullValue())); + assertThat(source, hasEntry("url$string", "http://127.0.0.1/roottxtfile.txt")); + } +} diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/WPSearchClientIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/WPSearchClientIT.java new file mode 100644 index 000000000..e698b42a4 --- /dev/null +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/workplacesearch/WPSearchClientIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.test.integration.workplacesearch; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.thirdparty.wpsearch.WPSearchClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Test Workplace Search HTTP client + */ +public class WPSearchClientIT extends AbstractWorkplaceSearchITCase { + + private WPSearchClient client; + + @Before + public void startClient() { + client = new WPSearchClient(testWorkplaceAccessToken, testWorkplaceKey) + .withHost(testWorkplaceUrl) + .withBulkSize(1); + client.start(); + } + + @After + public void stopClient() { + if (client != null) { + client.close(); + } + } + + @Before + public void cleanExistingIndex() throws IOException { + logger.info(" -> Removing existing index [.ent-search-engine-*]"); + documentService.getClient().deleteIndex(".ent-search-engine-*"); + } + + @Test + public void testSearch() throws Exception { + Map document = new HashMap<>(); + String uniqueId = RandomizedTest.randomAsciiLettersOfLength(10); + document.put("id", "testSearch"); + document.put("title", "To be searched " + uniqueId); + document.put("body", "Foo Bar Baz " + uniqueId); + client.indexDocument(document); + + // We need to wait until it's done + countTestHelper(new ESSearchRequest().withIndex(".ent-search-engine-*"), 1L, null); + client.search(uniqueId); + } + + @Test + public void testSendAndRemoveADocument() { + Map document = new HashMap<>(); + document.put("id", "testSendAndRemoveADocument"); + document.put("title", "To be deleted " + RandomizedTest.randomAsciiLettersOfLength(10)); + client.indexDocument(document); + client.destroyDocument("testSendAndRemoveADocument"); + } +} diff --git a/integration-tests/it-common/src/main/resources-binary/samples/test_ingest_pipeline_392/issue-392.pdf b/integration-tests/it-common/src/main/resources-binary/samples/test_ingest_pipeline_392/issue-392.pdf index 9d6922a7d..3c50d2414 100644 Binary files a/integration-tests/it-common/src/main/resources-binary/samples/test_ingest_pipeline_392/issue-392.pdf and b/integration-tests/it-common/src/main/resources-binary/samples/test_ingest_pipeline_392/issue-392.pdf differ diff --git a/integration-tests/it-v6/pom.xml b/integration-tests/it-v6/pom.xml index d7de237f5..19d868475 100644 --- a/integration-tests/it-v6/pom.xml +++ b/integration-tests/it-v6/pom.xml @@ -14,6 +14,7 @@ ${elasticsearch6.version} + ${project.basedir}/../../contrib @@ -68,7 +69,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.1.0 + 3.2.0 add-test-source @@ -103,10 +104,27 @@ - + - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin + + + elasticsearch + + + + + de.scravy + waitfor-maven-plugin + + + + http://localhost:9200/ + 401 + + + @@ -125,7 +143,7 @@ - skip-fabric8 + skip-docker-compose tests.cluster.url @@ -134,8 +152,15 @@ - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin + + true + + + + de.scravy + waitfor-maven-plugin true @@ -144,7 +169,7 @@ - run-fabric8 + run-docker-compose !tests.cluster.url @@ -153,50 +178,17 @@ - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin ${skipIntegTests} - - - - - security - - - io.fabric8 - docker-maven-plugin + de.scravy + waitfor-maven-plugin - - - fscrawler - dadoonet/fscrawler:${project.version} - - ${integ.elasticsearch.image}:${integ.elasticsearch.version} - - single-node - trial - true - ${tests.cluster.pass} - - - - - integ.elasticsearch.port:9200 - - - - http://localhost:${integ.elasticsearch.port}/ - 200..499 - - - - - - + ${skipIntegTests} diff --git a/integration-tests/it-v7/pom.xml b/integration-tests/it-v7/pom.xml index 8fbdbae9b..ad26bc364 100644 --- a/integration-tests/it-v7/pom.xml +++ b/integration-tests/it-v7/pom.xml @@ -14,6 +14,7 @@ ${elasticsearch7.version} + ${project.basedir}/../../contrib @@ -68,7 +69,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.1.0 + 3.2.0 add-test-source @@ -103,10 +104,36 @@ - + - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin + + + elasticsearch + + + + + + + de.scravy + waitfor-maven-plugin + + + + http://localhost:9200/ + 401 + + + + + @@ -125,7 +152,7 @@ - skip-fabric8 + skip-docker-compose tests.cluster.url @@ -134,8 +161,15 @@ - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin + + true + + + + de.scravy + waitfor-maven-plugin true @@ -144,7 +178,7 @@ - run-fabric8 + run-docker-compose !tests.cluster.url @@ -153,50 +187,17 @@ - io.fabric8 - docker-maven-plugin + com.dkanejs.maven.plugins + docker-compose-maven-plugin ${skipIntegTests} - - - - - security - - - io.fabric8 - docker-maven-plugin + de.scravy + waitfor-maven-plugin - - - fscrawler - dadoonet/fscrawler:${project.version} - - ${integ.elasticsearch.image}:${integ.elasticsearch.version} - - single-node - trial - true - ${tests.cluster.pass} - - - - - integ.elasticsearch.port:9200 - - - - http://localhost:${integ.elasticsearch.port}/ - 200..499 - - - - - - + ${skipIntegTests} diff --git a/lgtm.yml b/lgtm.yml index a7e89d835..2929ddcd0 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -6,3 +6,5 @@ extraction: java: before_index: export DOCKER_SKIP=true + index: + java_version: "14" diff --git a/pom.xml b/pom.xml index a15193abb..00751ce15 100644 --- a/pom.xml +++ b/pom.xml @@ -11,13 +11,14 @@ test-framework settings test-documents + beans + 3rdparty elasticsearch-client core integration-tests distribution cli tika - beans crawler rest docs @@ -36,37 +37,38 @@ - 7.6.2 - 6.8.8 + 7.10.1 + 6.8.13 ${elasticsearch7.version} - 1.24 - 2.10.3 - 2.13.1 - 1.18 - 2.30.1 + 1.25 + 2.12.0 + 2.14.0 + 2.1.1 + 3.0.0 1.2 - 1.14 - 1.26 - 4.5.12 + 1.15 + 1.27 + 4.5.13 4.1.4 - 4.4.13 + 4.4.14 - 2.0 + 3.0.3 1.4.0 - 1.3.0 + 1.4.0 false ${skipTests} ${skipTests} - + + ${project.basedir}/contrib docker.elastic.co/elasticsearch/elasticsearch ${elasticsearch.version} 9200 @@ -74,6 +76,11 @@ elastic changeme + docker.elastic.co/enterprise-search/enterprise-search + 3002 + http://127.0.0.1:3002 + + 8080 @@ -100,7 +107,7 @@ ${env.DOCKER_USERNAME} ${env.DOCKER_PASSWORD} - 1.8 + 14 UTF-8 UTF-8 @@ -134,8 +141,7 @@ maven-compiler-plugin 3.8.1 - ${java.compiler.version} - ${java.compiler.version} + ${java.compiler.version} UTF-8 true true @@ -156,12 +162,12 @@ org.apache.maven.plugins maven-resources-plugin - 3.1.0 + 3.2.0 org.codehaus.mojo versions-maven-plugin - 2.7 + 2.8.1 false @@ -182,7 +188,7 @@ com.carrotsearch.randomizedtesting junit4-maven-plugin - 2.7.7 + 2.7.8 10 pipe,ignore @@ -206,6 +212,9 @@ ${tests.cluster.cloud_id} ${tests.cluster.user} ${tests.cluster.pass} + ${tests.workplace.url} + ${tests.workplace.access_token} + ${tests.workplace.key} ${tests.rest.port} @@ -260,52 +269,84 @@ - + - io.fabric8 - docker-maven-plugin - 0.33.0 + com.dkanejs.maven.plugins + docker-compose-maven-plugin + 4.0.0 - - - fscrawler - dadoonet/fscrawler:${project.version} - - ${integ.elasticsearch.image}:${integ.elasticsearch.version} - - single-node - - - - - integ.elasticsearch.port:9200 - - - - http://localhost:${integ.elasticsearch.port}/ - 200..499 - - - - - - + ${contrib.dir}/docker-compose-it/.env + + ${integ.elasticsearch.version} + ${tests.cluster.pass} + ${integ.elasticsearch.image} + ${integ.elasticsearch.port} + ${integ.workplace.image} + ${integ.workplace.port} + + + ${contrib.dir}/docker-compose-it/docker-compose-elasticsearch.yml + ${contrib.dir}/docker-compose-it/docker-compose-enterprise-search.yml + + + elasticsearch + enterprisesearch + + true + true + true + + + + true - start-elasticsearch + up pre-integration-test - build - stop - start + down + up - stop-elasticsearch + down post-integration-test - stop + down + + + + + + de.scravy + waitfor-maven-plugin + 1.3 + + true + false + 600 + 1000 + + + + + + wait-for-environment-to-be-up + pre-integration-test + + waitfor @@ -315,7 +356,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.2.0 + 3.3.0 org.apache.maven.plugins @@ -468,6 +509,11 @@ fscrawler-elasticsearch-client-v7 2.7-SNAPSHOT + + fr.pilato.elasticsearch.crawler + fscrawler-workplacesearch-client + 2.7-SNAPSHOT + fr.pilato.elasticsearch.crawler fscrawler-cli @@ -552,13 +598,28 @@ snakeyaml ${snakeyaml.version} - + + + + com.sun.activation + jakarta.activation + 2.0.0 + - org.codehaus.woodstox - stax2-api - 4.2 + jakarta.activation + jakarta.activation-api + 2.0.0 + + + + jakarta.xml.bind + jakarta.xml.bind-api + 3.0.0 + + + javax.xml.bind + jaxb-api + 2.3.1 @@ -652,7 +713,7 @@ org.xerial sqlite-jdbc - 3.30.1 + 3.34.0 - com.levigo.jbig2 - levigo-jbig2-imageio - ${levigo.version} + org.apache.pdfbox + jbig2-imageio + ${imageio.version} true @@ -742,11 +803,21 @@ jersey-hk2 ${jersey.version} + + org.glassfish.jersey.core + jersey-client + ${jersey.version} + + + org.glassfish.jersey.core + jersey-common + ${jersey.version} + jakarta.annotation jakarta.annotation-api - 1.3.5 + 2.0.0 @@ -803,7 +874,7 @@ commons-io commons-io - 2.6 + 2.8.0 @@ -827,6 +898,12 @@ httpcore-nio ${httpcore.version} + + + org.apache.httpcomponents + httpmime + ${httpclient.version} + commons-codec commons-codec @@ -842,19 +919,24 @@ junit junit - 4.13 + 4.13.1 com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.7.7 + 2.7.8 - org.bouncycastle bcprov-jdk15on - 1.65 + 1.68 + + + + org.eclipse.jetty.websocket + websocket-client + 9.4.35.v20201120 @@ -887,11 +969,22 @@ false true + + + - skip-fabric8 + skip-docker-compose tests.cluster.url @@ -899,7 +992,7 @@ - run-fabric8 + run-docker-compose !tests.cluster.url diff --git a/rest/pom.xml b/rest/pom.xml index cb833ddd6..df8fb9dfa 100644 --- a/rest/pom.xml +++ b/rest/pom.xml @@ -46,34 +46,10 @@ - + fr.pilato.elasticsearch.crawler - fscrawler-framework - - - - - fr.pilato.elasticsearch.crawler - fscrawler-settings - - - - - fr.pilato.elasticsearch.crawler - fscrawler-beans - - - - - fr.pilato.elasticsearch.crawler - fscrawler-tika - - - - - fr.pilato.elasticsearch.crawler - fscrawler-elasticsearch-client-base + fscrawler-core @@ -93,6 +69,10 @@ org.glassfish.jersey.inject jersey-hk2 + + javax.xml.bind + jaxb-api + diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/CORSFilter.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/CORSFilter.java index 88b194c73..eb8e2557a 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/CORSFilter.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/CORSFilter.java @@ -1,15 +1,14 @@ package fr.pilato.elasticsearch.crawler.fs.rest; import fr.pilato.elasticsearch.crawler.fs.settings.Rest; - -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerResponseContext; -import javax.ws.rs.container.ContainerResponseFilter; -import javax.ws.rs.ext.Provider; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.ext.Provider; @Provider public class CORSFilter implements ContainerResponseFilter { - private Rest rest; + private final Rest rest; public CORSFilter(Rest rest) { this.rest = rest; diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java index 182b481e0..6304fe406 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java @@ -20,11 +20,10 @@ package fr.pilato.elasticsearch.crawler.fs.rest; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.ws.rs.ext.ContextResolver; +import jakarta.ws.rs.ext.Provider; import org.apache.logging.log4j.LogManager; -import javax.ws.rs.ext.ContextResolver; -import javax.ws.rs.ext.Provider; - import static fr.pilato.elasticsearch.crawler.fs.framework.MetaParser.mapper; import static fr.pilato.elasticsearch.crawler.fs.framework.MetaParser.prettyMapper; diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java index 926ebea74..8db5047f8 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java @@ -19,7 +19,8 @@ package fr.pilato.elasticsearch.crawler.fs.rest; -import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,17 +40,18 @@ public class RestServer { /** * Starts Grizzly HTTP server exposing JAX-RS resources defined in this application. * @param settings FSCrawler settings - * @param esClient Elasticsearch client + * @param managementService The management service + * @param documentService The document service */ - public static void start(FsSettings settings, ElasticsearchClient esClient) { + public static void start(FsSettings settings, FsCrawlerManagementService managementService, FsCrawlerDocumentService documentService) { // We create the service only one if (httpServer == null) { // create a resource config that scans for JAX-RS resources and providers // in fr.pilato.elasticsearch.crawler.fs.rest package final ResourceConfig rc = new ResourceConfig() .registerInstances( - new ServerStatusApi(esClient, settings), - new UploadApi(settings, esClient)) + new ServerStatusApi(managementService.getClient(), settings), + new UploadApi(settings, documentService.getClient())) .register(MultiPartFeature.class) .register(RestJsonProvider.class) .register(JacksonFeature.class) diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/ServerStatusApi.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/ServerStatusApi.java index 79a6cc103..9d477810b 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/ServerStatusApi.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/ServerStatusApi.java @@ -23,11 +23,11 @@ import fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient; import fr.pilato.elasticsearch.crawler.fs.framework.Version; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; import java.io.IOException; /** @@ -49,7 +49,7 @@ public class ServerStatusApi extends RestApi { public ServerStatusResponse getStatus() throws IOException { ServerStatusResponse status = new ServerStatusResponse(); status.setVersion(Version.getVersion()); - status.setElasticsearch(esClient.getVersion().toString()); + status.setElasticsearch(esClient.getVersion()); status.setOk(true); status.setSettings(settings); return status; diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/TimeBasedUUIDGenerator.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/TimeBasedUUIDGenerator.java index 20c4288c2..b107d5b2f 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/TimeBasedUUIDGenerator.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/TimeBasedUUIDGenerator.java @@ -38,15 +38,13 @@ public class TimeBasedUUIDGenerator { private static byte[] getMacAddress() throws SocketException { Enumeration en = NetworkInterface.getNetworkInterfaces(); - if (en != null) { - while (en.hasMoreElements()) { - NetworkInterface nint = en.nextElement(); - if (!nint.isLoopback()) { - // Pick the first valid non loopback address we find - byte[] address = nint.getHardwareAddress(); - if (isValidAddress(address)) { - return address; - } + while (en.hasMoreElements()) { + NetworkInterface nint = en.nextElement(); + if (!nint.isLoopback()) { + // Pick the first valid non loopback address we find + byte[] address = nint.getHardwareAddress(); + if (isValidAddress(address)) { + return address; } } } diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/UploadApi.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/UploadApi.java index 60dcfd43a..e1bc86bec 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/UploadApi.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/UploadApi.java @@ -29,17 +29,17 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.ServerUrl; import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; import org.apache.commons.io.FilenameUtils; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; -import javax.ws.rs.BadRequestException; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -123,11 +123,11 @@ public UploadResponse post( logger.debug("Simulate mode is on, so we skip sending document [{}] to elasticsearch at [{}].", filename, url); } else { logger.debug("Sending document [{}] to elasticsearch.", filename); - doc = this.getMergedJsonDoc(doc, tags); + final Doc mergedDoc = this.getMergedJsonDoc(doc, tags); esClient.index( index, id, - DocParser.toJson(doc), + mergedDoc, settings.getElasticsearch().getPipeline()); } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Elasticsearch.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Elasticsearch.java index 0b65dc985..f788de6a4 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Elasticsearch.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Elasticsearch.java @@ -47,6 +47,7 @@ public class Elasticsearch { private String password; private String pipeline; private String pathPrefix; + private boolean sslVerification = true; public Elasticsearch() { @@ -54,7 +55,7 @@ public Elasticsearch() { private Elasticsearch(List nodes, String index, String indexFolder, int bulkSize, TimeValue flushInterval, ByteSizeValue byteSize, String username, String password, String pipeline, - String pathPrefix) { + String pathPrefix, boolean sslVerification) { this.nodes = nodes; this.index = index; this.indexFolder = indexFolder; @@ -65,6 +66,7 @@ private Elasticsearch(List nodes, String index, String indexFolder, i this.password = password; this.pipeline = pipeline; this.pathPrefix = pathPrefix; + this.sslVerification = sslVerification; } public static Builder builder() { @@ -124,6 +126,10 @@ public String getPassword() { return password; } + public boolean getSslVerification() { + return sslVerification; + } + @JsonProperty public void setPassword(String password) { this.password = password; @@ -145,6 +151,11 @@ public void setPathPrefix(String pathPrefix) { this.pathPrefix = pathPrefix; } + public void setSslVerification(boolean sslVerification) { + this.sslVerification = sslVerification; + } + + @SuppressWarnings("UnusedReturnValue") public static class Builder { private List nodes; private String index; @@ -156,6 +167,7 @@ public static class Builder { private String password = null; private String pipeline = null; private String pathPrefix = null; + private boolean sslVerification = true; public Builder setNodes(List nodes) { this.nodes = nodes; @@ -215,8 +227,13 @@ public Builder setPathPrefix(String pathPrefix) { return this; } + public Builder setSslVerification(boolean sslVerification) { + this.sslVerification = sslVerification; + return this; + } + public Elasticsearch build() { - return new Elasticsearch(nodes, index, indexFolder, bulkSize, flushInterval, byteSize, username, password, pipeline, pathPrefix); + return new Elasticsearch(nodes, index, indexFolder, bulkSize, flushInterval, byteSize, username, password, pipeline, pathPrefix, sslVerification); } } @@ -235,6 +252,7 @@ public boolean equals(Object o) { // We can't really test the password as it may be obfuscated if (!Objects.equals(pipeline, that.pipeline)) return false; if (!Objects.equals(pathPrefix, that.pathPrefix)) return false; + if (!Objects.equals(sslVerification, that.sslVerification)) return false; return Objects.equals(flushInterval, that.flushInterval); } @@ -249,6 +267,7 @@ public int hashCode() { result = 31 * result + (pathPrefix != null ? pathPrefix.hashCode() : 0); result = 31 * result + bulkSize; result = 31 * result + (flushInterval != null ? flushInterval.hashCode() : 0); + result = 31 * result + (sslVerification? 1: 0); return result; } @@ -263,6 +282,7 @@ public String toString() { ", username='" + username + '\'' + ", pipeline='" + pipeline + '\'' + ", pathPrefix='" + pathPrefix + '\'' + + ", sslVerification='" + sslVerification + '\'' + '}'; } } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Fs.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Fs.java index 3aef0eb33..5f4da1199 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Fs.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Fs.java @@ -68,7 +68,7 @@ public static Builder builder() { public static final Fs DEFAULT = Fs.builder().setUrl(DEFAULT_DIR).setExcludes(DEFAULT_EXCLUDED).build(); public static class Builder { - private String url; + private String url = DEFAULT_DIR; private TimeValue updateRate = TimeValue.timeValueMinutes(15); private List includes = null; private List excludes = null; diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettings.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettings.java index 6897ec191..660b17147 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettings.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettings.java @@ -28,17 +28,19 @@ public class FsSettings { private Fs fs; private Server server; private Elasticsearch elasticsearch; + private WorkplaceSearch workplaceSearch; private Rest rest; public FsSettings() { } - private FsSettings(String name, Fs fs, Server server, Elasticsearch elasticsearch, Rest rest) { + private FsSettings(String name, Fs fs, Server server, Elasticsearch elasticsearch, WorkplaceSearch workplaceSearch, Rest rest) { this.name = name; this.fs = fs; this.server = server; this.elasticsearch = elasticsearch; + this.workplaceSearch = workplaceSearch; this.rest = rest; } @@ -51,6 +53,7 @@ public static class Builder { private Fs fs = Fs.DEFAULT; private Server server = null; private Elasticsearch elasticsearch = Elasticsearch.DEFAULT(); + private WorkplaceSearch workplaceSearch = null; private Rest rest = null; private Builder setName(String name) { @@ -73,13 +76,18 @@ public Builder setElasticsearch(Elasticsearch elasticsearch) { return this; } + public Builder setWorkplaceSearch(WorkplaceSearch workplaceSearch) { + this.workplaceSearch = workplaceSearch; + return this; + } + public Builder setRest(Rest rest) { this.rest = rest; return this; } public FsSettings build() { - return new FsSettings(name, fs, server, elasticsearch, rest); + return new FsSettings(name, fs, server, elasticsearch, workplaceSearch, rest); } } @@ -115,6 +123,14 @@ public void setElasticsearch(Elasticsearch elasticsearch) { this.elasticsearch = elasticsearch; } + public void setWorkplaceSearch(WorkplaceSearch workplaceSearch) { + this.workplaceSearch = workplaceSearch; + } + + public WorkplaceSearch getWorkplaceSearch() { + return workplaceSearch; + } + public Rest getRest() { return rest; } @@ -134,6 +150,7 @@ public boolean equals(Object o) { if (!Objects.equals(fs, that.fs)) return false; if (!Objects.equals(server, that.server)) return false; if (!Objects.equals(rest, that.rest)) return false; + if (!Objects.equals(workplaceSearch, that.workplaceSearch)) return false; return Objects.equals(elasticsearch, that.elasticsearch); } @@ -145,6 +162,7 @@ public int hashCode() { result = 31 * result + (server != null ? server.hashCode() : 0); result = 31 * result + (rest != null ? rest.hashCode() : 0); result = 31 * result + (elasticsearch != null ? elasticsearch.hashCode() : 0); + result = 31 * result + (workplaceSearch != null ? workplaceSearch.hashCode() : 0); return result; } @@ -154,6 +172,7 @@ public String toString() { ", fs=" + fs + ", server=" + server + ", elasticsearch=" + elasticsearch + + ", enterpriseSearch=" + workplaceSearch + ", rest=" + rest + '}'; } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java index e0fd3c6d4..525c331bb 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java @@ -142,7 +142,7 @@ public void setEnabled(boolean enabled) { } /** - * Get the PDF Strategy. Could be "no_ocr", "ocr_only" or "ocr_and_text" (default) + * Get the PDF Strategy. Could be "no_ocr", "auto", "ocr_only" or "ocr_and_text" (default) * @return the PDF Strategy */ public String getPdfStrategy() { @@ -151,7 +151,7 @@ public String getPdfStrategy() { /** * Set the PDF Strategy. - * @param pdfStrategy the PDF Strategy. Could be "no_ocr", "ocr_only" or "ocr_and_text" + * @param pdfStrategy the PDF Strategy. Could be "no_ocr", "auto", "ocr_only" or "ocr_and_text" */ public void setPdfStrategy(String pdfStrategy) { this.pdfStrategy = pdfStrategy; diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index 61c921583..270392e0e 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -22,6 +22,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; + public class Server { public static final class PROTOCOL { @@ -156,11 +158,11 @@ public boolean equals(Object o) { Server server = (Server) o; if (port != server.port) return false; - if (hostname != null ? !hostname.equals(server.hostname) : server.hostname != null) return false; - if (username != null ? !username.equals(server.username) : server.username != null) return false; + if (!Objects.equals(hostname, server.hostname)) return false; + if (!Objects.equals(username, server.username)) return false; // We can't really test the password as it may be obfuscated - if (protocol != null ? !protocol.equals(server.protocol) : server.protocol != null) return false; - return !(pemPath != null ? !pemPath.equals(server.pemPath) : server.pemPath != null); + if (!Objects.equals(protocol, server.protocol)) return false; + return Objects.equals(pemPath, server.pemPath); } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/ServerUrl.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/ServerUrl.java index d55ba3b49..27243f249 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/ServerUrl.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/ServerUrl.java @@ -58,7 +58,7 @@ public ServerUrl(String urlOrCloudId) { * The cloudId can be found from the cloud console. * * @param cloudId The cloud ID to decode. - * @return A Node running on https://address:443 + * @return A Node running on https://address */ public static String decodeCloudId(String cloudId) { // 1. Ignore anything before `:`. @@ -71,7 +71,7 @@ public static String decodeCloudId(String cloudId) { String[] words = decoded.split("\\$"); // 4. form the URLs - return "https://" + words[1] + "." + words[0] + ":443"; + return "https://" + words[1] + "." + words[0]; } /** diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/WorkplaceSearch.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/WorkplaceSearch.java new file mode 100644 index 000000000..de91a68ca --- /dev/null +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/WorkplaceSearch.java @@ -0,0 +1,174 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.settings; + +import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Objects; + +public class WorkplaceSearch { + + protected static final Logger logger = LogManager.getLogger(WorkplaceSearch.class); + public static final ServerUrl DEFAULT_SERVER = new ServerUrl("http://127.0.0.1:3002"); + public static final String DEFAULT_URL_PREFIX = "http://127.0.0.1"; + + private ServerUrl server = DEFAULT_SERVER; + private String key; + private String accessToken; + private String urlPrefix = DEFAULT_URL_PREFIX; + private int bulkSize = 100; + private TimeValue flushInterval = TimeValue.timeValueSeconds(5); + + public WorkplaceSearch() { + + } + + public WorkplaceSearch(ServerUrl server, String key, String accessToken, String urlPrefix, + int bulkSize, TimeValue flushInterval) { + this.server = server; + this.key = key; + this.accessToken = accessToken; + this.urlPrefix = urlPrefix; + this.bulkSize = bulkSize; + this.flushInterval = flushInterval; + } + + public static Builder builder() { + return new Builder(); + } + + public ServerUrl getServer() { + return server; + } + + public void setServer(ServerUrl server) { + this.server = server; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getAccessToken() { + return accessToken; + } + + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } + + public String getUrlPrefix() { + return urlPrefix; + } + + public void setUrlPrefix(String urlPrefix) { + this.urlPrefix = urlPrefix; + } + + public int getBulkSize() { + return bulkSize; + } + + public void setBulkSize(int bulkSize) { + this.bulkSize = bulkSize; + } + + public TimeValue getFlushInterval() { + return flushInterval; + } + + public void setFlushInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + } + + public static class Builder { + private ServerUrl server = DEFAULT_SERVER; + private String key; + private String accessToken; + private String urlPrefix = DEFAULT_URL_PREFIX; + private int bulkSize = 100; + private TimeValue flushInterval = TimeValue.timeValueSeconds(5); + + public Builder setServer(ServerUrl server) { + this.server = server; + return this; + } + + public Builder setKey(String key) { + this.key = key; + return this; + } + + public Builder setAccessToken(String accessToken) { + this.accessToken = accessToken; + return this; + } + + public Builder setUrlPrefix(String urlPrefix) { + this.urlPrefix = urlPrefix; + return this; + } + + public Builder setBulkSize(int bulkSize) { + this.bulkSize = bulkSize; + return this; + } + + public Builder setFlushInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + public WorkplaceSearch build() { + return new WorkplaceSearch(server, key, accessToken, urlPrefix, bulkSize, flushInterval); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WorkplaceSearch that = (WorkplaceSearch) o; + return Objects.equals(server, that.server) && + Objects.equals(key, that.key) && + Objects.equals(accessToken, that.accessToken) && + Objects.equals(urlPrefix, that.urlPrefix); + } + + @Override + public int hashCode() { + return Objects.hash(server, key, accessToken, urlPrefix); + } + + @Override + public String toString() { + return "WorkplaceSearch{" + "server=" + server + + ", key='" + key + '\'' + + ", accessToken='" + accessToken + '\'' + + ", urlPrefix='" + urlPrefix + '\'' + + '}'; + } +} diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCopyResourcesTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCopyResourcesTest.java index 55ca0345a..7129cb3a1 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCopyResourcesTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCopyResourcesTest.java @@ -50,28 +50,28 @@ public void testCopyResources() throws IOException { AtomicInteger dirCounter = new AtomicInteger(); Files.walkFileTree(target, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new FileVisitor() { + new FileVisitor<>() { @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { logger.info(" --> found directory [{}]", dir); dirCounter.incrementAndGet(); return FileVisitResult.CONTINUE; } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { logger.info(" --> found file [{}]", file); fileCounter.incrementAndGet(); return FileVisitResult.CONTINUE; } @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + public FileVisitResult visitFileFailed(Path file, IOException exc) { return FileVisitResult.CONTINUE; } @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { return FileVisitResult.CONTINUE; } }); diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java index 2b8217c6b..b59164255 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java @@ -34,8 +34,8 @@ public class FsCrawlerValidatorTest extends AbstractFSCrawlerTestCase { @Test public void testSettingsValidation() { // Checking default values - FsSettings settings = buildSettings(Fs.builder().build(), null, null, null); - assertThat(settings.getFs().getUrl(), nullValue()); + FsSettings settings = buildSettings(Fs.builder().build(), null); + assertThat(settings.getFs().getUrl(), is(Fs.DEFAULT_DIR)); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(false)); assertThat(settings.getFs().getUrl(), is(Fs.DEFAULT_DIR)); assertThat(settings.getElasticsearch().getNodes(), hasItem(Elasticsearch.NODE_DEFAULT)); @@ -45,7 +45,7 @@ public void testSettingsValidation() { assertThat(settings.getRest(), nullValue()); // Checking default values - settings = buildSettings(null, null, null, null); + settings = buildSettings(null, null); assertThat(settings.getFs().getUrl(), is(Fs.DEFAULT_DIR)); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(false)); assertThat(settings.getElasticsearch().getNodes(), hasItem(Elasticsearch.NODE_DEFAULT)); @@ -55,41 +55,39 @@ public void testSettingsValidation() { assertThat(settings.getRest(), nullValue()); // Checking Checksum Algorithm - settings = buildSettings(Fs.builder().setChecksum("FSCRAWLER").build(), null, null, null); + settings = buildSettings(Fs.builder().setChecksum("FSCRAWLER").build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking protocol - settings = buildSettings(null, null, Server.builder().setProtocol("FSCRAWLER").build(), null); + settings = buildSettings(null, Server.builder().setProtocol("FSCRAWLER").build()); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking username / password when SSH - settings = buildSettings(null, null, Server.builder().setProtocol(Server.PROTOCOL.SSH).build(), null); + settings = buildSettings(null, Server.builder().setProtocol(Server.PROTOCOL.SSH).build()); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking That we don't try to do both xml and json - settings = buildSettings(Fs.builder().setJsonSupport(true).setXmlSupport(true).build(), null, null, null); + settings = buildSettings(Fs.builder().setJsonSupport(true).setXmlSupport(true).build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking That we don't try to do index xml with not indexing the content - settings = buildSettings(Fs.builder().setIndexContent(false).setXmlSupport(true).build(), null, null, null); + settings = buildSettings(Fs.builder().setIndexContent(false).setXmlSupport(true).build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking That we don't try to do index json with not indexing the content - settings = buildSettings(Fs.builder().setIndexContent(false).setJsonSupport(true).build(), null, null, null); + settings = buildSettings(Fs.builder().setIndexContent(false).setJsonSupport(true).build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); // Checking with Rest but no Rest settings - settings = buildSettings(null, null, null, null); + settings = buildSettings(null, null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, true), is(false)); assertThat(settings.getRest(), notNullValue()); } - private FsSettings buildSettings(Fs fs, Elasticsearch elasticsearch, Server server, Rest rest) { + private FsSettings buildSettings(Fs fs, Server server) { FsSettings.Builder settingsBuilder = FsSettings.builder(getCurrentTestName()); settingsBuilder.setFs(fs == null ? Fs.DEFAULT : fs); - settingsBuilder.setElasticsearch(elasticsearch == null ? Elasticsearch.DEFAULT() : elasticsearch); settingsBuilder.setServer(server); - settingsBuilder.setRest(rest); return settingsBuilder.build(); } diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsMappingTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsMappingTest.java index c6cd73e73..aed614c6d 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsMappingTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsMappingTest.java @@ -26,7 +26,6 @@ import org.junit.Test; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Path; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SETTINGS_FILE; @@ -337,8 +336,8 @@ public void fsSettingsForDocVersionNotSupported() throws Exception { try { readJsonFile(rootTmpDir, metadataDir, "0", INDEX_SETTINGS_FILE); fail("We should have thrown an exception for an unknown elasticsearch version"); - } catch (IllegalArgumentException ignored) { - assertThat(ignored.getMessage(), containsString("does not exist for elasticsearch version")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("does not exist for elasticsearch version")); } } @@ -621,8 +620,8 @@ public void fsSettingsForFolderVersionNotSupported() throws Exception { try { readJsonFile(rootTmpDir, metadataDir, "0", INDEX_SETTINGS_FOLDER_FILE); fail("We should have thrown an exception for an unknown elasticsearch version"); - } catch (IllegalArgumentException ignored) { - assertThat(ignored.getMessage(), containsString("does not exist for elasticsearch version")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("does not exist for elasticsearch version")); } } } diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParserTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParserTest.java index fae2d6a7e..3dd58f25c 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParserTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParserTest.java @@ -30,13 +30,7 @@ import java.io.IOException; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.iterableWithSize; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class FsSettingsParserTest extends AbstractFSCrawlerTestCase { @@ -64,10 +58,19 @@ public class FsSettingsParserTest extends AbstractFSCrawlerTestCase { .setUsername("elastic") .setPassword("changeme") .setBulkSize(1000) + .setByteSize(ByteSizeValue.parseBytesSizeValue("10mb")) .setFlushInterval(TimeValue.timeValueSeconds(5)) .setIndex("docs") .setPipeline("pipeline-id-if-any") .build(); + private static final WorkplaceSearch WORKPLACE_SEARCH_EMPTY = WorkplaceSearch.builder().build(); + private static final WorkplaceSearch WORKPLACE_SEARCH_FULL = WorkplaceSearch.builder() + .setServer(new ServerUrl("https://127.0.0.1:3002")) + .setKey("KEY") + .setAccessToken("ACCESS_TOKEN") + .setUrlPrefix("https://127.0.0.1") + .setBulkSize(100) + .build(); private static final Server SERVER_EMPTY = Server.builder().build(); private static final Server SERVER_FULL = Server.builder() .setHostname("127.0.0.1") @@ -233,7 +236,7 @@ public void testParseSettingsElasticsearchWithPathPrefix() throws IOException { @Test public void testParseSettingsElasticsearchCloudId() throws IOException { - String cloudId = "fscrawler:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyQxZDFlYTk5Njg4Nzc0NWE2YTJiN2NiNzkzMTUzNDhhMyQyOTk1MDI3MzZmZGQ0OTI5OTE5M2UzNjdlOTk3ZmU3Nw=="; + String cloudId = "wpsearch:ZXUtd2VzdC0zLmF3cy5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJDg4NTYwNTJmNDBkNjQ4YjE5Mzk1ZDQ5YTViZjgwNTA4JDJhYjJmYTU5N2RiNDQwNjJhMGZmMjY2ZTBkZTEwY2Fm"; FsSettings fsSettings = FsSettings.builder(getCurrentTestName()) .setElasticsearch(Elasticsearch.builder() .addNode(new ServerUrl(cloudId)) @@ -241,9 +244,9 @@ public void testParseSettingsElasticsearchCloudId() throws IOException { .build(); settingsTester(fsSettings); - assertThat(fsSettings.getElasticsearch().getNodes().get(0).getCloudId(), is("fscrawler:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyQxZDFlYTk5Njg4Nzc0NWE2YTJiN2NiNzkzMTUzNDhhMyQyOTk1MDI3MzZmZGQ0OTI5OTE5M2UzNjdlOTk3ZmU3Nw==")); + assertThat(fsSettings.getElasticsearch().getNodes().get(0).getCloudId(), is("wpsearch:ZXUtd2VzdC0zLmF3cy5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJDg4NTYwNTJmNDBkNjQ4YjE5Mzk1ZDQ5YTViZjgwNTA4JDJhYjJmYTU5N2RiNDQwNjJhMGZmMjY2ZTBkZTEwY2Fm")); assertThat(fsSettings.getElasticsearch().getNodes().get(0).getUrl(), is(nullValue())); - assertThat(fsSettings.getElasticsearch().getNodes().get(0).decodedUrl(), is("https://1d1ea996887745a6a2b7cb79315348a3.europe-west1.gcp.cloud.es.io:443")); + assertThat(fsSettings.getElasticsearch().getNodes().get(0).decodedUrl(), is("https://8856052f40d648b19395d49a5bf80508.eu-west-3.aws.elastic-cloud.com:9243")); } @@ -256,6 +259,24 @@ public void testParseSettingsElasticsearchIndexSettings() throws IOException { ); } + @Test + public void testParseSettingsEmptyWorkplaceSearch() throws IOException { + settingsTester( + FsSettings.builder(getCurrentTestName()) + .setWorkplaceSearch(WORKPLACE_SEARCH_EMPTY) + .build() + ); + } + + @Test + public void testParseSettingsWorkplaceSearch() throws IOException { + settingsTester( + FsSettings.builder(getCurrentTestName()) + .setWorkplaceSearch(WORKPLACE_SEARCH_FULL) + .build() + ); + } + @Test public void testParseSettingsEmptyServer() throws IOException { settingsTester( @@ -279,6 +300,7 @@ public void testParseCompleteSettings() throws IOException { settingsTester( FsSettings.builder(getCurrentTestName()) .setElasticsearch(ELASTICSEARCH_FULL) + .setWorkplaceSearch(WORKPLACE_SEARCH_FULL) .setServer(SERVER_FULL) .setFs(FS_FULL) .setRest(REST_FULL) diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsTest.java index 6dacd50e8..7b30c5ddf 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsTest.java @@ -31,10 +31,11 @@ public class FsSettingsTest extends AbstractFSCrawlerTestCase { @Test public void testCloudId() { - String cloudId = "fscrawler:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyQxZDFlYTk5Njg4Nzc0NWE2YTJiN2NiNzkzMTUzNDhhMyQyOTk1MDI3MzZmZGQ0OTI5OTE5M2UzNjdlOTk3ZmU3Nw=="; + String cloudId = "foobar:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRmb29iYXJlbGFzdGljc2VhcmNoJGZvb2JhcmtpYmFuYQ=="; String httpHost = decodeCloudId(cloudId); - assertThat(httpHost, is("https://1d1ea996887745a6a2b7cb79315348a3.europe-west1.gcp.cloud.es.io:443")); + assertThat(httpHost, is("https://foobarelasticsearch.us-east-1.aws.found.io")); } } + diff --git a/test-documents/src/main/resources/documents/test-ocr-heb.pdf b/test-documents/src/main/resources/documents/test-ocr-heb.pdf new file mode 100644 index 000000000..328dd577a Binary files /dev/null and b/test-documents/src/main/resources/documents/test-ocr-heb.pdf differ diff --git a/test-documents/src/main/resources/documents/test-ocr-notext.pdf b/test-documents/src/main/resources/documents/test-ocr-notext.pdf new file mode 100644 index 000000000..0ba3e4861 Binary files /dev/null and b/test-documents/src/main/resources/documents/test-ocr-notext.pdf differ diff --git a/test-documents/src/main/resources/documents/test.eml b/test-documents/src/main/resources/documents/test.eml new file mode 100644 index 000000000..38af7df49 --- /dev/null +++ b/test-documents/src/main/resources/documents/test.eml @@ -0,0 +1,101 @@ +X-QQ-FEAT: nUNo11GR11/U6FbQCNwFDxz5RuSvMVGNGxEpRmUP0pnRiHfxyHH7wUwrzQh5W + yDUdu10znc8GTwdaOmkwH3K60FN/W0BBh7hsvcBhqJe4YBQoD33uPGiCLqB/EXQQgHZsKt6 + F/ZsbisizWEwZJV1mD7Y8yAHp4B6Uw/x4ne/29C3xVwMcLktqPia9gI/Lrl2vJzr4w9VBg+ + b/Sii+gxtjajGZDA+GFWTSe8dXOd/N0RVfP3vXfZoP0nlcDU1S7ZckP8SKTeQV3c= +X-QQ-SSF: 000000000000000000000000000000A +X-QQ-XMAILINFO: Na/7hYqAgZHnwE6NBGOaFx4Ph8LoIgfMjPtcuElAXe5orgl65aMrGbzfQXlCeA + +rCMGjbxwua38mglkeeeXAjACCi/E7hQqhw0cbWjcmxB0sCQ8yjX/zXr4AhlBQNH7muyt4qYMfHgg + l4cN4fHmLYIbRiZNMGEesgbvnFvQKMLUX5SJQ/d2gvHVO6PlizhYKpGp0fvJDTuu6Xe2dgt05g60x + SCdfsl2+dg8+ALjaiBotdYDT+gr04YQyQAixx/vOTfSoQJJyaM1V4+PFK2HVW36O3+o0eGa2GsHeW + EU+KaJGCnFgxFiyCZC9oDLzLL5X4pGdb9Nk2Oj8PodwwfCwrQnuRQbKEqaJdNmoTVb/PMLpUFXUy4 + EnS+hz1liezKJSnUWD0JBh8r7JRU2+tRedKHswUn2xuP+RavdM2uMtP89RbyiOrwbdChud0Xc7Aoh + 0JRYY2IUXyMdTOS42p4ZZpUadAzOzx8RZ59WRdLOJozsSGxVb8vNun0TSDF8FUfzZT2JVY2wbNLit + BgV8Ynqy6PdxJifnKe1fa3ITsY6YKNDLFrJt812FykUSJbZY04fgeuoGwjz6YWf/ +X-HAS-ATTACH: no +X-QQ-BUSINESS-ORIGIN: 2 +X-Originating-IP: 36.5.123.158 +X-QQ-STYLE: +X-QQ-mid: webmail824t1588854644t1277638 +From: "=?gb18030?B?9ui19A==?=" <2428617664@qq.com> +To: "=?gb18030?B?vqjC5A==?=" <2428617664@qq.com> +Subject: Test +Mime-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_NextPart_5EB3FF74_109736B8_0C149CAD" +Content-Transfer-Encoding: 8Bit +Date: Thu, 7 May 2020 20:30:44 +0800 +X-Priority: 3 +Message-ID: +X-QQ-MIME: TCMime 1.0 by Tencent +X-Mailer: QQMail 2.x +X-QQ-Mailer: QQMail 2.x + +This is a multi-part message in MIME format. + +------=_NextPart_5EB3FF74_109736B8_0C149CAD +Content-Type: multipart/alternative; + boundary="----=_NextPart_5EB3FF74_109736B8_4807DE12"; + +------=_NextPart_5EB3FF74_109736B8_4807DE12 +Content-Type: text/plain; + charset="gb18030" +Content-Transfer-Encoding: base64 + +dGVzdA== + +------=_NextPart_5EB3FF74_109736B8_4807DE12 +Content-Type: text/html; + charset="gb18030" +Content-Transfer-Encoding: base64 + +dGVzdA== + +------=_NextPart_5EB3FF74_109736B8_4807DE12-- + +------=_NextPart_5EB3FF74_109736B8_0C149CAD +Content-Type: application/octet-stream; + charset="gb18030"; + name="123.docx" +Content-Disposition: attachment; filename="123.docx" +Content-Transfer-Encoding: base64 + +UEsDBBQACAAIALCjp1AAAAAAAAAAAAAAAAALAAAAX3JlbHMvLnJlbHONjzsOwjAQRK9ibU82 +UCCE4qRBSGmjcADL3jhR4o9s87s9LigIoqAc7czbmap5mIXdKMTJWQ7bogRGVjo1Wc3h0p83 +B2jqqqNFpOyI4+QjyxEbOYwp+SNilCMZEQvnyebL4IIRKcug0Qs5C024K8s9hk8GrJmsF0FT +4nB3QaFy8mrIpiLjgLWKg59116rcrX96+uezG4ZJ0ukN+lHgywEM6wpXM+sXUEsHCE+L3Tym +AAAAHAEAAFBLAwQUAAgACACwo6dQAAAAAAAAAAAAAAAAHAAAAHdvcmQvX3JlbHMvZG9jdW1l +bnQueG1sLnJlbHOtkMsKwjAQRX8lzN6mdSEiTbsRodtSPyAm0wc2D5JU7N8bFMSKgguXw8y9 +5zB5eVUjuaDzg9EMsiQFgloYOeiOwbE5rLZQFnmNIw/xwveD9SRGtGfQh2B3lHrRo+I+MRZ1 +3LTGKR7i6DpquTjzDuk6TTfUvXbAspM03HUYGOhJndBFeBKrgFSSgTSirmT0amaLv1BN2w4C +90ZMCnX4AKdPChD6RcSHeUT/bpH90+KBuCvQxYOLG1BLBwjc8wDAtgAAAJYBAABQSwMEFAAI +AAgAsKOnUAAAAAAAAAAAAAAAABEAAAB3b3JkL2RvY3VtZW50LnhtbN1SwU7DMAz9lSp3llEh +hKp10y47ceAAH5Cl3hqUxpWdrYwT4sqR/+AP+BrgP0jSdgM+ASlK4ufnZzvxbPHQ2GwPxAZd +Kc4nU5GB01gZty3F3e3q7Epk7JWrlEUHpTgAi8V81hUV6l0DzmdBwHHRlaL2vi2kZF1Do3iC +Lbjg2yA1ygeTtrJDqlpCDcxBv7Eyn04vZaOME1FyjdUhnm3abige3CoduFnwQpAKFYQKu0Jt +PNBwtyYWll8EQ8YQ46qAktnWfmTAZrxuDLG/ThEj/14HfK9sKdbo6wGkPj2t0HmOCVkbU4qv +t5fP99coVC8d/wI0/7RAsV+yUScsyWq0SGM6tfM44Kc2k2csrUU2PvzMX5wfRyTPR6iuUnv2 +KDwwtAVFiSSHtuTxdem/t9oVaUBT0vDlLQED7UHMP56e+xWpvg/oXybt/SjK05jPvwFQSwcI +OrhPSU4BAAAqAwAAUEsDBBQACAAIALCjp1AAAAAAAAAAAAAAAAASAAAAd29yZC9udW1iZXJp +bmcueG1sDYxBDsIwDAS/EvlOXTggFDXtrS+AB4TEtJUau4oDgd+T42pmdpi+aTcfyroJOzh3 +PRjiIHHjxcHjPp9uYLR4jn4XJgc/UpjGoVp+pyflppn2wGqrg7WUwyJqWCl57eQgbuwlOfnS +Zl6wSo5HlkCqrUw7Xvr+islvDAbHP1BLBwgsMb9xfAAAAI0AAABQSwMEFAAIAAgAsKOnUAAA +AAAAAAAAAAAAAA8AAAB3b3JkL3N0eWxlcy54bWwNjEEOwiAQAL9C9m5BD8aQ0t58gT6AwNqS +wG7DErG/l+NkMjOvv5LVF6skJgfXyYBCChwTbQ7er+flAUqap+gzEzo4UWBd5m6lnRlFjZzE +dgd7a4fVWsKOxcvEB9JwH67Ft4F1051rPCoHFBn3kvXNmLsuPhEovfwBUEsHCK9TyEF5AAAA +igAAAFBLAwQUAAgACACwo6dQAAAAAAAAAAAAAAAAEwAAAFtDb250ZW50X1R5cGVzXS54bWyt +kr1OxDAQhF/FcotiBwqEUJIr+CmB4ngAY28S6/wnr3PcvT2bBKVASAjpSntm9hutttmdvGNH +yGhjaPm1qDmDoKOxYWj5+/65uuO7rtmfEyAja8CWj6WkeylRj+AVipggkNLH7FWhZx5kUvqg +BpA3dX0rdQwFQqnKPIN3zSP0anKFPZ3oe8VmcMjZw2qcWS1XKTmrVSFdHoP5Qam+CYKSiwdH +m/CKDJzJXxGL9B9C7HurwUQ9eYqIz5hNylEDIq3GO7EpXtmwkV9pldkaYG8qlxfliSPnqAyT +/4BMUXHxJtvov1tgOTvAy1dY5258udxL9wVQSwcI/2JruvMAAABdAgAAUEsBAi0AFAAIAAgA +sKOnUE+L3TymAAAAHAEAAAsAAAAAAAAAAAAAAAAAAAAAAF9yZWxzLy5yZWxzUEsBAi0AFAAI +AAgAsKOnUNzzAMC2AAAAlgEAABwAAAAAAAAAAAAAAAAA3wAAAHdvcmQvX3JlbHMvZG9jdW1l +bnQueG1sLnJlbHNQSwECLQAUAAgACACwo6dQOrhPSU4BAAAqAwAAEQAAAAAAAAAAAAAAAADf +AQAAd29yZC9kb2N1bWVudC54bWxQSwECLQAUAAgACACwo6dQLDG/cXwAAACNAAAAEgAAAAAA +AAAAAAAAAABsAwAAd29yZC9udW1iZXJpbmcueG1sUEsBAi0AFAAIAAgAsKOnUK9TyEF5AAAA +igAAAA8AAAAAAAAAAAAAAAAAKAQAAHdvcmQvc3R5bGVzLnhtbFBLAQItABQACAAIALCjp1D/ +Ymu68wAAAF0CAAATAAAAAAAAAAAAAAAAAN4EAABbQ29udGVudF9UeXBlc10ueG1sUEsFBgAA +AAAGAAYAgAEAABIGAAAAAA== + +------=_NextPart_5EB3FF74_109736B8_0C149CAD-- + diff --git a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerMetadataTestCase.java b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerMetadataTestCase.java index 84199d5cd..d436ca54e 100644 --- a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerMetadataTestCase.java +++ b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerMetadataTestCase.java @@ -29,7 +29,7 @@ import static fr.pilato.elasticsearch.crawler.fs.test.framework.FsCrawlerUtilForTests.copyDefaultResources; -public class AbstractFSCrawlerMetadataTestCase extends AbstractFSCrawlerTestCase { +public abstract class AbstractFSCrawlerMetadataTestCase extends AbstractFSCrawlerTestCase { protected static Path metadataDir; diff --git a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java index ed26b7756..f3a4be893 100644 --- a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java +++ b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java @@ -64,7 +64,7 @@ public abstract class AbstractFSCrawlerTestCase { public TestName name = new TestName(); @ClassRule - public static TemporaryFolder folder = new TemporaryFolder(); + public static final TemporaryFolder folder = new TemporaryFolder(); protected static Path rootTmpDir; @BeforeClass @@ -147,9 +147,6 @@ public static long awaitBusy(LongSupplier breakSupplier, Long expected, long max while (sum + timeInMillis < maxTimeInMillis) { long current = breakSupplier.getAsLong(); - if (current < 0) { - return current; - } if (expected == null && current >= 1) { return current; } else if (expected != null && current == expected) { @@ -177,16 +174,13 @@ public static String toUnderscoreCase(String value) { sb.append(value.charAt(j)); } changed = true; - if (i == 0) { - sb.append(Character.toLowerCase(c)); - } else { + if (i != 0) { sb.append('_'); - sb.append(Character.toLowerCase(c)); } } else { sb.append('_'); - sb.append(Character.toLowerCase(c)); } + sb.append(Character.toLowerCase(c)); } else { if (changed) { sb.append(c); @@ -212,13 +206,14 @@ public static File URLtoFile(URL url) { * @param exceptionClass Expected error * @param function Function to be executed */ - public static T expectThrows(Class exceptionClass, Supplier function) { + public static T expectThrows(Class exceptionClass, Supplier function) { try { Object o = function.get(); fail("We should have caught a " + exceptionClass.getName() + ". " + "But we returned " + o + "."); } catch (Throwable t) { assertThat(t, instanceOf(exceptionClass)); + //noinspection unchecked return (T) t; } return null; diff --git a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/FSCrawlerReproduceInfoPrinter.java b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/FSCrawlerReproduceInfoPrinter.java index 53464d0f4..266711ddc 100644 --- a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/FSCrawlerReproduceInfoPrinter.java +++ b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/FSCrawlerReproduceInfoPrinter.java @@ -46,17 +46,17 @@ public class FSCrawlerReproduceInfoPrinter extends RunListener { private static final Logger logger = LogManager.getLogger(); @Override - public void testStarted(Description description) throws Exception { + public void testStarted(Description description) { logger.trace("Test {} started", description.getDisplayName()); } @Override - public void testFinished(Description description) throws Exception { + public void testFinished(Description description) { logger.trace("Test {} finished", description.getDisplayName()); } @Override - public void testFailure(Failure failure) throws Exception { + public void testFailure(Failure failure) { // Ignore assumptions. if (failure.getException() instanceof AssumptionViolatedException) { return; @@ -99,7 +99,7 @@ public ReproduceErrorMessageBuilder appendAllOpts(Description description) { List properties = new ArrayList<>(); scanProperties(description.getTestClass(), properties); - appendProperties(properties.toArray(new String[properties.size()])); + appendProperties(properties.toArray(new String[0])); return appendESProperties(); } @@ -109,7 +109,7 @@ public ReproduceErrorMessageBuilder appendAllOpts(Description description) { * superclass to subclass. */ private void scanProperties(Class c, List properties) { - if (Object.class.equals(c) == false) { + if (!Object.class.equals(c)) { scanProperties(c.getSuperclass(), properties); } Properties extraParameterAnnotation = c.getAnnotation(Properties.class); diff --git a/test-framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCaseTest.java b/test-framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCaseTest.java index b89342c82..603858987 100644 --- a/test-framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCaseTest.java +++ b/test-framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCaseTest.java @@ -18,14 +18,20 @@ */ package fr.pilato.elasticsearch.crawler.fs.test.framework; +import org.hamcrest.Matchers; import org.junit.Test; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; public class AbstractFSCrawlerTestCaseTest extends AbstractFSCrawlerTestCase { + @SuppressWarnings("ConstantConditions") @Test public void testExpectThrows() { NullPointerException npe = expectThrows(NullPointerException.class, () -> { @@ -38,4 +44,21 @@ public void testExpectThrows() { assertThat(assertionError.getMessage(), containsString("Expected: an instance of " + NullPointerException.class.getName())); } + @Test + public void testSimulateElasticsearchException() throws InterruptedException { + AtomicLong l = new AtomicLong(-1); + + long hits = awaitBusy(() -> { + // Let's search for entries + try { + throw new RuntimeException("foo bar"); + } catch (RuntimeException e) { + staticLogger.warn("error caught", e); + return l.getAndIncrement(); + } + }, null, 1, TimeUnit.SECONDS); + + assertThat(hits, Matchers.greaterThan(-1L)); + } + } diff --git a/tika/pom.xml b/tika/pom.xml index 4e34aac8f..61872ba5d 100644 --- a/tika/pom.xml +++ b/tika/pom.xml @@ -79,8 +79,8 @@ to use them have to provided them manually in the lib dir. --> - com.levigo.jbig2 - levigo-jbig2-imageio + org.apache.pdfbox + jbig2-imageio true @@ -102,7 +102,7 @@ com.google.guava guava - 28.2-jre + 29.0-jre diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParser.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParser.java index bcb4e855c..49998eb7b 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParser.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParser.java @@ -20,7 +20,9 @@ package fr.pilato.elasticsearch.crawler.fs.tika; import fr.pilato.elasticsearch.crawler.fs.beans.Doc; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import org.apache.commons.io.input.TeeInputStream; import org.apache.logging.log4j.LogManager; @@ -35,13 +37,14 @@ import java.io.InputStream; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; -import static fr.pilato.elasticsearch.crawler.fs.framework.StreamsUtil.copy; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; import static fr.pilato.elasticsearch.crawler.fs.tika.TikaInstance.extractText; import static fr.pilato.elasticsearch.crawler.fs.tika.TikaInstance.langDetector; @@ -97,12 +100,20 @@ public static void generate(FsSettings fsSettings, InputStream inputStream, Stri Throwable current = e; StringBuilder sb = new StringBuilder(); while (current != null) { - sb.append(" -> "); sb.append(current.getMessage()); current = current.getCause(); + if (current != null) { + sb.append(" -> "); + } } - logger.warn("Failed to extract [{}] characters of text for [{}] {}", indexedChars, fullFilename, sb.toString()); + try { + FSCrawlerLogger.documentError( + fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(fullFilename), + computeVirtualPathName(fsSettings.getFs().getUrl(), fullFilename), + sb.toString()); + } catch (NoSuchAlgorithmException ignored) { } + logger.warn("Failed to extract [{}] characters of text for [{}]: {}", indexedChars, fullFilename, sb.toString()); logger.debug("Failed to extract [" + indexedChars + "] characters of text for [" + fullFilename + "]", e); } @@ -195,12 +206,11 @@ public static void generate(FsSettings fsSettings, InputStream inputStream, Stri } else if (fsSettings.getFs().isStoreSource()) { // We don't extract content but just store the binary file // We need to create the ByteArrayOutputStream which has not been created then - copy(inputStream, bos); + inputStream.transferTo(bos); } // Doc as binary attachment if (fsSettings.getFs().isStoreSource()) { - //noinspection ConstantConditions doc.setAttachment(Base64.getEncoder().encodeToString(bos.toByteArray())); } logger.trace("End document generation"); diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java index 45acca127..ccb902e07 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java @@ -134,15 +134,13 @@ static String extractText(FsSettings fsSettings, int indexedChars, InputStream s TikaException { initTika(fsSettings.getFs()); WriteOutContentHandler handler = new WriteOutContentHandler(indexedChars); - try { + try (stream) { parser.parse(stream, new BodyContentHandler(handler), metadata, context); } catch (SAXException e) { if (!handler.isWriteLimitReached(e)) { // This should never happen with BodyContentHandler... throw new TikaException("Unexpected SAX processing failure", e); } - } finally { - stream.close(); } return handler.toString(); } diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java index 4909cb80d..bf1646260 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java @@ -19,21 +19,14 @@ package fr.pilato.elasticsearch.crawler.fs.tika; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.deser.std.UntypedObjectDeserializer; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import static fr.pilato.elasticsearch.crawler.fs.framework.MetaParser.mapper; @@ -41,6 +34,7 @@ /** * Parse a XML document and generate a FSCrawler Doc */ +@SuppressWarnings("unchecked") public class XmlDocParser { private final static Logger logger = LogManager.getLogger(XmlDocParser.class); @@ -48,22 +42,24 @@ public class XmlDocParser { static { xmlMapper = new XmlMapper(); - xmlMapper.registerModule(new SimpleModule() - .addDeserializer(Object.class, new FixedUntypedObjectDeserializer())); } - public static String generate(InputStream inputStream) throws IOException { - logger.trace("Converting XML document [{}]"); + public static String generate(InputStream inputStream) { + logger.trace("Converting XML document"); // Extracting XML content // See #185: https://github.com/dadoonet/fscrawler/issues/185 Map map = generateMap(inputStream); // Serialize to JSON - String json = mapper.writeValueAsString(map); - - logger.trace("Generated JSON: {}", json); - return json; + try { + String json = mapper.writeValueAsString(map); + logger.trace("Generated JSON: {}", json); + return json; + } catch (JsonProcessingException e) { + // TODO Fix that code. We should log here and return null. + throw new RuntimeException(e); + } } /** @@ -72,7 +68,7 @@ public static String generate(InputStream inputStream) throws IOException { * @return The XML Content as a map */ public static Map generateMap(InputStream inputStream) { - logger.trace("Converting XML document [{}]"); + logger.trace("Converting XML document"); Map map = asMap(inputStream); logger.trace("Generated JSON: {}", map); @@ -86,57 +82,4 @@ private static Map asMap(InputStream stream) { throw new RuntimeException(e); } } - - // This code is coming from the gist provided at https://github.com/FasterXML/jackson-dataformat-xml/issues/205 - @SuppressWarnings({"deprecation", "serial"}) - public static class FixedUntypedObjectDeserializer extends UntypedObjectDeserializer { - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - protected Object mapObject(JsonParser p, DeserializationContext ctx) throws IOException { - String firstKey; - - JsonToken t = p.getCurrentToken(); - - if (t == JsonToken.START_OBJECT) { - firstKey = p.nextFieldName(); - } else if (t == JsonToken.FIELD_NAME) { - firstKey = p.getCurrentName(); - } else { - if (t != JsonToken.END_OBJECT) { - throw ctx.mappingException(handledType(), p.getCurrentToken()); - } - firstKey = null; - } - - // empty map might work; but caller may want to modify... so better - // just give small modifiable - Map resultMap = new LinkedHashMap<>(2); - if (firstKey == null) { - return resultMap; - } - - p.nextToken(); - resultMap.put(firstKey, deserialize(p, ctx)); - - String nextKey; - while ((nextKey = p.nextFieldName()) != null) { - p.nextToken(); - if (resultMap.containsKey(nextKey)) { - Object listObject = resultMap.get(nextKey); - - if (!(listObject instanceof List)) { - listObject = new ArrayList<>(); - ((List) listObject).add(resultMap.get(nextKey)); - resultMap.put(nextKey, listObject); - } - - ((List) listObject).add(deserialize(p, ctx)); - } else { - resultMap.put(nextKey, deserialize(p, ctx)); - } - } - - return resultMap; - } - } } diff --git a/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParserTest.java b/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParserTest.java index 5c4fc9f14..1b8acc9c1 100644 --- a/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParserTest.java +++ b/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaDocParserTest.java @@ -224,7 +224,7 @@ public void testExtractFromDocx() throws IOException { assertThat(doc.getMeta().getTitle(), is("Test Tika title")); Map raw = doc.getMeta().getRaw(); - assertThat(raw.entrySet(), iterableWithSize(59)); + assertThat(raw.entrySet(), iterableWithSize(60)); assertThat(raw, hasEntry("date", "2016-07-07T08:36:00Z")); assertThat(raw, hasEntry("Total-Time", "6")); assertThat(raw, hasEntry("extended-properties:AppVersion", "15.0000")); @@ -281,6 +281,7 @@ public void testExtractFromDocx() throws IOException { assertThat(raw, hasEntry("meta:last-author", "David Pilato")); assertThat(raw, hasEntry("xmpTPg:NPages", "2")); assertThat(raw, hasEntry("Revision-Number", "4")); + assertThat(raw, hasEntry("extended-properties:DocSecurityString", "None")); assertThat(raw, hasEntry("meta:keyword", "keyword1, keyword2")); assertThat(raw, hasEntry("cp:category", "test")); assertThat(raw, hasEntry("dc:publisher", "elastic")); @@ -703,6 +704,18 @@ public void testOcr() throws IOException { assertThat(doc.getContent(), containsString("This file contains some words.")); assertThat(doc.getContent(), containsString("This file also contains text.")); + // Test with OCR On and PDF Strategy set to auto (meaning that PDF will be only OCRed if less than 10 characters are found) + fsSettings = FsSettings.builder(getCurrentTestName()) + .setFs(Fs.builder().setOcr(Ocr.builder() + .setPdfStrategy("auto") + .build()).build()) + .build(); + doc = extractFromFile("test-ocr.pdf", fsSettings); + assertThat(doc.getContent(), not(containsString("This file contains some words."))); + assertThat(doc.getContent(), containsString("This file also contains text.")); + doc = extractFromFile("test-ocr-notext.pdf", fsSettings); + assertThat(doc.getContent(), containsString("This file contains some words.")); + // Test with OCR Off fsSettings = FsSettings.builder(getCurrentTestName()) .setFs(Fs.builder().setOcr(Ocr.builder() @@ -736,6 +749,18 @@ public void testOcr() throws IOException { assertThat(doc.getContent(), stringContainsInOrder(Arrays.asList("This", "file", "contains", "some", "words."))); doc = extractFromFile("test-ocr.pdf", fsSettings); assertThat(doc.getContent(), stringContainsInOrder(Arrays.asList("This", "file", "contains", "some", "words."))); + + // Test with heb language + fsSettings = FsSettings.builder(getCurrentTestName()) + .setFs(Fs.builder().setOcr(Ocr.builder().setLanguage("heb").build()).build()) + .build(); + doc = extractFromFile("test-ocr-heb.pdf", fsSettings); + try { + // This test requires to have the hebrew language pack so we don't fail the test but just log + assertThat(doc.getContent(), containsString("המבודדים מתקבלים")); + } catch (AssertionError e) { + logger.info("We were not able to get the Hebrew content with OCR. May be the language pack was not installed?"); + } } @Test @@ -754,6 +779,58 @@ public void testProtectedDocument() throws IOException { assertThat(doc.getFile().getContentType(), is("application/x-tika-ooxml-protected")); } + @Test + public void testExtractFromEml() throws IOException { + Doc doc = extractFromFileExtension("eml"); + + // Extracted content + assertThat(doc.getContent(), containsString("test")); + + // Content Type + assertThat(doc.getFile().getContentType(), is("message/rfc822")); + + // Meta data + assertThat(doc.getMeta().getAuthor(), is("鲨掉 <2428617664@qq.com>")); + assertThat(doc.getMeta().getCreated(), is(localDateTimeToDate(LocalDateTime.of(2020, 5, 7, 12, 30, 44)))); + assertThat(doc.getMeta().getTitle(), is("Test")); + + Map raw = doc.getMeta().getRaw(); + assertThat(raw.entrySet(), iterableWithSize(33)); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-XMAILINFO", "Na/7hYqAgZHnwE6NBGOaFx4Ph8LoIgfMjPtcuElAXe5orgl65aMrGbzfQXlCeA +rCMGjbxwua38mglkeeeXAjACCi/E7hQqhw0cbWjcmxB0sCQ8yjX/zXr4AhlBQNH7muyt4qYMfHgg l4cN4fHmLYIbRiZNMGEesgbvnFvQKMLUX5SJQ/d2gvHVO6PlizhYKpGp0fvJDTuu6Xe2dgt05g60x SCdfsl2+dg8+ALjaiBotdYDT+gr04YQyQAixx/vOTfSoQJJyaM1V4+PFK2HVW36O3+o0eGa2GsHeW EU+KaJGCnFgxFiyCZC9oDLzLL5X4pGdb9Nk2Oj8PodwwfCwrQnuRQbKEqaJdNmoTVb/PMLpUFXUy4 EnS+hz1liezKJSnUWD0JBh8r7JRU2+tRedKHswUn2xuP+RavdM2uMtP89RbyiOrwbdChud0Xc7Aoh 0JRYY2IUXyMdTOS42p4ZZpUadAzOzx8RZ59WRdLOJozsSGxVb8vNun0TSDF8FUfzZT2JVY2wbNLit BgV8Ynqy6PdxJifnKe1fa3ITsY6YKNDLFrJt812FykUSJbZY04fgeuoGwjz6YWf/")); + assertThat(raw, hasEntry("Message:Raw-Header:X-HAS-ATTACH", "no")); + assertThat(raw, hasEntry("subject", "Test")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-FEAT", "nUNo11GR11/U6FbQCNwFDxz5RuSvMVGNGxEpRmUP0pnRiHfxyHH7wUwrzQh5W yDUdu10znc8GTwdaOmkwH3K60FN/W0BBh7hsvcBhqJe4YBQoD33uPGiCLqB/EXQQgHZsKt6 F/ZsbisizWEwZJV1mD7Y8yAHp4B6Uw/x4ne/29C3xVwMcLktqPia9gI/Lrl2vJzr4w9VBg+ b/Sii+gxtjajGZDA+GFWTSe8dXOd/N0RVfP3vXfZoP0nlcDU1S7ZckP8SKTeQV3c=")); + assertThat(raw, hasEntry("dc:creator", "鲨掉 <2428617664@qq.com>")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-SSF", "000000000000000000000000000000A")); + assertThat(raw, hasEntry("Message:From-Email", "2428617664@qq.com")); + assertThat(raw, hasEntry("dcterms:created", "2020-05-07T12:30:44Z")); + assertThat(raw, hasEntry("Message-To", "鲸落 <2428617664@qq.com>")); + assertThat(raw, hasEntry("Multipart-Boundary", "----=_NextPart_5EB3FF74_109736B8_0C149CAD")); + assertThat(raw, hasEntry("dc:title", "Test")); + assertThat(raw, hasEntry("Message:Raw-Header:Message-ID", "")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-STYLE", " ")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-BUSINESS-ORIGIN", "2")); + assertThat(raw, hasEntry("Content-Type", "message/rfc822")); + assertThat(raw, hasEntry("X-Parsed-By", "org.apache.tika.parser.DefaultParser")); + assertThat(raw, hasEntry("creator", "鲨掉 <2428617664@qq.com>")); + assertThat(raw, hasEntry("meta:author", "鲨掉 <2428617664@qq.com>")); + assertThat(raw, hasEntry("meta:creation-date", "2020-05-07T12:30:44Z")); + assertThat(raw, hasEntry("Message:Raw-Header:Mime-Version", "1.0")); + assertThat(raw, hasEntry("Message:Raw-Header:Content-Transfer-Encoding", "8Bit")); + assertThat(raw, hasEntry("Creation-Date", "2020-05-07T12:30:44Z")); + assertThat(raw, hasEntry("resourceName", "test.eml")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-mid", "webmail824t1588854644t1277638")); + assertThat(raw, hasEntry("Message:Raw-Header:X-Priority", "3")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-Mailer", "QQMail 2.x")); + assertThat(raw, hasEntry("Message:Raw-Header:X-QQ-MIME", "TCMime 1.0 by Tencent")); + assertThat(raw, hasEntry("Message:From-Name", "鲨掉")); + assertThat(raw, hasEntry("Message:Raw-Header:X-Originating-IP", "36.5.123.158")); + assertThat(raw, hasEntry("Author", "鲨掉 <2428617664@qq.com>")); + assertThat(raw, hasEntry("Multipart-Subtype", "mixed")); + assertThat(raw, hasEntry("Message:Raw-Header:X-Mailer", "QQMail 2.x")); + assertThat(raw, hasEntry("Message-From", "鲨掉 <2428617664@qq.com>")); + } + private Doc extractFromFileExtension(String extension) throws IOException { FsSettings fsSettings = FsSettings.builder(getCurrentTestName()) .setFs(Fs.builder().setRawMetadata(true).build())