From cc8b55169570cc29cc2d60ba5b05dc3c608a1fe3 Mon Sep 17 00:00:00 2001 From: otodat Date: Sat, 28 Sep 2019 02:36:38 +0900 Subject: [PATCH 01/28] Generate FSCrawler docker images --- distribution/es5/pom.xml | 9 +++ distribution/es6/pom.xml | 8 +++ distribution/es7/pom.xml | 11 ++++ distribution/pom.xml | 86 +++++++++++++++++++++++++ distribution/src/main/docker/Dockerfile | 16 +++++ 5 files changed, 130 insertions(+) create mode 100644 distribution/src/main/docker/Dockerfile diff --git a/distribution/es5/pom.xml b/distribution/es5/pom.xml index 61fb184b5..6de6fa5ab 100644 --- a/distribution/es5/pom.xml +++ b/distribution/es5/pom.xml @@ -12,6 +12,10 @@ fscrawler-es5 FSCrawler ZIP Distribution for Elasticsearch 5.x + + es5 + + fr.pilato.elasticsearch.crawler @@ -42,6 +46,11 @@ + + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es6/pom.xml b/distribution/es6/pom.xml index 995246c58..dc3ecdef3 100644 --- a/distribution/es6/pom.xml +++ b/distribution/es6/pom.xml @@ -12,6 +12,10 @@ fscrawler-es6 FSCrawler ZIP Distribution for Elasticsearch 6.x + + es6 + + fr.pilato.elasticsearch.crawler @@ -25,6 +29,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es7/pom.xml b/distribution/es7/pom.xml index 45e98b766..2376f48c3 100644 --- a/distribution/es7/pom.xml +++ b/distribution/es7/pom.xml @@ -12,6 +12,13 @@ fscrawler-es7 FSCrawler ZIP Distribution for Elasticsearch 7.x + + es7 + + latest + ${project.version} + + fr.pilato.elasticsearch.crawler @@ -25,6 +32,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/pom.xml b/distribution/pom.xml index fe5b73716..bcee067e1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,6 +19,49 @@ es5 + + build + dadoonet + + es7 + + + + ${project.artifactId}-nolang + ${docker.username}/fscrawler:${project.version}-${module.name}-nolang + ${docker.nolang.name} + ${project.basedir}/../src/main/docker/Dockerfile + ${project.basedir}/../src/main/assembly/assembly.xml + tgz + + + ${project.artifactId}-eng + ${docker.username}/fscrawler:${project.version}-${module.name}-eng + ${docker.eng.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-eng + + + ${project.artifactId}-fra + ${docker.username}/fscrawler:${project.version}-${module.name}-fra + ${docker.fra.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-fra + + + ${project.artifactId}-jpn + ${docker.username}/fscrawler:${project.version}-${module.name}-jpn + ${docker.jpn.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-jpn + + fr.pilato.elasticsearch.crawler @@ -110,6 +153,49 @@ + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile new file mode 100644 index 000000000..e0de41cf7 --- /dev/null +++ b/distribution/src/main/docker/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8-jdk + +ARG langsPkg +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + gettext-base \ + ${langsPkg} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY maven /usr/share/fscrawler +RUN set -ex \ + && ln -sn /usr/share/fscrawler/bin/fscrawler /usr/bin/ + +WORKDIR /usr/share/fscrawler From bd3b54ab6ac338ad73dcbb63bc98f861dbaeafe0 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:37:25 +0900 Subject: [PATCH 02/28] Use apt command and Specify libs version --- distribution/pom.xml | 7 ++++--- distribution/src/main/docker/Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index bcee067e1..62d99b0df 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,6 +20,7 @@ + 3.04.* build dadoonet @@ -41,7 +42,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-eng + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} ${project.artifactId}-fra @@ -50,7 +51,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-fra + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} ${project.artifactId}-jpn @@ -59,7 +60,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-jpn + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index e0de41cf7..eb52cb973 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -2,11 +2,11 @@ FROM openjdk:8-jdk ARG langsPkg RUN set -ex \ - && apt-get update \ - && apt-get install -y \ - gettext-base \ + && apt update \ + && apt install -y --no-install-recommends \ + "gettext-base=0.19.*" \ ${langsPkg} \ - && apt-get clean \ + && apt clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From 07e19f1c7edddd12b7f3439ee6e7fde24eee2cee Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:38:25 +0900 Subject: [PATCH 03/28] Skip the executions for IT --- distribution/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 62d99b0df..2553c680b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -195,6 +195,16 @@ build + + + start-elasticsearch + none + + + + stop-elasticsearch + none + From ce84005f9a648dc7ff2a13af64a014e06e0c09f9 Mon Sep 17 00:00:00 2001 From: otodat Date: Wed, 2 Oct 2019 01:38:36 +0900 Subject: [PATCH 04/28] Add a parameter to skip to build docker images. --- distribution/pom.xml | 4 ++++ lgtm.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 2553c680b..cd1008b61 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,6 +21,10 @@ 3.04.* + + + + ${env.DOCKER_SKIP} build dadoonet diff --git a/lgtm.yml b/lgtm.yml index 74d5e471e..a7e89d835 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -3,3 +3,6 @@ extraction: python_setup: version: 3 setup_py: false + java: + before_index: + export DOCKER_SKIP=true From 261b202e2f67381dab812e0007a7a2f4e90a7998 Mon Sep 17 00:00:00 2001 From: otodat Date: Sun, 6 Oct 2019 20:42:27 +0900 Subject: [PATCH 05/28] Add a new section about docker in getting_started --- docs/source/user/getting_started.rst | 116 +++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index a2f406ce1..c8d9614a8 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -86,3 +86,119 @@ If you would like to ignore some folders to be scanned, just add a ``.fscrawleri The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. + +Using docker +^^^^^^^^^^^^^ + +You can also run FSCrawler using docker. `The docker image is here `__. + +The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). + +.. code:: sh + + docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name + +Using with Elasticsearch installed by Docker +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you `installing Elasticsearch with docker `__, you need to communicate between each container. + +In the following, the following directory arrangement is assumed. + +.. code-block:: none + + . + ├── config + │ └── job_name + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch elasticsearch2 + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + From b40bd93ae33930c1bc907b559774aac9ea2fe92c Mon Sep 17 00:00:00 2001 From: otodat Date: Sat, 28 Sep 2019 02:36:38 +0900 Subject: [PATCH 06/28] Generate FSCrawler docker images --- distribution/es6/pom.xml | 8 +++ distribution/es7/pom.xml | 11 ++++ distribution/pom.xml | 86 +++++++++++++++++++++++++ distribution/src/main/docker/Dockerfile | 16 +++++ 4 files changed, 121 insertions(+) create mode 100644 distribution/src/main/docker/Dockerfile diff --git a/distribution/es6/pom.xml b/distribution/es6/pom.xml index 995246c58..dc3ecdef3 100644 --- a/distribution/es6/pom.xml +++ b/distribution/es6/pom.xml @@ -12,6 +12,10 @@ fscrawler-es6 FSCrawler ZIP Distribution for Elasticsearch 6.x + + es6 + + fr.pilato.elasticsearch.crawler @@ -25,6 +29,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es7/pom.xml b/distribution/es7/pom.xml index 45e98b766..2376f48c3 100644 --- a/distribution/es7/pom.xml +++ b/distribution/es7/pom.xml @@ -12,6 +12,13 @@ fscrawler-es7 FSCrawler ZIP Distribution for Elasticsearch 7.x + + es7 + + latest + ${project.version} + + fr.pilato.elasticsearch.crawler @@ -25,6 +32,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/pom.xml b/distribution/pom.xml index 09659dba6..c1b97b6d6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -18,6 +18,49 @@ es6 + + build + dadoonet + + es7 + + + + ${project.artifactId}-nolang + ${docker.username}/fscrawler:${project.version}-${module.name}-nolang + ${docker.nolang.name} + ${project.basedir}/../src/main/docker/Dockerfile + ${project.basedir}/../src/main/assembly/assembly.xml + tgz + + + ${project.artifactId}-eng + ${docker.username}/fscrawler:${project.version}-${module.name}-eng + ${docker.eng.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-eng + + + ${project.artifactId}-fra + ${docker.username}/fscrawler:${project.version}-${module.name}-fra + ${docker.fra.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-fra + + + ${project.artifactId}-jpn + ${docker.username}/fscrawler:${project.version}-${module.name}-jpn + ${docker.jpn.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-jpn + + fr.pilato.elasticsearch.crawler @@ -109,6 +152,49 @@ + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile new file mode 100644 index 000000000..e0de41cf7 --- /dev/null +++ b/distribution/src/main/docker/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8-jdk + +ARG langsPkg +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + gettext-base \ + ${langsPkg} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY maven /usr/share/fscrawler +RUN set -ex \ + && ln -sn /usr/share/fscrawler/bin/fscrawler /usr/bin/ + +WORKDIR /usr/share/fscrawler From 425877ea76cd27e6f830f044c5faf7a21b42f9d6 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:37:25 +0900 Subject: [PATCH 07/28] Use apt command and Specify libs version --- distribution/pom.xml | 7 ++++--- distribution/src/main/docker/Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index c1b97b6d6..6a3c4551d 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,6 +19,7 @@ + 3.04.* build dadoonet @@ -40,7 +41,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-eng + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} ${project.artifactId}-fra @@ -49,7 +50,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-fra + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} ${project.artifactId}-jpn @@ -58,7 +59,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-jpn + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index e0de41cf7..eb52cb973 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -2,11 +2,11 @@ FROM openjdk:8-jdk ARG langsPkg RUN set -ex \ - && apt-get update \ - && apt-get install -y \ - gettext-base \ + && apt update \ + && apt install -y --no-install-recommends \ + "gettext-base=0.19.*" \ ${langsPkg} \ - && apt-get clean \ + && apt clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From bb64a77f9ffca471a681c3807e53477544314010 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:38:25 +0900 Subject: [PATCH 08/28] Skip the executions for IT --- distribution/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 6a3c4551d..93a2a15c1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -194,6 +194,16 @@ build + + + start-elasticsearch + none + + + + stop-elasticsearch + none + From e0467cc05c185d8184580e90727463afd4dab7ba Mon Sep 17 00:00:00 2001 From: otodat Date: Wed, 2 Oct 2019 01:38:36 +0900 Subject: [PATCH 09/28] Add a parameter to skip to build docker images. --- distribution/pom.xml | 4 ++++ lgtm.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 93a2a15c1..5c2b3ea7e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,6 +20,10 @@ 3.04.* + + + + ${env.DOCKER_SKIP} build dadoonet diff --git a/lgtm.yml b/lgtm.yml index 74d5e471e..a7e89d835 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -3,3 +3,6 @@ extraction: python_setup: version: 3 setup_py: false + java: + before_index: + export DOCKER_SKIP=true From bda976b86e2e9411d45f9256831bf0358e1a0745 Mon Sep 17 00:00:00 2001 From: otodat Date: Sun, 6 Oct 2019 20:42:27 +0900 Subject: [PATCH 10/28] Add a new section about docker in getting_started --- docs/source/user/getting_started.rst | 116 +++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index a2f406ce1..c8d9614a8 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -86,3 +86,119 @@ If you would like to ignore some folders to be scanned, just add a ``.fscrawleri The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. + +Using docker +^^^^^^^^^^^^^ + +You can also run FSCrawler using docker. `The docker image is here `__. + +The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). + +.. code:: sh + + docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name + +Using with Elasticsearch installed by Docker +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you `installing Elasticsearch with docker `__, you need to communicate between each container. + +In the following, the following directory arrangement is assumed. + +.. code-block:: none + + . + ├── config + │ └── job_name + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch elasticsearch2 + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + From 9ebf3d468ee6fd5383f0cc08be766c2c530a07c9 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 11 Feb 2020 18:04:00 +0900 Subject: [PATCH 11/28] Specify minor version to avoid OS version upgrade. --- distribution/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index eb52cb973..ba2b2d784 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8-jdk +FROM openjdk:8u222-jdk ARG langsPkg RUN set -ex \ From ae6c226ed9aa43ac01a906f7571b23c457cc153d Mon Sep 17 00:00:00 2001 From: otodat Date: Sat, 28 Sep 2019 02:36:38 +0900 Subject: [PATCH 12/28] Generate FSCrawler docker images --- distribution/es6/pom.xml | 8 +++ distribution/es7/pom.xml | 11 ++++ distribution/pom.xml | 86 +++++++++++++++++++++++++ distribution/src/main/docker/Dockerfile | 16 +++++ 4 files changed, 121 insertions(+) create mode 100644 distribution/src/main/docker/Dockerfile diff --git a/distribution/es6/pom.xml b/distribution/es6/pom.xml index 995246c58..dc3ecdef3 100644 --- a/distribution/es6/pom.xml +++ b/distribution/es6/pom.xml @@ -12,6 +12,10 @@ fscrawler-es6 FSCrawler ZIP Distribution for Elasticsearch 6.x + + es6 + + fr.pilato.elasticsearch.crawler @@ -25,6 +29,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es7/pom.xml b/distribution/es7/pom.xml index 45e98b766..2376f48c3 100644 --- a/distribution/es7/pom.xml +++ b/distribution/es7/pom.xml @@ -12,6 +12,13 @@ fscrawler-es7 FSCrawler ZIP Distribution for Elasticsearch 7.x + + es7 + + latest + ${project.version} + + fr.pilato.elasticsearch.crawler @@ -25,6 +32,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/pom.xml b/distribution/pom.xml index 09659dba6..c1b97b6d6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -18,6 +18,49 @@ es6 + + build + dadoonet + + es7 + + + + ${project.artifactId}-nolang + ${docker.username}/fscrawler:${project.version}-${module.name}-nolang + ${docker.nolang.name} + ${project.basedir}/../src/main/docker/Dockerfile + ${project.basedir}/../src/main/assembly/assembly.xml + tgz + + + ${project.artifactId}-eng + ${docker.username}/fscrawler:${project.version}-${module.name}-eng + ${docker.eng.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-eng + + + ${project.artifactId}-fra + ${docker.username}/fscrawler:${project.version}-${module.name}-fra + ${docker.fra.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-fra + + + ${project.artifactId}-jpn + ${docker.username}/fscrawler:${project.version}-${module.name}-jpn + ${docker.jpn.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr tesseract-ocr-jpn + + fr.pilato.elasticsearch.crawler @@ -109,6 +152,49 @@ + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile new file mode 100644 index 000000000..e0de41cf7 --- /dev/null +++ b/distribution/src/main/docker/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8-jdk + +ARG langsPkg +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + gettext-base \ + ${langsPkg} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY maven /usr/share/fscrawler +RUN set -ex \ + && ln -sn /usr/share/fscrawler/bin/fscrawler /usr/bin/ + +WORKDIR /usr/share/fscrawler From 0062321aa55ea597065e74510bc119bfb6150424 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:37:25 +0900 Subject: [PATCH 13/28] Use apt command and Specify libs version --- distribution/pom.xml | 7 ++++--- distribution/src/main/docker/Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index c1b97b6d6..6a3c4551d 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,6 +19,7 @@ + 3.04.* build dadoonet @@ -40,7 +41,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-eng + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} ${project.artifactId}-fra @@ -49,7 +50,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-fra + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} ${project.artifactId}-jpn @@ -58,7 +59,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-jpn + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index e0de41cf7..eb52cb973 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -2,11 +2,11 @@ FROM openjdk:8-jdk ARG langsPkg RUN set -ex \ - && apt-get update \ - && apt-get install -y \ - gettext-base \ + && apt update \ + && apt install -y --no-install-recommends \ + "gettext-base=0.19.*" \ ${langsPkg} \ - && apt-get clean \ + && apt clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From c6edf10c736e51adcca2ec30e066bd3213d4fb59 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:38:25 +0900 Subject: [PATCH 14/28] Skip the executions for IT --- distribution/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 6a3c4551d..93a2a15c1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -194,6 +194,16 @@ build + + + start-elasticsearch + none + + + + stop-elasticsearch + none + From 9252aa4c8811184c31d5f277bde3472fe9b8e94d Mon Sep 17 00:00:00 2001 From: otodat Date: Wed, 2 Oct 2019 01:38:36 +0900 Subject: [PATCH 15/28] Add a parameter to skip to build docker images. --- distribution/pom.xml | 4 ++++ lgtm.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 93a2a15c1..5c2b3ea7e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,6 +20,10 @@ 3.04.* + + + + ${env.DOCKER_SKIP} build dadoonet diff --git a/lgtm.yml b/lgtm.yml index 74d5e471e..a7e89d835 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -3,3 +3,6 @@ extraction: python_setup: version: 3 setup_py: false + java: + before_index: + export DOCKER_SKIP=true From fa62ba9ae0eedf8fc2f21a262018ca58e27d37dd Mon Sep 17 00:00:00 2001 From: otodat Date: Sun, 6 Oct 2019 20:42:27 +0900 Subject: [PATCH 16/28] Add a new section about docker in getting_started --- docs/source/user/getting_started.rst | 116 +++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index a2f406ce1..c8d9614a8 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -86,3 +86,119 @@ If you would like to ignore some folders to be scanned, just add a ``.fscrawleri The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. + +Using docker +^^^^^^^^^^^^^ + +You can also run FSCrawler using docker. `The docker image is here `__. + +The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). + +.. code:: sh + + docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name + +Using with Elasticsearch installed by Docker +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you `installing Elasticsearch with docker `__, you need to communicate between each container. + +In the following, the following directory arrangement is assumed. + +.. code-block:: none + + . + ├── config + │ └── job_name + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch elasticsearch2 + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + From 42e611aeb5c841cc3ba8f3dd2e360190f309c7e3 Mon Sep 17 00:00:00 2001 From: otodat Date: Sat, 28 Sep 2019 02:36:38 +0900 Subject: [PATCH 17/28] Generate FSCrawler docker images --- distribution/pom.xml | 21 +++------------------ distribution/src/main/docker/Dockerfile | 8 ++++---- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 5c2b3ea7e..c1b97b6d6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,11 +19,6 @@ - 3.04.* - - - - ${env.DOCKER_SKIP} build dadoonet @@ -45,7 +40,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} + tesseract-ocr tesseract-ocr-eng ${project.artifactId}-fra @@ -54,7 +49,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} + tesseract-ocr tesseract-ocr-fra ${project.artifactId}-jpn @@ -63,7 +58,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} + tesseract-ocr tesseract-ocr-jpn @@ -198,16 +193,6 @@ build - - - start-elasticsearch - none - - - - stop-elasticsearch - none - diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index eb52cb973..e0de41cf7 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -2,11 +2,11 @@ FROM openjdk:8-jdk ARG langsPkg RUN set -ex \ - && apt update \ - && apt install -y --no-install-recommends \ - "gettext-base=0.19.*" \ + && apt-get update \ + && apt-get install -y \ + gettext-base \ ${langsPkg} \ - && apt clean \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From aa01afc97cb7578f648b1ab211a1f267c7781280 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:37:25 +0900 Subject: [PATCH 18/28] Use apt command and Specify libs version --- distribution/pom.xml | 7 ++++--- distribution/src/main/docker/Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index c1b97b6d6..6a3c4551d 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,6 +19,7 @@ + 3.04.* build dadoonet @@ -40,7 +41,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-eng + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} ${project.artifactId}-fra @@ -49,7 +50,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-fra + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} ${project.artifactId}-jpn @@ -58,7 +59,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr tesseract-ocr-jpn + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index e0de41cf7..eb52cb973 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -2,11 +2,11 @@ FROM openjdk:8-jdk ARG langsPkg RUN set -ex \ - && apt-get update \ - && apt-get install -y \ - gettext-base \ + && apt update \ + && apt install -y --no-install-recommends \ + "gettext-base=0.19.*" \ ${langsPkg} \ - && apt-get clean \ + && apt clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From 535f82d7b9b3123951471d5e3549ffd556ed3d92 Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 1 Oct 2019 08:38:25 +0900 Subject: [PATCH 19/28] Skip the executions for IT --- distribution/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 6a3c4551d..93a2a15c1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -194,6 +194,16 @@ build + + + start-elasticsearch + none + + + + stop-elasticsearch + none + From 8d8c7f85016331c8f83e041e9738f90184cadba6 Mon Sep 17 00:00:00 2001 From: otodat Date: Wed, 2 Oct 2019 01:38:36 +0900 Subject: [PATCH 20/28] Add a parameter to skip to build docker images. --- distribution/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 93a2a15c1..5c2b3ea7e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,6 +20,10 @@ 3.04.* + + + + ${env.DOCKER_SKIP} build dadoonet From f8b049373f3c216a2399c1c4c63166d3bf3655fe Mon Sep 17 00:00:00 2001 From: otodat Date: Tue, 11 Feb 2020 18:04:00 +0900 Subject: [PATCH 21/28] Specify minor version to avoid OS version upgrade. --- distribution/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index eb52cb973..ba2b2d784 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8-jdk +FROM openjdk:8u222-jdk ARG langsPkg RUN set -ex \ From af91b8af9f697bb42b8bc4417c54019e67ebec73 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 10 Mar 2020 18:22:32 +0100 Subject: [PATCH 22/28] Add documentation, fix the release process for Docker images Still WIP --- distribution/pom.xml | 13 +++++++++++++ docs/source/dev/build.rst | 21 +++++++++++++++++++++ docs/source/dev/release.rst | 3 +++ docs/source/user/tips.rst | 3 +++ pom.xml | 6 ++++++ release.sh | 13 +++++++++++++ 6 files changed, 59 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 5c2b3ea7e..7353fc1ad 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -162,6 +162,12 @@ io.fabric8 docker-maven-plugin + + + ${docker.push.username} + ${docker.push.password} + + @@ -198,6 +204,13 @@ build + + docker-push + deploy + + push + + start-elasticsearch diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index 40e10839c..e9141796f 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -150,3 +150,24 @@ If you want to skip the check, you can run with ``-Dossindex.fail=false``:: mvn clean install -Dossindex.fail=false +DockerHub publication +^^^^^^^^^^^^^^^^^^^^^ + +To publish the latest build to `DockerHub `_ you can manually +call ``docker:push`` maven task and provide credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn -f distribution/pom.xml docker:push \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +Otherwise, if you call the maven ``deploy`` phase, it will be done automatically. +Note that it will still require that you provide the credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn deploy \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +You can also provide the settings as environment variables: + +* ``env.DOCKER_USERNAME`` or ``DOCKER_USERNAME`` +* ``env.DOCKER_PASSWORD`` or ``DOCKER_PASSWORD`` diff --git a/docs/source/dev/release.rst b/docs/source/dev/release.rst index 9e5e43464..7fc012f17 100644 --- a/docs/source/dev/release.rst +++ b/docs/source/dev/release.rst @@ -11,3 +11,6 @@ And follow the instructions. Only developers with write rights to the sonatype repository under ``fr.pilato`` space can perform the release. + + Only developers with write rights to the `DockerHub repository `_ + can push the Docker images. diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index a23ba26d6..393e3095f 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -38,3 +38,6 @@ To use FSCrawler with `docker `__, check `docker-fscrawler `__ recipe. +TODO: replace with the docker documentation as we are now building all this from +this repository. + diff --git a/pom.xml b/pom.xml index 482f0ae32..3ec2d8421 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,12 @@ ${env.SONATYPE_USER} ${env.SONATYPE_PASS} + + ${DOCKER_USERNAME} + ${DOCKER_PASSWORD} + ${env.DOCKER_USERNAME} + ${env.DOCKER_PASSWORD} + 1.8 UTF-8 UTF-8 diff --git a/release.sh b/release.sh index 295feed82..1fe4ef86c 100755 --- a/release.sh +++ b/release.sh @@ -201,6 +201,19 @@ then fi fi +if [ ${DRY_RUN} -eq 0 ] +then + echo "Inspect DockerHub" + open https://hub.docker.com/r/dadoonet/fscrawler/tags + if promptyn "Is the DockerHub repository ok?" + then + echo "We can continue the release process." + else + echo "We cancel the release process." + RELEASE=0 + fi +fi + # We are releasing, so let's merge into the original branch if [ ${RELEASE} -eq 1 ] then From c465b04d8852e08fa4272cdf8a51d908a4c49092 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 11 Mar 2020 12:12:22 +0100 Subject: [PATCH 23/28] Add more documentation --- docs/source/user/tips.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index 393e3095f..5dca1d7d0 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -41,3 +41,21 @@ recipe. TODO: replace with the docker documentation as we are now building all this from this repository. +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Run it: + +.. code:: sh + + docker run dadoonet/fscrawler job + +Using docker compose +-------------------- + +TODO fill this with docker compose instructions + + From 5ffab0dffd4ac686dfe27543d1202a55538169a5 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 12 Mar 2020 11:59:02 +0100 Subject: [PATCH 24/28] Move doc from tips to installation guide --- docs/source/installation.rst | 28 ++++++++++++++++++++++++++++ docs/source/user/tips.rst | 27 --------------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 158a5a2c4..5e91e2c8d 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -58,6 +58,34 @@ The distribution contains: ├── ... All needed jars +Using docker +------------ + +To use FSCrawler with `docker `__, check +`docker-fscrawler `__ +recipe. + +TODO: replace with the docker documentation as we are now building all this from +this repository. + +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Run it: + +.. code:: sh + + docker run dadoonet/fscrawler job + +Using docker compose +-------------------- + +TODO fill this with docker compose instructions + + Running as a Service on Windows ------------------------------- diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index 5dca1d7d0..5022e71a9 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -31,31 +31,4 @@ machine `__ and run FS crawler on this mount point. You can also read details about `HDFS NFS Gateway `__. -Using docker ------------- - -To use FSCrawler with `docker `__, check -`docker-fscrawler `__ -recipe. - -TODO: replace with the docker documentation as we are now building all this from -this repository. - -Pull the Docker image: - -.. code:: sh - - docker pull dadoonet/fscrawler - -Run it: - -.. code:: sh - - docker run dadoonet/fscrawler job - -Using docker compose --------------------- - -TODO fill this with docker compose instructions - From cbfd77d512a1b4cc9bcd1fc1c8b0f8ea957e7f22 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 2 Apr 2020 16:34:06 +0200 Subject: [PATCH 25/28] Move docker documentation in the installation guide --- docs/source/installation.rst | 136 +++++++++++++++++++++++++-- docs/source/user/getting_started.rst | 115 ---------------------- 2 files changed, 126 insertions(+), 125 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 5e91e2c8d..597d4aa87 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -61,29 +61,145 @@ The distribution contains: Using docker ------------ -To use FSCrawler with `docker `__, check -`docker-fscrawler `__ -recipe. - -TODO: replace with the docker documentation as we are now building all this from -this repository. - Pull the Docker image: .. code:: sh docker pull dadoonet/fscrawler -Run it: +Let say your documents are located in ``~/tmp`` dir and you want to store your fscrawler jobs in ``~/.fscrawler``. +You can run FSCrawler with: .. code:: sh - docker run dadoonet/fscrawler job + docker run -it --rm -v ~/.fscrawler:/root/.fscrawler -v ~/tmp:/tmp/es:ro dadoonet/fscrawler fscrawler job_name + +On the first run, if the job does not exist yet in ``~/.fscrawler``, FSCrawler will ask you if you want to create it: + +:: + + 10:16:53,880 INFO [f.p.e.c.f.c.BootstrapChecks] Memory [Free/Total=Percent]: HEAP [67.3mb/876.5mb=7.69%], RAM [2.1gb/3.8gb=55.43%], Swap [1023.9mb/1023.9mb=100.0%]. + 10:16:53,899 WARN [f.p.e.c.f.c.FsCrawlerCli] job [job_name] does not exist + 10:16:53,900 INFO [f.p.e.c.f.c.FsCrawlerCli] Do you want to create it (Y/N)? + y + 10:16:56,745 INFO [f.p.e.c.f.c.FsCrawlerCli] Settings have been created in [/root/.fscrawler/job_name/_settings.yaml]. Please review and edit before relaunch + +.. note:: + + The configuration file is actually stored on your machine in ``~/.fscrawler/job_name/_settings.yaml``. + Remember to change the URL of your elasticsearch instance as the container won't be able to see it + running under the default ``127.0.0.1``. You will need to use the actual IP address of the host. + Using docker compose -------------------- -TODO fill this with docker compose instructions +In this section, the following directory layout is assumed: + +.. code-block:: none + + . + ├── config + │ └── job + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml``. + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch + docker-compose logs -f elasticsearch + +Wait for elasticsearch to be started: + +:: + + + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + Running as a Service on Windows diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index c8d9614a8..b77481aa2 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -87,118 +87,3 @@ The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. -Using docker -^^^^^^^^^^^^^ - -You can also run FSCrawler using docker. `The docker image is here `__. - -The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). - -.. code:: sh - - docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name - -Using with Elasticsearch installed by Docker -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you `installing Elasticsearch with docker `__, you need to communicate between each container. - -In the following, the following directory arrangement is assumed. - -.. code-block:: none - - . - ├── config - │ └── job_name - │ └── _settings.yaml - ├── data - │ └── - └── docker-compose.yml - -For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). - -.. code:: yaml - - name: "test" - elasticsearch: - nodes: - - url: "http://elasticsearch:9200" - -And, prepare the following ``docker-compose.yml``. - -.. code:: yaml - - version: '2.2' - services: - # FSCrawler - fscrawler: - image: toto1310/fscrawler - container_name: fscrawler - volumes: - - ${PWD}/config:/root/.fscrawler - - ${PWD}/data:/tmp/es - networks: - - esnet - command: fscrawler job_name - - # Elasticsearch Cluster - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 - container_name: elasticsearch - environment: - - node.name=elasticsearch - - discovery.seed_hosts=elasticsearch2 - - cluster.initial_master_nodes=elasticsearch,elasticsearch2 - - cluster.name=docker-cluster - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - esdata01:/usr/share/elasticsearch/data - ports: - - 9200:9200 - networks: - - esnet - elasticsearch2: - image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 - container_name: elasticsearch2 - environment: - - node.name=elasticsearch2 - - discovery.seed_hosts=elasticsearch - - cluster.initial_master_nodes=elasticsearch,elasticsearch2 - - cluster.name=docker-cluster - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - esdata02:/usr/share/elasticsearch/data - networks: - - esnet - - volumes: - esdata01: - driver: local - esdata02: - driver: local - - networks: - esnet: - -Then, you can run Elasticsearch. - -.. code:: sh - - docker-compose up -d elasticsearch elasticsearch2 - -After starting Elasticsearch, you can run FSCrawler. - -.. code:: sh - - docker-compose up fscrawler - From 29c747ed57cae04bef942814928b17ad81f5fcd0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:21:33 +0100 Subject: [PATCH 26/28] Fix typo --- distribution/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 7353fc1ad..dfed7d65b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -47,7 +47,7 @@ ${docker.nolang.assembly.mode} tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} - + ${project.artifactId}-fra ${docker.username}/fscrawler:${project.version}-${module.name}-fra ${docker.fra.name} From 1df150b531ccfec339f3e49ccc7937796d0c3ff0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:24:39 +0100 Subject: [PATCH 27/28] Let's try to use openjdk 15 As we upgraded the needed JDK --- distribution/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index ba2b2d784..86d898cd0 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8u222-jdk +FROM openjdk:15.0.1 ARG langsPkg RUN set -ex \ From e501e34f43e5187f260e86b6bd1fefd32f117769 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:26:24 +0100 Subject: [PATCH 28/28] Update image name --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index e54237b8b..ae9e147b1 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -125,7 +125,7 @@ And, prepare the following ``docker-compose.yml``. services: # FSCrawler fscrawler: - image: toto1310/fscrawler + image: dadoonet/fscrawler container_name: fscrawler volumes: - ${PWD}/config:/root/.fscrawler