From 68b29cb9a7aa8058c7a8bf248dac6a6dab80f687 Mon Sep 17 00:00:00 2001 From: otodat Date: Sat, 28 Sep 2019 02:36:38 +0900 Subject: [PATCH 01/11] Generate FSCrawler docker images * Generate FSCrawler docker images * Use apt command and Specify libs version * Skip the executions for IT * Add a parameter to skip to build docker images. * Add a new section about docker in getting_started * Specify minor version to avoid OS version upgrade. --- distribution/es6/pom.xml | 8 + distribution/es7/pom.xml | 11 ++ distribution/pom.xml | 207 ++++++++++++++++++++++++ distribution/src/main/docker/Dockerfile | 16 ++ docs/source/user/getting_started.rst | 116 +++++++++++++ lgtm.yml | 2 + 6 files changed, 360 insertions(+) create mode 100644 distribution/src/main/docker/Dockerfile diff --git a/distribution/es6/pom.xml b/distribution/es6/pom.xml index 995246c58..dc3ecdef3 100644 --- a/distribution/es6/pom.xml +++ b/distribution/es6/pom.xml @@ -12,6 +12,10 @@ fscrawler-es6 FSCrawler ZIP Distribution for Elasticsearch 6.x + + es6 + + fr.pilato.elasticsearch.crawler @@ -25,6 +29,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es7/pom.xml b/distribution/es7/pom.xml index 45e98b766..2376f48c3 100644 --- a/distribution/es7/pom.xml +++ b/distribution/es7/pom.xml @@ -12,6 +12,13 @@ fscrawler-es7 FSCrawler ZIP Distribution for Elasticsearch 7.x + + es7 + + latest + ${project.version} + + fr.pilato.elasticsearch.crawler @@ -25,6 +32,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/pom.xml b/distribution/pom.xml index 09659dba6..8d9ab29c1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -18,6 +18,54 @@ es6 + + 3.04.* + + + + ${env.DOCKER_SKIP} + build + dadoonet + + es7 + + + + ${project.artifactId}-nolang + ${docker.username}/fscrawler:${project.version}-${module.name}-nolang + ${docker.nolang.name} + ${project.basedir}/../src/main/docker/Dockerfile + ${project.basedir}/../src/main/assembly/assembly.xml + tgz + + + ${project.artifactId}-eng + ${docker.username}/fscrawler:${project.version}-${module.name}-eng + ${docker.eng.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} + + + ${project.artifactId}-fra + ${docker.username}/fscrawler:${project.version}-${module.name}-fra + ${docker.fra.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} + + + ${project.artifactId}-jpn + ${docker.username}/fscrawler:${project.version}-${module.name}-jpn + ${docker.jpn.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} + + fr.pilato.elasticsearch.crawler @@ -109,6 +157,165 @@ + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + + start-elasticsearch + none + + + + stop-elasticsearch + none + + + + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + + start-elasticsearch + none + + + + stop-elasticsearch + none + + + + + + io.fabric8 + docker-maven-plugin + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + + start-elasticsearch + none + + + + stop-elasticsearch + none + + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile new file mode 100644 index 000000000..ba2b2d784 --- /dev/null +++ b/distribution/src/main/docker/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8u222-jdk + +ARG langsPkg +RUN set -ex \ + && apt update \ + && apt install -y --no-install-recommends \ + "gettext-base=0.19.*" \ + ${langsPkg} \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* + +COPY maven /usr/share/fscrawler +RUN set -ex \ + && ln -sn /usr/share/fscrawler/bin/fscrawler /usr/bin/ + +WORKDIR /usr/share/fscrawler diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index 6a4428a28..e89bde20b 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -86,3 +86,119 @@ If you would like to ignore some folders to be scanned, just add a ``.fscrawleri The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. + +Using docker +^^^^^^^^^^^^^ + +You can also run FSCrawler using docker. `The docker image is here `__. + +The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). + +.. code:: sh + + docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name + +Using with Elasticsearch installed by Docker +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you `installing Elasticsearch with docker `__, you need to communicate between each container. + +In the following, the following directory arrangement is assumed. + +.. code-block:: none + + . + ├── config + │ └── job_name + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch elasticsearch2 + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + diff --git a/lgtm.yml b/lgtm.yml index 7637c4dd6..2929ddcd0 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -4,5 +4,7 @@ extraction: version: 3 setup_py: false java: + before_index: + export DOCKER_SKIP=true index: java_version: "14" From 546b1e4b41baeb3c44d6350928dc3925a2ad2eaa Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 10 Mar 2020 18:22:32 +0100 Subject: [PATCH 02/11] Add documentation, fix the release process for Docker images Still WIP --- distribution/pom.xml | 13 +++++++++++++ docs/source/dev/build.rst | 21 +++++++++++++++++++++ docs/source/dev/release.rst | 3 +++ pom.xml | 6 ++++++ release.sh | 13 +++++++++++++ 5 files changed, 56 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 8d9ab29c1..d2d47a983 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -162,6 +162,12 @@ io.fabric8 docker-maven-plugin + + + ${docker.push.username} + ${docker.push.password} + + @@ -198,6 +204,13 @@ build + + docker-push + deploy + + push + + start-elasticsearch diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index bb6a7ee4c..8ffa34f64 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -229,3 +229,24 @@ If you want to skip the check, you can run with ``-Dossindex.fail=false``:: mvn clean install -Dossindex.fail=false +DockerHub publication +^^^^^^^^^^^^^^^^^^^^^ + +To publish the latest build to `DockerHub `_ you can manually +call ``docker:push`` maven task and provide credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn -f distribution/pom.xml docker:push \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +Otherwise, if you call the maven ``deploy`` phase, it will be done automatically. +Note that it will still require that you provide the credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn deploy \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +You can also provide the settings as environment variables: + +* ``env.DOCKER_USERNAME`` or ``DOCKER_USERNAME`` +* ``env.DOCKER_PASSWORD`` or ``DOCKER_PASSWORD`` diff --git a/docs/source/dev/release.rst b/docs/source/dev/release.rst index 4b1957d3a..d3667c4c4 100644 --- a/docs/source/dev/release.rst +++ b/docs/source/dev/release.rst @@ -31,3 +31,6 @@ the tests while building the release. Only developers with write rights to the sonatype repository under ``fr.pilato`` space can perform the release. + + Only developers with write rights to the `DockerHub repository `_ + can push the Docker images. diff --git a/pom.xml b/pom.xml index bb4c3556d..e7b435995 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,12 @@ ${env.SONATYPE_USER} ${env.SONATYPE_PASS} + + ${DOCKER_USERNAME} + ${DOCKER_PASSWORD} + ${env.DOCKER_USERNAME} + ${env.DOCKER_PASSWORD} + 14 UTF-8 UTF-8 diff --git a/release.sh b/release.sh index 295feed82..1fe4ef86c 100755 --- a/release.sh +++ b/release.sh @@ -201,6 +201,19 @@ then fi fi +if [ ${DRY_RUN} -eq 0 ] +then + echo "Inspect DockerHub" + open https://hub.docker.com/r/dadoonet/fscrawler/tags + if promptyn "Is the DockerHub repository ok?" + then + echo "We can continue the release process." + else + echo "We cancel the release process." + RELEASE=0 + fi +fi + # We are releasing, so let's merge into the original branch if [ ${RELEASE} -eq 1 ] then From 88734292652331f7f22d47be2f5d8ddd5e3c3274 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 11 Mar 2020 12:12:22 +0100 Subject: [PATCH 03/11] Add more documentation --- docs/source/user/tips.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index 0859cb20f..b26c858ac 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -91,3 +91,21 @@ Then expose the docker container you've created by changing the IP of the REST U url: "http://fscrawler:8080" +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Run it: + +.. code:: sh + + docker run dadoonet/fscrawler job + +Using docker compose +-------------------- + +TODO fill this with docker compose instructions + + From 9b512277032285188b4d1722c5eb5d447a17d4a1 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 12 Mar 2020 11:59:02 +0100 Subject: [PATCH 04/11] Move doc from tips to installation guide --- docs/source/installation.rst | 28 ++++++++++++++++++++++++++++ docs/source/user/tips.rst | 5 ----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4c1f89f6d..2c73bfd02 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -60,6 +60,34 @@ The distribution contains: ├── ... All needed jars +Using docker +------------ + +To use FSCrawler with `docker `__, check +`docker-fscrawler `__ +recipe. + +TODO: replace with the docker documentation as we are now building all this from +this repository. + +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Run it: + +.. code:: sh + + docker run dadoonet/fscrawler job + +Using docker compose +-------------------- + +TODO fill this with docker compose instructions + + Running as a Service on Windows ------------------------------- diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index b26c858ac..8ae04ea75 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -103,9 +103,4 @@ Run it: docker run dadoonet/fscrawler job -Using docker compose --------------------- - -TODO fill this with docker compose instructions - From 9bed8aa33189826b63bce84692a4e0104325aa88 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 2 Apr 2020 16:34:06 +0200 Subject: [PATCH 05/11] Move docker documentation in the installation guide --- docs/source/installation.rst | 136 +++++++++++++++++++++++++-- docs/source/user/getting_started.rst | 115 ---------------------- 2 files changed, 126 insertions(+), 125 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 2c73bfd02..e54237b8b 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -63,29 +63,145 @@ The distribution contains: Using docker ------------ -To use FSCrawler with `docker `__, check -`docker-fscrawler `__ -recipe. - -TODO: replace with the docker documentation as we are now building all this from -this repository. - Pull the Docker image: .. code:: sh docker pull dadoonet/fscrawler -Run it: +Let say your documents are located in ``~/tmp`` dir and you want to store your fscrawler jobs in ``~/.fscrawler``. +You can run FSCrawler with: .. code:: sh - docker run dadoonet/fscrawler job + docker run -it --rm -v ~/.fscrawler:/root/.fscrawler -v ~/tmp:/tmp/es:ro dadoonet/fscrawler fscrawler job_name + +On the first run, if the job does not exist yet in ``~/.fscrawler``, FSCrawler will ask you if you want to create it: + +:: + + 10:16:53,880 INFO [f.p.e.c.f.c.BootstrapChecks] Memory [Free/Total=Percent]: HEAP [67.3mb/876.5mb=7.69%], RAM [2.1gb/3.8gb=55.43%], Swap [1023.9mb/1023.9mb=100.0%]. + 10:16:53,899 WARN [f.p.e.c.f.c.FsCrawlerCli] job [job_name] does not exist + 10:16:53,900 INFO [f.p.e.c.f.c.FsCrawlerCli] Do you want to create it (Y/N)? + y + 10:16:56,745 INFO [f.p.e.c.f.c.FsCrawlerCli] Settings have been created in [/root/.fscrawler/job_name/_settings.yaml]. Please review and edit before relaunch + +.. note:: + + The configuration file is actually stored on your machine in ``~/.fscrawler/job_name/_settings.yaml``. + Remember to change the URL of your elasticsearch instance as the container won't be able to see it + running under the default ``127.0.0.1``. You will need to use the actual IP address of the host. + Using docker compose -------------------- -TODO fill this with docker compose instructions +In this section, the following directory layout is assumed: + +.. code-block:: none + + . + ├── config + │ └── job + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml``. + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: toto1310/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch + docker-compose logs -f elasticsearch + +Wait for elasticsearch to be started: + +:: + + + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + Running as a Service on Windows diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index e89bde20b..a77a1b392 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -87,118 +87,3 @@ The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. -Using docker -^^^^^^^^^^^^^ - -You can also run FSCrawler using docker. `The docker image is here `__. - -The following command let FSCrawler read its configuration files from ``/root/.fscrawler`` (i.e. ``--config_dir``) and its target files from ``/tmp/es`` (i.e. ``fs.url``). - -.. code:: sh - - docker run -it --rm -v ${PWD}/config:/root/.fscrawler -v ${PWD}/data:/tmp/es:ro toto1310/fscrawler fscrawler job_name - -Using with Elasticsearch installed by Docker -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you `installing Elasticsearch with docker `__, you need to communicate between each container. - -In the following, the following directory arrangement is assumed. - -.. code-block:: none - - . - ├── config - │ └── job_name - │ └── _settings.yaml - ├── data - │ └── - └── docker-compose.yml - -For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml`` (or ``_settings.json``). - -.. code:: yaml - - name: "test" - elasticsearch: - nodes: - - url: "http://elasticsearch:9200" - -And, prepare the following ``docker-compose.yml``. - -.. code:: yaml - - version: '2.2' - services: - # FSCrawler - fscrawler: - image: toto1310/fscrawler - container_name: fscrawler - volumes: - - ${PWD}/config:/root/.fscrawler - - ${PWD}/data:/tmp/es - networks: - - esnet - command: fscrawler job_name - - # Elasticsearch Cluster - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 - container_name: elasticsearch - environment: - - node.name=elasticsearch - - discovery.seed_hosts=elasticsearch2 - - cluster.initial_master_nodes=elasticsearch,elasticsearch2 - - cluster.name=docker-cluster - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - esdata01:/usr/share/elasticsearch/data - ports: - - 9200:9200 - networks: - - esnet - elasticsearch2: - image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 - container_name: elasticsearch2 - environment: - - node.name=elasticsearch2 - - discovery.seed_hosts=elasticsearch - - cluster.initial_master_nodes=elasticsearch,elasticsearch2 - - cluster.name=docker-cluster - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - esdata02:/usr/share/elasticsearch/data - networks: - - esnet - - volumes: - esdata01: - driver: local - esdata02: - driver: local - - networks: - esnet: - -Then, you can run Elasticsearch. - -.. code:: sh - - docker-compose up -d elasticsearch elasticsearch2 - -After starting Elasticsearch, you can run FSCrawler. - -.. code:: sh - - docker-compose up fscrawler - From 354a08fa637dc6825bfd8cd36626d660ec2b74ca Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:21:33 +0100 Subject: [PATCH 06/11] Fix typo --- distribution/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index d2d47a983..fa10b2611 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -47,7 +47,7 @@ ${docker.nolang.assembly.mode} tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} - + ${project.artifactId}-fra ${docker.username}/fscrawler:${project.version}-${module.name}-fra ${docker.fra.name} From e745d61929cb7fd0b0a7ed42a5c26432106f11de Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:24:39 +0100 Subject: [PATCH 07/11] Let's try to use openjdk 15 As we upgraded the needed JDK --- distribution/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index ba2b2d784..86d898cd0 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8u222-jdk +FROM openjdk:15.0.1 ARG langsPkg RUN set -ex \ From 28463e5036b346429ab841c8453d7893303077b8 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Dec 2020 11:26:24 +0100 Subject: [PATCH 08/11] Update image name --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index e54237b8b..ae9e147b1 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -125,7 +125,7 @@ And, prepare the following ``docker-compose.yml``. services: # FSCrawler fscrawler: - image: toto1310/fscrawler + image: dadoonet/fscrawler container_name: fscrawler volumes: - ${PWD}/config:/root/.fscrawler From cee9a432286ebdd5cc5dab8a29287917fc4b44f3 Mon Sep 17 00:00:00 2001 From: Erwan Arzur Date: Tue, 30 Mar 2021 12:56:32 +0200 Subject: [PATCH 09/11] update base image to openjdk:15.0.2-jdk-slim-buster and replace apt commands with apt-get (hadolint recommandations) --- distribution/src/main/docker/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile index 86d898cd0..c169bf084 100644 --- a/distribution/src/main/docker/Dockerfile +++ b/distribution/src/main/docker/Dockerfile @@ -1,12 +1,13 @@ -FROM openjdk:15.0.1 +FROM openjdk:15.0.2-jdk-slim-buster ARG langsPkg + RUN set -ex \ - && apt update \ - && apt install -y --no-install-recommends \ - "gettext-base=0.19.*" \ - ${langsPkg} \ - && apt clean \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + "gettext-base=0.19.*" \ + ${langsPkg} \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* COPY maven /usr/share/fscrawler From 2140533ead253c697048a99fc8ab7b7289bb6558 Mon Sep 17 00:00:00 2001 From: Erwan Arzur Date: Tue, 30 Mar 2021 12:58:33 +0200 Subject: [PATCH 10/11] upgrade tesseract-ocr to 4.0.0 - use a different version spec for language packs --- distribution/pom.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index fa10b2611..9f895bddb 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -19,7 +19,8 @@ - 3.04.* + 4.0.0* + 1:4.00* @@ -45,7 +46,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.lang.version} ${project.artifactId}-fra @@ -54,7 +55,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.lang.version} ${project.artifactId}-jpn @@ -63,7 +64,7 @@ ${docker.nolang.dockerFile} ${docker.nolang.assembly.descriptor} ${docker.nolang.assembly.mode} - tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.version} + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.lang.version} From 691ddf1c287db15d01fdf8e95fe10a67f6355d2d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 23 Apr 2021 11:02:58 +0200 Subject: [PATCH 11/11] Fix the deploy phase We had 3 times the same plugin and deploy did not deploy to dockerhub because the configuration was overwritten. --- distribution/pom.xml | 106 ------------------------------------------- 1 file changed, 106 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 9f895bddb..24662f3b6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -224,112 +224,6 @@ - - - io.fabric8 - docker-maven-plugin - - - - - - properties - docker.nolang - - - - - properties - docker.eng - - - - - properties - docker.fra - - - - - properties - docker.jpn - - - - - - - docker-build - package - - build - - - - - start-elasticsearch - none - - - - stop-elasticsearch - none - - - - - - io.fabric8 - docker-maven-plugin - - - - - - properties - docker.nolang - - - - - properties - docker.eng - - - - - properties - docker.fra - - - - - properties - docker.jpn - - - - - - - docker-build - package - - build - - - - - start-elasticsearch - none - - - - stop-elasticsearch - none - - -