diff --git a/distribution/es6/pom.xml b/distribution/es6/pom.xml index 995246c58..dc3ecdef3 100644 --- a/distribution/es6/pom.xml +++ b/distribution/es6/pom.xml @@ -12,6 +12,10 @@ fscrawler-es6 FSCrawler ZIP Distribution for Elasticsearch 6.x + + es6 + + fr.pilato.elasticsearch.crawler @@ -25,6 +29,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/es7/pom.xml b/distribution/es7/pom.xml index 45e98b766..2376f48c3 100644 --- a/distribution/es7/pom.xml +++ b/distribution/es7/pom.xml @@ -12,6 +12,13 @@ fscrawler-es7 FSCrawler ZIP Distribution for Elasticsearch 7.x + + es7 + + latest + ${project.version} + + fr.pilato.elasticsearch.crawler @@ -25,6 +32,10 @@ org.apache.maven.plugins maven-assembly-plugin + + io.fabric8 + docker-maven-plugin + diff --git a/distribution/pom.xml b/distribution/pom.xml index 09659dba6..24662f3b6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -18,6 +18,55 @@ es6 + + 4.0.0* + 1:4.00* + + + + ${env.DOCKER_SKIP} + build + dadoonet + + es7 + + + + ${project.artifactId}-nolang + ${docker.username}/fscrawler:${project.version}-${module.name}-nolang + ${docker.nolang.name} + ${project.basedir}/../src/main/docker/Dockerfile + ${project.basedir}/../src/main/assembly/assembly.xml + tgz + + + ${project.artifactId}-eng + ${docker.username}/fscrawler:${project.version}-${module.name}-eng + ${docker.eng.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-eng=${tesseract.lang.version} + + + ${project.artifactId}-fra + ${docker.username}/fscrawler:${project.version}-${module.name}-fra + ${docker.fra.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-fra=${tesseract.lang.version} + + + ${project.artifactId}-jpn + ${docker.username}/fscrawler:${project.version}-${module.name}-jpn + ${docker.jpn.name} + ${docker.nolang.dockerFile} + ${docker.nolang.assembly.descriptor} + ${docker.nolang.assembly.mode} + tesseract-ocr=${tesseract.version} tesseract-ocr-jpn=${tesseract.lang.version} + + fr.pilato.elasticsearch.crawler @@ -109,6 +158,72 @@ + + + io.fabric8 + docker-maven-plugin + + + + ${docker.push.username} + ${docker.push.password} + + + + + + + properties + docker.nolang + + + + + properties + docker.eng + + + + + properties + docker.fra + + + + + properties + docker.jpn + + + + + + + docker-build + package + + build + + + + docker-push + deploy + + push + + + + + start-elasticsearch + none + + + + stop-elasticsearch + none + + + diff --git a/distribution/src/main/docker/Dockerfile b/distribution/src/main/docker/Dockerfile new file mode 100644 index 000000000..c169bf084 --- /dev/null +++ b/distribution/src/main/docker/Dockerfile @@ -0,0 +1,17 @@ +FROM openjdk:15.0.2-jdk-slim-buster + +ARG langsPkg + +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + "gettext-base=0.19.*" \ + ${langsPkg} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY maven /usr/share/fscrawler +RUN set -ex \ + && ln -sn /usr/share/fscrawler/bin/fscrawler /usr/bin/ + +WORKDIR /usr/share/fscrawler diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index bb6a7ee4c..8ffa34f64 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -229,3 +229,24 @@ If you want to skip the check, you can run with ``-Dossindex.fail=false``:: mvn clean install -Dossindex.fail=false +DockerHub publication +^^^^^^^^^^^^^^^^^^^^^ + +To publish the latest build to `DockerHub `_ you can manually +call ``docker:push`` maven task and provide credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn -f distribution/pom.xml docker:push \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +Otherwise, if you call the maven ``deploy`` phase, it will be done automatically. +Note that it will still require that you provide the credentials ``docker.push.username`` and ``docker.push.password``:: + + mvn deploy \ + -Ddocker.push.username=yourdockerhubaccount \ + -Ddocker.push.password=yourverysecuredpassword + +You can also provide the settings as environment variables: + +* ``env.DOCKER_USERNAME`` or ``DOCKER_USERNAME`` +* ``env.DOCKER_PASSWORD`` or ``DOCKER_PASSWORD`` diff --git a/docs/source/dev/release.rst b/docs/source/dev/release.rst index 4b1957d3a..d3667c4c4 100644 --- a/docs/source/dev/release.rst +++ b/docs/source/dev/release.rst @@ -31,3 +31,6 @@ the tests while building the release. Only developers with write rights to the sonatype repository under ``fr.pilato`` space can perform the release. + + Only developers with write rights to the `DockerHub repository `_ + can push the Docker images. diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4c1f89f6d..ae9e147b1 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -60,6 +60,150 @@ The distribution contains: ├── ... All needed jars +Using docker +------------ + +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Let say your documents are located in ``~/tmp`` dir and you want to store your fscrawler jobs in ``~/.fscrawler``. +You can run FSCrawler with: + +.. code:: sh + + docker run -it --rm -v ~/.fscrawler:/root/.fscrawler -v ~/tmp:/tmp/es:ro dadoonet/fscrawler fscrawler job_name + +On the first run, if the job does not exist yet in ``~/.fscrawler``, FSCrawler will ask you if you want to create it: + +:: + + 10:16:53,880 INFO [f.p.e.c.f.c.BootstrapChecks] Memory [Free/Total=Percent]: HEAP [67.3mb/876.5mb=7.69%], RAM [2.1gb/3.8gb=55.43%], Swap [1023.9mb/1023.9mb=100.0%]. + 10:16:53,899 WARN [f.p.e.c.f.c.FsCrawlerCli] job [job_name] does not exist + 10:16:53,900 INFO [f.p.e.c.f.c.FsCrawlerCli] Do you want to create it (Y/N)? + y + 10:16:56,745 INFO [f.p.e.c.f.c.FsCrawlerCli] Settings have been created in [/root/.fscrawler/job_name/_settings.yaml]. Please review and edit before relaunch + +.. note:: + + The configuration file is actually stored on your machine in ``~/.fscrawler/job_name/_settings.yaml``. + Remember to change the URL of your elasticsearch instance as the container won't be able to see it + running under the default ``127.0.0.1``. You will need to use the actual IP address of the host. + + +Using docker compose +-------------------- + +In this section, the following directory layout is assumed: + +.. code-block:: none + + . + ├── config + │ └── job + │ └── _settings.yaml + ├── data + │ └── + └── docker-compose.yml + +For example, to connect to a docker container named ``elasticsearch``, modify your ``_settings.yaml``. + +.. code:: yaml + + name: "test" + elasticsearch: + nodes: + - url: "http://elasticsearch:9200" + +And, prepare the following ``docker-compose.yml``. + +.. code:: yaml + + version: '2.2' + services: + # FSCrawler + fscrawler: + image: dadoonet/fscrawler + container_name: fscrawler + volumes: + - ${PWD}/config:/root/.fscrawler + - ${PWD}/data:/tmp/es + networks: + - esnet + command: fscrawler job_name + + # Elasticsearch Cluster + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.seed_hosts=elasticsearch2 + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + container_name: elasticsearch2 + environment: + - node.name=elasticsearch2 + - discovery.seed_hosts=elasticsearch + - cluster.initial_master_nodes=elasticsearch,elasticsearch2 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + networks: + - esnet + + volumes: + esdata01: + driver: local + esdata02: + driver: local + + networks: + esnet: + +Then, you can run Elasticsearch. + +.. code:: sh + + docker-compose up -d elasticsearch + docker-compose logs -f elasticsearch + +Wait for elasticsearch to be started: + +:: + + + +After starting Elasticsearch, you can run FSCrawler. + +.. code:: sh + + docker-compose up fscrawler + + + Running as a Service on Windows ------------------------------- diff --git a/docs/source/user/getting_started.rst b/docs/source/user/getting_started.rst index 6a4428a28..a77a1b392 100644 --- a/docs/source/user/getting_started.rst +++ b/docs/source/user/getting_started.rst @@ -86,3 +86,4 @@ If you would like to ignore some folders to be scanned, just add a ``.fscrawleri The folder content and all sub folders will be ignored. For more information, read :ref:`includes_excludes`. + diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index 0859cb20f..8ae04ea75 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -91,3 +91,16 @@ Then expose the docker container you've created by changing the IP of the REST U url: "http://fscrawler:8080" +Pull the Docker image: + +.. code:: sh + + docker pull dadoonet/fscrawler + +Run it: + +.. code:: sh + + docker run dadoonet/fscrawler job + + diff --git a/lgtm.yml b/lgtm.yml index 7637c4dd6..2929ddcd0 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -4,5 +4,7 @@ extraction: version: 3 setup_py: false java: + before_index: + export DOCKER_SKIP=true index: java_version: "14" diff --git a/pom.xml b/pom.xml index bb4c3556d..e7b435995 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,12 @@ ${env.SONATYPE_USER} ${env.SONATYPE_PASS} + + ${DOCKER_USERNAME} + ${DOCKER_PASSWORD} + ${env.DOCKER_USERNAME} + ${env.DOCKER_PASSWORD} + 14 UTF-8 UTF-8 diff --git a/release.sh b/release.sh index 295feed82..1fe4ef86c 100755 --- a/release.sh +++ b/release.sh @@ -201,6 +201,19 @@ then fi fi +if [ ${DRY_RUN} -eq 0 ] +then + echo "Inspect DockerHub" + open https://hub.docker.com/r/dadoonet/fscrawler/tags + if promptyn "Is the DockerHub repository ok?" + then + echo "We can continue the release process." + else + echo "We cancel the release process." + RELEASE=0 + fi +fi + # We are releasing, so let's merge into the original branch if [ ${RELEASE} -eq 1 ] then