From 17391981eb785ade226cd87c7276fdb5878f20d3 Mon Sep 17 00:00:00 2001 From: Aram Mirzadeh Date: Thu, 16 Jul 2020 08:51:59 -0400 Subject: [PATCH 1/2] Added a contrib section with a sample docker-compose file --- .../Dockerfile-fscrawler | 27 +++ .../config/_default/6/_settings.json | 214 ++++++++++++++++++ .../config/_default/6/_settings_folder.json | 32 +++ .../config/_default/7/_settings.json | 214 ++++++++++++++++++ .../config/_default/7/_settings_folder.json | 32 +++ .../config/idx/_settings.yaml | 30 +++ .../config/idx/_status.json | 6 + .../docker-compose-example/docker-compose.yml | 84 +++++++ contrib/docker-compose-example/wait-for-it.sh | 182 +++++++++++++++ 9 files changed, 821 insertions(+) create mode 100644 contrib/docker-compose-example/Dockerfile-fscrawler create mode 100644 contrib/docker-compose-example/config/_default/6/_settings.json create mode 100644 contrib/docker-compose-example/config/_default/6/_settings_folder.json create mode 100644 contrib/docker-compose-example/config/_default/7/_settings.json create mode 100644 contrib/docker-compose-example/config/_default/7/_settings_folder.json create mode 100644 contrib/docker-compose-example/config/idx/_settings.yaml create mode 100644 contrib/docker-compose-example/config/idx/_status.json create mode 100644 contrib/docker-compose-example/docker-compose.yml create mode 100755 contrib/docker-compose-example/wait-for-it.sh diff --git a/contrib/docker-compose-example/Dockerfile-fscrawler b/contrib/docker-compose-example/Dockerfile-fscrawler new file mode 100644 index 000000000..8d02d38fc --- /dev/null +++ b/contrib/docker-compose-example/Dockerfile-fscrawler @@ -0,0 +1,27 @@ +FROM openjdk:8-alpine AS builder +ENV LANG C.UTF-8 +ENV MAVEN_OPTS "-Xmx256m -Xms256m" +RUN apk add --update --no-cache openssl wget maven +WORKDIR /usr/src/ +RUN wget https://github.com/dadoonet/fscrawler/archive/master.zip && unzip master.zip +WORKDIR /usr/src/fscrawler-master +RUN mvn clean package + +FROM openjdk:8-jre-alpine + +ENV ES es7 +ENV VERSION 2.7-SNAPSHOT +ENV FSCRAWLER_FNAME fscrawler-$ES-$VERSION-SNAPSHOT + +RUN apk add --update \ + && addgroup -S fscrawler && adduser -S -G fscrawler fscrawler +WORKDIR /usr/app +COPY --from=builder /usr/src/fscrawler-master/distribution/es7/target/$FSCRAWLER_FNAME.zip . +RUN unzip $FSCRAWLER_FNAME.zip && rm $FSCRAWLER_FNAME.zip \ + && mkdir -p data/idx \ + && mkdir config \ + && chown -R fscrawler:fscrawler . + +VOLUME [ "/usr/app/config", "/usr/app/data" ] + +ENTRYPOINT [ "$FSCRAWLER_FNAME/bin/fscrawler", "--config_dir", "/usr/app/config","idx", "--rest" ] diff --git a/contrib/docker-compose-example/config/_default/6/_settings.json b/contrib/docker-compose-example/config/_default/6/_settings.json new file mode 100644 index 000000000..3fa999288 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/6/_settings.json @@ -0,0 +1,214 @@ +{ + "settings": { + "number_of_shards": 1, + "index.mapping.total_fields.limit": 2000, + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "dynamic_templates": [ + { + "raw_as_text": { + "path_match": "meta.raw.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "attachment": { + "type": "binary", + "doc_values": false + }, + "attributes": { + "properties": { + "group": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + } + } + }, + "content": { + "type": "text" + }, + "file": { + "properties": { + "content_type": { + "type": "keyword" + }, + "filename": { + "type": "keyword", + "store": true + }, + "extension": { + "type": "keyword" + }, + "filesize": { + "type": "long" + }, + "indexed_chars": { + "type": "long" + }, + "indexing_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_modified": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_accessed": { + "type": "date", + "format": "dateOptionalTime" + }, + "checksum": { + "type": "keyword" + }, + "url": { + "type": "keyword", + "index": false + } + } + }, + "meta": { + "properties": { + "author": { + "type": "text" + }, + "date": { + "type": "date", + "format": "dateOptionalTime" + }, + "keywords": { + "type": "text" + }, + "title": { + "type": "text" + }, + "language": { + "type": "keyword" + }, + "format": { + "type": "text" + }, + "identifier": { + "type": "text" + }, + "contributor": { + "type": "text" + }, + "coverage": { + "type": "text" + }, + "modifier": { + "type": "text" + }, + "creator_tool": { + "type": "keyword" + }, + "publisher": { + "type": "text" + }, + "relation": { + "type": "text" + }, + "rights": { + "type": "text" + }, + "source": { + "type": "text" + }, + "type": { + "type": "text" + }, + "description": { + "type": "text" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "print_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "metadata_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "latitude": { + "type": "text" + }, + "longitude": { + "type": "text" + }, + "altitude": { + "type": "text" + }, + "rating": { + "type": "byte" + }, + "comments": { + "type": "text" + } + } + }, + "path": { + "properties": { + "real": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + }, + "root": { + "type": "keyword" + }, + "virtual": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/6/_settings_folder.json b/contrib/docker-compose-example/config/_default/6/_settings_folder.json new file mode 100644 index 000000000..2b63ec7f2 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/6/_settings_folder.json @@ -0,0 +1,32 @@ +{ + "settings": { + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "properties" : { + "real" : { + "type" : "keyword", + "store" : true + }, + "root" : { + "type" : "keyword", + "store" : true + }, + "virtual" : { + "type" : "keyword", + "store" : true + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/7/_settings.json b/contrib/docker-compose-example/config/_default/7/_settings.json new file mode 100644 index 000000000..3fa999288 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/7/_settings.json @@ -0,0 +1,214 @@ +{ + "settings": { + "number_of_shards": 1, + "index.mapping.total_fields.limit": 2000, + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "dynamic_templates": [ + { + "raw_as_text": { + "path_match": "meta.raw.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "attachment": { + "type": "binary", + "doc_values": false + }, + "attributes": { + "properties": { + "group": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + } + } + }, + "content": { + "type": "text" + }, + "file": { + "properties": { + "content_type": { + "type": "keyword" + }, + "filename": { + "type": "keyword", + "store": true + }, + "extension": { + "type": "keyword" + }, + "filesize": { + "type": "long" + }, + "indexed_chars": { + "type": "long" + }, + "indexing_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_modified": { + "type": "date", + "format": "dateOptionalTime" + }, + "last_accessed": { + "type": "date", + "format": "dateOptionalTime" + }, + "checksum": { + "type": "keyword" + }, + "url": { + "type": "keyword", + "index": false + } + } + }, + "meta": { + "properties": { + "author": { + "type": "text" + }, + "date": { + "type": "date", + "format": "dateOptionalTime" + }, + "keywords": { + "type": "text" + }, + "title": { + "type": "text" + }, + "language": { + "type": "keyword" + }, + "format": { + "type": "text" + }, + "identifier": { + "type": "text" + }, + "contributor": { + "type": "text" + }, + "coverage": { + "type": "text" + }, + "modifier": { + "type": "text" + }, + "creator_tool": { + "type": "keyword" + }, + "publisher": { + "type": "text" + }, + "relation": { + "type": "text" + }, + "rights": { + "type": "text" + }, + "source": { + "type": "text" + }, + "type": { + "type": "text" + }, + "description": { + "type": "text" + }, + "created": { + "type": "date", + "format": "dateOptionalTime" + }, + "print_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "metadata_date": { + "type": "date", + "format": "dateOptionalTime" + }, + "latitude": { + "type": "text" + }, + "longitude": { + "type": "text" + }, + "altitude": { + "type": "text" + }, + "rating": { + "type": "byte" + }, + "comments": { + "type": "text" + } + } + }, + "path": { + "properties": { + "real": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + }, + "root": { + "type": "keyword" + }, + "virtual": { + "type": "keyword", + "fields": { + "tree": { + "type": "text", + "analyzer": "fscrawler_path", + "fielddata": true + }, + "fulltext": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/contrib/docker-compose-example/config/_default/7/_settings_folder.json b/contrib/docker-compose-example/config/_default/7/_settings_folder.json new file mode 100644 index 000000000..2b63ec7f2 --- /dev/null +++ b/contrib/docker-compose-example/config/_default/7/_settings_folder.json @@ -0,0 +1,32 @@ +{ + "settings": { + "analysis": { + "analyzer": { + "fscrawler_path": { + "tokenizer": "fscrawler_path" + } + }, + "tokenizer": { + "fscrawler_path": { + "type": "path_hierarchy" + } + } + } + }, + "mappings": { + "properties" : { + "real" : { + "type" : "keyword", + "store" : true + }, + "root" : { + "type" : "keyword", + "store" : true + }, + "virtual" : { + "type" : "keyword", + "store" : true + } + } + } +} diff --git a/contrib/docker-compose-example/config/idx/_settings.yaml b/contrib/docker-compose-example/config/idx/_settings.yaml new file mode 100644 index 000000000..95b7e774c --- /dev/null +++ b/contrib/docker-compose-example/config/idx/_settings.yaml @@ -0,0 +1,30 @@ +--- +name: "idx" +fs: + url: "/usr/app/data" + update_rate: "15m" + indexed_chars: 100% + json_support: false + filename_as_id: false + add_filesize: true + remove_deleted: true + add_as_inner_object: false + store_source: false + index_content: true + attributes_support: false + raw_metadata: false + xml_support: false + index_folders: true + lang_detect: true + continue_on_error: false + ocr: + language: "eng" + enabled: true + pdf_strategy: "ocr_and_text" + follow_symlinks: false +elasticsearch: + nodes: + - url: "http://es01:9200" + bulk_size: 100 + flush_interval: "5s" + byte_size: "10mb" diff --git a/contrib/docker-compose-example/config/idx/_status.json b/contrib/docker-compose-example/config/idx/_status.json new file mode 100644 index 000000000..e697ddb83 --- /dev/null +++ b/contrib/docker-compose-example/config/idx/_status.json @@ -0,0 +1,6 @@ +{ + "name" : "idx", + "lastrun" : "2020-07-15T23:52:08.174", + "indexed" : 2048, + "deleted" : 0 +} \ No newline at end of file diff --git a/contrib/docker-compose-example/docker-compose.yml b/contrib/docker-compose-example/docker-compose.yml new file mode 100644 index 000000000..389d78ba5 --- /dev/null +++ b/contrib/docker-compose-example/docker-compose.yml @@ -0,0 +1,84 @@ +version: '3' +services: + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0 + container_name: es01 + environment: + - node.name=es01 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es02 + - cluster.initial_master_nodes=es01,es02 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms768m -Xmx768m" + restart: always + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - fscrawler_net + + es02: + image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0 + container_name: es02 + environment: + - node.name=es02 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01 + - cluster.initial_master_nodes=es01,es02 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms768m -Xmx768m" + restart: always + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data02:/usr/share/elasticsearch/data + ports: + - 9201:9200 + networks: + - fscrawler_net + + kib01: + image: docker.elastic.co/kibana/kibana:7.8.0 + container_name: kib01 + ports: + - 5601:5601 + environment: + ELASTICSEARCH_URL: http://es01:9200 + ELASTICSEARCH_HOSTS: http://es01:9200 + restart: always + networks: + - fscrawler_net + + fscrawler: + build: + context: ${PWD} + dockerfile: Dockerfile-fscrawler + container_name: fscrawler + restart: always + volumes: + - path_to_files_to_scan:/usr/app/data/:ro + - ./config/:/usr/app/config/ + depends_on: + - es01 + - es02 + - kib01 + command: [ "./wait-for-it.sh", "-h es01", "-t 30", "-p 9200" ] + networks: + - fscrawler_net + +volumes: + data01: + driver: local + data02: + driver: local + +networks: + fscrawler_net: + driver: bridge diff --git a/contrib/docker-compose-example/wait-for-it.sh b/contrib/docker-compose-example/wait-for-it.sh new file mode 100755 index 000000000..5e8679e54 --- /dev/null +++ b/contrib/docker-compose-example/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi From 1484109e43795a2ca4d6070cd3492abf662eafd1 Mon Sep 17 00:00:00 2001 From: Aram Mirzadeh Date: Thu, 16 Jul 2020 20:45:19 -0400 Subject: [PATCH 2/2] Updated documentation for instructions on how to use the contrib/docker-compose-example files --- docs/source/user/tips.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/user/tips.rst b/docs/source/user/tips.rst index a23ba26d6..2e06e60c4 100644 --- a/docs/source/user/tips.rst +++ b/docs/source/user/tips.rst @@ -38,3 +38,32 @@ To use FSCrawler with `docker `__, check `docker-fscrawler `__ recipe. +Using docker-compose +-------------------- +To standup a full environment you can use docker-compose from the contrib directory. +This environment will setup a node ElasticSearch cluster, a copy of Kibana +for searching and FSCrawler as containers. No other installs are neeeded, aside form Docker and docker-compose. + +Steps: + + 1. Download and install `docker `__. + 2. Download and install `docker-compose `__. + 3. Copy the contrib directory into your home directory. + 4. Edit the docker-compose.yaml + 1. Edit the line (somewhere around 66) that points to the "files to be scanned". + This is the path on the host machine prior to the colon. (ex: /fs/resume) + 2. In the ./config/ directory exists the name of the index name that FSCrawler will use. + By default, it's set to 'idx'. You can change it by renaming this directory, and changing the _settings.yaml file. + Check the ./config/idx/_settings.yaml to update any changes you like. + If you have multiple directories that you like to scan, I would suggest linking them under a single directory and + changing the "follow_links" option. + 5. Check the Dockerfile-fscrawler file. This is where the version of the package is determined. By default I have set to + download the 'master' branch which is currently producing a es7-2.7-SNAPSHOT version but you can lock this into a + specific version to make it more reliable. Update (DO NOT MOVE) the ENV variables to match what you want the build to be. + 6. Issue `docker-compose up -d` in that directory and it'll download and create the containers. It'll also compile and build a + custom container for fscrawler. + 7. After the containers are up and running, wait about 30 seconds for everything to start syncing. You can now access Kibana and + build your index (just need to do it once). After that the search will be available via Kibana. + +TODO: Build a more robust link to a specific version in the Dockerfile so it's a little more specific about what it downloads and builds.0:w +