From fc5e1c6ac35bc7f75afbd30cfd1e1874b41f4456 Mon Sep 17 00:00:00 2001 From: Juan Jose Lopez Martin Date: Mon, 19 Dec 2016 08:16:38 +0100 Subject: [PATCH 01/22] [WIP][DCS-67] Docker entry point refactor (#56) [DCS-67] Docker entry point refactor --- docker/catalog-config.sh | 30 +++++++------- docker/crossdata-config.sh | 16 ++++---- docker/crossdata-security.sh | 14 +++++++ docker/docker-entrypoint.sh | 4 +- docker/gosec-config.sh | 66 +++++++++++++++++++++++++++++++ docker/kerberos-server-config.sh | 45 +++++++++++++++++++++ docker/security-config.sh | 59 +++++++++++++++++++++++++++ docker/shell-config.sh | 24 +++++------ docker/streaming-config.sh | 8 ++-- docker/tls-config.sh | 19 +++++++++ docker/truststore-config.sh | 29 ++++++++++++++ events.csv/._SUCCESS.crc | Bin 0 -> 8 bytes events.csv/.part-00000.crc | Bin 0 -> 12 bytes events.csv/_SUCCESS | 0 events.csv/part-00000 | 11 ++++++ 15 files changed, 284 insertions(+), 41 deletions(-) create mode 100644 docker/crossdata-security.sh create mode 100644 docker/gosec-config.sh create mode 100644 docker/kerberos-server-config.sh create mode 100644 docker/security-config.sh create mode 100644 docker/tls-config.sh create mode 100644 docker/truststore-config.sh create mode 100644 events.csv/._SUCCESS.crc create mode 100644 events.csv/.part-00000.crc create mode 100644 events.csv/_SUCCESS create mode 100644 events.csv/part-00000 diff --git a/docker/catalog-config.sh b/docker/catalog-config.sh index 138891280..ac130e268 100644 --- a/docker/catalog-config.sh +++ b/docker/catalog-config.sh @@ -1,35 +1,35 @@ #!/bin/bash -xe function jdbcCatalog() { - crossdata_core_catalog_jdbc_driver=${1:-3306} - crossdata_core_catalog_jdbc_url=$2 - crossdata_core_catalog_jdbc_name=$3 - crossdata_core_catalog_jdbc_user=$4 - crossdata_core_catalog_jdbc_pass=$5 + export crossdata_core_catalog_jdbc_driver=${1:-3306} + export crossdata_core_catalog_jdbc_url=$2 + export crossdata_core_catalog_jdbc_name=$3 + export crossdata_core_catalog_jdbc_user=$4 + export crossdata_core_catalog_jdbc_pass=$5 } function zookeeperCatalog() { - crossdata_core_catalog_zookeeper_connectionString=${1:-localhost:2181} - crossdata_core_catalog_zookeeper_connectionTimeout=${2:-15000} - crossdata_core_catalog_zookeeper_sessionTimeout=${3:-60000} - crossdata_core_catalog_zookeeper_retryAttempts=${4:-5} - crossdata_core_catalog_zookeeper_retryInterval=${5:-10000} + export crossdata_core_catalog_zookeeper_connectionString=${1:-localhost:2181} + export crossdata_core_catalog_zookeeper_connectionTimeout=${2:-15000} + export crossdata_core_catalog_zookeeper_sessionTimeout=${3:-60000} + export crossdata_core_catalog_zookeeper_retryAttempts=${4:-5} + export crossdata_core_catalog_zookeeper_retryInterval=${5:-10000} } if [$# > 0 ]; then if [ "x$1x" != "xx" ]; then - crossdata_core_catalog_class="\"org.apache.spark.sql.crossdata.catalog.persistent.$1Catalog\"" + export crossdata_core_catalog_class="\"org.apache.spark.sql.crossdata.catalog.persistent.$1Catalog\"" if [ "$1" == "MySQL" ]; then - jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} + export jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi if [ "$1" == "PostgreSQL" ]; then - jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} + export jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi if [ "$1" == "Zookeeper" ]; then - zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} + export zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} fi if [ "x$2x" != "xx" ]; then - crossdata_core_catalog_prefix=${2:-crossdataCluster} + export crossdata_core_catalog_prefix=${2:-crossdataCluster} fi fi fi \ No newline at end of file diff --git a/docker/crossdata-config.sh b/docker/crossdata-config.sh index 1e348979e..c137c1941 100644 --- a/docker/crossdata-config.sh +++ b/docker/crossdata-config.sh @@ -21,12 +21,12 @@ function setDriverConfig() { } function standaloneConfig() { - AKKAIP="akka.tcp://CrossdataServerCluster@${DOCKER_HOST}:13420" + export AKKAIP="akka.tcp://CrossdataServerCluster@${DOCKER_HOST}:13420" #TODO: Test instead of XD_SEED : CROSSDATA_SERVER_AKKA_CLUSTER_SEED_NODES if [ -z "$XD_SEED" ]; then export CROSSDATA_SERVER_AKKA_CLUSTER_SEED_NODES=${AKKAIP} else - SEED_IP="akka.tcp://CrossdataServerCluster@${XD_SEED}:13420" + export SEED_IP="akka.tcp://CrossdataServerCluster@${XD_SEED}:13420" export CROSSDATA_SERVER_AKKA_CLUSTER_SEED_NODES=${SEED_IP},${AKKAIP} # TODO: Study whether it is worth of making hazelcast nodes available when auto discovery is disabled. # If so, find a better way of editing hazelcast.xml. The method commented below is as flimsy as it gets. @@ -43,9 +43,9 @@ function standaloneConfig() { export CROSSDATA_SERVER_AKKA_REMOTE_NETTY_TCP_BIND_HOSTNAME=${DOCKER_HOST} if [ -z "$XD_SEED" ]; then - crossdata_driver_config_cluster_hosts="\[${DOCKER_HOST}:13420\]" + export crossdata_driver_config_cluster_hosts="\[${DOCKER_HOST}:13420\]" else - crossdata_driver_config_cluster_hosts="\[${DOCKER_HOST}:13420, ${XD_SEED}\]" + export crossdata_driver_config_cluster_hosts="\[${DOCKER_HOST}:13420, ${XD_SEED}\]" fi } @@ -54,8 +54,8 @@ function marathonConfig() { #################################################### #Memory #################################################### - RAM_AVAIL=$(echo $MARATHON_APP_RESOURCE_MEM | cut -d "." -f1) - CROSSDATA_JAVA_OPT="-Xmx${RAM_AVAIL}m -Xms${RAM_AVAIL}m" + export RAM_AVAIL=$(echo $MARATHON_APP_RESOURCE_MEM | cut -d "." -f1) + export CROSSDATA_JAVA_OPT="-Xmx${RAM_AVAIL}m -Xms${RAM_AVAIL}m" sed -i "s|# CROSSDATA_LIB|#CROSSDATA_JAVA_OPTS\nCROSSDATA_JAVA_OPTS=\"${CROSSDATA_JAVA_OPT}\"\n# CROSSDATA_LIB|" /etc/sds/crossdata/server/crossdata-env.sh #Spark UI port @@ -88,9 +88,9 @@ function marathonConfig() { fi # When using ClusterClient External IP, the hosts-files get updated in order to keep a consistent # binding address in AKKA. - NAMEADDR="$(hostname -i)" + export NAMEADDR="$(hostname -i)" if [ -n "$HAPROXY_SERVER_INTERNAL_ADDRESS" ]; then - NAMEADDR=$HAPROXY_SERVER_INTERNAL_ADDRESS + export NAMEADDR=$HAPROXY_SERVER_INTERNAL_ADDRESS fi echo -e "$NAMEADDR\t$XD_EXTERNAL_IP" >> /etc/hosts fi diff --git a/docker/crossdata-security.sh b/docker/crossdata-security.sh new file mode 100644 index 000000000..110bd2533 --- /dev/null +++ b/docker/crossdata-security.sh @@ -0,0 +1,14 @@ +#!/bin/bash -xe +export CROSSDATA_SERVER_AKKA_REMOTE_NETTY_SSL_ENABLE_SSL=true +export CROSSDATA_SERVER_AKKA_HTTP_SSL_ENABLE=true +export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE=${XD_TRUST_JKS_NAME} +export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE_PASSWORD=${XD_TRUSTSTORE_PASSWORD} +export CROSSDATA_SERVER_AKKA_HTTP_SSL_KEYSTORE=${XD_TLS_JKS_NAME} +export CROSSDATA_SERVER_AKKA_HTTP_SSL_KEYSTORE_PASSWORD=${XD_TLS_PASSWORD} + +#Configure kerberos keytab +export CROSSDATA_SERVER_CONFIG_KERBEROS_ENABLED=true + +## Configure HDFS Kerberos keytab +export CROSSDATA_SERVER_CONFIG_KERBEROS_PRINCIPAL=${XD_PRINCIPAL} +export CROSSDATA_SERVER_CONFIG_KERBEROS_KEYTAB=${XD_KEYTAB_NAME} \ No newline at end of file diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6d0fa0c13..08de7b235 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -2,9 +2,9 @@ DOCKER_HOST="hostname -f" if [[ "$(hostname -f)" =~ \. ]]; then - DOCKER_HOST="$(hostname -f)" + export DOCKER_HOST="$(hostname -f)" else - DOCKER_HOST="$(hostname -i)" + export DOCKER_HOST="$(hostname -i)" fi #################################################### diff --git a/docker/gosec-config.sh b/docker/gosec-config.sh new file mode 100644 index 000000000..fafab1b08 --- /dev/null +++ b/docker/gosec-config.sh @@ -0,0 +1,66 @@ +#!/bin/bash -xe + +####################################################### +## Get Gosec-plugin LDAP user and pass and set XD vars +####################################################### + +### Get LDAP user and pass +export XD_GOSEC_PLUGIN_LDAP_USER=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"user\"") +export XD_GOSEC_PLUGIN_LDAP_PASS=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"pass\"") + + +############################################################################# +## Get XD Gosec-plugin x509 client cert and set XD_GOSEC_PLUGIN_JKS_PASSWORD +############################################################################# + +### Get certificate from KMS +curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/x509-auth/x509-client-cert" -s | jq -r ".data .certificate_chain" > "$UUID.crt" +curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/x509-auth/x509-client-cert" -s | jq -r ".data .private_key" > "$UUID.key" + +### Get keystore password +export XD_GOSEC_PLUGIN_JKS_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/keystore | jq -r ".data .keystore_pass") + +## Generating PKCS12 +openssl pkcs12 -inkey "$UUID.key" -name "$SERVER_CERT_ALIAS" -in "$UUID.crt" \ + -export -out "$GOSEC_PLUGIN_JKS_NAME.pkcs12" -password "env:XD_GOSEC_PLUGIN_JKS_PASSWORD" + +## Generating JKS +keytool -importkeystore -srckeystore "$GOSEC_PLUGIN_JKS_NAME.pkcs12" -srcalias "$SERVER_CERT_ALIAS" \ + -srcstorepass "$XD_GOSEC_PLUGIN_JKS_PASSWORD" -srcstoretype PKCS12 -destkeystore "$GOSEC_PLUGIN_JKS_NAME" -deststorepass "$XD_GOSEC_PLUGIN_JKS_PASSWORD" + +## Cleaning +rm -f $GOSEC_PLUGIN_JKS_NAME.pkcs12 $UUID.crt $UUID.key + + #Set JAAS config + cat > /etc/sds/crossdata/security/jaas.conf<#$XD_PRINCIPAL#" $XD_PLUGIN_CLIENT_JAAS_PATH \ + && echo "[JAAS_CONF] ZK principal configured as $XD_PRINCIPAL" \ + || echo "[JAAS_CONF-ERROR] ZK principal was NOT configured" + sed -i "s#<__KEYTAB__>#$XD_KEYTAB_NAME#" $XD_PLUGIN_CLIENT_JAAS_PATH\ + && echo "[JAAS_CONF] ZK keytab configured as $XD_KEYTAB_NAME" \ + || echo "[JAAS_CONF-ERROR] ZK keytab was NOT configured" + + + #Set LDAP config + export XD_PLUGIN_LDAP_PRINCIPAL=$XD_GOSEC_PLUGIN_LDAP_USER + export XD_PLUGIN_LDAP_CREDENTIALS=$XD_GOSEC_PLUGIN_LDAP_PASS + + #Set Kafka config + export XD_PLUGIN_KAFKA_TRUSTSTORE_PASSWORD=$XD_TRUSTSTORE_PASSWORD + export XD_PLUGIN_KAFKA_TRUSTSTORE=$XD_TRUST_JKS_NAME + export XD_PLUGIN_KAFKA_KEYSTORE=$GOSEC_PLUGIN_JKS_NAME + export XD_PLUGIN_KAFKA_KEYSTORE_PASSWORD=$XD_GOSEC_PLUGIN_JKS_PASSWORD + export XD_PLUGIN_KAFKA_KEY_PASSWOR=D$XD_GOSEC_PLUGIN_JKS_PASSWOR diff --git a/docker/kerberos-server-config.sh b/docker/kerberos-server-config.sh new file mode 100644 index 000000000..0a4402092 --- /dev/null +++ b/docker/kerberos-server-config.sh @@ -0,0 +1,45 @@ +#!/bin/bash -xe + +### Get keytab +export BASE64_KEYTAB=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/krb" -s | jq -r ".data .keytab_base64") +export XD_PRINCIPAL=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/krb" -s | jq -r ".data .principal") + +## Generating keytab +echo $BASE64_KEYTAB | base64 -d > $XD_KEYTAB_NAME + + +#Set krb5.conf +cat > $JAVA_HOME/jre/lib/security/krb5.conf <__ + dns_lookup_realm = false + udp_preference_limit = 1 + [realms] + ____ = { + kdc = ____ + admin_server = ____ + default_domain = ____ + } + [domain_realm] + .____ = ____ + ____ = ____ +EOF + + +lw_realm=$(echo $REALM | tr '[:upper:]' '[:lower:]') +sed -i "s#____#$REALM#" $JAVA_HOME/jre/lib/security/krb5.conf \ +&& echo "[KRB-CONF] Realm configured in krb5.conf" \ +|| echo "[KRB-CONF-ERROR] Something went wrong when REALM was configured in krb5.conf" + +sed -i "s#____#$lw_realm#" $JAVA_HOME/jre/lib/security/krb5.conf \ +&& echo "[KRB-CONF] Domain configured in krb5.conf" \ +|| echo "[KRB-CONF-ERROR] Something went wrong when DOMAIN was configured in krb5.conf" + +sed -i "s#____#$KDC_HOST#" $JAVA_HOME/jre/lib/security/krb5.conf \ +&& echo "[KRB-CONF] kdc host configured in krb5.conf" \ +|| echo "[KRB-CONF-ERROR] Something went wrong when kdc host was configured in krb5.conf" + +sed -i "s#____#$KADMIN_HOST#" $JAVA_HOME/jre/lib/security/krb5.conf \ +&& echo "[KRB-CONF] kadmin host configured in krb5.conf" \ +|| echo "[KRB-CONF-ERROR] Something went wrong when kadmin host was configured in krb5.conf" + diff --git a/docker/security-config.sh b/docker/security-config.sh new file mode 100644 index 000000000..913757bb1 --- /dev/null +++ b/docker/security-config.sh @@ -0,0 +1,59 @@ +#!/bin/bash -xe + +#Ensure security folder is created +mkdir -p /etc/sds/crossdata/security + +#Get from vault the different data +#TODO: REMOVE -k FROM CURL WHEN TRUSTSTORE FOR VAULT IS INSIDE THE DOCKER!!!!!!!!!!! +# Get params +#VAULT_TOKEN='4d6cafd2-f5a4-abad-88f0-48eae1c24904' # This is the only env. value from entry point + + +# Main execution + +## Init +#TENANT_NAME="crossdata1" # MARATHON_APP_ID without slash +#VAULT_HOST='127.0.0.1' # It should be a predefined hostname +#VAULT_PORT='8200' # It should be a predefined port + +## Configure constant +export SERVER_CERT_ALIAS="crossdata-server" +export XD_TLS_JKS_NAME="/etc/sds/crossdata/security/server.jks" +export XD_TRUST_JKS_NAME="/etc/sds/crossdata/security/truststore.jks" +export XD_KEYTAB_NAME="/etc/sds/crossdata/security/crossdata.keytab" +export GOSEC_PLUGIN_JKS_NAME="/etc/sds/crossdata/security/gosec-plugin.jks" + +## Generating uuid +export UUID=$(uuidgen) + +#################################################### +## Get XD TLS Server Info and set XD_TLS_PASSWORD +#################################################### +source tls-config.sh + +####################################################### +## Create XD Truststore and set XD_TRUSTSTORE_PASSWORD +####################################################### +source truststore-config.sh + +#################################################### +## Kerberos config +#################################################### +source kerberos-server-config.sh + +####################################################### +## Gosec-plugin config +####################################################### +source gosec-config.sh + +####################################################### +## HDFS security +####################################################### +source hdfs-security.sh + +####################################################### +## Crossdata security +####################################################### +source crossdata-security.sh + + diff --git a/docker/shell-config.sh b/docker/shell-config.sh index bae2bffcd..023d3d6a3 100644 --- a/docker/shell-config.sh +++ b/docker/shell-config.sh @@ -1,30 +1,30 @@ #!/bin/bash -xe -CROSSDATA_DRIVER_CONFIG_HTTP_SERVER_HOST=$SHELL_SERVERADDR -CROSSDATA_DRIVER_CONFIG_HTTP_SERVER_PORT=$SHELL_SERVERPORT +export CROSSDATA_DRIVER_CONFIG_HTTP_SERVER_HOST=$SHELL_SERVERADDR +export CROSSDATA_DRIVER_CONFIG_HTTP_SERVER_PORT=$SHELL_SERVERPORT # Prepare options string from docker environment settings -OPTIONS="" +export OPTIONS="" if [ -n "$SHELL_USER" ]; then - OPTIONS="$OPTIONS --user $SHELL_USER" + export OPTIONS="$OPTIONS --user $SHELL_USER" fi; if [ "$CONNECTION_TIMEOUT" ]; then - OPTIONS="$OPTIONS --timeout $CONNECTION_TIMEOUT" + export OPTIONS="$OPTIONS --timeout $CONNECTION_TIMEOUT" fi; -OPTIONS="$OPTIONS --http" +export OPTIONS="$OPTIONS --http" if [ "$SHELL_MODE" == "async" ]; then - OPTIONS="$OPTIONS --async" + export OPTIONS="$OPTIONS --async" fi; if [ "$SHELL_CERTIFICATE_PATH" -a "$SHELL_CERTIFICATE_PASSWORD" -a "$SHELL_TRUSTSTORE_PATH" -a "$SHELL_TRUSTSTORE_PASSWORD" ]; then - CROSSDATA_DRIVER_AKKA_HTTP_SSL_ENABLE="true" - CROSSDATA_DRIVER_AKKA_HTTP_SSL_KEYSTORE=$SHELL_CERTIFICATE_PATH - CROSSDATA_DRIVER_AKKA_HTTP_SSL_KEYSTORE_PASSWORD=$SHELL_CERTIFICATE_PASSWORD - CROSSDATA_DRIVER_AKKA_HTTP_SSL_TRUSTSTORE=$SHELL_TRUSTSTORE_PATH - CROSSDATA_DRIVER_AKKA_HTTP_SSL_TRUSTSTORE_PASSWORD=$SHELL_TRUSTSTORE_PASSWORD + export CROSSDATA_DRIVER_AKKA_HTTP_SSL_ENABLE="true" + export CROSSDATA_DRIVER_AKKA_HTTP_SSL_KEYSTORE=$SHELL_CERTIFICATE_PATH + export CROSSDATA_DRIVER_AKKA_HTTP_SSL_KEYSTORE_PASSWORD=$SHELL_CERTIFICATE_PASSWORD + export CROSSDATA_DRIVER_AKKA_HTTP_SSL_TRUSTSTORE=$SHELL_TRUSTSTORE_PATH + export CROSSDATA_DRIVER_AKKA_HTTP_SSL_TRUSTSTORE_PASSWORD=$SHELL_TRUSTSTORE_PASSWORD fi \ No newline at end of file diff --git a/docker/streaming-config.sh b/docker/streaming-config.sh index b3fa93dad..2fa06ed58 100644 --- a/docker/streaming-config.sh +++ b/docker/streaming-config.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe sed -i "s|//crossdata-core.streaming*|crossdata-core.streaming|" /etc/sds/crossdata/server/core-application.conf -crossdata_core_catalog_zookeeper_connectionString=$1 -crossdata_core_streaming_receiver_zk_connection=$2 -crossdata_core_streaming_receiver_kafka_connection=$3 -crossdata_core_streaming_spark_master=$4 \ No newline at end of file +export crossdata_core_catalog_zookeeper_connectionString=$1 +export crossdata_core_streaming_receiver_zk_connection=$2 +export crossdata_core_streaming_receiver_kafka_connection=$3 +export crossdata_core_streaming_spark_master=$4 \ No newline at end of file diff --git a/docker/tls-config.sh b/docker/tls-config.sh new file mode 100644 index 000000000..493bcac2e --- /dev/null +++ b/docker/tls-config.sh @@ -0,0 +1,19 @@ +#!/bin/bash -xe + +### Get certificate from KMS +curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/x509-server-cert" -s | jq -r ".data .certificate_chain" > "$UUID.crt" +curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/x509-server-cert" -s | jq -r ".data .private_key" > "$UUID.key" + +### Get keystore password +export XD_TLS_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/keystore | jq -r ".data .keystore_pass") + +## Generating PKCS12 +openssl pkcs12 -inkey "$UUID.key" -name "$SERVER_CERT_ALIAS" -in "$UUID.crt" \ + -export -out "$XD_TLS_JKS_NAME.pkcs12" -password "env:XD_TLS_PASSWORD" + +## Generating JKS +keytool -importkeystore -srckeystore "$XD_TLS_JKS_NAME.pkcs12" -srcalias "$SERVER_CERT_ALIAS" \ + -srcstorepass "$XD_TLS_PASSWORD" -srcstoretype PKCS12 -destkeystore "$XD_TLS_JKS_NAME" -deststorepass "$XD_TLS_PASSWORD" + +## Cleaning +rm -f $XD_TLS_JKS_NAME.pkcs12 $UUID.crt $UUID.key \ No newline at end of file diff --git a/docker/truststore-config.sh b/docker/truststore-config.sh new file mode 100644 index 000000000..b6c9c9dbb --- /dev/null +++ b/docker/truststore-config.sh @@ -0,0 +1,29 @@ +#!/bin/bash -xe + +### Get keystore password +export XD_TRUSTSTORE_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/keystore | jq -r ".data .keystore_pass") + +export XD_JVMCA_PASS="changeit" #TODO: This password should be provided by VAULT service + +## Prepare Truststore +counter=1 +code=$(curl -k -L -s -o /dev/null -w "%{http_code}" -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter) +while [ $code -eq 200 ]; do + curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter | jq -r ".data .certificate" > "$UUID.crt" + + # Create der file for root CA + openssl x509 -outform der -in "$UUID.crt" -out $UUID.der + + # Create keystore + keytool -import -noprompt -alias $counter -keystore $XD_TRUST_JKS_NAME -storepass $XD_TRUSTSTORE_PASSWORD -file $UUID.der + + # Add CA to JVM Keystore + keytool -importcert -noprompt -alias $counter -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass $XD_JVMCA_PASS -file $UUID.der + + # Update counter + let counter=counter+1 + code=$(curl -k -L -s -o /dev/null -w "%{http_code}" -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter) + + # Clean + rm -f $UUID.der "$UUID.crt" +done \ No newline at end of file diff --git a/events.csv/._SUCCESS.crc b/events.csv/._SUCCESS.crc new file mode 100644 index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c GIT binary patch literal 8 PcmYc;N@ieSU}69O2$TUk literal 0 HcmV?d00001 diff --git a/events.csv/.part-00000.crc b/events.csv/.part-00000.crc new file mode 100644 index 0000000000000000000000000000000000000000..9c46cabe47227cd55389ab439d5e0913c52a64fd GIT binary patch literal 12 TcmYc;N@ieSU}CU&QCtoH5`_av literal 0 HcmV?d00001 diff --git a/events.csv/_SUCCESS b/events.csv/_SUCCESS new file mode 100644 index 000000000..e69de29bb diff --git a/events.csv/part-00000 b/events.csv/part-00000 new file mode 100644 index 000000000..653ed6980 --- /dev/null +++ b/events.csv/part-00000 @@ -0,0 +1,11 @@ +ident,money +5,15.2 +1,11.2 +8,18.2 +0,10.2 +2,12.2 +4,14.2 +7,17.2 +6,16.2 +9,19.2 +3,13.2 From 15f6181a9a9631478b14a9cef70b01fb993e92c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Francisco=20P=C3=A9rez=20Hidalgo?= Date: Wed, 21 Dec 2016 16:37:16 +0100 Subject: [PATCH 02/22] Docker entry points fixes for standalone mode with vault (#61) - Docker entry point: Added secrets retreival phase to standalone mode. - Fixed typo error. - Catalog config fix --- docker/catalog-config.sh | 10 +++++++--- docker/crossdata-config.sh | 9 ++++++++- docker/gosec-config.sh | 31 +++++++++++++++---------------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docker/catalog-config.sh b/docker/catalog-config.sh index ac130e268..b083fa3c5 100644 --- a/docker/catalog-config.sh +++ b/docker/catalog-config.sh @@ -1,6 +1,10 @@ #!/bin/bash -xe + +CATALOG_CLASS_PREFIX="org.apache.spark.sql.crossdata.catalog.persistent" + function jdbcCatalog() { - export crossdata_core_catalog_jdbc_driver=${1:-3306} + DEFAULT_CATALOG="$CATALOG_CLASS_PREFIX.DerbyCatalog" + export crossdata_core_catalog_jdbc_driver=${1:-$DEFAULT_CATALOG} export crossdata_core_catalog_jdbc_url=$2 export crossdata_core_catalog_jdbc_name=$3 export crossdata_core_catalog_jdbc_user=$4 @@ -16,9 +20,9 @@ function zookeeperCatalog() { } -if [$# > 0 ]; then +if [ $# > 0 ]; then if [ "x$1x" != "xx" ]; then - export crossdata_core_catalog_class="\"org.apache.spark.sql.crossdata.catalog.persistent.$1Catalog\"" + export crossdata_core_catalog_class="\"${CATALOG_CLASS_PREFIX}.$1Catalog\"" if [ "$1" == "MySQL" ]; then export jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi diff --git a/docker/crossdata-config.sh b/docker/crossdata-config.sh index c137c1941..7751a4cf8 100644 --- a/docker/crossdata-config.sh +++ b/docker/crossdata-config.sh @@ -94,12 +94,19 @@ function marathonConfig() { fi echo -e "$NAMEADDR\t$XD_EXTERNAL_IP" >> /etc/hosts fi + } #################################################### -## Main +## Vault and secrets (configured if enabled) #################################################### +if [ ! -z ${VAULT_HOST} ]; then + source security-config.sh +fi +#################################################### +## Main +#################################################### if [ -z ${MARATHON_APP_ID} ]; then standaloneConfig else diff --git a/docker/gosec-config.sh b/docker/gosec-config.sh index fafab1b08..c6aeb1d83 100644 --- a/docker/gosec-config.sh +++ b/docker/gosec-config.sh @@ -46,21 +46,20 @@ EOF export XD_JAAS_FILE=/etc/sds/crossdata/security/jaas.conf export XD_PLUGIN_CLIENT_JAAS_PATH=$XD_JAAS_FILE - sed -i "s#<__PRINCIPAL__>#$XD_PRINCIPAL#" $XD_PLUGIN_CLIENT_JAAS_PATH \ - && echo "[JAAS_CONF] ZK principal configured as $XD_PRINCIPAL" \ - || echo "[JAAS_CONF-ERROR] ZK principal was NOT configured" - sed -i "s#<__KEYTAB__>#$XD_KEYTAB_NAME#" $XD_PLUGIN_CLIENT_JAAS_PATH\ - && echo "[JAAS_CONF] ZK keytab configured as $XD_KEYTAB_NAME" \ - || echo "[JAAS_CONF-ERROR] ZK keytab was NOT configured" +sed -i "s#<__PRINCIPAL__>#$XD_PRINCIPAL#" $XD_PLUGIN_CLIENT_JAAS_PATH \ + && echo "[JAAS_CONF] ZK principal configured as $XD_PRINCIPAL" \ + || echo "[JAAS_CONF-ERROR] ZK principal was NOT configured" +sed -i "s#<__KEYTAB__>#$XD_KEYTAB_NAME#" $XD_PLUGIN_CLIENT_JAAS_PATH\ + && echo "[JAAS_CONF] ZK keytab configured as $XD_KEYTAB_NAME" \ + || echo "[JAAS_CONF-ERROR] ZK keytab was NOT configured" +#Set LDAP config +export XD_PLUGIN_LDAP_PRINCIPAL=$XD_GOSEC_PLUGIN_LDAP_USER +export XD_PLUGIN_LDAP_CREDENTIALS=$XD_GOSEC_PLUGIN_LDAP_PASS - #Set LDAP config - export XD_PLUGIN_LDAP_PRINCIPAL=$XD_GOSEC_PLUGIN_LDAP_USER - export XD_PLUGIN_LDAP_CREDENTIALS=$XD_GOSEC_PLUGIN_LDAP_PASS - - #Set Kafka config - export XD_PLUGIN_KAFKA_TRUSTSTORE_PASSWORD=$XD_TRUSTSTORE_PASSWORD - export XD_PLUGIN_KAFKA_TRUSTSTORE=$XD_TRUST_JKS_NAME - export XD_PLUGIN_KAFKA_KEYSTORE=$GOSEC_PLUGIN_JKS_NAME - export XD_PLUGIN_KAFKA_KEYSTORE_PASSWORD=$XD_GOSEC_PLUGIN_JKS_PASSWORD - export XD_PLUGIN_KAFKA_KEY_PASSWOR=D$XD_GOSEC_PLUGIN_JKS_PASSWOR +#Set Kafka config +export XD_PLUGIN_KAFKA_TRUSTSTORE_PASSWORD=$XD_TRUSTSTORE_PASSWORD +export XD_PLUGIN_KAFKA_TRUSTSTORE=$XD_TRUST_JKS_NAME +export XD_PLUGIN_KAFKA_KEYSTORE=$GOSEC_PLUGIN_JKS_NAME +export XD_PLUGIN_KAFKA_KEYSTORE_PASSWORD=$XD_GOSEC_PLUGIN_JKS_PASSWORD +export XD_PLUGIN_KAFKA_KEY_PASSWOR=$XD_GOSEC_PLUGIN_JKS_PASSWORD \ No newline at end of file From 66f49899e4cac8e9081bb8f3943fa99bd45496a9 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Wed, 21 Dec 2016 22:35:54 +0100 Subject: [PATCH 03/22] [CROSSDATA] [DOCKER] Scripts for generation dockers locally (#58) * [CROSSDATA] [DOCKER] Scripts for generation dockers locally * [CROSSDATA] [DOCKER] Scripts for generating dockers ready * [CROSSDATA] [DOCKER] Possibility of using qa docker+ * [CROSSDATA] [DOCKER] Some refactor * [CROSSDATA] [DOCKER] Temporary directory removed --- docker/Dockerfile | 24 +++++++++++++ docker/commons.sh | 12 +++++++ docker/dependencyfix.sh | 12 +++++++ docker/generate.sh | 26 ++++++++++++++ docker/generateWithGosec.sh | 72 +++++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/commons.sh create mode 100755 docker/dependencyfix.sh create mode 100755 docker/generate.sh create mode 100755 docker/generateWithGosec.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..8de507ff8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM qa.stratio.com/stratio/ubuntu-base:16.04 +MAINTAINER Stratio Crossdata team "crossdata@stratio.com" + +# USAGE: build --build-arg PKG= -t +# USAGE Example: docker build --build-arg PKG=1.8.0-RC2-SNAPSHOT -t crossdata-enterprise . + +COPY . / + +VOLUME /usr/lib/mesos + +ARG PKG + +RUN wget -q "http://apt.repository.stratio.com/pool/trusty/1.7/main/stratio-release_1.0.0_all.deb" \ + && dpkg -i stratio-release_1.0.0_all.deb \ + && rm -rf stratio-release_1.0.0_all.deb \ + && apt-get update \ + && ./dependencyfix.sh stratio-crossdata-mesosphere-scala211-${PKG}.all.deb \ + && dpkg -i stratio-crossdata-mesosphere-scala211-${PKG}.all.deb + +ENTRYPOINT ["/docker-entrypoint.sh"] +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 + +CMD tail -f /var/log/* + diff --git a/docker/commons.sh b/docker/commons.sh new file mode 100644 index 000000000..fb2de6ba4 --- /dev/null +++ b/docker/commons.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +function changedir() { + cd $1 + echo "Current dir: $PWD" +} + +#Get Crossdata version from parent pom +tmp1=`grep -m2 "" ../pom.xml | tail -n1` +tmp2=${tmp1//} +tmp3=${tmp2/<\/version>/} +XD_VERSION=${tmp3// } diff --git a/docker/dependencyfix.sh b/docker/dependencyfix.sh new file mode 100755 index 000000000..79e2ecd18 --- /dev/null +++ b/docker/dependencyfix.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +deps=`dpkg --info $1 | grep -i depends | sed -n "s/.*depends: //ip"` + +IFS=',' read -ra dep <<< "$deps" +for i in "${dep[@]}"; do + echo $i + elver=`echo "$i" | sed -e "s/[() ]//gp"` + echo $elver + apt-get -y install $elver +done + diff --git a/docker/generate.sh b/docker/generate.sh new file mode 100755 index 000000000..a2091bcb7 --- /dev/null +++ b/docker/generate.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +. commons.sh + +changedir "../" + +echo ">>> DOCKER GENERATION: Installing Crossdata before generating $XD_VERSION" + +mvn clean install -DskipUTs -DskipITs + +echo ">>> DOCKER GENERATION: Generating $XD_VERSION package" + +mvn package -Ppackage -DskipUTs -DskipITs + +echo ">>> DOCKER GENERATION: Moving $XD_VERSION to docker directory" + +cp dist/target/2.11/stratio-crossdata-mesosphere-scala211-$XD_VERSION.all.deb docker/ + +changedir "docker" + +echo ">>> DOCKER GENERATION: Executing Dockerfile using $XD_VERSION" + +docker build --build-arg PKG=$XD_VERSION -t crossdata-enterprise:$XD_VERSION . + +echo ">>> DOCKER GENERATION: Docker generated using $XD_VERSION" + diff --git a/docker/generateWithGosec.sh b/docker/generateWithGosec.sh new file mode 100755 index 000000000..cf6de58bf --- /dev/null +++ b/docker/generateWithGosec.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +. commons.sh + +if [ -z "$1" ]; then + echo ">>> DOCKER GENERATION: Usage: ./generateWithGosec.sh [skipXDDocker]" + echo ">>> DOCKER GENERATION: Usage example: ./generateWithGosec.sh branch-0.4 skipXDDocker" + exit 1 +fi + +if [ -z "$2" ]; then + echo ">>> DOCKER GENERATION: Generating Crossdata $XD_VERSION docker without security" + . generate.sh + echo ">>> DOCKER GENERATION: Crossdata $XD_VERSION docker without security created" +fi + +echo ">>> DOCKER GENERATION: Cloning dyplon-gosec project" + +if [ -d "gosec-dyplon" ]; then + rm -rf gosec-dyplon +fi + +git clone https://github.com/Stratio/gosec-dyplon.git + +echo ">>> DOCKER GENERATION: dyplon-gosec cloned" + +changedir "gosec-dyplon" + +echo ">>> DOCKER GENERATION: checkout to branch $1" + +git checkout $1 + +echo ">>> DOCKER GENERATION: Modifying Gosec plugin for Crossdata" + +if [ -z "$2" ]; then + #Modify Gosec Crossdata plugin Dockerfile to put $XD_VERSION and docker image created + sed -i .tmp "s|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:.*|FROM crossdata-enterprise:$XD_VERSION|" plugins/crossdata/Dockerfile +else + #Modify Gosec Crossdata plugin Dockerfile to put $XD_VERSION and docker image from qa.stratio.com + sed -i .tmp "s|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:.*|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:$XD_VERSION|" plugins/crossdata/Dockerfile +fi + +#Modify Crossdata pom to put $XD_VERSION +sed -i .tmp "s|.*|$XD_VERSION|" plugins/crossdata/pom.xml +sed -i .tmp "s|COPY plugins/crossdata/|COPY |" plugins/crossdata/Dockerfile +sed -i .tmp '7i\ +RUN chmod +x /secured-docker-entrypoint.sh' plugins/crossdata/Dockerfile + +#Get Crossdata plugin version from pom +tmp4=`grep -m1 "" plugins/crossdata/pom.xml` +tmp5=${tmp4//} +tmp6=${tmp5/<\/version>/} +XD_GOSEC_VERSION=${tmp6// } + +echo ">>> DOCKER GENERATION: Installing gosec-dyplon (including Crossdata $XD_GOSEC_VERSION)" + +mvn clean install -DskipUTs -DskipITs + +echo ">>> DOCKER GENERATION: gosec-dyplon installed" + +changedir "plugins/crossdata" + +echo ">>> DOCKER GENERATION: Generating docker $XD_VERSION with Gosec" + +docker build --build-arg VERSION=$XD_GOSEC_VERSION -t crossdata-gosec:$XD_VERSION . + +echo ">>> DOCKER GENERATION: Docker generated for $XD_VERSION with Gosec generated" + +changedir "../../../" + +rm -rf gosec-dyplon + From 038a9920e4cb49ccfb013feed691f19b24d76cf5 Mon Sep 17 00:00:00 2001 From: David Arroyo Cazorla Date: Thu, 22 Dec 2016 09:35:40 +0100 Subject: [PATCH 04/22] [docker-entrypoint] fix catalog configuration (#65) --- docker/catalog-config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/catalog-config.sh b/docker/catalog-config.sh index b083fa3c5..2185f1008 100644 --- a/docker/catalog-config.sh +++ b/docker/catalog-config.sh @@ -24,13 +24,13 @@ if [ $# > 0 ]; then if [ "x$1x" != "xx" ]; then export crossdata_core_catalog_class="\"${CATALOG_CLASS_PREFIX}.$1Catalog\"" if [ "$1" == "MySQL" ]; then - export jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} + jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi if [ "$1" == "PostgreSQL" ]; then - export jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} + jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi if [ "$1" == "Zookeeper" ]; then - export zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} + zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} fi if [ "x$2x" != "xx" ]; then export crossdata_core_catalog_prefix=${2:-crossdataCluster} From a1ed836e02cef6ab1fecb97b118537643deb6e22 Mon Sep 17 00:00:00 2001 From: David Arroyo Cazorla Date: Thu, 22 Dec 2016 11:33:07 +0100 Subject: [PATCH 05/22] remove quotation marks (#67) --- docker/catalog-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/catalog-config.sh b/docker/catalog-config.sh index 2185f1008..26d7ed80a 100644 --- a/docker/catalog-config.sh +++ b/docker/catalog-config.sh @@ -22,7 +22,7 @@ function zookeeperCatalog() { if [ $# > 0 ]; then if [ "x$1x" != "xx" ]; then - export crossdata_core_catalog_class="\"${CATALOG_CLASS_PREFIX}.$1Catalog\"" + export crossdata_core_catalog_class="${CATALOG_CLASS_PREFIX}.$1Catalog" if [ "$1" == "MySQL" ]; then jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} fi From ba586c37b01f30b84fd1a1765c2cf7048eaeec64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Francisco=20P=C3=A9rez=20Hidalgo?= Date: Thu, 22 Dec 2016 12:28:24 +0100 Subject: [PATCH 06/22] RowSerializer: Safe guard against double types with Decimal schema tags (#66) From 8df501e2cd2db59d26871732f83104511f34e69e Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Thu, 22 Dec 2016 15:03:18 +0100 Subject: [PATCH 07/22] [CROSSDATA] [SERVICE DISCOVERY] Local member from config (#64) --- .../server/ServiceDiscoveryProvider.scala | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index d67f22952..e67e4bccf 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -48,11 +48,6 @@ trait ServiceDiscoveryProvider { private def getLocalSeed: String = s"${Try(serverConfig.getString("akka.remote.netty.tcp.hostname")).getOrElse("127.0.0.1")}:${Try(serverConfig.getInt("akka.remote.netty.tcp.port")).getOrElse("13420")}" - private def getLocalSeed(xdCluster: Cluster): String = { - val selfAddress = xdCluster.selfAddress - s"${selfAddress.host.getOrElse("127.0.0.1")}:${selfAddress.port.getOrElse("13420")}" - } - private def getLocalMember: String = { val defaultAddr = "127.0.0.1" val defaultPort = "5701" @@ -63,12 +58,6 @@ trait ServiceDiscoveryProvider { } getOrElse s"$defaultAddr:$defaultPort" } - private def getLocalMember(hsp: HazelcastSessionProvider): String = { - val selfAddress = hsp.gelLocalMember.getAddress - s"${selfAddress.getHost}:${selfAddress.getPort}" - } - - protected def startServiceDiscovery(sdch: SDCH) = { // Start ZK connection val curatorClient = startDiscoveryClient(sdch) @@ -185,6 +174,8 @@ trait ServiceDiscoveryProvider { val newSeeds = (Set(localSeed) ++ currentSeeds.split(",").toSet).map(m => m.trim).filter(_.nonEmpty) dClient.setData.forPath(pathForSeeds, newSeeds.mkString(",").getBytes) + logger.info(s"Service discovery config - Cluster seeds: ${newSeeds.mkString(",")}") + val protocol = s"akka.${ if (Try(serverConfig.getBoolean("akka.remote.netty.ssl.enable-ssl")).getOrElse(false)) "ssl." else "" }tcp" @@ -211,6 +202,8 @@ trait ServiceDiscoveryProvider { Set(localMember) }).map(m => m.trim).filter(_.nonEmpty) + logger.info(s"Service discovery config - Provider members: ${newMembers.mkString(",")}") + dClient.setData.forPath(pathForMembers, newMembers.mkString(",").getBytes) val modifiedHzConfig = hzConfig.setNetworkConfig( hzConfig.getNetworkConfig.setJoin( @@ -226,7 +219,7 @@ trait ServiceDiscoveryProvider { val pathForMembers = h.sdch.getOrElse(SDCH.ProviderPath, SDCH.DefaultProviderPath) ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForMembers) - val updatedMembers = Set(getLocalMember(hsp)) ++ sessionProviderOpt.map { + val updatedMembers = Set(getLocalMember) ++ sessionProviderOpt.map { case hzSP: HazelcastSessionProvider => hzSP.getHzMembers.to[Set].map { m => s"${m.getAddress.getHost}:${m.getAddress.getPort}" @@ -248,8 +241,8 @@ trait ServiceDiscoveryProvider { } private def updateClusterSeeds(xCluster: Cluster, h: SDH) = { - val currentSeeds = xCluster.state.members.filter(_.roles.contains("server")).map( - m => s"${m.address.host.getOrElse("127.0.0.1")}:${m.address.port.getOrElse("13420")}") + getLocalSeed(xCluster) + val currentSeeds = getLocalSeed + xCluster.state.members.filter(_.roles.contains("server")).map( + m => s"${m.address.host.getOrElse("127.0.0.1")}:${m.address.port.getOrElse("13420")}") val pathForSeeds = h.sdch.getOrElse(SDCH.SeedsPath, SDCH.DefaultSeedsPath) ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForSeeds) logger.info(s"Updating seeds: ${currentSeeds.mkString(",")}") From c4c1ac385155cc3ce5e30baa27fa8737d606adb4 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Thu, 22 Dec 2016 19:58:11 +0100 Subject: [PATCH 08/22] [CROSSDATA] [SERVICE DISCOVERY] Scaladoc added (#62) * [CROSSDATA] [SERVICE DISCOVERY] Scaladoc added * [CROSSDATA] [SERVICE DISCOVERY] Improvements in scaladoc * [CROSSDATA] [SERVICE DISCOVERY] Some errors fixed * [CROSSDATA] [SERVICE DISCOVERY] Another error fixed --- .../crossdata/server/CrossdataServer.scala | 6 +- .../server/ServiceDiscoveryProvider.scala | 111 ++++++++++++------ 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala b/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala index 517752338..19f61b514 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala @@ -22,18 +22,18 @@ import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} import akka.actor.{ActorSystem, Props} import akka.cluster.Cluster import akka.cluster.client.ClusterClientReceptionist +import akka.cluster.pubsub.DistributedPubSub +import akka.cluster.pubsub.DistributedPubSubMediator.Put import akka.http.scaladsl.Http.ServerBinding import akka.http.scaladsl.{Http, HttpsConnectionContext} import akka.routing.{DefaultResizer, RoundRobinPool} import akka.stream.{ActorMaterializer, TLSClientAuth} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator.Put import com.stratio.crossdata.common.security.KeyStoreUtils import com.stratio.crossdata.common.util.akka.keepalive.KeepAliveMaster import com.stratio.crossdata.server.actors.{ResourceManagerActor, ServerActor} import com.stratio.crossdata.server.config.ServerConfig import com.stratio.crossdata.server.discovery.{ServiceDiscoveryConfigHelper => SDCH, ServiceDiscoveryHelper => SDH} -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.ConfigFactory import org.apache.log4j.Logger import org.apache.spark.sql.crossdata import org.apache.spark.sql.crossdata.session.{BasicSessionProvider, HazelcastSessionProvider} diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index e67e4bccf..f592885c5 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -15,20 +15,22 @@ */ package com.stratio.crossdata.server +import java.util import java.util.UUID import java.util.concurrent.TimeUnit -import akka.actor.{ActorSystem, Address} +import akka.actor.{ActorSystem, Address, Cancellable} import akka.cluster.Cluster import com.hazelcast.config.{XmlConfigBuilder, Config => HzConfig} import com.stratio.crossdata.server.discovery.{ServiceDiscoveryConfigHelper => SDCH, ServiceDiscoveryHelper => SDH} -import com.typesafe.config.ConfigValueFactory +import com.typesafe.config.{Config, ConfigValueFactory} import org.apache.curator.framework.recipes.leader.LeaderLatch import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} import org.apache.curator.retry.ExponentialBackoffRetry import org.apache.curator.utils.ZKPaths import org.apache.log4j.Logger import org.apache.spark.sql.crossdata.session.{HazelcastSessionProvider, XDSessionProvider} +import org.apache.zookeeper.data.Stat import scala.collection.JavaConversions._ import scala.concurrent.duration.FiniteDuration @@ -45,9 +47,17 @@ trait ServiceDiscoveryProvider { protected val hzConfig: HzConfig = new XmlConfigBuilder().build() protected[crossdata] var sessionProviderOpt: Option[XDSessionProvider] = None //TODO Remove [crossdata] + /** + * Get public address according to the initial configuration of the servers' cluster. + * + */ private def getLocalSeed: String = s"${Try(serverConfig.getString("akka.remote.netty.tcp.hostname")).getOrElse("127.0.0.1")}:${Try(serverConfig.getInt("akka.remote.netty.tcp.port")).getOrElse("13420")}" + /** + * Get public address according to the initial configuration of the service provider. + * + */ private def getLocalMember: String = { val defaultAddr = "127.0.0.1" val defaultPort = "5701" @@ -58,7 +68,12 @@ trait ServiceDiscoveryProvider { } getOrElse s"$defaultAddr:$defaultPort" } - protected def startServiceDiscovery(sdch: SDCH) = { + /** + * It starts the subscription to the Crossdata cluster according to the initial configuration + * and the information provided in the remote server of the service discovery. + * + */ + protected def startServiceDiscovery(sdch: SDCH): SDH = { // Start ZK connection val curatorClient = startDiscoveryClient(sdch) @@ -74,10 +89,8 @@ trait ServiceDiscoveryProvider { } /** - * Create and start the curator client + * Create and start the curator client. * - * @param sdConfig - * @return */ private def startDiscoveryClient(sdConfig: SDCH): CuratorFramework = { val curatorClient = CuratorFrameworkFactory.newClient( @@ -91,13 +104,11 @@ trait ServiceDiscoveryProvider { } /** - * Waiting to try to be the leader??? + * Non-blocking call to acquire cluster leadership. In every moment, there is only one cluster leader. + * This cluster leader updates the list of current members of the cluster every x seconds (300 by default). * - * @param dClient - * @param sdc - * @return */ - private def requestClusterLeadership(dClient: CuratorFramework, sdc: SDCH) = { + private def requestClusterLeadership(dClient: CuratorFramework, sdc: SDCH): Future[Unit] = { val cLeaderPath = sdc.getOrElse(SDCH.ClusterLeaderPath, SDCH.DefaultClusterLeaderPath) logger.debug(s"Service discovery - cluster leadership path: $cLeaderPath") @@ -128,15 +139,12 @@ trait ServiceDiscoveryProvider { leadershipFuture } - /** - * Trying to get who is the leader??? + * Wait until subscription leadership is acquired in order to: write down this node as part of the seeds + * and join to the cluster. * - * @param dClient - * @param sdc - * @return */ - private def requestSubscriptionLeadership(dClient: CuratorFramework, sdc: SDCH) = { + private def requestSubscriptionLeadership(dClient: CuratorFramework, sdc: SDCH): Try[LeaderLatch] = { val sLeaderPath = sdc.getOrElse(SDCH.SubscriptionPath, SDCH.DefaultSubscriptionPath) @@ -160,7 +168,12 @@ trait ServiceDiscoveryProvider { } } - private def generateFinalConfig(dClient: CuratorFramework, sdc: SDCH) = { + /** + * Generate contact points in the config according to the content of the remote server of the service discovery. + * In addition, it adds itself to the content of the contact points before generating the final config. + * + */ + private def generateFinalConfig(dClient: CuratorFramework, sdc: SDCH): (Config, HzConfig) = { val pathForSeeds = sdc.getOrElse(SDCH.SeedsPath, SDCH.DefaultSeedsPath) logger.debug(s"Service Discovery - seeds path: $pathForSeeds") @@ -214,24 +227,26 @@ trait ServiceDiscoveryProvider { (modifiedAkkaConfig, modifiedHzConfig) } - private def updateClusterMembers(h: SDH, hsp: HazelcastSessionProvider) = { - - val pathForMembers = h.sdch.getOrElse(SDCH.ProviderPath, SDCH.DefaultProviderPath) - ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForMembers) - - val updatedMembers = Set(getLocalMember) ++ sessionProviderOpt.map { - case hzSP: HazelcastSessionProvider => - hzSP.getHzMembers.to[Set].map { m => - s"${m.getAddress.getHost}:${m.getAddress.getPort}" - } - case _ => Set.empty - }.getOrElse(Set.empty) + /** + * It creates a scheduled task (every x seconds, 300 by default) that updates the members of the cluster + * on the remote server of the server discovery. + * + */ + protected def updateServiceDiscovery(xCluster: Cluster, hsp: HazelcastSessionProvider, s: SDH, aSystem: ActorSystem): Cancellable = { + val delay = new FiniteDuration( + s.sdch.getOrElse(SDCH.ClusterDelayPath, SDCH.DefaultClusterDelay.toString).toLong, TimeUnit.SECONDS) - logger.info(s"Updating members: ${updatedMembers.mkString(",")}") - h.curatorClient.setData.forPath(pathForMembers, updatedMembers.mkString(",").getBytes) + import scala.concurrent.ExecutionContext.Implicits.global + aSystem.scheduler.schedule(delay, delay)(updateSeeds(xCluster, hsp, s)) } - private def updateSeeds(xCluster: Cluster, hsp: HazelcastSessionProvider, h: SDH) = { + /** + * It acquires the subscription leadership (in order to avoid race conditions with new members + * joining to the cluster at the same time) and triggers the methods to update the contact points for + * the members of the Crossdata cluster and the members of the service provider. + * + */ + private def updateSeeds(xCluster: Cluster, hsp: HazelcastSessionProvider, h: SDH): Unit = { val sll = new LeaderLatch(h.curatorClient, h.sdch.getOrElse(SDCH.SubscriptionPath, SDCH.DefaultSubscriptionPath)) sll.start sll.await @@ -240,7 +255,11 @@ trait ServiceDiscoveryProvider { sll.close } - private def updateClusterSeeds(xCluster: Cluster, h: SDH) = { + /** + * Overrides the cluster seeds on the remote server of the service discovery according to the cluster state. + * + */ + private def updateClusterSeeds(xCluster: Cluster, h: SDH): String = { val currentSeeds = getLocalSeed + xCluster.state.members.filter(_.roles.contains("server")).map( m => s"${m.address.host.getOrElse("127.0.0.1")}:${m.address.port.getOrElse("13420")}") val pathForSeeds = h.sdch.getOrElse(SDCH.SeedsPath, SDCH.DefaultSeedsPath) @@ -250,12 +269,26 @@ trait ServiceDiscoveryProvider { currentSeeds } - protected def updateServiceDiscovery(xCluster: Cluster, hsp: HazelcastSessionProvider, s: SDH, aSystem: ActorSystem) = { - val delay = new FiniteDuration( - s.sdch.getOrElse(SDCH.ClusterDelayPath, SDCH.DefaultClusterDelay.toString).toLong, TimeUnit.SECONDS) + /** + * Overrides the service provider members on the remote server of the service discovery according to + * the current members. + * + */ + private def updateClusterMembers(h: SDH, hsp: HazelcastSessionProvider): Stat = { - import scala.concurrent.ExecutionContext.Implicits.global - aSystem.scheduler.schedule(delay, delay)(updateSeeds(xCluster, hsp, s)) + val pathForMembers = h.sdch.getOrElse(SDCH.ProviderPath, SDCH.DefaultProviderPath) + ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForMembers) + + val updatedMembers = Set(getLocalMember) ++ sessionProviderOpt.map { + case hzSP: HazelcastSessionProvider => + hzSP.getHzMembers.to[Set].map { m => + s"${m.getAddress.getHost}:${m.getAddress.getPort}" + } + case _ => Set.empty + }.getOrElse(Set.empty) + + logger.info(s"Updating members: ${updatedMembers.mkString(",")}") + h.curatorClient.setData.forPath(pathForMembers, updatedMembers.mkString(",").getBytes) } } From 74838bcebec53cbd2891963ee66fe840aea7c781 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Fri, 23 Dec 2016 09:13:31 +0100 Subject: [PATCH 09/22] [DCS-556] [CROSSDATA] [DOCKER] Gosec security manager activated (#69) * [DCS-556] [CROSSDATA] [DOCKER] Gosec security manager activated * Using the right dependency --- dist/pom.xml | 5 +++++ docker/crossdata-config.sh | 2 +- docker/docker-entrypoint.sh | 2 +- docker/gosec-config.sh | 5 +++++ docker/security-config.sh | 2 +- pom.xml | 1 + 6 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dist/pom.xml b/dist/pom.xml index b35056a6e..c08882cc0 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -35,6 +35,11 @@ ${project.parent.version} jar-with-dependencies + + com.stratio.gosec.dyplon.plugins + crossdata + ${gosec.version} + diff --git a/docker/crossdata-config.sh b/docker/crossdata-config.sh index 7751a4cf8..7e3470656 100644 --- a/docker/crossdata-config.sh +++ b/docker/crossdata-config.sh @@ -101,7 +101,7 @@ function marathonConfig() { ## Vault and secrets (configured if enabled) #################################################### if [ ! -z ${VAULT_HOST} ]; then - source security-config.sh + source security-config.sh $1 fi #################################################### diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 08de7b235..4ba1bd568 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -23,7 +23,7 @@ source catalog-config.sh $XD_CATALOG $XD_CATALOG_PREFIX #################################################### ## Crossdata Config #################################################### -source crossdata-config.sh +source crossdata-config.sh $1 case "$SERVER_MODE" in "shell") # This mode will launch a crossdata shell instead of a crossdata server if [ "$SHELL_SERVERADDR" -a "$SHELL_SERVERPORT" ]; then diff --git a/docker/gosec-config.sh b/docker/gosec-config.sh index c6aeb1d83..7e314021b 100644 --- a/docker/gosec-config.sh +++ b/docker/gosec-config.sh @@ -4,6 +4,11 @@ ## Get Gosec-plugin LDAP user and pass and set XD vars ####################################################### +if ["$1" != "skipSecManager"]; then + export CROSSDATA_SECURITY_MANAGER_CLASS=com.stratio.gosec.dyplon.plugins.crossdata.GoSecCrossdataSecurityManager + export CROSSDATA_SECURITY_MANAGER_ENABLED=true +fi + ### Get LDAP user and pass export XD_GOSEC_PLUGIN_LDAP_USER=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"user\"") export XD_GOSEC_PLUGIN_LDAP_PASS=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"pass\"") diff --git a/docker/security-config.sh b/docker/security-config.sh index 913757bb1..0bb42f66e 100644 --- a/docker/security-config.sh +++ b/docker/security-config.sh @@ -44,7 +44,7 @@ source kerberos-server-config.sh ####################################################### ## Gosec-plugin config ####################################################### -source gosec-config.sh +source gosec-config.sh $1 ####################################################### ## HDFS security diff --git a/pom.xml b/pom.xml index f7723a94d..e64656794 100644 --- a/pom.xml +++ b/pom.xml @@ -107,6 +107,7 @@ 1.6.2 + 0.5.0-SNAPSHOT 2.2.5 3.4 ${project.build.outputDirectory} From bf5889f07b72e978adc2235c9d17812364004d8c Mon Sep 17 00:00:00 2001 From: jjlopezm Date: Wed, 18 Jan 2017 17:19:30 +0100 Subject: [PATCH 10/22] remove gosec dependency due to community version --- dist/pom.xml | 5 ----- pom.xml | 1 - 2 files changed, 6 deletions(-) diff --git a/dist/pom.xml b/dist/pom.xml index c08882cc0..b35056a6e 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -35,11 +35,6 @@ ${project.parent.version} jar-with-dependencies - - com.stratio.gosec.dyplon.plugins - crossdata - ${gosec.version} - diff --git a/pom.xml b/pom.xml index e64656794..f7723a94d 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,6 @@ 1.6.2 - 0.5.0-SNAPSHOT 2.2.5 3.4 ${project.build.outputDirectory} From 65e61f6518fa3cbc12a1e7aac08f0f8b56d6f32e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Francisco=20P=C3=A9rez=20Hidalgo?= Date: Fri, 23 Dec 2016 10:41:47 +0100 Subject: [PATCH 11/22] RowSerializers: Added additional types as candidates to be serialized as Decimal values (#71) * RowSerializer: Safe guard against double types with Decimal schema tags * RowSerializer: Added additional types as decimal values. --- .../common/serializers/RowSerializer.scala | 4 ++++ .../common/serializers/RowSerializerSpec.scala | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala b/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala index abb29e775..147e5739d 100644 --- a/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala +++ b/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala @@ -94,9 +94,13 @@ case class RowSerializer(providedSchema: StructType) extends Serializer[Row] { case (FloatType, v: Float) => JDouble(v) case (DoubleType, v: Double) => JDouble(v) case (LongType, v: Long) => JInt(v) + case (_: DecimalType, v: BigDecimal) => JDecimal(v) case (_: DecimalType, v: Decimal) => JDecimal(v.toBigDecimal) + case (_: DecimalType, v: String) => JDecimal(BigDecimal(v)) case (_: DecimalType, v: Double) => JDecimal(BigDecimal(v)) case (_: DecimalType, v: Float) => JDecimal(BigDecimal(v)) + case (_: DecimalType, v: Long) => JDecimal(BigDecimal(v)) + case (_: DecimalType, v: Int) => JDecimal(BigDecimal(v)) case (ByteType, v: Byte) => JInt(v.toInt) case (BinaryType, v: Array[Byte]) => JString(new String(v)) case (BooleanType, v: Boolean) => JBool(v) diff --git a/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala b/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala index cbf5018e1..7e56f54cb 100644 --- a/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala +++ b/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala @@ -117,8 +117,15 @@ class RowSerializerSpec extends XDSerializationTest[Row] with CrossdataCommonSer it should " be able to recover Double values when their schema type is misleading" in { - val schema = StructType(List(StructField("decimaldouble", DecimalType(10,1),true))) - val row = Row.fromSeq(Array(32.1)) + val row = Row.fromSeq( + Array(32.0, 32.0F, BigDecimal(32.0), "32.0", 32L, 32) + ) + + val schema = StructType ( + (0 until row.size) map { idx => + StructField(s"decimaldouble$idx", DecimalType(10,1), true) + } toList + ) val formats = json4sJacksonFormats + new RowSerializer(schema) @@ -126,7 +133,9 @@ class RowSerializerSpec extends XDSerializationTest[Row] with CrossdataCommonSer val extracted = parse(serialized, false).extract[Row](formats, implicitly[Manifest[Row]]) inside(extracted) { - case r: Row => r.get(0) shouldBe Decimal(32.1) + case r: Row => r.toSeq foreach { cellValue => + cellValue shouldBe Decimal(32.0) + } } } From a2fdefdf0e384a815f6e127791f9f81299c4ccbc Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Fri, 23 Dec 2016 12:15:28 +0100 Subject: [PATCH 12/22] [CROSSDATA] [DOCKER] XD_CATALOG set to Derby by default (#72) * [CROSSDATA] [DOCKER] XD_CATALOG set to Derby by default * [CROSSDATA] [DOCKER] Redundant information removed * [CROSSDATA] Implicits global for scheduler added --- .../catalog/persistent/DerbyCatalog.scala | 5 ++- docker/catalog-config.sh | 31 ++++++++----------- docker/docker-entrypoint.sh | 3 +- .../server/ServiceDiscoveryProvider.scala | 1 - 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/DerbyCatalog.scala b/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/DerbyCatalog.scala index cc5fdabca..a360910ac 100644 --- a/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/DerbyCatalog.scala +++ b/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/DerbyCatalog.scala @@ -18,12 +18,11 @@ package org.apache.spark.sql.crossdata.catalog.persistent import java.sql.{Connection, DriverManager, PreparedStatement, ResultSet} import com.stratio.crossdata.util.using -import org.apache.spark.sql.catalyst.{CatalystConf, TableIdentifier} +import org.apache.spark.sql.catalyst.CatalystConf import org.apache.spark.sql.crossdata.CrossdataVersion -import org.apache.spark.sql.crossdata.catalog.{IndexIdentifierNormalized, TableIdentifierNormalized, StringNormalized, XDCatalog, persistent} +import org.apache.spark.sql.crossdata.catalog._ import scala.annotation.tailrec -import scala.util.Try // TODO refactor SQL catalog implementations object DerbyCatalog { diff --git a/docker/catalog-config.sh b/docker/catalog-config.sh index 26d7ed80a..fae238e3d 100644 --- a/docker/catalog-config.sh +++ b/docker/catalog-config.sh @@ -3,8 +3,7 @@ CATALOG_CLASS_PREFIX="org.apache.spark.sql.crossdata.catalog.persistent" function jdbcCatalog() { - DEFAULT_CATALOG="$CATALOG_CLASS_PREFIX.DerbyCatalog" - export crossdata_core_catalog_jdbc_driver=${1:-$DEFAULT_CATALOG} + export crossdata_core_catalog_jdbc_driver=$1 export crossdata_core_catalog_jdbc_url=$2 export crossdata_core_catalog_jdbc_name=$3 export crossdata_core_catalog_jdbc_user=$4 @@ -20,20 +19,16 @@ function zookeeperCatalog() { } -if [ $# > 0 ]; then -if [ "x$1x" != "xx" ]; then - export crossdata_core_catalog_class="${CATALOG_CLASS_PREFIX}.$1Catalog" - if [ "$1" == "MySQL" ]; then - jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} - fi - if [ "$1" == "PostgreSQL" ]; then - jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} - fi - if [ "$1" == "Zookeeper" ]; then - zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} - fi - if [ "x$2x" != "xx" ]; then - export crossdata_core_catalog_prefix=${2:-crossdataCluster} - fi +export crossdata_core_catalog_class="${CATALOG_CLASS_PREFIX}.$1Catalog" +if [ "$1" == "MySQL" ]; then + jdbcCatalog "org.mariadb.jdbc.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} +fi +if [ "$1" == "PostgreSQL" ]; then + jdbcCatalog "org.postgresql.Driver" ${XD_CATALOG_HOST} ${XD_CATALOG_DB_NAME} ${XD_CATALOG_DB_USER} ${XD_CATALOG_DB_PASS} +fi +if [ "$1" == "Zookeeper" ]; then + zookeeperCatalog ${XD_CATALOG_ZOOKEEPER_CONNECTION_STRING} ${XD_CATALOG_ZOOKEEPER_CONNECTION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_SESSION_TIMEOUT} ${XD_CATALOG_ZOOKEEPER_RETRY_ATTEMPS} ${XD_CATALOG_ZOOKEEPER_RETRY_INTERVAL} +fi +if [ "x$2x" != "xx" ]; then + export crossdata_core_catalog_prefix=${2:-crossdataCluster} fi -fi \ No newline at end of file diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4ba1bd568..af3bfe52b 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -10,7 +10,8 @@ fi #################################################### ## XD Catalog #################################################### -source catalog-config.sh $XD_CATALOG $XD_CATALOG_PREFIX +CROSSDATA_CATALOG=${XD_CATALOG:-Derby} +source catalog-config.sh $CROSSDATA_CATALOG $XD_CATALOG_PREFIX #################################################### diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index f592885c5..909f1935c 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -15,7 +15,6 @@ */ package com.stratio.crossdata.server -import java.util import java.util.UUID import java.util.concurrent.TimeUnit From 7fe0c9d60794d4b7f4a468b01c1151681668cb73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Francisco=20P=C3=A9rez=20Hidalgo?= Date: Tue, 27 Dec 2016 12:06:44 +0100 Subject: [PATCH 13/22] RowSerializer: Added Java's big decimal type as valid source type. (#74) --- .../stratio/crossdata/common/serializers/RowSerializer.scala | 5 +++++ .../crossdata/common/serializers/RowSerializerSpec.scala | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala b/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala index 147e5739d..4a2a64547 100644 --- a/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala +++ b/common/src/main/scala/com/stratio/crossdata/common/serializers/RowSerializer.scala @@ -16,6 +16,7 @@ package com.stratio.crossdata.common.serializers import java.sql.Timestamp +import java.math.{BigDecimal => JBigDecimal} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} @@ -50,6 +51,7 @@ case class RowSerializer(providedSchema: StructType) extends Serializer[Row] { case (_: DecimalType, v: JNumber) => v match { case JInt(v) => Decimal(v.toString) + case JLong(v) => Decimal(v.toString) case JDecimal(v) => Decimal(v) case JDouble(v) => Decimal(v) } @@ -94,6 +96,9 @@ case class RowSerializer(providedSchema: StructType) extends Serializer[Row] { case (FloatType, v: Float) => JDouble(v) case (DoubleType, v: Double) => JDouble(v) case (LongType, v: Long) => JInt(v) + case (_: DecimalType, v: JBigDecimal) => + import scala.collection.JavaConverters._ + JDecimal(v) case (_: DecimalType, v: BigDecimal) => JDecimal(v) case (_: DecimalType, v: Decimal) => JDecimal(v.toBigDecimal) case (_: DecimalType, v: String) => JDecimal(BigDecimal(v)) diff --git a/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala b/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala index 7e56f54cb..ea0c406d8 100644 --- a/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala +++ b/common/src/test/scala/com/stratio/crossdata/common/serializers/RowSerializerSpec.scala @@ -118,7 +118,7 @@ class RowSerializerSpec extends XDSerializationTest[Row] with CrossdataCommonSer it should " be able to recover Double values when their schema type is misleading" in { val row = Row.fromSeq( - Array(32.0, 32.0F, BigDecimal(32.0), "32.0", 32L, 32) + Array(32.0, 32.0F, BigDecimal(32.0), new java.math.BigDecimal(32.0), "32.0", 32L, 32) ) val schema = StructType ( From cc9da02d4bc46c917dcbcb1824c3f41b4193d6b6 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Tue, 27 Dec 2016 15:11:49 +0100 Subject: [PATCH 14/22] [CROSSDATA] [DOCKER] Deactivation of SSL in TCP connections (#73) * [CROSSDATA] [DOCKER] Deactivation of SSL in TCP connections * [CROSSDATA] [DOCKER] Logic for TCP access from outside removed --- docker/crossdata-config.sh | 39 +++++++++--------------------------- docker/crossdata-security.sh | 1 - 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/docker/crossdata-config.sh b/docker/crossdata-config.sh index 7e3470656..7e5bb00dd 100644 --- a/docker/crossdata-config.sh +++ b/docker/crossdata-config.sh @@ -63,37 +63,16 @@ function marathonConfig() { ######################################################################################################## - #If XD_EXTERNAL_IP and MARATHON_APP_LABEL_HAPROXY_1_PORT are not specified assume we are working in HTTP mode - #Scenary: HAProxy exposing Akka http port, and creating an internal cluster using netty and autodiscovery through Zookeeper + # Working in HTTP mode + # Scenary: HAProxy exposing Akka http port, and creating an internal cluster using netty and + # autodiscovery through Zookeeper ######################################################################################################## - if [ -z ${XD_EXTERNAL_IP} ] && [ -z ${MARATHON_APP_LABEL_HAPROXY_1_PORT} ]; then - setCrossdataDir ${HOST} ${PORT_13420} - setCrossdataBindHost ${HOST} ${PORT_13420} - setHazelcastConfig ${HOST} ${PORT_5701} - setDriverConfig ${HOST} ${PORT_13420} - # CROSSDATA_SERVER_CONFIG_HTTP_SERVER_PORT is set with the port provided by Marathon-LB - export CROSSDATA_SERVER_CONFIG_HTTP_SERVER_PORT=$PORT_13422 - else - #Scenary: HAProxy exposing the akka netty port with the external IP. Supported only for one instance of Crossdata - if [ -z ${XD_EXTERNAL_IP} ] || [ -z ${MARATHON_APP_LABEL_HAPROXY_1_PORT} ]; then - echo "ERROR: Env var XD_EXTERNAL_IP and label HAPROXY_1_PORT must be provided together using Marathon&Haproxy in TCP mode" 1>&2 - exit 1 # terminate and indicate error - else - #Hostname and port of haproxy - setCrossdataDir ${XD_EXTERNAL_IP} ${MARATHON_APP_LABEL_HAPROXY_1_PORT} - #Bind address for local - setCrossdataBindHost ${DOCKER_HOST} ${PORT_13420} - #Driver - setDriverConfig ${XD_EXTERNAL_IP} ${MARATHON_APP_LABEL_HAPROXY_1_PORT} - fi - # When using ClusterClient External IP, the hosts-files get updated in order to keep a consistent - # binding address in AKKA. - export NAMEADDR="$(hostname -i)" - if [ -n "$HAPROXY_SERVER_INTERNAL_ADDRESS" ]; then - export NAMEADDR=$HAPROXY_SERVER_INTERNAL_ADDRESS - fi - echo -e "$NAMEADDR\t$XD_EXTERNAL_IP" >> /etc/hosts - fi + setCrossdataDir ${HOST} ${PORT_13420} + setCrossdataBindHost ${HOST} ${PORT_13420} + setHazelcastConfig ${HOST} ${PORT_5701} + setDriverConfig ${HOST} ${PORT_13420} + # CROSSDATA_SERVER_CONFIG_HTTP_SERVER_PORT is set with the port provided by Marathon-LB + export CROSSDATA_SERVER_CONFIG_HTTP_SERVER_PORT=$PORT_13422 } diff --git a/docker/crossdata-security.sh b/docker/crossdata-security.sh index 110bd2533..810f42362 100644 --- a/docker/crossdata-security.sh +++ b/docker/crossdata-security.sh @@ -1,5 +1,4 @@ #!/bin/bash -xe -export CROSSDATA_SERVER_AKKA_REMOTE_NETTY_SSL_ENABLE_SSL=true export CROSSDATA_SERVER_AKKA_HTTP_SSL_ENABLE=true export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE=${XD_TRUST_JKS_NAME} export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE_PASSWORD=${XD_TRUSTSTORE_PASSWORD} From f2c5c9443128c81d845c8dc8d9723d7dcd524b6a Mon Sep 17 00:00:00 2001 From: ca-flores Date: Wed, 28 Dec 2016 15:48:51 +0100 Subject: [PATCH 15/22] curator dependency has been shaded as well zookeeper (#79) --- core/pom.xml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/pom.xml b/core/pom.xml index ea790d65a..5367edd4a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -131,6 +131,16 @@ reference.conf + + + org.apache.curator + shaded.org.apache.curator + + + org.apache.zookeeper + shaded.org.apache.zookeeper + + @@ -145,4 +155,3 @@ - From 1ac4cccdd94ada1e2aada20a0e1b78d9a951106d Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Thu, 29 Dec 2016 15:06:51 +0100 Subject: [PATCH 16/22] Update of Spark Compatibility table (#81) --- README.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d3b190487..8e5a31393 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,8 @@ Spark Compatibility | Crossdata Version | Spark Version | |-------------------|:--------------| -| 1.7.X | 1.6.X | -| 1.6.X | 1.6.X | -| 1.5.X | 1.6.X | -| 1.4.X | 1.6.X | -| 1.3.X | 1.6.X | -| 1.2.X | 1.5.X | -| 1.1.X | 1.5.X | -| 1.0.X | 1.5.X | - +| 1.3.X - 1.9.X | 1.6.X | +| 1.0.X - 1.2.X | 1.5.X | ============= Documentation From 8e1011e8be65f6e7098e182e1515f0090de88fa1 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Thu, 29 Dec 2016 16:13:57 +0100 Subject: [PATCH 17/22] [CROSSDATA] [DOCKER] Script for building docker locally (#80) --- docker/generate.sh | 26 --------- docker/generateWithGosec.sh | 72 ------------------------ {docker => scripts/dockerGen}/Dockerfile | 2 +- {docker => scripts/dockerGen}/commons.sh | 2 +- scripts/dockerGen/dockerGen.sh | 36 ++++++++++++ 5 files changed, 38 insertions(+), 100 deletions(-) delete mode 100755 docker/generate.sh delete mode 100755 docker/generateWithGosec.sh rename {docker => scripts/dockerGen}/Dockerfile (97%) rename {docker => scripts/dockerGen}/commons.sh (77%) create mode 100755 scripts/dockerGen/dockerGen.sh diff --git a/docker/generate.sh b/docker/generate.sh deleted file mode 100755 index a2091bcb7..000000000 --- a/docker/generate.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -. commons.sh - -changedir "../" - -echo ">>> DOCKER GENERATION: Installing Crossdata before generating $XD_VERSION" - -mvn clean install -DskipUTs -DskipITs - -echo ">>> DOCKER GENERATION: Generating $XD_VERSION package" - -mvn package -Ppackage -DskipUTs -DskipITs - -echo ">>> DOCKER GENERATION: Moving $XD_VERSION to docker directory" - -cp dist/target/2.11/stratio-crossdata-mesosphere-scala211-$XD_VERSION.all.deb docker/ - -changedir "docker" - -echo ">>> DOCKER GENERATION: Executing Dockerfile using $XD_VERSION" - -docker build --build-arg PKG=$XD_VERSION -t crossdata-enterprise:$XD_VERSION . - -echo ">>> DOCKER GENERATION: Docker generated using $XD_VERSION" - diff --git a/docker/generateWithGosec.sh b/docker/generateWithGosec.sh deleted file mode 100755 index cf6de58bf..000000000 --- a/docker/generateWithGosec.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -. commons.sh - -if [ -z "$1" ]; then - echo ">>> DOCKER GENERATION: Usage: ./generateWithGosec.sh [skipXDDocker]" - echo ">>> DOCKER GENERATION: Usage example: ./generateWithGosec.sh branch-0.4 skipXDDocker" - exit 1 -fi - -if [ -z "$2" ]; then - echo ">>> DOCKER GENERATION: Generating Crossdata $XD_VERSION docker without security" - . generate.sh - echo ">>> DOCKER GENERATION: Crossdata $XD_VERSION docker without security created" -fi - -echo ">>> DOCKER GENERATION: Cloning dyplon-gosec project" - -if [ -d "gosec-dyplon" ]; then - rm -rf gosec-dyplon -fi - -git clone https://github.com/Stratio/gosec-dyplon.git - -echo ">>> DOCKER GENERATION: dyplon-gosec cloned" - -changedir "gosec-dyplon" - -echo ">>> DOCKER GENERATION: checkout to branch $1" - -git checkout $1 - -echo ">>> DOCKER GENERATION: Modifying Gosec plugin for Crossdata" - -if [ -z "$2" ]; then - #Modify Gosec Crossdata plugin Dockerfile to put $XD_VERSION and docker image created - sed -i .tmp "s|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:.*|FROM crossdata-enterprise:$XD_VERSION|" plugins/crossdata/Dockerfile -else - #Modify Gosec Crossdata plugin Dockerfile to put $XD_VERSION and docker image from qa.stratio.com - sed -i .tmp "s|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:.*|FROM qa.stratio.com/stratio/crossdata-mesosphere-scala211:$XD_VERSION|" plugins/crossdata/Dockerfile -fi - -#Modify Crossdata pom to put $XD_VERSION -sed -i .tmp "s|.*|$XD_VERSION|" plugins/crossdata/pom.xml -sed -i .tmp "s|COPY plugins/crossdata/|COPY |" plugins/crossdata/Dockerfile -sed -i .tmp '7i\ -RUN chmod +x /secured-docker-entrypoint.sh' plugins/crossdata/Dockerfile - -#Get Crossdata plugin version from pom -tmp4=`grep -m1 "" plugins/crossdata/pom.xml` -tmp5=${tmp4//} -tmp6=${tmp5/<\/version>/} -XD_GOSEC_VERSION=${tmp6// } - -echo ">>> DOCKER GENERATION: Installing gosec-dyplon (including Crossdata $XD_GOSEC_VERSION)" - -mvn clean install -DskipUTs -DskipITs - -echo ">>> DOCKER GENERATION: gosec-dyplon installed" - -changedir "plugins/crossdata" - -echo ">>> DOCKER GENERATION: Generating docker $XD_VERSION with Gosec" - -docker build --build-arg VERSION=$XD_GOSEC_VERSION -t crossdata-gosec:$XD_VERSION . - -echo ">>> DOCKER GENERATION: Docker generated for $XD_VERSION with Gosec generated" - -changedir "../../../" - -rm -rf gosec-dyplon - diff --git a/docker/Dockerfile b/scripts/dockerGen/Dockerfile similarity index 97% rename from docker/Dockerfile rename to scripts/dockerGen/Dockerfile index 8de507ff8..1b8bc909b 100644 --- a/docker/Dockerfile +++ b/scripts/dockerGen/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Stratio Crossdata team "crossdata@stratio.com" # USAGE: build --build-arg PKG= -t # USAGE Example: docker build --build-arg PKG=1.8.0-RC2-SNAPSHOT -t crossdata-enterprise . -COPY . / +COPY dockerfiles/* / VOLUME /usr/lib/mesos diff --git a/docker/commons.sh b/scripts/dockerGen/commons.sh similarity index 77% rename from docker/commons.sh rename to scripts/dockerGen/commons.sh index fb2de6ba4..acb861f52 100644 --- a/docker/commons.sh +++ b/scripts/dockerGen/commons.sh @@ -6,7 +6,7 @@ function changedir() { } #Get Crossdata version from parent pom -tmp1=`grep -m2 "" ../pom.xml | tail -n1` +tmp1=`grep -m2 "" ../../pom.xml | tail -n1` tmp2=${tmp1//} tmp3=${tmp2/<\/version>/} XD_VERSION=${tmp3// } diff --git a/scripts/dockerGen/dockerGen.sh b/scripts/dockerGen/dockerGen.sh new file mode 100755 index 000000000..f6318c68f --- /dev/null +++ b/scripts/dockerGen/dockerGen.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +. commons.sh + +echo " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> " +echo " >>> NOTE: .deb PACKAGE MUST BE UP-TO-DATE! >>> " +echo " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> " +echo " " + +echo " >>> Executing script for generation Crossdata $XD_VERSION Docker" +echo " " + +debFile="../../dist/target/2.11/stratio-crossdata-mesosphere-scala211-$XD_VERSION.all.deb" + +if [[ ! -f $debFile ]]; then + echo "$debFile must be created previously" + exit 1 +fi + +mkdir dockerfiles + +echo " >>> Copying $debFile" +cp $debFile dockerfiles + +echo " >>> Copying docker scripts" +cp ../../docker/* dockerfiles + +echo " >>> Building Crossdata $XD_VERSION Docker" +docker build --build-arg PKG=$XD_VERSION -t crossdata-enterprise:$XD_VERSION . +echo " >>> Crossdata $XD_VERSION Docker generated" + +echo " >>> Cleaning some stuff..." +rm -rf dockerfiles + +echo " >>> Start Crossdata Docker with the Gosec security Manager: docker run [OPTIONS] crossdata-enterprise[:TAG] [COMMAND]" +echo " >>> Start Crossdata Docker with the default security Manager: docker run [OPTIONS] crossdata-enterprise[:TAG] [COMMAND] skipSecManager" From 4d5d454646b5ab15e0f0c7f76465d881d01ce709 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Thu, 29 Dec 2016 21:20:04 +0100 Subject: [PATCH 18/22] [CROSSDATA] [SERVICE DISCOVERY] Keeping order of seeds (#75) * [CROSSDATA] [SERVICE DISCOVERY] Keeping order of seeds * [CROSSDATA] [SERVICE DISCOVERY] Unnecessary code removed * [CROSSDATA] [SERVICE DISCOVERY] Improvements --- .../server/ServiceDiscoveryProvider.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index 909f1935c..9d6ca496d 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -183,7 +183,7 @@ trait ServiceDiscoveryProvider { val localSeed = getLocalSeed ZKPaths.mkdirs(dClient.getZookeeperClient.getZooKeeper, pathForSeeds) val currentSeeds = new String(dClient.getData.forPath(pathForSeeds)) - val newSeeds = (Set(localSeed) ++ currentSeeds.split(",").toSet).map(m => m.trim).filter(_.nonEmpty) + val newSeeds = (localSeed +: currentSeeds.split(",")).map(m => m.trim).filter(_.nonEmpty) dClient.setData.forPath(pathForSeeds, newSeeds.mkString(",").getBytes) logger.info(s"Service discovery config - Cluster seeds: ${newSeeds.mkString(",")}") @@ -194,7 +194,7 @@ trait ServiceDiscoveryProvider { val modifiedAkkaConfig = serverConfig.withValue( "akka.cluster.seed-nodes", - ConfigValueFactory.fromIterable(newSeeds.map { s => + ConfigValueFactory.fromIterable(newSeeds.toSeq.map { s => val hostPort = s.split(":") new Address(protocol, serverConfig.getString("config.cluster.name"), @@ -209,9 +209,9 @@ trait ServiceDiscoveryProvider { val currentMembers = new String(dClient.getData.forPath(pathForMembers)) val newMembers = (if (localMember.split(":").head != "127.0.0.1") { - currentMembers.split(",").toSet + localMember + localMember +: currentMembers.split(",") } else { - Set(localMember) + Array(localMember) }).map(m => m.trim).filter(_.nonEmpty) logger.info(s"Service discovery config - Provider members: ${newMembers.mkString(",")}") @@ -278,13 +278,13 @@ trait ServiceDiscoveryProvider { val pathForMembers = h.sdch.getOrElse(SDCH.ProviderPath, SDCH.DefaultProviderPath) ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForMembers) - val updatedMembers = Set(getLocalMember) ++ sessionProviderOpt.map { + val updatedMembers = getLocalMember +: sessionProviderOpt.map { case hzSP: HazelcastSessionProvider => - hzSP.getHzMembers.to[Set].map { m => + hzSP.getHzMembers.to[Seq].map { m => s"${m.getAddress.getHost}:${m.getAddress.getPort}" } - case _ => Set.empty - }.getOrElse(Set.empty) + case _ => Seq.empty + }.getOrElse(Seq.empty) logger.info(s"Updating members: ${updatedMembers.mkString(",")}") h.curatorClient.setData.forPath(pathForMembers, updatedMembers.mkString(",").getBytes) From 8bb9eea6312791b94ae6c91704835ee0bc51437c Mon Sep 17 00:00:00 2001 From: Juan Jose Lopez Martin Date: Mon, 9 Jan 2017 17:24:49 +0100 Subject: [PATCH 19/22] [DCS-769] Tables should not be cached when persistence fails (#87) * Order changed in cache persistent catalogs --- .../catalog/persistent/PersistentCatalogWithCache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/PersistentCatalogWithCache.scala b/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/PersistentCatalogWithCache.scala index 036b82fc8..88deb4cd5 100644 --- a/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/PersistentCatalogWithCache.scala +++ b/core/src/main/scala/org/apache/spark/sql/crossdata/catalog/persistent/PersistentCatalogWithCache.scala @@ -78,8 +78,8 @@ abstract class PersistentCatalogWithCache(catalystConf: CatalystConf) extends XD throw new UnsupportedOperationException(msg) } else { logInfo(s"Persisting view ${viewIdentifier.unquotedString}") - viewCache.put(viewIdentifier, plan) persistViewMetadata(viewIdentifier, sqlText) + viewCache.put(viewIdentifier, plan) } } @@ -91,8 +91,8 @@ abstract class PersistentCatalogWithCache(catalystConf: CatalystConf) extends XD throw new UnsupportedOperationException(s"The table $tableIdentifier already exists") } else { logInfo(s"Persisting table ${crossdataTable.tableIdentifier.table}") - tableCache.put(tableIdentifier, table) persistTableMetadata(crossdataTable.copy(schema = Option(table.schema))) + tableCache.put(tableIdentifier, table) } } @@ -105,8 +105,8 @@ abstract class PersistentCatalogWithCache(catalystConf: CatalystConf) extends XD throw new UnsupportedOperationException(s"The index $indexIdentifier already exists") } else { logInfo(s"Persisting index ${crossdataIndex.indexIdentifier}") - indexCache.put(crossdataIndex.tableIdentifier, crossdataIndex) persistIndexMetadata(crossdataIndex) + indexCache.put(crossdataIndex.tableIdentifier, crossdataIndex) } } From 236710ac0cdc9db6a11732437e2a579451b5412e Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Mon, 9 Jan 2017 19:14:41 +0100 Subject: [PATCH 20/22] [SERVICE DISCOVERY] Fix Update of cluster seeds (#88) * [SERVICE DISCOVERY] Fix Update of cluster seeds * [SERVICE DISCOVERY] More readable code * [SERVICE DISCOVERY] Fix problem in updateClusterSeeds --- .../crossdata/server/ServiceDiscoveryProvider.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index 9d6ca496d..a9d0d0d8d 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -259,13 +259,15 @@ trait ServiceDiscoveryProvider { * */ private def updateClusterSeeds(xCluster: Cluster, h: SDH): String = { - val currentSeeds = getLocalSeed + xCluster.state.members.filter(_.roles.contains("server")).map( + val currentSeeds: Set[String] = xCluster.state.members.filter(_.roles.contains("server")).map( m => s"${m.address.host.getOrElse("127.0.0.1")}:${m.address.port.getOrElse("13420")}") + val newSeeds: Seq[String] = (getLocalSeed +: currentSeeds.toSeq) distinct val pathForSeeds = h.sdch.getOrElse(SDCH.SeedsPath, SDCH.DefaultSeedsPath) ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForSeeds) - logger.info(s"Updating seeds: ${currentSeeds.mkString(",")}") - h.curatorClient.setData.forPath(pathForSeeds, currentSeeds.mkString(",").getBytes) - currentSeeds + val newSeedsStr = newSeeds.mkString(",") + logger.info(s"Updating seeds: $newSeedsStr") + h.curatorClient.setData.forPath(pathForSeeds, newSeedsStr.getBytes) + newSeedsStr } /** From 352dd9dc9a9173f2ec776255de7f6e7940a1b4e6 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Tue, 10 Jan 2017 11:03:05 +0100 Subject: [PATCH 21/22] [SERVICE DISCOVERY] Duplicated members removed from Hz members (#91) --- .../server/ServiceDiscoveryProvider.scala | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala index a9d0d0d8d..177843516 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/ServiceDiscoveryProvider.scala @@ -258,7 +258,7 @@ trait ServiceDiscoveryProvider { * Overrides the cluster seeds on the remote server of the service discovery according to the cluster state. * */ - private def updateClusterSeeds(xCluster: Cluster, h: SDH): String = { + private def updateClusterSeeds(xCluster: Cluster, h: SDH): Seq[String] = { val currentSeeds: Set[String] = xCluster.state.members.filter(_.roles.contains("server")).map( m => s"${m.address.host.getOrElse("127.0.0.1")}:${m.address.port.getOrElse("13420")}") val newSeeds: Seq[String] = (getLocalSeed +: currentSeeds.toSeq) distinct @@ -267,7 +267,7 @@ trait ServiceDiscoveryProvider { val newSeedsStr = newSeeds.mkString(",") logger.info(s"Updating seeds: $newSeedsStr") h.curatorClient.setData.forPath(pathForSeeds, newSeedsStr.getBytes) - newSeedsStr + newSeeds } /** @@ -275,7 +275,7 @@ trait ServiceDiscoveryProvider { * the current members. * */ - private def updateClusterMembers(h: SDH, hsp: HazelcastSessionProvider): Stat = { + private def updateClusterMembers(h: SDH, hsp: HazelcastSessionProvider): Seq[String] = { val pathForMembers = h.sdch.getOrElse(SDCH.ProviderPath, SDCH.DefaultProviderPath) ZKPaths.mkdirs(h.curatorClient.getZookeeperClient.getZooKeeper, pathForMembers) @@ -288,8 +288,12 @@ trait ServiceDiscoveryProvider { case _ => Seq.empty }.getOrElse(Seq.empty) - logger.info(s"Updating members: ${updatedMembers.mkString(",")}") - h.curatorClient.setData.forPath(pathForMembers, updatedMembers.mkString(",").getBytes) + val newMembers = updatedMembers distinct + val newMembersStr = newMembers.mkString(",") + + logger.info(s"Updating members: $newMembersStr") + h.curatorClient.setData.forPath(pathForMembers, newMembersStr.getBytes) + newMembers } } From 9cc6b3e8254b90af5a7a834eddca8f92e54d2a3e Mon Sep 17 00:00:00 2001 From: jjlopezm Date: Wed, 18 Jan 2017 17:37:15 +0100 Subject: [PATCH 22/22] remove security references --- docker/crossdata-config.sh | 7 ---- docker/crossdata-security.sh | 13 ------ docker/dependencyfix.sh | 12 ------ docker/gosec-config.sh | 70 -------------------------------- docker/kerberos-server-config.sh | 45 -------------------- docker/security-config.sh | 59 --------------------------- docker/tls-config.sh | 19 --------- docker/truststore-config.sh | 29 ------------- 8 files changed, 254 deletions(-) delete mode 100644 docker/crossdata-security.sh delete mode 100755 docker/dependencyfix.sh delete mode 100644 docker/gosec-config.sh delete mode 100644 docker/kerberos-server-config.sh delete mode 100644 docker/security-config.sh delete mode 100644 docker/tls-config.sh delete mode 100644 docker/truststore-config.sh diff --git a/docker/crossdata-config.sh b/docker/crossdata-config.sh index 7e5bb00dd..e68e03d17 100644 --- a/docker/crossdata-config.sh +++ b/docker/crossdata-config.sh @@ -76,13 +76,6 @@ function marathonConfig() { } -#################################################### -## Vault and secrets (configured if enabled) -#################################################### -if [ ! -z ${VAULT_HOST} ]; then - source security-config.sh $1 -fi - #################################################### ## Main #################################################### diff --git a/docker/crossdata-security.sh b/docker/crossdata-security.sh deleted file mode 100644 index 810f42362..000000000 --- a/docker/crossdata-security.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe -export CROSSDATA_SERVER_AKKA_HTTP_SSL_ENABLE=true -export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE=${XD_TRUST_JKS_NAME} -export CROSSDATA_SERVER_AKKA_HTTP_SSL_TRUSTSTORE_PASSWORD=${XD_TRUSTSTORE_PASSWORD} -export CROSSDATA_SERVER_AKKA_HTTP_SSL_KEYSTORE=${XD_TLS_JKS_NAME} -export CROSSDATA_SERVER_AKKA_HTTP_SSL_KEYSTORE_PASSWORD=${XD_TLS_PASSWORD} - -#Configure kerberos keytab -export CROSSDATA_SERVER_CONFIG_KERBEROS_ENABLED=true - -## Configure HDFS Kerberos keytab -export CROSSDATA_SERVER_CONFIG_KERBEROS_PRINCIPAL=${XD_PRINCIPAL} -export CROSSDATA_SERVER_CONFIG_KERBEROS_KEYTAB=${XD_KEYTAB_NAME} \ No newline at end of file diff --git a/docker/dependencyfix.sh b/docker/dependencyfix.sh deleted file mode 100755 index 79e2ecd18..000000000 --- a/docker/dependencyfix.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -deps=`dpkg --info $1 | grep -i depends | sed -n "s/.*depends: //ip"` - -IFS=',' read -ra dep <<< "$deps" -for i in "${dep[@]}"; do - echo $i - elver=`echo "$i" | sed -e "s/[() ]//gp"` - echo $elver - apt-get -y install $elver -done - diff --git a/docker/gosec-config.sh b/docker/gosec-config.sh deleted file mode 100644 index 7e314021b..000000000 --- a/docker/gosec-config.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -xe - -####################################################### -## Get Gosec-plugin LDAP user and pass and set XD vars -####################################################### - -if ["$1" != "skipSecManager"]; then - export CROSSDATA_SECURITY_MANAGER_CLASS=com.stratio.gosec.dyplon.plugins.crossdata.GoSecCrossdataSecurityManager - export CROSSDATA_SECURITY_MANAGER_ENABLED=true -fi - -### Get LDAP user and pass -export XD_GOSEC_PLUGIN_LDAP_USER=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"user\"") -export XD_GOSEC_PLUGIN_LDAP_PASS=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/ldap" -s | jq -r ".data .\"pass\"") - - -############################################################################# -## Get XD Gosec-plugin x509 client cert and set XD_GOSEC_PLUGIN_JKS_PASSWORD -############################################################################# - -### Get certificate from KMS -curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/x509-auth/x509-client-cert" -s | jq -r ".data .certificate_chain" > "$UUID.crt" -curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/gosec-plugin/x509-auth/x509-client-cert" -s | jq -r ".data .private_key" > "$UUID.key" - -### Get keystore password -export XD_GOSEC_PLUGIN_JKS_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/keystore | jq -r ".data .keystore_pass") - -## Generating PKCS12 -openssl pkcs12 -inkey "$UUID.key" -name "$SERVER_CERT_ALIAS" -in "$UUID.crt" \ - -export -out "$GOSEC_PLUGIN_JKS_NAME.pkcs12" -password "env:XD_GOSEC_PLUGIN_JKS_PASSWORD" - -## Generating JKS -keytool -importkeystore -srckeystore "$GOSEC_PLUGIN_JKS_NAME.pkcs12" -srcalias "$SERVER_CERT_ALIAS" \ - -srcstorepass "$XD_GOSEC_PLUGIN_JKS_PASSWORD" -srcstoretype PKCS12 -destkeystore "$GOSEC_PLUGIN_JKS_NAME" -deststorepass "$XD_GOSEC_PLUGIN_JKS_PASSWORD" - -## Cleaning -rm -f $GOSEC_PLUGIN_JKS_NAME.pkcs12 $UUID.crt $UUID.key - - #Set JAAS config - cat > /etc/sds/crossdata/security/jaas.conf<#$XD_PRINCIPAL#" $XD_PLUGIN_CLIENT_JAAS_PATH \ - && echo "[JAAS_CONF] ZK principal configured as $XD_PRINCIPAL" \ - || echo "[JAAS_CONF-ERROR] ZK principal was NOT configured" -sed -i "s#<__KEYTAB__>#$XD_KEYTAB_NAME#" $XD_PLUGIN_CLIENT_JAAS_PATH\ - && echo "[JAAS_CONF] ZK keytab configured as $XD_KEYTAB_NAME" \ - || echo "[JAAS_CONF-ERROR] ZK keytab was NOT configured" - -#Set LDAP config -export XD_PLUGIN_LDAP_PRINCIPAL=$XD_GOSEC_PLUGIN_LDAP_USER -export XD_PLUGIN_LDAP_CREDENTIALS=$XD_GOSEC_PLUGIN_LDAP_PASS - -#Set Kafka config -export XD_PLUGIN_KAFKA_TRUSTSTORE_PASSWORD=$XD_TRUSTSTORE_PASSWORD -export XD_PLUGIN_KAFKA_TRUSTSTORE=$XD_TRUST_JKS_NAME -export XD_PLUGIN_KAFKA_KEYSTORE=$GOSEC_PLUGIN_JKS_NAME -export XD_PLUGIN_KAFKA_KEYSTORE_PASSWORD=$XD_GOSEC_PLUGIN_JKS_PASSWORD -export XD_PLUGIN_KAFKA_KEY_PASSWOR=$XD_GOSEC_PLUGIN_JKS_PASSWORD \ No newline at end of file diff --git a/docker/kerberos-server-config.sh b/docker/kerberos-server-config.sh deleted file mode 100644 index 0a4402092..000000000 --- a/docker/kerberos-server-config.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -xe - -### Get keytab -export BASE64_KEYTAB=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/krb" -s | jq -r ".data .keytab_base64") -export XD_PRINCIPAL=$(curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/krb" -s | jq -r ".data .principal") - -## Generating keytab -echo $BASE64_KEYTAB | base64 -d > $XD_KEYTAB_NAME - - -#Set krb5.conf -cat > $JAVA_HOME/jre/lib/security/krb5.conf <__ - dns_lookup_realm = false - udp_preference_limit = 1 - [realms] - ____ = { - kdc = ____ - admin_server = ____ - default_domain = ____ - } - [domain_realm] - .____ = ____ - ____ = ____ -EOF - - -lw_realm=$(echo $REALM | tr '[:upper:]' '[:lower:]') -sed -i "s#____#$REALM#" $JAVA_HOME/jre/lib/security/krb5.conf \ -&& echo "[KRB-CONF] Realm configured in krb5.conf" \ -|| echo "[KRB-CONF-ERROR] Something went wrong when REALM was configured in krb5.conf" - -sed -i "s#____#$lw_realm#" $JAVA_HOME/jre/lib/security/krb5.conf \ -&& echo "[KRB-CONF] Domain configured in krb5.conf" \ -|| echo "[KRB-CONF-ERROR] Something went wrong when DOMAIN was configured in krb5.conf" - -sed -i "s#____#$KDC_HOST#" $JAVA_HOME/jre/lib/security/krb5.conf \ -&& echo "[KRB-CONF] kdc host configured in krb5.conf" \ -|| echo "[KRB-CONF-ERROR] Something went wrong when kdc host was configured in krb5.conf" - -sed -i "s#____#$KADMIN_HOST#" $JAVA_HOME/jre/lib/security/krb5.conf \ -&& echo "[KRB-CONF] kadmin host configured in krb5.conf" \ -|| echo "[KRB-CONF-ERROR] Something went wrong when kadmin host was configured in krb5.conf" - diff --git a/docker/security-config.sh b/docker/security-config.sh deleted file mode 100644 index 0bb42f66e..000000000 --- a/docker/security-config.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -xe - -#Ensure security folder is created -mkdir -p /etc/sds/crossdata/security - -#Get from vault the different data -#TODO: REMOVE -k FROM CURL WHEN TRUSTSTORE FOR VAULT IS INSIDE THE DOCKER!!!!!!!!!!! -# Get params -#VAULT_TOKEN='4d6cafd2-f5a4-abad-88f0-48eae1c24904' # This is the only env. value from entry point - - -# Main execution - -## Init -#TENANT_NAME="crossdata1" # MARATHON_APP_ID without slash -#VAULT_HOST='127.0.0.1' # It should be a predefined hostname -#VAULT_PORT='8200' # It should be a predefined port - -## Configure constant -export SERVER_CERT_ALIAS="crossdata-server" -export XD_TLS_JKS_NAME="/etc/sds/crossdata/security/server.jks" -export XD_TRUST_JKS_NAME="/etc/sds/crossdata/security/truststore.jks" -export XD_KEYTAB_NAME="/etc/sds/crossdata/security/crossdata.keytab" -export GOSEC_PLUGIN_JKS_NAME="/etc/sds/crossdata/security/gosec-plugin.jks" - -## Generating uuid -export UUID=$(uuidgen) - -#################################################### -## Get XD TLS Server Info and set XD_TLS_PASSWORD -#################################################### -source tls-config.sh - -####################################################### -## Create XD Truststore and set XD_TRUSTSTORE_PASSWORD -####################################################### -source truststore-config.sh - -#################################################### -## Kerberos config -#################################################### -source kerberos-server-config.sh - -####################################################### -## Gosec-plugin config -####################################################### -source gosec-config.sh $1 - -####################################################### -## HDFS security -####################################################### -source hdfs-security.sh - -####################################################### -## Crossdata security -####################################################### -source crossdata-security.sh - - diff --git a/docker/tls-config.sh b/docker/tls-config.sh deleted file mode 100644 index 493bcac2e..000000000 --- a/docker/tls-config.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -xe - -### Get certificate from KMS -curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/x509-server-cert" -s | jq -r ".data .certificate_chain" > "$UUID.crt" -curl -k -L -H "X-Vault-Token:$VAULT_TOKEN" "https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/x509-server-cert" -s | jq -r ".data .private_key" > "$UUID.key" - -### Get keystore password -export XD_TLS_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/tls/keystore | jq -r ".data .keystore_pass") - -## Generating PKCS12 -openssl pkcs12 -inkey "$UUID.key" -name "$SERVER_CERT_ALIAS" -in "$UUID.crt" \ - -export -out "$XD_TLS_JKS_NAME.pkcs12" -password "env:XD_TLS_PASSWORD" - -## Generating JKS -keytool -importkeystore -srckeystore "$XD_TLS_JKS_NAME.pkcs12" -srcalias "$SERVER_CERT_ALIAS" \ - -srcstorepass "$XD_TLS_PASSWORD" -srcstoretype PKCS12 -destkeystore "$XD_TLS_JKS_NAME" -deststorepass "$XD_TLS_PASSWORD" - -## Cleaning -rm -f $XD_TLS_JKS_NAME.pkcs12 $UUID.crt $UUID.key \ No newline at end of file diff --git a/docker/truststore-config.sh b/docker/truststore-config.sh deleted file mode 100644 index b6c9c9dbb..000000000 --- a/docker/truststore-config.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -xe - -### Get keystore password -export XD_TRUSTSTORE_PASSWORD=$(curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/keystore | jq -r ".data .keystore_pass") - -export XD_JVMCA_PASS="changeit" #TODO: This password should be provided by VAULT service - -## Prepare Truststore -counter=1 -code=$(curl -k -L -s -o /dev/null -w "%{http_code}" -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter) -while [ $code -eq 200 ]; do - curl -k -L -s -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter | jq -r ".data .certificate" > "$UUID.crt" - - # Create der file for root CA - openssl x509 -outform der -in "$UUID.crt" -out $UUID.der - - # Create keystore - keytool -import -noprompt -alias $counter -keystore $XD_TRUST_JKS_NAME -storepass $XD_TRUSTSTORE_PASSWORD -file $UUID.der - - # Add CA to JVM Keystore - keytool -importcert -noprompt -alias $counter -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass $XD_JVMCA_PASS -file $UUID.der - - # Update counter - let counter=counter+1 - code=$(curl -k -L -s -o /dev/null -w "%{http_code}" -H "X-Vault-Token:$VAULT_TOKEN" https://$VAULT_HOST:$VAULT_PORT/v1/crossdata/$TENANT_NAME/truststore/certs/$counter) - - # Clean - rm -f $UUID.der "$UUID.crt" -done \ No newline at end of file