From 6559ed5b031e0de33f2bedfb256215e5bfdffea7 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 31 Jan 2017 11:16:06 -0500 Subject: [PATCH 01/43] mapr52-code checkin mapr52-code checkin for docker container --- teradatalabs/mapr52-base/Dockerfile | 55 ++++++++++++++++ teradatalabs/mapr52-base/files/maprtech.repo | 13 ++++ teradatalabs/mapr52-hive/Dockerfile | 42 ++++++++++++ teradatalabs/mapr52-hive/files/bootstrap.sh | 41 ++++++++++++ .../mapr52-hive/files/conf/core-site.xml | 31 +++++++++ .../mapr52-hive/files/conf/hive-site.xml | 65 +++++++++++++++++++ teradatalabs/mapr52-hive/files/setup.sh | 9 +++ teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 + teradatalabs/mapr52-hive/files/startup.sh | 3 + .../mapr52-hive/files/supervisord.conf | 22 +++++++ .../files/supervisord.d/bootstrap.conf | 1 + .../files/supervisord.d/mysql-metastore.conf | 1 + .../files/supervisord.d/socks-proxy.conf | 1 + .../mapr52-hive/files/wardenTracker.sh | 54 +++++++++++++++ 14 files changed, 339 insertions(+) create mode 100644 teradatalabs/mapr52-base/Dockerfile create mode 100644 teradatalabs/mapr52-base/files/maprtech.repo create mode 100644 teradatalabs/mapr52-hive/Dockerfile create mode 100644 teradatalabs/mapr52-hive/files/bootstrap.sh create mode 100644 teradatalabs/mapr52-hive/files/conf/core-site.xml create mode 100644 teradatalabs/mapr52-hive/files/conf/hive-site.xml create mode 100644 teradatalabs/mapr52-hive/files/setup.sh create mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh create mode 100644 teradatalabs/mapr52-hive/files/startup.sh create mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf create mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile new file mode 100644 index 0000000..906dc09 --- /dev/null +++ b/teradatalabs/mapr52-base/Dockerfile @@ -0,0 +1,55 @@ +FROM teradatalabs/centos6-java8-oracle +MAINTAINER Teradata Docker Team + + +#Add repo for mapr +ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo +RUN yum update -y + +#install utility softwares +RUN yum install iputils vim openssh-server openssh-clients sudo -y + +#configure sshh +RUN chkconfig sshd on +RUN service sshd start + +#get maprgpg key +RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key + +#add user and password +RUN adduser mapr +RUN adduser hive-user +RUN usermod --password hive hive-user +RUN adduser hdfs-user +RUN usermod --password hdfs hdfs-user +RUN touch /home/hdfs-user +RUN touch /home/hive-user +RUN echo "cd /home/hive-user" >> /home/hive-user/.bashrc +RUN echo "cd /home/hdfs-user" >> /home/hdfs-user/.bashrc + +#install mapr +RUN yum install mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb mapr-zookeeper mapr-resourcemanager mapr-historyserver mapr-webserver mapr-gateway -y + +#necessary configuration for mapr +RUN rm -rf /opt/mapr/zkdata +RUN mkdir /opt/mapr/zkdata +RUN chmod 777 /opt/mapr/zkdata +RUN mkdir -p /mapr +RUN echo "/home/mapr/storagefile" > /root/disk.txt + + +#install python +RUN yum install -y python-setuptools \ + && easy_install pip \ + && pip install supervisor \ + && mkdir /etc/supervisord.d/ \ +# ... and its missing dependency + && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ + +# Cleanup + && yum -y clean all && rm -rf /tmp/* /var/tmp/* + +RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa +RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr52-base/files/maprtech.repo new file mode 100644 index 0000000..bbb4997 --- /dev/null +++ b/teradatalabs/mapr52-base/files/maprtech.repo @@ -0,0 +1,13 @@ +[maprtech] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/v5.2.0/redhat/ +enabled=1 +gpgcheck=0 +protect=1 + +[maprecosystem] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat +enabled=1 +gpgcheck=0 +protect=1 diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile new file mode 100644 index 0000000..69675ad --- /dev/null +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -0,0 +1,42 @@ +FROM teradatalabs/mapr52-base +MAINTAINER Teradata Docker Team + +#add all required scripts to root +ADD files/*.sh /root/ +ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml +ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml +ADD files/conf/supervisord.conf /etc/supervisord.conf +RUN chmod 777 /root/*.sh +COPY files/supervisord.d/* /etc/supervisord.d/ +RUN yum install openssh -y + +#install mysql server and required software +RUN yum install -y mysql-server mysql-connector-java \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* + +#install lsof +RUN yum install -y lsof + +# RUN SETUP script +RUN /root/setup.sh + + +#install hive +RUN yum install mapr-hive mapr-hiveserver2 mapr-hivemetastore -y +RUN yum -y clean all && rm -rf /tmp/* /var/tmp/* + + +# HDFS PORTS +EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 + +# YARN PORTS +EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 + +# HIVE PORT +EXPOSE 9083 10000 + +# SOCKS PORT +EXPOSE 1180 + +#run startup script +CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh new file mode 100644 index 0000000..14bc12a --- /dev/null +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +#configure mapr +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart + +#setup disk for mapr +dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 +/opt/mapr/server/disksetup -M -F /root/disk.txt + +#create proxy setup for hive users +chmod 755 /opt/mapr/conf/proxy +touch /opt/mapr/conf/proxy/hive-user +touch /opt/mapr/conf/proxy/hdfs-user + +#start zookeeper +service mapr-zookeeper start + +#start warden +service mapr-warden start + + +#configure hive +/opt/mapr/server/configure.sh -R + +#wait for warden to start all the services + +sh /root/wardenTracker.sh + + +#run hdfs commands +hadoop fs -mkdir /user/hive-user +hadoop fs -chmod 777 /user/hive-user +hadoop fs -mkdir /user/hdfs-user +hadoop fs -chmod 777 /user/hdfs-user +hadoop fs -mkdir /user/root +hadoop fs -mkdir /user/hive +hadoop fs -mkdir /user/hive/warehouse +hadoop fs -chmod 777 /user/hive +hadoop fs -chmod 777 /user/hive/warehouse + + diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr52-hive/files/conf/core-site.xml new file mode 100644 index 0000000..e08397b --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/core-site.xml @@ -0,0 +1,31 @@ + + + + + + + + + hadoop.proxyuser.mapr.groups + * + Allow the superuser mapr to impersonate any member of any group + + + hadoop.proxyuser.mapr.hosts + * + The superuser can connect from any host to impersonate a user + + + diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr52-hive/files/conf/hive-site.xml new file mode 100644 index 0000000..1bf86c3 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/hive-site.xml @@ -0,0 +1,65 @@ + + + + + + + datanucleus.schema.autoCreateAll + true + creates necessary schema on a startup if one doesn't exist. set + this to false, after creating it once + + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + + + + hive.metastore.uris + thrift://localhost:9083 + + + diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr52-hive/files/setup.sh new file mode 100644 index 0000000..ff085f9 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +mysql_install_db +/usr/bin/mysqld_safe & +sleep 20 +/usr/bin/mysqladmin -u root password 'root' +killall mysqld +sleep 10 +mkdir /var/log/mysql diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh new file mode 100644 index 0000000..852d770 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/socks-proxy.sh @@ -0,0 +1 @@ +../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh new file mode 100644 index 0000000..afd8798 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/startup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf new file mode 100644 index 0000000..0bb8beb --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.conf @@ -0,0 +1,22 @@ +[supervisord] +logfile = /var/log/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /var/run/supervisord.pid +nodaemon = true +directory = /tmp +strip_ansi = false + +[unix_http_server] +file = /tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock +prompt = cdh-pseudo-distributed + +[include] +files = /etc/supervisord.d/*.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf new file mode 100644 index 0000000..16a8759 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf new file mode 100644 index 0000000..b48213c --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf new file mode 100644 index 0000000..905e977 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh new file mode 100644 index 0000000..1f1578f --- /dev/null +++ b/teradatalabs/mapr52-hive/files/wardenTracker.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +hname=$(hostname) + +Services=0 + +maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + +#wait for CLDB to start +while [ $Services -ne 2 ] +do +if [ $maprcliReady == 1 ] +then + maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + Services=0 +else + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +fi +done + +#wait for NodeManager to start + +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +#wait for ResourceManager to start + +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +#wait for Hive Metastore to start + +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +#wait for HiveServer2 to start + +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + + + From 4e58395a2e127efbe6c0a13df25ada51bb1447c9 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 14 Feb 2017 15:51:35 -0500 Subject: [PATCH 02/43] Add docker container for MapR Add docker container for MapR --- teradatalabs/mapr52-base/Dockerfile | 92 ++++++++++--------- teradatalabs/mapr52-base/README.md | 9 ++ teradatalabs/mapr52-hive/Dockerfile | 46 ++++++---- teradatalabs/mapr52-hive/README.md | 17 ++++ teradatalabs/mapr52-hive/capabilities.txt | 4 + teradatalabs/mapr52-hive/files/bootstrap.sh | 34 ++----- .../mapr52-hive/files/conf/core-site.xml | 2 + .../mapr52-hive/files/conf/hive-site.xml | 52 ++++++----- teradatalabs/mapr52-hive/files/setup.sh | 13 ++- .../mapr52-hive/files/supervisord.conf | 23 +---- .../mapr52-hive/files/wardenTracker.sh | 23 ++--- 11 files changed, 170 insertions(+), 145 deletions(-) create mode 100644 teradatalabs/mapr52-base/README.md create mode 100644 teradatalabs/mapr52-hive/README.md create mode 100644 teradatalabs/mapr52-hive/capabilities.txt diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 906dc09..3e9bcd0 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -1,55 +1,61 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM teradatalabs/centos6-java8-oracle MAINTAINER Teradata Docker Team - -#Add repo for mapr +# ADD REPO FOR MAPR ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo -RUN yum update -y - -#install utility softwares -RUN yum install iputils vim openssh-server openssh-clients sudo -y - -#configure sshh -RUN chkconfig sshd on -RUN service sshd start - -#get maprgpg key -RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key - -#add user and password -RUN adduser mapr -RUN adduser hive-user -RUN usermod --password hive hive-user -RUN adduser hdfs-user -RUN usermod --password hdfs hdfs-user -RUN touch /home/hdfs-user -RUN touch /home/hive-user -RUN echo "cd /home/hive-user" >> /home/hive-user/.bashrc -RUN echo "cd /home/hdfs-user" >> /home/hdfs-user/.bashrc - -#install mapr -RUN yum install mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb mapr-zookeeper mapr-resourcemanager mapr-historyserver mapr-webserver mapr-gateway -y - -#necessary configuration for mapr -RUN rm -rf /opt/mapr/zkdata -RUN mkdir /opt/mapr/zkdata -RUN chmod 777 /opt/mapr/zkdata -RUN mkdir -p /mapr -RUN echo "/home/mapr/storagefile" > /root/disk.txt - - -#install python -RUN yum install -y python-setuptools \ +RUN yum update -y \ +# ... get maprgpg key + && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ + +# INSTALL UTILITY SOFTWARE + && yum install -y iputils vim openssh-server openssh-clients sudo \ + +# CONFIGURE SSH + && chkconfig sshd on \ + && service sshd start \ + +# INSTALL MAPR + && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ + && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ + && yum install -y mapr-webserver mapr-gateway \ + +# NECESSARY CONFIGURATION FOR MAPR + && rm -rf /opt/mapr/zkdata \ + && mkdir /opt/mapr/zkdata \ + && chmod 777 /opt/mapr/zkdata \ + && mkdir -p /mapr \ + && echo "/home/mapr/storagefile" > /root/disk.txt \ + +# INSTALL PYTHON + && yum install -y python-setuptools \ && easy_install pip \ && pip install supervisor \ && mkdir /etc/supervisord.d/ \ -# ... and its missing dependency +# ... AND ITS MISSING DEPENDENCY && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ -# Cleanup - && yum -y clean all && rm -rf /tmp/* /var/tmp/* +# CLEANUP + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ -RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa -RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys +# ADD USERS + && adduser mapr \ + && touch /home/mapr \ + && echo "cd /home/mapr" >> /home/mapr/.bashrc \ diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md new file mode 100644 index 0000000..8cc5226 --- /dev/null +++ b/teradatalabs/mapr52-base/README.md @@ -0,0 +1,9 @@ +# mapr52-base + + +Docker image with Hive installed from MAPR repositories. + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index 69675ad..82002d6 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -1,30 +1,42 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM teradatalabs/mapr52-base MAINTAINER Teradata Docker Team -#add all required scripts to root +# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY ADD files/*.sh /root/ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml -ADD files/conf/supervisord.conf /etc/supervisord.conf -RUN chmod 777 /root/*.sh +ADD files/supervisord.conf /etc/supervisord.conf COPY files/supervisord.d/* /etc/supervisord.d/ -RUN yum install openssh -y - -#install mysql server and required software -RUN yum install -y mysql-server mysql-connector-java \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* -#install lsof -RUN yum install -y lsof +RUN chmod 777 /root/*.sh \ +# INSTALL UTILITY SOFTWARE + && yum install -y openssh \ + && yum install -y lsof \ + +# INSTALL MYSQL SERVER + && yum install -y mysql-server mysql-connector-java \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # RUN SETUP script -RUN /root/setup.sh - - -#install hive -RUN yum install mapr-hive mapr-hiveserver2 mapr-hivemetastore -y -RUN yum -y clean all && rm -rf /tmp/* /var/tmp/* + && /root/setup.sh \ +# INSTALL HIVE + && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # HDFS PORTS EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 @@ -38,5 +50,5 @@ EXPOSE 9083 10000 # SOCKS PORT EXPOSE 1180 -#run startup script +# RUN STARTUP SCRIPT CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md new file mode 100644 index 0000000..ebf092a --- /dev/null +++ b/teradatalabs/mapr52-hive/README.md @@ -0,0 +1,17 @@ +# mapr52-hive + +Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt new file mode 100644 index 0000000..6892a21 --- /dev/null +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -0,0 +1,4 @@ +exposes_hive +allows_creating_a_table_in_hive +allows_selecting_from_the_table +exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 14bc12a..e1e2bfe 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -1,41 +1,27 @@ #!/bin/sh -#configure mapr +# CONFIGURE MAPR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -#setup disk for mapr +# SETUP DISK FOR MAPR dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 /opt/mapr/server/disksetup -M -F /root/disk.txt -#create proxy setup for hive users +# CREATE HIVE PROXY USERS chmod 755 /opt/mapr/conf/proxy -touch /opt/mapr/conf/proxy/hive-user -touch /opt/mapr/conf/proxy/hdfs-user -#start zookeeper +# START ZOOKEEPER service mapr-zookeeper start -#start warden +# START WARDEN service mapr-warden start - -#configure hive +# CONFIGURE HIVE /opt/mapr/server/configure.sh -R -#wait for warden to start all the services - +# WAIT FOR WARDEN TO START ALL THE SERVICES sh /root/wardenTracker.sh - -#run hdfs commands -hadoop fs -mkdir /user/hive-user -hadoop fs -chmod 777 /user/hive-user -hadoop fs -mkdir /user/hdfs-user -hadoop fs -chmod 777 /user/hdfs-user -hadoop fs -mkdir /user/root -hadoop fs -mkdir /user/hive -hadoop fs -mkdir /user/hive/warehouse -hadoop fs -chmod 777 /user/hive -hadoop fs -chmod 777 /user/hive/warehouse - - +# RUN HDFS COMMANDS +hadoop fs -mkdir /user/root /user/hive /user/hive/warehouse +hadoop fs -chmod 777 /user/hive /user/hive/warehouse diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr52-hive/files/conf/core-site.xml index e08397b..6549e0b 100644 --- a/teradatalabs/mapr52-hive/files/conf/core-site.xml +++ b/teradatalabs/mapr52-hive/files/conf/core-site.xml @@ -22,10 +22,12 @@ * Allow the superuser mapr to impersonate any member of any group + hadoop.proxyuser.mapr.hosts * The superuser can connect from any host to impersonate a user + diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr52-hive/files/conf/hive-site.xml index 1bf86c3..3518b61 100644 --- a/teradatalabs/mapr52-hive/files/conf/hive-site.xml +++ b/teradatalabs/mapr52-hive/files/conf/hive-site.xml @@ -22,44 +22,46 @@ creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once + hive.server2.enable.doAs true Set this property to enable impersonation in Hive Server 2 + hive.metastore.execute.setugi true Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + - - javax.jdo.option.ConnectionURL - jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true - JDBC connect string for a JDBC metastore - - - - javax.jdo.option.ConnectionDriverName - com.mysql.jdbc.Driver - Driver class name for a JDBC metastore - + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + - - javax.jdo.option.ConnectionUserName - root - username to use against metastore database - + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + - - javax.jdo.option.ConnectionPassword - root - password to use against metastore database - + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + - - hive.metastore.uris - thrift://localhost:9083 - + + hive.metastore.uris + thrift://localhost:9083 + diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr52-hive/files/setup.sh index ff085f9..2781708 100644 --- a/teradatalabs/mapr52-hive/files/setup.sh +++ b/teradatalabs/mapr52-hive/files/setup.sh @@ -1,9 +1,16 @@ #!/bin/bash +# SETUP METASTORE mysql_install_db + /usr/bin/mysqld_safe & -sleep 20 +sleep 10s + +echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql +echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql /usr/bin/mysqladmin -u root password 'root' + killall mysqld -sleep 10 -mkdir /var/log/mysql +sleep 10s +mkdir /var/log/mysql/ +chown mysql:mysql /var/log/mysql/ diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf index 0bb8beb..c821a2d 100644 --- a/teradatalabs/mapr52-hive/files/supervisord.conf +++ b/teradatalabs/mapr52-hive/files/supervisord.conf @@ -1,22 +1 @@ -[supervisord] -logfile = /var/log/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /var/run/supervisord.pid -nodaemon = true -directory = /tmp -strip_ansi = false - -[unix_http_server] -file = /tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock -prompt = cdh-pseudo-distributed - -[include] -files = /etc/supervisord.d/*.conf +../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh index 1f1578f..a52068f 100644 --- a/teradatalabs/mapr52-hive/files/wardenTracker.sh +++ b/teradatalabs/mapr52-hive/files/wardenTracker.sh @@ -6,7 +6,7 @@ Services=0 maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) -#wait for CLDB to start +# WAIT FOR CLDB TO START while [ $Services -ne 2 ] do if [ $maprcliReady == 1 ] @@ -14,41 +14,42 @@ then maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) Services=0 else - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) + Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) fi done -#wait for NodeManager to start - +# WAIT FOR NODEMANAGER TO START Services=0 while [ $Services -ne 2 ] do Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) done -#wait for ResourceManager to start - +# WAIT FOR RESOURCEMANAGER TO START Services=0 while [ $Services -ne 2 ] do Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) done -#wait for Hive Metastore to start - +# WAIT FOR HIVE METASTORE TO START Services=0 while [ $Services -ne 2 ] do Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) done -#wait for HiveServer2 to start - +# WAIT FOR HIVESERVER2 TO START Services=0 while [ $Services -ne 2 ] do Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) done - +# WAIT FOR JOBHISTORYSERVER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done From 9a9b8137ae07ec12ef56d4fc30e0caa588cbce03 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 14 Feb 2017 16:47:38 -0500 Subject: [PATCH 03/43] Add review changes for MapR docker container Add review changes for MapR docker container --- teradatalabs/mapr52-base/Dockerfile | 12 ++++++------ teradatalabs/mapr52-base/README.md | 2 +- teradatalabs/mapr52-hive/Dockerfile | 6 ++---- teradatalabs/mapr52-hive/files/bootstrap.sh | 10 +++++----- teradatalabs/mapr52-hive/files/wardenTracker.sh | 1 - 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 3e9bcd0..306f368 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -15,10 +15,10 @@ FROM teradatalabs/centos6-java8-oracle MAINTAINER Teradata Docker Team -# ADD REPO FOR MAPR +# ADD REPO FOR MapR ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo RUN yum update -y \ -# ... get maprgpg key +# ... GET MapRGPG KEY && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ # INSTALL UTILITY SOFTWARE @@ -28,19 +28,20 @@ RUN yum update -y \ && chkconfig sshd on \ && service sshd start \ -# INSTALL MAPR +# INSTALL MapR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ && yum install -y mapr-webserver mapr-gateway \ -# NECESSARY CONFIGURATION FOR MAPR +# CONFIGURE ZOOKEEPER'S DATA DIRECTORY && rm -rf /opt/mapr/zkdata \ && mkdir /opt/mapr/zkdata \ && chmod 777 /opt/mapr/zkdata \ && mkdir -p /mapr \ +# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK && echo "/home/mapr/storagefile" > /root/disk.txt \ -# INSTALL PYTHON +# INSTALL PYTHON AND SUPERVISORD && yum install -y python-setuptools \ && easy_install pip \ && pip install supervisor \ @@ -55,7 +56,6 @@ RUN yum update -y \ && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ -# ADD USERS && adduser mapr \ && touch /home/mapr \ && echo "cd /home/mapr" >> /home/mapr/.bashrc \ diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md index 8cc5226..e9d1cd8 100644 --- a/teradatalabs/mapr52-base/README.md +++ b/teradatalabs/mapr52-base/README.md @@ -1,7 +1,7 @@ # mapr52-base -Docker image with Hive installed from MAPR repositories. +Docker image with Hive installed from MapR repositories. ## Oracle license diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index 82002d6..c340530 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -31,7 +31,6 @@ RUN chmod 777 /root/*.sh \ && yum install -y mysql-server mysql-connector-java \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ -# RUN SETUP script && /root/setup.sh \ # INSTALL HIVE @@ -39,10 +38,10 @@ RUN chmod 777 /root/*.sh \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # HDFS PORTS -EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 +EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 # YARN PORTS -EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 +EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888 # HIVE PORT EXPOSE 9083 10000 @@ -50,5 +49,4 @@ EXPOSE 9083 10000 # SOCKS PORT EXPOSE 1180 -# RUN STARTUP SCRIPT CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index e1e2bfe..497e99a 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -1,19 +1,19 @@ #!/bin/sh -# CONFIGURE MAPR +# CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -# SETUP DISK FOR MAPR +# SETUP FLAT FILE /home/mapr/storagefile dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 + +# SETUP DISK FOR MAPR BY RUNNING disksetup /opt/mapr/server/disksetup -M -F /root/disk.txt # CREATE HIVE PROXY USERS chmod 755 /opt/mapr/conf/proxy -# START ZOOKEEPER +# START SERVICES service mapr-zookeeper start - -# START WARDEN service mapr-warden start # CONFIGURE HIVE diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh index a52068f..5124a4d 100644 --- a/teradatalabs/mapr52-hive/files/wardenTracker.sh +++ b/teradatalabs/mapr52-hive/files/wardenTracker.sh @@ -52,4 +52,3 @@ while [ $Services -ne 2 ] do Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) done - From 2c4ab2631c761cecb0c30aec34691c360113b054 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 15 Feb 2017 11:27:07 -0500 Subject: [PATCH 04/43] Rebasing the branch to master Rebasing the branch to master for splitting mapr commits into two --- teradatalabs/mapr52-base/Dockerfile | 61 ----------------- teradatalabs/mapr52-base/README.md | 9 --- teradatalabs/mapr52-base/files/maprtech.repo | 13 ---- teradatalabs/mapr52-hive/Dockerfile | 52 -------------- teradatalabs/mapr52-hive/README.md | 17 ----- teradatalabs/mapr52-hive/capabilities.txt | 4 -- teradatalabs/mapr52-hive/files/bootstrap.sh | 27 -------- .../mapr52-hive/files/conf/core-site.xml | 33 --------- .../mapr52-hive/files/conf/hive-site.xml | 67 ------------------- teradatalabs/mapr52-hive/files/setup.sh | 16 ----- teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 - teradatalabs/mapr52-hive/files/startup.sh | 3 - .../mapr52-hive/files/supervisord.conf | 1 - .../files/supervisord.d/bootstrap.conf | 1 - .../files/supervisord.d/mysql-metastore.conf | 1 - .../files/supervisord.d/socks-proxy.conf | 1 - .../mapr52-hive/files/wardenTracker.sh | 54 --------------- 17 files changed, 361 deletions(-) delete mode 100644 teradatalabs/mapr52-base/Dockerfile delete mode 100644 teradatalabs/mapr52-base/README.md delete mode 100644 teradatalabs/mapr52-base/files/maprtech.repo delete mode 100644 teradatalabs/mapr52-hive/Dockerfile delete mode 100644 teradatalabs/mapr52-hive/README.md delete mode 100644 teradatalabs/mapr52-hive/capabilities.txt delete mode 100644 teradatalabs/mapr52-hive/files/bootstrap.sh delete mode 100644 teradatalabs/mapr52-hive/files/conf/core-site.xml delete mode 100644 teradatalabs/mapr52-hive/files/conf/hive-site.xml delete mode 100644 teradatalabs/mapr52-hive/files/setup.sh delete mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh delete mode 100644 teradatalabs/mapr52-hive/files/startup.sh delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf delete mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile deleted file mode 100644 index 306f368..0000000 --- a/teradatalabs/mapr52-base/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 Teradata -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM teradatalabs/centos6-java8-oracle -MAINTAINER Teradata Docker Team - -# ADD REPO FOR MapR -ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo -RUN yum update -y \ -# ... GET MapRGPG KEY - && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ - -# INSTALL UTILITY SOFTWARE - && yum install -y iputils vim openssh-server openssh-clients sudo \ - -# CONFIGURE SSH - && chkconfig sshd on \ - && service sshd start \ - -# INSTALL MapR - && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ - && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ - && yum install -y mapr-webserver mapr-gateway \ - -# CONFIGURE ZOOKEEPER'S DATA DIRECTORY - && rm -rf /opt/mapr/zkdata \ - && mkdir /opt/mapr/zkdata \ - && chmod 777 /opt/mapr/zkdata \ - && mkdir -p /mapr \ -# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK - && echo "/home/mapr/storagefile" > /root/disk.txt \ - -# INSTALL PYTHON AND SUPERVISORD - && yum install -y python-setuptools \ - && easy_install pip \ - && pip install supervisor \ - && mkdir /etc/supervisord.d/ \ -# ... AND ITS MISSING DEPENDENCY - && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ - && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ - && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ - -# CLEANUP - && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ - && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ - && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ - - && adduser mapr \ - && touch /home/mapr \ - && echo "cd /home/mapr" >> /home/mapr/.bashrc \ diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md deleted file mode 100644 index e9d1cd8..0000000 --- a/teradatalabs/mapr52-base/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# mapr52-base - - -Docker image with Hive installed from MapR repositories. - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr52-base/files/maprtech.repo deleted file mode 100644 index bbb4997..0000000 --- a/teradatalabs/mapr52-base/files/maprtech.repo +++ /dev/null @@ -1,13 +0,0 @@ -[maprtech] -name=MapR Technologies -baseurl=http://package.mapr.com/releases/v5.2.0/redhat/ -enabled=1 -gpgcheck=0 -protect=1 - -[maprecosystem] -name=MapR Technologies -baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat -enabled=1 -gpgcheck=0 -protect=1 diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile deleted file mode 100644 index c340530..0000000 --- a/teradatalabs/mapr52-hive/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2017 Teradata -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM teradatalabs/mapr52-base -MAINTAINER Teradata Docker Team - -# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY -ADD files/*.sh /root/ -ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml -ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml -ADD files/supervisord.conf /etc/supervisord.conf -COPY files/supervisord.d/* /etc/supervisord.d/ - -RUN chmod 777 /root/*.sh \ -# INSTALL UTILITY SOFTWARE - && yum install -y openssh \ - && yum install -y lsof \ - -# INSTALL MYSQL SERVER - && yum install -y mysql-server mysql-connector-java \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ - - && /root/setup.sh \ - -# INSTALL HIVE - && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ - -# HDFS PORTS -EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 - -# YARN PORTS -EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888 - -# HIVE PORT -EXPOSE 9083 10000 - -# SOCKS PORT -EXPOSE 1180 - -CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md deleted file mode 100644 index ebf092a..0000000 --- a/teradatalabs/mapr52-hive/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# mapr52-hive - -Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. -For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. -If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal -for general correctness testing. - -## Run - -``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive -``` - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt deleted file mode 100644 index 6892a21..0000000 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ /dev/null @@ -1,4 +0,0 @@ -exposes_hive -allows_creating_a_table_in_hive -allows_selecting_from_the_table -exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh deleted file mode 100644 index 497e99a..0000000 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# CONFIGURE MapR -/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart - -# SETUP FLAT FILE /home/mapr/storagefile -dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 - -# SETUP DISK FOR MAPR BY RUNNING disksetup -/opt/mapr/server/disksetup -M -F /root/disk.txt - -# CREATE HIVE PROXY USERS -chmod 755 /opt/mapr/conf/proxy - -# START SERVICES -service mapr-zookeeper start -service mapr-warden start - -# CONFIGURE HIVE -/opt/mapr/server/configure.sh -R - -# WAIT FOR WARDEN TO START ALL THE SERVICES -sh /root/wardenTracker.sh - -# RUN HDFS COMMANDS -hadoop fs -mkdir /user/root /user/hive /user/hive/warehouse -hadoop fs -chmod 777 /user/hive /user/hive/warehouse diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr52-hive/files/conf/core-site.xml deleted file mode 100644 index 6549e0b..0000000 --- a/teradatalabs/mapr52-hive/files/conf/core-site.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - hadoop.proxyuser.mapr.groups - * - Allow the superuser mapr to impersonate any member of any group - - - - hadoop.proxyuser.mapr.hosts - * - The superuser can connect from any host to impersonate a user - - - - diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr52-hive/files/conf/hive-site.xml deleted file mode 100644 index 3518b61..0000000 --- a/teradatalabs/mapr52-hive/files/conf/hive-site.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - datanucleus.schema.autoCreateAll - true - creates necessary schema on a startup if one doesn't exist. set - this to false, after creating it once - - - - hive.server2.enable.doAs - true - Set this property to enable impersonation in Hive Server 2 - - - - hive.metastore.execute.setugi - true - Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. - - - - javax.jdo.option.ConnectionURL - jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true - JDBC connect string for a JDBC metastore - - - - javax.jdo.option.ConnectionDriverName - com.mysql.jdbc.Driver - Driver class name for a JDBC metastore - - - - javax.jdo.option.ConnectionUserName - root - username to use against metastore database - - - - javax.jdo.option.ConnectionPassword - root - password to use against metastore database - - - - hive.metastore.uris - thrift://localhost:9083 - - - diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr52-hive/files/setup.sh deleted file mode 100644 index 2781708..0000000 --- a/teradatalabs/mapr52-hive/files/setup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# SETUP METASTORE -mysql_install_db - -/usr/bin/mysqld_safe & -sleep 10s - -echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql -echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql -/usr/bin/mysqladmin -u root password 'root' - -killall mysqld -sleep 10s -mkdir /var/log/mysql/ -chown mysql:mysql /var/log/mysql/ diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh deleted file mode 100644 index 852d770..0000000 --- a/teradatalabs/mapr52-hive/files/socks-proxy.sh +++ /dev/null @@ -1 +0,0 @@ -../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh deleted file mode 100644 index afd8798..0000000 --- a/teradatalabs/mapr52-hive/files/startup.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf deleted file mode 100644 index c821a2d..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.conf +++ /dev/null @@ -1 +0,0 @@ -../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf deleted file mode 100644 index 16a8759..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf deleted file mode 100644 index b48213c..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf deleted file mode 100644 index 905e977..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh deleted file mode 100644 index 5124a4d..0000000 --- a/teradatalabs/mapr52-hive/files/wardenTracker.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -hname=$(hostname) - -Services=0 - -maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - -# WAIT FOR CLDB TO START -while [ $Services -ne 2 ] -do -if [ $maprcliReady == 1 ] -then - maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - Services=0 -else - Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -fi -done - -# WAIT FOR NODEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR RESOURCEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVE METASTORE TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVESERVER2 TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR JOBHISTORYSERVER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done From c09dafbb2c65ece03133ea12c23c97ed1348eef1 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 15 Feb 2017 11:29:01 -0500 Subject: [PATCH 05/43] Add docker container for MapR without hive Add docker container code for MapR-Base --- teradatalabs/mapr52-base/Dockerfile | 61 ++++++++++++++++++++ teradatalabs/mapr52-base/README.md | 9 +++ teradatalabs/mapr52-base/files/maprtech.repo | 13 +++++ 3 files changed, 83 insertions(+) create mode 100644 teradatalabs/mapr52-base/Dockerfile create mode 100644 teradatalabs/mapr52-base/README.md create mode 100644 teradatalabs/mapr52-base/files/maprtech.repo diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile new file mode 100644 index 0000000..306f368 --- /dev/null +++ b/teradatalabs/mapr52-base/Dockerfile @@ -0,0 +1,61 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/centos6-java8-oracle +MAINTAINER Teradata Docker Team + +# ADD REPO FOR MapR +ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo +RUN yum update -y \ +# ... GET MapRGPG KEY + && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ + +# INSTALL UTILITY SOFTWARE + && yum install -y iputils vim openssh-server openssh-clients sudo \ + +# CONFIGURE SSH + && chkconfig sshd on \ + && service sshd start \ + +# INSTALL MapR + && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ + && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ + && yum install -y mapr-webserver mapr-gateway \ + +# CONFIGURE ZOOKEEPER'S DATA DIRECTORY + && rm -rf /opt/mapr/zkdata \ + && mkdir /opt/mapr/zkdata \ + && chmod 777 /opt/mapr/zkdata \ + && mkdir -p /mapr \ +# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK + && echo "/home/mapr/storagefile" > /root/disk.txt \ + +# INSTALL PYTHON AND SUPERVISORD + && yum install -y python-setuptools \ + && easy_install pip \ + && pip install supervisor \ + && mkdir /etc/supervisord.d/ \ +# ... AND ITS MISSING DEPENDENCY + && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ + +# CLEANUP + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + + && adduser mapr \ + && touch /home/mapr \ + && echo "cd /home/mapr" >> /home/mapr/.bashrc \ diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md new file mode 100644 index 0000000..e9d1cd8 --- /dev/null +++ b/teradatalabs/mapr52-base/README.md @@ -0,0 +1,9 @@ +# mapr52-base + + +Docker image with Hive installed from MapR repositories. + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr52-base/files/maprtech.repo new file mode 100644 index 0000000..bbb4997 --- /dev/null +++ b/teradatalabs/mapr52-base/files/maprtech.repo @@ -0,0 +1,13 @@ +[maprtech] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/v5.2.0/redhat/ +enabled=1 +gpgcheck=0 +protect=1 + +[maprecosystem] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat +enabled=1 +gpgcheck=0 +protect=1 From 4093ab7a0784f17b27a2f48c25b0371da5c0e84f Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 15 Feb 2017 11:30:38 -0500 Subject: [PATCH 06/43] Add docker container for MapR with hive Add docker container code for MapR-Hive --- teradatalabs/mapr52-hive/Dockerfile | 52 ++++++++++++++ teradatalabs/mapr52-hive/README.md | 17 +++++ teradatalabs/mapr52-hive/capabilities.txt | 4 ++ teradatalabs/mapr52-hive/files/bootstrap.sh | 27 ++++++++ .../mapr52-hive/files/conf/core-site.xml | 33 +++++++++ .../mapr52-hive/files/conf/hive-site.xml | 67 +++++++++++++++++++ teradatalabs/mapr52-hive/files/setup.sh | 16 +++++ teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 + teradatalabs/mapr52-hive/files/startup.sh | 3 + .../mapr52-hive/files/supervisord.conf | 1 + .../files/supervisord.d/bootstrap.conf | 1 + .../files/supervisord.d/mysql-metastore.conf | 1 + .../files/supervisord.d/socks-proxy.conf | 1 + .../mapr52-hive/files/wardenTracker.sh | 54 +++++++++++++++ 14 files changed, 278 insertions(+) create mode 100644 teradatalabs/mapr52-hive/Dockerfile create mode 100644 teradatalabs/mapr52-hive/README.md create mode 100644 teradatalabs/mapr52-hive/capabilities.txt create mode 100644 teradatalabs/mapr52-hive/files/bootstrap.sh create mode 100644 teradatalabs/mapr52-hive/files/conf/core-site.xml create mode 100644 teradatalabs/mapr52-hive/files/conf/hive-site.xml create mode 100644 teradatalabs/mapr52-hive/files/setup.sh create mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh create mode 100644 teradatalabs/mapr52-hive/files/startup.sh create mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf create mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile new file mode 100644 index 0000000..c340530 --- /dev/null +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -0,0 +1,52 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/mapr52-base +MAINTAINER Teradata Docker Team + +# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY +ADD files/*.sh /root/ +ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml +ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml +ADD files/supervisord.conf /etc/supervisord.conf +COPY files/supervisord.d/* /etc/supervisord.d/ + +RUN chmod 777 /root/*.sh \ +# INSTALL UTILITY SOFTWARE + && yum install -y openssh \ + && yum install -y lsof \ + +# INSTALL MYSQL SERVER + && yum install -y mysql-server mysql-connector-java \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + + && /root/setup.sh \ + +# INSTALL HIVE + && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + +# HDFS PORTS +EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 + +# YARN PORTS +EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888 + +# HIVE PORT +EXPOSE 9083 10000 + +# SOCKS PORT +EXPOSE 1180 + +CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md new file mode 100644 index 0000000..ebf092a --- /dev/null +++ b/teradatalabs/mapr52-hive/README.md @@ -0,0 +1,17 @@ +# mapr52-hive + +Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt new file mode 100644 index 0000000..6892a21 --- /dev/null +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -0,0 +1,4 @@ +exposes_hive +allows_creating_a_table_in_hive +allows_selecting_from_the_table +exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh new file mode 100644 index 0000000..497e99a --- /dev/null +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# CONFIGURE MapR +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart + +# SETUP FLAT FILE /home/mapr/storagefile +dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 + +# SETUP DISK FOR MAPR BY RUNNING disksetup +/opt/mapr/server/disksetup -M -F /root/disk.txt + +# CREATE HIVE PROXY USERS +chmod 755 /opt/mapr/conf/proxy + +# START SERVICES +service mapr-zookeeper start +service mapr-warden start + +# CONFIGURE HIVE +/opt/mapr/server/configure.sh -R + +# WAIT FOR WARDEN TO START ALL THE SERVICES +sh /root/wardenTracker.sh + +# RUN HDFS COMMANDS +hadoop fs -mkdir /user/root /user/hive /user/hive/warehouse +hadoop fs -chmod 777 /user/hive /user/hive/warehouse diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr52-hive/files/conf/core-site.xml new file mode 100644 index 0000000..6549e0b --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/core-site.xml @@ -0,0 +1,33 @@ + + + + + + + + + hadoop.proxyuser.mapr.groups + * + Allow the superuser mapr to impersonate any member of any group + + + + hadoop.proxyuser.mapr.hosts + * + The superuser can connect from any host to impersonate a user + + + + diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr52-hive/files/conf/hive-site.xml new file mode 100644 index 0000000..3518b61 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/hive-site.xml @@ -0,0 +1,67 @@ + + + + + + + datanucleus.schema.autoCreateAll + true + creates necessary schema on a startup if one doesn't exist. set + this to false, after creating it once + + + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + + + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + + + + hive.metastore.uris + thrift://localhost:9083 + + + diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr52-hive/files/setup.sh new file mode 100644 index 0000000..2781708 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# SETUP METASTORE +mysql_install_db + +/usr/bin/mysqld_safe & +sleep 10s + +echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql +echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql +/usr/bin/mysqladmin -u root password 'root' + +killall mysqld +sleep 10s +mkdir /var/log/mysql/ +chown mysql:mysql /var/log/mysql/ diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh new file mode 100644 index 0000000..852d770 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/socks-proxy.sh @@ -0,0 +1 @@ +../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh new file mode 100644 index 0000000..afd8798 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/startup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf new file mode 100644 index 0000000..c821a2d --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.conf @@ -0,0 +1 @@ +../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf new file mode 100644 index 0000000..16a8759 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf new file mode 100644 index 0000000..b48213c --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf new file mode 100644 index 0000000..905e977 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh new file mode 100644 index 0000000..5124a4d --- /dev/null +++ b/teradatalabs/mapr52-hive/files/wardenTracker.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +hname=$(hostname) + +Services=0 + +maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + +# WAIT FOR CLDB TO START +while [ $Services -ne 2 ] +do +if [ $maprcliReady == 1 ] +then + maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + Services=0 +else + Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +fi +done + +# WAIT FOR NODEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR RESOURCEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVE METASTORE TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVESERVER2 TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR JOBHISTORYSERVER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done From fceaa6e9164dfd429f38f777b5ee49120b95478a Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 16 Mar 2017 16:35:56 -0400 Subject: [PATCH 07/43] Add Kerberize docker container for MapR with hive Add Kerberize docker container for MapR with hive --- teradatalabs/mapr52-base/Dockerfile | 11 ++- .../mapr52-hive-kerberized/Dockerfile | 84 +++++++++++++++++ teradatalabs/mapr52-hive-kerberized/README.md | 17 ++++ .../mapr52-hive-kerberized/capabilities.txt | 4 + .../mapr52-hive-kerberized/files/bootstrap.sh | 44 +++++++++ .../files/conf/hive-site.xml | 94 +++++++++++++++++++ teradatalabs/mapr52-hive/Dockerfile | 2 +- teradatalabs/mapr52-hive/files/bootstrap.sh | 4 +- 8 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 teradatalabs/mapr52-hive-kerberized/Dockerfile create mode 100644 teradatalabs/mapr52-hive-kerberized/README.md create mode 100644 teradatalabs/mapr52-hive-kerberized/capabilities.txt create mode 100644 teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh create mode 100644 teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 306f368..5a75197 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -26,6 +26,8 @@ RUN yum update -y \ # CONFIGURE SSH && chkconfig sshd on \ + && ssh root@hadoop-master 'bash -s' < /root/wardenTracker.sh + && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ && service sshd start \ # INSTALL MapR @@ -57,5 +59,12 @@ RUN yum update -y \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ && adduser mapr \ - && touch /home/mapr \ + && adduser hive \ + && adduser hdfs \ + && touch /home/mapr /home/hive /home/hdfs \ && echo "cd /home/mapr" >> /home/mapr/.bashrc \ + && echo "cd /home/hive" >> /home/hive/.bashrc \ + && echo "cd /home/hdfs" >> /home/hdfs/.bashrc \ + && chown mapr:mapr /home/mapr \ + && chown hive:hive /home/hive \ + && chown hdfs:hdfs /home/hdfs diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile new file mode 100644 index 0000000..5ffb495 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/mapr52-hive +MAINTAINER Teradata Docker Team + +# REMOVE THESE FILES IF IT IS ALREADY THERE +RUN rm -rf /opt/mapr/conf/ssl_truststore \ + && rm -rf /opt/mapr/conf/maprserverticket \ + && rm -rf /opt/mapr/conf/cldb.key \ + && rm -rf /opt/mapr/conf/ssl_keystore \ + && rm -rf /root/bootstrap.sh \ + +# INSTALL KERBEROS + && yum install -y krb5-libs krb5-server krb5-workstation + +# ADD KERBEROS CONFIGURATION +ADD files/bootstrap.sh /root/ +ADD files/kerberos/krb5.conf /etc/krb5.conf +ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf +ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl +ADD files/jceJars/local_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/local_policy.jar +ADD files/jceJars/US_export_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/US_export_policy.jar +ADD files/conf/env.sh /opt/mapr/conf/env.sh + +# ENABLE HIVE SECURITY +ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml + +# CREATE KERBEROS DATABASE +RUN /usr/sbin/kdb5_util create -s -P password \ + && usermod -g root hdfs + && usermod -g mapr hdfs +# CREATE MAPR AND HIVE PRINCIPALS AND KEYTABS + && /usr/sbin/kadmin.local -q "addprinc -randkey mapr/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey hive/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/hive.keytab hive/mycluster@LABS.TERADATA.COM" \ + +# CREATE HDFS USER + && /usr/sbin/kadmin.local -q "addprinc -randkey hdfs/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/hdfs.keytab hdfs/mycluster@LABS.TERADATA.COM" \ + +# CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS + && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ + && chmod 777 /root/bootstrap.sh \ + && chown mapr:mapr /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab \ + && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ + +# CREATE PRESTO PRINCIPAL AND KEYTAB + && /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey presto-client/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey hive/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && mkdir -p /etc/presto/conf \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-server.keytab presto-server/presto-master.docker.cluster" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" \ + && chmod 644 /etc/presto/conf/*.keytab + +# CREATE SSL KEYSTORE +RUN keytool -genkeypair \ + -alias presto \ + -keyalg RSA \ + -keystore /etc/presto/conf/keystore.jks \ + -keypass password \ + -storepass password \ + -dname "CN=presto-master, OU=, O=, L=, S=, C=" +RUN chmod 644 /etc/presto/conf/keystore.jks + +# EXPOSE KERBEROS PORTS +EXPOSE 88 +EXPOSE 749 + +CMD /root/startup.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive-kerberized/README.md b/teradatalabs/mapr52-hive-kerberized/README.md new file mode 100644 index 0000000..ebf092a --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/README.md @@ -0,0 +1,17 @@ +# mapr52-hive + +Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/capabilities.txt b/teradatalabs/mapr52-hive-kerberized/capabilities.txt new file mode 100644 index 0000000..6892a21 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/capabilities.txt @@ -0,0 +1,4 @@ +exposes_hive +allows_creating_a_table_in_hive +allows_selecting_from_the_table +exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh new file mode 100644 index 0000000..88cc860 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# CONFIGURE MAPR +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart + +# SETUP FLAT FILE /home/mapr/storagefile +dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 + +# SETUP DISK FOR MAPR BY RUNNING disksetup +/opt/mapr/server/disksetup -M -F /root/disk.txt + +# CREATE HIVE PROXY USERS +chmod 755 /opt/mapr/conf/proxy + +# CONFIGURE HIVE +/opt/mapr/server/configure.sh -R + +# CONFIGURE KERBEROS +/opt/mapr/server/configure.sh -secure -genkeys -C localhost -Z localhost -N mycluster -no-autostart + +# START KERBEROS SERVICES +/sbin/service krb5kdc start +/sbin/service kadmin start + +# START MAPR SERVICES +service mapr-zookeeper start +service mapr-warden start + +# WAIT FOR WARDEN TO START ALL THE SERVICES +sh /root/wardenTracker.sh + +# CREATE KERBEROS TICKET +kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + +# CREATE MAPR TICKET +maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + +# RUN HDFS COMMANDS +hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive + +# REMOVE MAPR TICKET AND KERBEROS TICKET +kdestroy +rm -rf /tmp/* diff --git a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml new file mode 100644 index 0000000..2031be7 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml @@ -0,0 +1,94 @@ + + + + + + + datanucleus.schema.autoCreateAll + true + creates necessary schema on a startup if one doesn't exist. set + this to false, after creating it once + + + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + + + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + + + + hive.metastore.uris + thrift://localhost:9083 + + + + + + hive.metastore.kerberos.keytab.file + /opt/mapr/conf/hive.keytab + The path to the Kerberos Keytab file containing the metastore thrift server's service principal. + + + hive.metastore.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct hostname. + + + hive.server2.authentication + KERBEROS + authenticationtype + + + hive.server2.authentication.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + HiveServer2 principal. If _HOST is used as the FQDN portion, it will be replaced with the actual hostname of the running instance. + + + hive.server2.authentication.kerberos.keytab + /opt/mapr/conf/hive.keytab + Keytab file for HiveServer2 principal + + diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index c340530..709bfa9 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -35,7 +35,7 @@ RUN chmod 777 /root/*.sh \ # INSTALL HIVE && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* # HDFS PORTS EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 497e99a..424e643 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -23,5 +23,5 @@ service mapr-warden start sh /root/wardenTracker.sh # RUN HDFS COMMANDS -hadoop fs -mkdir /user/root /user/hive /user/hive/warehouse -hadoop fs -chmod 777 /user/hive /user/hive/warehouse +hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive From 614519cef9fbbfc50f20c93f9eebed39f5427bbe Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Mon, 20 Mar 2017 16:26:08 -0400 Subject: [PATCH 08/43] Add modification for docker-base Add modification for docker-base --- teradatalabs/mapr52-base/Dockerfile | 3 ++- teradatalabs/mapr52-base/files/id_rsa.pub | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 teradatalabs/mapr52-base/files/id_rsa.pub diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 5a75197..67ea40f 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -17,6 +17,7 @@ MAINTAINER Teradata Docker Team # ADD REPO FOR MapR ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo +COPY files/id_rsa.pub /root/ RUN yum update -y \ # ... GET MapRGPG KEY && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ @@ -26,7 +27,6 @@ RUN yum update -y \ # CONFIGURE SSH && chkconfig sshd on \ - && ssh root@hadoop-master 'bash -s' < /root/wardenTracker.sh && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ && service sshd start \ @@ -57,6 +57,7 @@ RUN yum update -y \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys \ && adduser mapr \ && adduser hive \ diff --git a/teradatalabs/mapr52-base/files/id_rsa.pub b/teradatalabs/mapr52-base/files/id_rsa.pub new file mode 100644 index 0000000..cdd0869 --- /dev/null +++ b/teradatalabs/mapr52-base/files/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 From 7bdc127b23880c26823ef3ce401bd4579106952b Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 21 Mar 2017 13:39:41 -0400 Subject: [PATCH 09/43] Modification for kerberized MapR container Modification for kerberized MapR container --- .../files/jceJars/US_export_policy.jar | Bin 0 -> 3023 bytes .../files/jceJars/local_policy.jar | Bin 0 -> 3035 bytes teradatalabs/mapr52-hive/Dockerfile | 2 +- .../mapr52-hive/files/wait_for_mapr_hive.sh | 15 +++++++++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar create mode 100644 teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar create mode 100644 teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar b/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar new file mode 100644 index 0000000000000000000000000000000000000000..251b102c57c076504ba818330e4fceb4509753d8 GIT binary patch literal 3023 zcma)82Q-^|AC9d?+fPt?)UHutUz8wm36X=Jzvp*;&-**idH&~l;4nIRP5?7AGoaZf)c|m! zIH_kdh!t4g_%igO8Thg>6k=(mU63ZbT|Xr|6!~>UD!I2VF6jX z$GM}7FXJ9BPm2h}ojWt>yzYF*AENmL?3EQ|Wx57%(kk_(%)j)Cic4=M)t!R(J5+n{ z6$dc|v+dl`NQ~&BI0-uuCFC9Sltr~f2|@(eczH?8s;HjHm+~0Lipyi%ar)Y|xLM8d zgf5x~dtR1v-C-vN!j_A zBA~o%?BN|j-OQ?+M_GtCHZ)<>9(L9HX^LET3f(5v1J|pC_m~=Ion4DznxFKYNIyXtR_FP>=OeFUeypaYVAgaACTrFc1u(snGASKw~TE*&3zwyjZ;=B%rw!d*yxp^U4g=c3R-(0 zzeAyJuJ94(vP#zsK~%nR|C4jBlhXXoxrHTI+0!+$uahveIIxwCP0IEH z1HZCIOK!=UVlemj8S8zrPWcL92xK`^43Uj-g83eX6+a4M$7gWU2JF+iOF|_jCEZ~U zS#CUh5X3D<)?0`NvemL{jAa^=LbNCqiBG|Y z_4vd)CFD39@kP^_ILB0kqrGD?-hnuc!}>h-dB~i+$eq4!A_uc#?lYH_F_$y3M)jE+ zTQm2W_VZwY<)5^GVb8iLSfFxO_XxsaJ#n|<2S%9EbKkz%uT$@BJdz3X>p9{1^)q6Jupn2BLqZ z5q@b~zK_8ig2RWI7|TU238Q(?FJ5(y$g@oq>;VO&__z!e!h*mv#3xbOKVQW`TJ?xv z@y!8jbqe;+OQv7YcBz}2P{?aybT8X$Jr;WVd3HDcM=1|DEC<)o7GHWSVSyWH3&Hgk za1k~u7oA+3vB}F8oMEVcF7Rq7ue<>KQ!#O;7^d*m zV?64S?oVx0ft&#Z2#P0xSlI5V`~om7fEF??GU^>27=NB6qO^~i-?+c(!QkWH=;1!$w&I$ShnSgc-Z{Mf|?GzdYV3#ep@rAehfKf zFRXhQp;@->(khqcTuq>;MX{V=?{kPzmruDct8)`)oSFw2+MFrPmQAs-ZTwAbbO0y+ zXHy;LeC3c7tQa=g6!Zqqwc;Idlh)x%r{SGd2|O}1Z`gKZx3KfImwlNW3~?h5H8i+9 zE-WZF>l7G}Hu1+?`539adOMfSg%jYm6wzlkE6rl`^4T6&Kfj%h;5lG`H87 z_AZA?Ea)#uG&Fw5(Qrz=)qe31`@Evz#VR69d{E`|Thryz!6DsL#cqdrL`=481az=b z;7ormYyI5SB@>o?&Jgxgh5GgEumC)m1#07QO?{dPY32gA@ld_SXc42VoUhn?AztE) zcZ9CLvd)%Ahg!mNf`i|B-JbRsDAu@r)5l9PclP3#cc!X(!@i%K_x^TbkqfW@sAx7u z<1G>0`94~a=~r3Z$~PQUsUBWF83z#`2GZ{qt>`q&E&;s{#9!2!tiSOkXKcXj<{YP$ zi?E$NMv}R!w@PPyD{Iyd&Wl#OGwN~BkBC(5B4J0>i0+L#A#XPN{rc54e01TSV&%Tp z0nHS$7>D_^8T0uP66nTbHG6l06_O`_$VpR9=)h_?2dmQ=Am}oknS!a zKEViUO9b*3%0Dnz0p;)Gb^W#uTIqDFA{%5oYc^LzQ%|ZL)85QLl=h9c&NkaZ$BuuX)zs}6&dry{L*4SVuKuMd2Q0Yx7_F1tzMf*;`yCuh6s7YUc+ z^BFgXc;EA59pR$1UdR#5aL8(tefVH&OC7#F>+7ZAMi0W}vb|BgwM#jddX03tCya&! zlDB1Cy@7)UJed=kDDf=3KW1+_Oz^7{93w=p&+19}3fcno*Cw4Sl#aWqh+VH(6t~MW zv;fyDOo>HTv%vPK*33<`QG!0fc*BH?{COX=rxqC35)^2r z-QD}moPF9jJ7}oJT=31DX=phC$I0Q6xrftNh(6rP{{me?m0K~zvuqGbH4L_=RNQ9zTb16-}^jhBoD6`00aU7o;Z;# z06R*IJwA?bfNNQwu+%;dKVfZ&uy=qSx5V|rZDV0#ypz8Qu&}Ap596*oh>D8%{Y10# zO@f=G;lgjrRBm*Xt9_hD*8lm@uoKn5xUL=)H}p*StOlCTYqq-b(U67bwIa*d5{9~B zPWrT_Fll(th}gS7s#Bucd))HIKN!3g?T`+QrBV`WA8Cz6;P9Yh0#-xT8ORybFJPNy z{xO@^$~oGLK58ZJ;jmm7k_8=;lu!M=dRfqkKI^|PwdDY|ZUS*Cc`V}cl)mZas8$%& z!vE;Nmg;N!ymGosjHxdsT*Z{oP*?6>xU~_L@nLWZ+-z$qxP0e&;|w1f$q#Z&Y`{;k z2@v4|0MOqlYWXjUZ0+C{CV@qs=20OZeX}FNyV@B*Sp#*=Se!y6@Ba8PdM_66pxCt&ZocdEOb^Fh~TdTY4IL1 zjTUkB<>^&71*MP+y)IYjamSVlXtX^t8Y3gx!!$AiiUU-g?F{{i2eONoq1INhTaD#~ zb%s?LbBog0!1FS*(Z(7PZ~dtvX##EC*Bq9DT$tB~un%kXp@<8G4v{s{5@T9!7O-%# z9RPW&I?gbTJDT-L4OuRBQSh+M)f5=%JVq4MS=$OBEFrZ)Cn; zSpL-z2~7sY1E|+lL)})>#xbzh*4h!& zi|XrUAdN`6*SOU^BqUaY!w_k9fnf&MAa4&D80c_$==1PEUaKh%3wIp47z`GrO#^*+ ztgpz#Z6=9kguu!Xd|wfX%9O9+z~oQxWd*IAdEI;Dc&od0-C4;`s8cEWy|3twDlf$Q)SMRSo6^F-krJsxH z^Y)2piinU|YpF`eF=Na)Yj&?(n!2+HVfrG|bIUq|6Bbcb<9SXWOMZXXn=(3AIn~}1 z>e!MMl;?NGaXiX^b>d*(T4?=gqsJ7ol+->+qg5$`Fd2~fbi+ad^F1ulhW zC(Ad9L>Bz~5I?Y7(NrO6UsA_F+pw1KMBzF_R#jU3C^RwzK1IKK@yM~$6vPu49j?TD zo>iHV_01IZIq4LM$+SegpvQI#zkmtyI?sT;WNv3%6}9hQNwWLgWseO0lw>Ej(hM)o z%FatlFCjCfgwK+Xno9=XDWwD!IR}2a+?szb8NZS&ujM7WO!d=}MK6`mzmy=M8~Cw{ zx6OWOVGBeA5G1FRrY9&IZ}0~IRcKiRoCU%-1K_do=65qHZ3{3ukI*SgMJstyF6O=w zKJ8a=wJfQ}Q}(Q?d$}v^*BM^*r8SLSkwYgtWrA&Q#RTg1QkNUunFsGbsLY=1iILAe z^FenraOTtHMLZuNpaJha?!T(u|KWuJB(-eS1C!r$>y4oax^j{?fp_h2Zrvzu!dbzL z6?M351^YxZ^Y=&(kL7DAuH#h(YF5X^uvQ)5(S_h9l1@THhLL z#bzo7RgopgSB)Vrsp21eqt0=={ML3fdQq8*i?}&-a`}KF#|NNMO zoaVG!a8Ty>t-T6-z+Qxmv7BP?F_B9B27CTO&6oBMnvrxmU1l(GIUZ4#{SC4r zeR)E4uTQ_V5!@6D1(Tq4D>;!tRJfp}6W-luk{@>*i*~{vatGSQ>FXEhH2s{cEaDqw z7NBpkihpXDGN0n&zf!w?WK=J~x|K=rQOTRu9`(I($f$nPU(r<^=pz!|htL65TmL$IRm=E2wu*1y-hioed(m)%XFW7#jbi;S6C%=g z{BVe$?V@x@^5(07HhEo~tAcl^S(N1YiT9_&ADHL%Ik8mpzM@d2-)ysDyf+D3cjq7N zM?8$Swz@rd52}`zTjRY^P-#?yAI`m-vlhLYRxm9SLmrTu+wu>fnlarM^2D1<>?XK4 zw=MkWs}&7C0AQYfr@bAL+MNId0KD4?V5`)SNBV$B-yc-Zl*sKoXb{R zhA!#p?{}xKd6ef={5G8kIo*?Oh{_hQ7r({hZg*rBq2lLR`b~aZR-@=5nvS#_Jbb%lFUL*rb-ui4>;M>9vrmFITi#^Bc?6A-!2h;y6NfxOkp4Q1)c{rgG{9cpY@&VC=(Y`KO8w41`%ZPf_dDS3_sz zw&J-TRDlFr&&S%PB_54zW084QD%|bQBT971w}~MW*9kd7 zZ=P`p-C!nJrI^#KKiP;gmN)g*4TBS#LxPVV>FieE2!YTd_LPh5*M)z3HW#-TU^_|f z3{V^??5=;ym1yKP0w-wgOa!|T4zlO%pWlqV^xs-zCjv)*JNf14Z@cyg>{25d$p_?20@+U# Lo19~UoUi`@yDZ_Z literal 0 HcmV?d00001 diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index 709bfa9..5f9cebb 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -44,7 +44,7 @@ EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888 # HIVE PORT -EXPOSE 9083 10000 +EXPOSE 9083 10000 22 # SOCKS PORT EXPOSE 1180 diff --git a/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh b/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh new file mode 100644 index 0000000..eef4670 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +nohup hadoop fs -ls /user/hive/warehouse > hivecliReady.log 2>&1 & +wait +hivecliReady=$(cat hivecliReady.log | grep 'No such file or directory' | wc -l) + +# WAIT FOR hive directories to be created +while [ $hivecliReady == 1 ] +do + nohup hadoop fs -ls /user/hive/warehouse > hivecliReady.log 2>&1 & + wait + hivecliReady=$(cat hivecliReady.log | grep 'No such file or directory' | wc -l +) +done +rm -rf hivecliReady.log From 586ebc734fac5509e64ad3b143e2807f15d985e7 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 21 Mar 2017 13:49:50 -0400 Subject: [PATCH 10/43] Adding modifications to test cluster for mapr Adding modifications to test cluster for mapr --- test/Dockerfile | 15 +++++++++++++-- test/conf/id_rsa | 27 +++++++++++++++++++++++++++ test/conf/id_rsa.pub | 1 + test/conf/known_hosts | 2 ++ test/files/expose_mapr_hive.sh | 23 +++++++++++++++++++++++ test/image_tests/image_tests.bats.sh | 10 ++++++++-- 6 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 test/conf/id_rsa create mode 100644 test/conf/id_rsa.pub create mode 100644 test/conf/known_hosts create mode 100644 test/files/expose_mapr_hive.sh diff --git a/test/Dockerfile b/test/Dockerfile index 17dfffe..5a231c0 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -15,12 +15,22 @@ FROM teradatalabs/centos6-java8-oracle ENV DOCKERIZE_VERSION v0.3.0 - +RUN mkdir /root/.ssh +COPY conf/* /root/.ssh/ +COPY files/* /root/ RUN \ yum install -y \ wget \ unzip \ +# INSTALL UTILITY SOFTWARE + && yum install -y openssh-server openssh-clients \ + +# CONFIGURE SSH + && chkconfig sshd on \ + && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + && service sshd start \ + # setup CDH repo, pin the CDH distribution to a concrete version && wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ @@ -45,7 +55,8 @@ RUN \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # create 'image_tests' volume mount path - && mkdir /image_tests + && mkdir /image_tests \ + && chmod 777 /root/*.sh VOLUME /image_tests diff --git a/test/conf/id_rsa b/test/conf/id_rsa new file mode 100644 index 0000000..2d01c1c --- /dev/null +++ b/test/conf/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6 +pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9s +fmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7 +yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqO +q2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5E +KHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQIBIwKCAQAYVkg8sAouiNeg7HDJ +VaMa9h8v2Z7uRlq1RrOB05BVaQ6j/kE3V+oa9Vu9DnpV5sroPNV8jXh1z2gR8ov0 +m0lEdVLq3AZbZDvv9sebDcomBTwz5lWKyesr4S81G+w3gA5f+j1Zy2Wn81l+4pu+ +vLhPxfM4tzUKrWU61OIDRCrRFhHaFiDxBMsMhOSBfGT5OuIIHOAcjvB+YxKgB644 +ANoOnnExaukOOX0ZLgnwIG7YG0Cf6MVJpbK+gx7ER85SnMV/wjzsAedOfF+tD8Z1 +P0nfRXKr9X+CE2m3JMAb4QxXphsVHLAtGjPmzEz/2JNu7mnevJW1iQ0X19giSdyE +0+U7AoGBAPgsgpFrGKf0vy3XKMkIl8uwn+GGKob3/Fc6sGYaGu2fgR/BeMnKf4IT +BsDBgPJPz/ozQaWXWVBCZc5Wqm7CPCiccHzUF8G0PZFMNkR2ayO854co82Sj+sWW +XwS/kdTRRQ6HHr67KKSZFZO60HaWWSJ9Z0IlrVe+k3s0FtMV72pXAoGBANuqF4oi +fmHawxXgIdUcESzJlt3HUjOC32e7rEcnskjrNQxM4ZUkmQrf6IhhFrSdCk5cNCrP +pdaw87YtIEtVO3LvAT8E+dRuIO8LV4VufaBBHUq/TwntldF1/fN41ZpgpehDBH/i +rTXAMncd2junSpsyt800W7KtXsdca0YKODOnAoGAHFzbuNkKIdLUBT0pO4v0F0di +urA/X+HTsjKX0SeODIdB9QArD79fB40lV9uZtUrzMomD2GkRhYPuYLlykFC2awqQ +g0twbei2lEM5Zuj2TTopFsLZ/N+KX7lp8eoCCbGEPCyzDnvKIXCUwG0fI37lnYqs +txo/skGqdHr7S1L2ylMCgYEA1WNnUvzvzMXiFUAg3aY8kehmq45tHBi7w9rh3rjZ +E6Koj5skR70m9J7/JWWhBzl3udXpiKx1NvUKAW2qV88jzrxKWnncv7twcy+W2WQF +AhNeSJyV7GMj0spkaNvIL5EWMhU3kiyoQtf2gleDimf/VO9wuLaE9rcLnRfzLhiO +Xg8CgYEAk6QP8i01oPfY4IzE9JTHbVIglllbygxL7CKid6kH2GtbcnT8NU9aj4YY +XdGRiDRw7/O6sHSKbLkSQ1TUAVil8Xw6tLQddbRI+p93RVw9haa8bCBtzDV7eB4m +M5/MrINmIquLaSBd+l3HA36GCxLzBUsT87UZIQnJu7YauFVJYsU= +-----END RSA PRIVATE KEY----- diff --git a/test/conf/id_rsa.pub b/test/conf/id_rsa.pub new file mode 100644 index 0000000..cdd0869 --- /dev/null +++ b/test/conf/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 diff --git a/test/conf/known_hosts b/test/conf/known_hosts new file mode 100644 index 0000000..400481d --- /dev/null +++ b/test/conf/known_hosts @@ -0,0 +1,2 @@ +hadoop-master,172.19.0.2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv28akj3lavwQlF/Q1hkF9beEinFechrUwWNqq5f+gawUUba9AubecRX3VN0u4ix3BKQvckjGTXUUUd3wGWTz/2BfBgojXaz5LDPBxHG83t0OgIfO5W07kyDxgKPiHXm4JRGKs/TeOtVPiURbxv1Br7c9n9wMuYHQYIZhNXRQ8ETj+sjg6OiWYXahgmaErlGq+A6A5pk4lkFDnx8ESqEWnO7kEucdhUlCYqtStyCStRPci3WjHxS28RSVQoard/zap4BL7Squ0yqrIAR4UqmQNgIoGKUZv/Kh9ykw9dCwJYjMrxiuMvp/TbAR7KOYeuz5yebtLAPTHgCGbTTok6aBFQ== +sdl03545.labs.teradata.com,10.25.47.3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAsH14tOfK0dR04fuvDmY/InP+G3/FMArIFyDl1DyV8Qg8COomAukPcUNMFCqAg8wt37RZKzQ63HUlRPqxyKX6YZ1htkKXUhM+xM/gbQ4Z+OtfnINflFGhiU4gk0ClpccUdq1hmWQUE7eBu1XBQJdejwlDNmYI4rToAJf3KgSsDIPXPpJjrw3/3dtvp5UBIdcdlZNL6Xz0AY1h6Z3I1D/9z83zIX8oCCMHW9O7/zWgwd4TBay6embbkj34x8IR/9uZ7v64g9qYGugPcvfzoLAldA57VD3nbFePy+Rrl1YAfsqcBhTaP7QfBcMofgqXX776euJA9aF5j/cQM6lO72fcww== diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh new file mode 100644 index 0000000..eb8c5d4 --- /dev/null +++ b/test/files/expose_mapr_hive.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +conn=0 +while [ $conn -ne 1 ] +do + ping -c 4 -q hadoop-master + if [ "$?" -eq 0 ]; then + conn=1 + else + conn=0 + fi +done + +hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) + +# WAIT FOR hive directories to be created +while [ $hiveportReady -ne 1 ] +do + hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) +done + +ssh root@hadoop-master './wardenTracker.sh' +ssh root@hadoop-master './wait_for_mapr_hive.sh' diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 4f0af22..a67d573 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -22,19 +22,25 @@ assert_output_contains() { printf '%s\n' "${lines[@]}" | grep -q $1 } +function wait_for_mapr { + skip_if_needed + assert_run sh /root/expose_mapr_hive.sh +} + function exposes_hive { skip_if_needed assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } function allows_creating_a_table_in_hive { + skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From 4426b3bc0f23449376192a5dcda35ec626cc80c0 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 21 Mar 2017 13:50:16 -0400 Subject: [PATCH 11/43] Revert "Adding modifications to test cluster for mapr" This reverts commit 586ebc734fac5509e64ad3b143e2807f15d985e7. --- test/Dockerfile | 15 ++------------- test/conf/id_rsa | 27 --------------------------- test/conf/id_rsa.pub | 1 - test/conf/known_hosts | 2 -- test/files/expose_mapr_hive.sh | 23 ----------------------- test/image_tests/image_tests.bats.sh | 10 ++-------- 6 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 test/conf/id_rsa delete mode 100644 test/conf/id_rsa.pub delete mode 100644 test/conf/known_hosts delete mode 100644 test/files/expose_mapr_hive.sh diff --git a/test/Dockerfile b/test/Dockerfile index 5a231c0..17dfffe 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -15,22 +15,12 @@ FROM teradatalabs/centos6-java8-oracle ENV DOCKERIZE_VERSION v0.3.0 -RUN mkdir /root/.ssh -COPY conf/* /root/.ssh/ -COPY files/* /root/ + RUN \ yum install -y \ wget \ unzip \ -# INSTALL UTILITY SOFTWARE - && yum install -y openssh-server openssh-clients \ - -# CONFIGURE SSH - && chkconfig sshd on \ - && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ - && service sshd start \ - # setup CDH repo, pin the CDH distribution to a concrete version && wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ @@ -55,8 +45,7 @@ RUN \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # create 'image_tests' volume mount path - && mkdir /image_tests \ - && chmod 777 /root/*.sh + && mkdir /image_tests VOLUME /image_tests diff --git a/test/conf/id_rsa b/test/conf/id_rsa deleted file mode 100644 index 2d01c1c..0000000 --- a/test/conf/id_rsa +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6 -pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9s -fmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7 -yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqO -q2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5E -KHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQIBIwKCAQAYVkg8sAouiNeg7HDJ -VaMa9h8v2Z7uRlq1RrOB05BVaQ6j/kE3V+oa9Vu9DnpV5sroPNV8jXh1z2gR8ov0 -m0lEdVLq3AZbZDvv9sebDcomBTwz5lWKyesr4S81G+w3gA5f+j1Zy2Wn81l+4pu+ -vLhPxfM4tzUKrWU61OIDRCrRFhHaFiDxBMsMhOSBfGT5OuIIHOAcjvB+YxKgB644 -ANoOnnExaukOOX0ZLgnwIG7YG0Cf6MVJpbK+gx7ER85SnMV/wjzsAedOfF+tD8Z1 -P0nfRXKr9X+CE2m3JMAb4QxXphsVHLAtGjPmzEz/2JNu7mnevJW1iQ0X19giSdyE -0+U7AoGBAPgsgpFrGKf0vy3XKMkIl8uwn+GGKob3/Fc6sGYaGu2fgR/BeMnKf4IT -BsDBgPJPz/ozQaWXWVBCZc5Wqm7CPCiccHzUF8G0PZFMNkR2ayO854co82Sj+sWW -XwS/kdTRRQ6HHr67KKSZFZO60HaWWSJ9Z0IlrVe+k3s0FtMV72pXAoGBANuqF4oi -fmHawxXgIdUcESzJlt3HUjOC32e7rEcnskjrNQxM4ZUkmQrf6IhhFrSdCk5cNCrP -pdaw87YtIEtVO3LvAT8E+dRuIO8LV4VufaBBHUq/TwntldF1/fN41ZpgpehDBH/i -rTXAMncd2junSpsyt800W7KtXsdca0YKODOnAoGAHFzbuNkKIdLUBT0pO4v0F0di -urA/X+HTsjKX0SeODIdB9QArD79fB40lV9uZtUrzMomD2GkRhYPuYLlykFC2awqQ -g0twbei2lEM5Zuj2TTopFsLZ/N+KX7lp8eoCCbGEPCyzDnvKIXCUwG0fI37lnYqs -txo/skGqdHr7S1L2ylMCgYEA1WNnUvzvzMXiFUAg3aY8kehmq45tHBi7w9rh3rjZ -E6Koj5skR70m9J7/JWWhBzl3udXpiKx1NvUKAW2qV88jzrxKWnncv7twcy+W2WQF -AhNeSJyV7GMj0spkaNvIL5EWMhU3kiyoQtf2gleDimf/VO9wuLaE9rcLnRfzLhiO -Xg8CgYEAk6QP8i01oPfY4IzE9JTHbVIglllbygxL7CKid6kH2GtbcnT8NU9aj4YY -XdGRiDRw7/O6sHSKbLkSQ1TUAVil8Xw6tLQddbRI+p93RVw9haa8bCBtzDV7eB4m -M5/MrINmIquLaSBd+l3HA36GCxLzBUsT87UZIQnJu7YauFVJYsU= ------END RSA PRIVATE KEY----- diff --git a/test/conf/id_rsa.pub b/test/conf/id_rsa.pub deleted file mode 100644 index cdd0869..0000000 --- a/test/conf/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 diff --git a/test/conf/known_hosts b/test/conf/known_hosts deleted file mode 100644 index 400481d..0000000 --- a/test/conf/known_hosts +++ /dev/null @@ -1,2 +0,0 @@ -hadoop-master,172.19.0.2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv28akj3lavwQlF/Q1hkF9beEinFechrUwWNqq5f+gawUUba9AubecRX3VN0u4ix3BKQvckjGTXUUUd3wGWTz/2BfBgojXaz5LDPBxHG83t0OgIfO5W07kyDxgKPiHXm4JRGKs/TeOtVPiURbxv1Br7c9n9wMuYHQYIZhNXRQ8ETj+sjg6OiWYXahgmaErlGq+A6A5pk4lkFDnx8ESqEWnO7kEucdhUlCYqtStyCStRPci3WjHxS28RSVQoard/zap4BL7Squ0yqrIAR4UqmQNgIoGKUZv/Kh9ykw9dCwJYjMrxiuMvp/TbAR7KOYeuz5yebtLAPTHgCGbTTok6aBFQ== -sdl03545.labs.teradata.com,10.25.47.3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAsH14tOfK0dR04fuvDmY/InP+G3/FMArIFyDl1DyV8Qg8COomAukPcUNMFCqAg8wt37RZKzQ63HUlRPqxyKX6YZ1htkKXUhM+xM/gbQ4Z+OtfnINflFGhiU4gk0ClpccUdq1hmWQUE7eBu1XBQJdejwlDNmYI4rToAJf3KgSsDIPXPpJjrw3/3dtvp5UBIdcdlZNL6Xz0AY1h6Z3I1D/9z83zIX8oCCMHW9O7/zWgwd4TBay6embbkj34x8IR/9uZ7v64g9qYGugPcvfzoLAldA57VD3nbFePy+Rrl1YAfsqcBhTaP7QfBcMofgqXX776euJA9aF5j/cQM6lO72fcww== diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh deleted file mode 100644 index eb8c5d4..0000000 --- a/test/files/expose_mapr_hive.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -conn=0 -while [ $conn -ne 1 ] -do - ping -c 4 -q hadoop-master - if [ "$?" -eq 0 ]; then - conn=1 - else - conn=0 - fi -done - -hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) - -# WAIT FOR hive directories to be created -while [ $hiveportReady -ne 1 ] -do - hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) -done - -ssh root@hadoop-master './wardenTracker.sh' -ssh root@hadoop-master './wait_for_mapr_hive.sh' diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index a67d573..4f0af22 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -22,25 +22,19 @@ assert_output_contains() { printf '%s\n' "${lines[@]}" | grep -q $1 } -function wait_for_mapr { - skip_if_needed - assert_run sh /root/expose_mapr_hive.sh -} - function exposes_hive { skip_if_needed assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } function allows_creating_a_table_in_hive { - skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From dfcb0094da29842a43de3885f2b30744d09a8181 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 21 Mar 2017 14:03:02 -0400 Subject: [PATCH 12/43] Adding test container changes for mapr Adding test container changes to accomodate mapr --- teradatalabs/mapr52-hive/capabilities.txt | 1 + teradatalabs/mapr52-hive/conf/id_rsa | 27 +++++++++++++++++++++++ teradatalabs/mapr52-hive/conf/id_rsa.pub | 1 + teradatalabs/mapr52-hive/conf/known_hosts | 2 ++ test/Dockerfile | 15 +++++++++++-- test/files/expose_mapr_hive.sh | 23 +++++++++++++++++++ test/image_tests/image_tests.bats.sh | 10 +++++++-- 7 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 teradatalabs/mapr52-hive/conf/id_rsa create mode 100644 teradatalabs/mapr52-hive/conf/id_rsa.pub create mode 100644 teradatalabs/mapr52-hive/conf/known_hosts create mode 100644 test/files/expose_mapr_hive.sh diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt index 6892a21..ba21a5b 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -1,3 +1,4 @@ +wait_for_mapr exposes_hive allows_creating_a_table_in_hive allows_selecting_from_the_table diff --git a/teradatalabs/mapr52-hive/conf/id_rsa b/teradatalabs/mapr52-hive/conf/id_rsa new file mode 100644 index 0000000..2d01c1c --- /dev/null +++ b/teradatalabs/mapr52-hive/conf/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6 +pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9s +fmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7 +yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqO +q2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5E +KHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQIBIwKCAQAYVkg8sAouiNeg7HDJ +VaMa9h8v2Z7uRlq1RrOB05BVaQ6j/kE3V+oa9Vu9DnpV5sroPNV8jXh1z2gR8ov0 +m0lEdVLq3AZbZDvv9sebDcomBTwz5lWKyesr4S81G+w3gA5f+j1Zy2Wn81l+4pu+ +vLhPxfM4tzUKrWU61OIDRCrRFhHaFiDxBMsMhOSBfGT5OuIIHOAcjvB+YxKgB644 +ANoOnnExaukOOX0ZLgnwIG7YG0Cf6MVJpbK+gx7ER85SnMV/wjzsAedOfF+tD8Z1 +P0nfRXKr9X+CE2m3JMAb4QxXphsVHLAtGjPmzEz/2JNu7mnevJW1iQ0X19giSdyE +0+U7AoGBAPgsgpFrGKf0vy3XKMkIl8uwn+GGKob3/Fc6sGYaGu2fgR/BeMnKf4IT +BsDBgPJPz/ozQaWXWVBCZc5Wqm7CPCiccHzUF8G0PZFMNkR2ayO854co82Sj+sWW +XwS/kdTRRQ6HHr67KKSZFZO60HaWWSJ9Z0IlrVe+k3s0FtMV72pXAoGBANuqF4oi +fmHawxXgIdUcESzJlt3HUjOC32e7rEcnskjrNQxM4ZUkmQrf6IhhFrSdCk5cNCrP +pdaw87YtIEtVO3LvAT8E+dRuIO8LV4VufaBBHUq/TwntldF1/fN41ZpgpehDBH/i +rTXAMncd2junSpsyt800W7KtXsdca0YKODOnAoGAHFzbuNkKIdLUBT0pO4v0F0di +urA/X+HTsjKX0SeODIdB9QArD79fB40lV9uZtUrzMomD2GkRhYPuYLlykFC2awqQ +g0twbei2lEM5Zuj2TTopFsLZ/N+KX7lp8eoCCbGEPCyzDnvKIXCUwG0fI37lnYqs +txo/skGqdHr7S1L2ylMCgYEA1WNnUvzvzMXiFUAg3aY8kehmq45tHBi7w9rh3rjZ +E6Koj5skR70m9J7/JWWhBzl3udXpiKx1NvUKAW2qV88jzrxKWnncv7twcy+W2WQF +AhNeSJyV7GMj0spkaNvIL5EWMhU3kiyoQtf2gleDimf/VO9wuLaE9rcLnRfzLhiO +Xg8CgYEAk6QP8i01oPfY4IzE9JTHbVIglllbygxL7CKid6kH2GtbcnT8NU9aj4YY +XdGRiDRw7/O6sHSKbLkSQ1TUAVil8Xw6tLQddbRI+p93RVw9haa8bCBtzDV7eB4m +M5/MrINmIquLaSBd+l3HA36GCxLzBUsT87UZIQnJu7YauFVJYsU= +-----END RSA PRIVATE KEY----- diff --git a/teradatalabs/mapr52-hive/conf/id_rsa.pub b/teradatalabs/mapr52-hive/conf/id_rsa.pub new file mode 100644 index 0000000..cdd0869 --- /dev/null +++ b/teradatalabs/mapr52-hive/conf/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 diff --git a/teradatalabs/mapr52-hive/conf/known_hosts b/teradatalabs/mapr52-hive/conf/known_hosts new file mode 100644 index 0000000..400481d --- /dev/null +++ b/teradatalabs/mapr52-hive/conf/known_hosts @@ -0,0 +1,2 @@ +hadoop-master,172.19.0.2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv28akj3lavwQlF/Q1hkF9beEinFechrUwWNqq5f+gawUUba9AubecRX3VN0u4ix3BKQvckjGTXUUUd3wGWTz/2BfBgojXaz5LDPBxHG83t0OgIfO5W07kyDxgKPiHXm4JRGKs/TeOtVPiURbxv1Br7c9n9wMuYHQYIZhNXRQ8ETj+sjg6OiWYXahgmaErlGq+A6A5pk4lkFDnx8ESqEWnO7kEucdhUlCYqtStyCStRPci3WjHxS28RSVQoard/zap4BL7Squ0yqrIAR4UqmQNgIoGKUZv/Kh9ykw9dCwJYjMrxiuMvp/TbAR7KOYeuz5yebtLAPTHgCGbTTok6aBFQ== +sdl03545.labs.teradata.com,10.25.47.3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAsH14tOfK0dR04fuvDmY/InP+G3/FMArIFyDl1DyV8Qg8COomAukPcUNMFCqAg8wt37RZKzQ63HUlRPqxyKX6YZ1htkKXUhM+xM/gbQ4Z+OtfnINflFGhiU4gk0ClpccUdq1hmWQUE7eBu1XBQJdejwlDNmYI4rToAJf3KgSsDIPXPpJjrw3/3dtvp5UBIdcdlZNL6Xz0AY1h6Z3I1D/9z83zIX8oCCMHW9O7/zWgwd4TBay6embbkj34x8IR/9uZ7v64g9qYGugPcvfzoLAldA57VD3nbFePy+Rrl1YAfsqcBhTaP7QfBcMofgqXX776euJA9aF5j/cQM6lO72fcww== diff --git a/test/Dockerfile b/test/Dockerfile index 17dfffe..5a231c0 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -15,12 +15,22 @@ FROM teradatalabs/centos6-java8-oracle ENV DOCKERIZE_VERSION v0.3.0 - +RUN mkdir /root/.ssh +COPY conf/* /root/.ssh/ +COPY files/* /root/ RUN \ yum install -y \ wget \ unzip \ +# INSTALL UTILITY SOFTWARE + && yum install -y openssh-server openssh-clients \ + +# CONFIGURE SSH + && chkconfig sshd on \ + && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + && service sshd start \ + # setup CDH repo, pin the CDH distribution to a concrete version && wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ @@ -45,7 +55,8 @@ RUN \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # create 'image_tests' volume mount path - && mkdir /image_tests + && mkdir /image_tests \ + && chmod 777 /root/*.sh VOLUME /image_tests diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh new file mode 100644 index 0000000..eb8c5d4 --- /dev/null +++ b/test/files/expose_mapr_hive.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +conn=0 +while [ $conn -ne 1 ] +do + ping -c 4 -q hadoop-master + if [ "$?" -eq 0 ]; then + conn=1 + else + conn=0 + fi +done + +hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) + +# WAIT FOR hive directories to be created +while [ $hiveportReady -ne 1 ] +do + hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) +done + +ssh root@hadoop-master './wardenTracker.sh' +ssh root@hadoop-master './wait_for_mapr_hive.sh' diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 4f0af22..a67d573 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -22,19 +22,25 @@ assert_output_contains() { printf '%s\n' "${lines[@]}" | grep -q $1 } +function wait_for_mapr { + skip_if_needed + assert_run sh /root/expose_mapr_hive.sh +} + function exposes_hive { skip_if_needed assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } function allows_creating_a_table_in_hive { + skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From 3ee5f1a539b01b4fff75d27b9b0408978e019b1d Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 21 Mar 2017 14:22:12 -0400 Subject: [PATCH 13/43] Adding ssh keys for mapr hadoop-master Adding ssh keys for mapr hadoop-master --- {teradatalabs/mapr52-hive => test/files}/conf/id_rsa | 0 {teradatalabs/mapr52-hive => test/files}/conf/id_rsa.pub | 0 {teradatalabs/mapr52-hive => test/files}/conf/known_hosts | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {teradatalabs/mapr52-hive => test/files}/conf/id_rsa (100%) rename {teradatalabs/mapr52-hive => test/files}/conf/id_rsa.pub (100%) rename {teradatalabs/mapr52-hive => test/files}/conf/known_hosts (100%) diff --git a/teradatalabs/mapr52-hive/conf/id_rsa b/test/files/conf/id_rsa similarity index 100% rename from teradatalabs/mapr52-hive/conf/id_rsa rename to test/files/conf/id_rsa diff --git a/teradatalabs/mapr52-hive/conf/id_rsa.pub b/test/files/conf/id_rsa.pub similarity index 100% rename from teradatalabs/mapr52-hive/conf/id_rsa.pub rename to test/files/conf/id_rsa.pub diff --git a/teradatalabs/mapr52-hive/conf/known_hosts b/test/files/conf/known_hosts similarity index 100% rename from teradatalabs/mapr52-hive/conf/known_hosts rename to test/files/conf/known_hosts From 5c3e653e00078928f11b04be6164e1f2aeb952c0 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 22 Mar 2017 15:58:23 -0400 Subject: [PATCH 14/43] Changes to accomodate tests for mapr Changes to accomodate tests for mapr --- teradatalabs/mapr52-hive/files/bootstrap.sh | 3 +++ test/Dockerfile | 5 +++-- test/files/conf/known_hosts | 2 -- test/files/expose_mapr_hive.sh | 20 +++++++++++++++----- 4 files changed, 21 insertions(+), 9 deletions(-) delete mode 100644 test/files/conf/known_hosts diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 424e643..09bfa3f 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -1,5 +1,8 @@ #!/bin/sh +# START SSH SERVICE +service sshd start + # CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart diff --git a/test/Dockerfile b/test/Dockerfile index 5a231c0..89a0168 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -16,7 +16,7 @@ FROM teradatalabs/centos6-java8-oracle ENV DOCKERIZE_VERSION v0.3.0 RUN mkdir /root/.ssh -COPY conf/* /root/.ssh/ +COPY /files/conf/* /root/.ssh/ COPY files/* /root/ RUN \ yum install -y \ @@ -56,7 +56,8 @@ RUN \ # create 'image_tests' volume mount path && mkdir /image_tests \ - && chmod 777 /root/*.sh + && chmod 777 /root/*.sh \ + && chmod 600 /root/.ssh/id_rsa.pub /root/.ssh/id_rsa VOLUME /image_tests diff --git a/test/files/conf/known_hosts b/test/files/conf/known_hosts deleted file mode 100644 index 400481d..0000000 --- a/test/files/conf/known_hosts +++ /dev/null @@ -1,2 +0,0 @@ -hadoop-master,172.19.0.2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv28akj3lavwQlF/Q1hkF9beEinFechrUwWNqq5f+gawUUba9AubecRX3VN0u4ix3BKQvckjGTXUUUd3wGWTz/2BfBgojXaz5LDPBxHG83t0OgIfO5W07kyDxgKPiHXm4JRGKs/TeOtVPiURbxv1Br7c9n9wMuYHQYIZhNXRQ8ETj+sjg6OiWYXahgmaErlGq+A6A5pk4lkFDnx8ESqEWnO7kEucdhUlCYqtStyCStRPci3WjHxS28RSVQoard/zap4BL7Squ0yqrIAR4UqmQNgIoGKUZv/Kh9ykw9dCwJYjMrxiuMvp/TbAR7KOYeuz5yebtLAPTHgCGbTTok6aBFQ== -sdl03545.labs.teradata.com,10.25.47.3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAsH14tOfK0dR04fuvDmY/InP+G3/FMArIFyDl1DyV8Qg8COomAukPcUNMFCqAg8wt37RZKzQ63HUlRPqxyKX6YZ1htkKXUhM+xM/gbQ4Z+OtfnINflFGhiU4gk0ClpccUdq1hmWQUE7eBu1XBQJdejwlDNmYI4rToAJf3KgSsDIPXPpJjrw3/3dtvp5UBIdcdlZNL6Xz0AY1h6Z3I1D/9z83zIX8oCCMHW9O7/zWgwd4TBay6embbkj34x8IR/9uZ7v64g9qYGugPcvfzoLAldA57VD3nbFePy+Rrl1YAfsqcBhTaP7QfBcMofgqXX776euJA9aF5j/cQM6lO72fcww== diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh index eb8c5d4..264c6e3 100644 --- a/test/files/expose_mapr_hive.sh +++ b/test/files/expose_mapr_hive.sh @@ -11,13 +11,23 @@ do fi done -hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) +hiveportReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) -# WAIT FOR hive directories to be created +# WAIT FOR HIVE PORT TO OPEN while [ $hiveportReady -ne 1 ] do - hiveportReady=$(ssh root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) + sleep 5s + hiveportReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) done -ssh root@hadoop-master './wardenTracker.sh' -ssh root@hadoop-master './wait_for_mapr_hive.sh' +maprReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'ps -ef | grep wardenTracker.sh' | wc -l) + +# WAIT FOR WARDEN TO START ALL SERVICES +while [ $maprReady -ne 2 ] +do + maprReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'ps -ef | grep wardenTracker.sh' | wc -l) +done + +# WAIT FOR HIVE DIRECTORIES TO BE CREATED +ssh -o StrictHostKeyChecking=no root@hadoop-master './wait_for_mapr_hive.sh' + From 806199818cb22d81c827efdf5728ca64dcd6f29c Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 13:34:33 -0400 Subject: [PATCH 15/43] Adding Symlinks for the common files Adding Symlinks for the common files --- teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 - .../mapr52-hive/files/socks-proxy.sh.lnk | Bin 0 -> 1097 bytes teradatalabs/mapr52-hive/files/supervisord.conf | 1 - .../mapr52-hive/files/supervisord.conf.lnk | Bin 0 -> 1109 bytes .../files/supervisord.d/bootstrap.conf | 1 - .../files/supervisord.d/bootstrap.conf.lnk | Bin 0 -> 1273 bytes .../files/supervisord.d/mysql-metastore.conf | 1 - .../files/supervisord.d/mysql-metastore.conf.lnk | Bin 0 -> 1303 bytes .../files/supervisord.d/socks-proxy.conf | 1 - .../files/supervisord.d/socks-proxy.conf.lnk | Bin 0 -> 1283 bytes .../files/supervisord.d/sshd.conf.lnk | Bin 0 -> 1246 bytes 11 files changed, 5 deletions(-) delete mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh create mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk create mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf.lnk diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh deleted file mode 100644 index 852d770..0000000 --- a/teradatalabs/mapr52-hive/files/socks-proxy.sh +++ /dev/null @@ -1 +0,0 @@ -../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk b/teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk new file mode 100644 index 0000000000000000000000000000000000000000..910c9ac2fc3bff0af26802936fff249435a4dbee GIT binary patch literal 1097 zcma)4OK1~O6g~Ycl9kgMq<)}vVe!Mnr1%NJwx(%5jA@h3xQOx=jcI8^m`=#FHpLqJ zAVL>vsS81bAn01$`a#^O;HR64tKvp*p{QFIo|jBxQbc{>zI)%i`_4T%_a*?ecv`sw z&#F73x8Xt$Y!sdhe9%R`IfH(adMgYMLN@@>!OU{1OF-h~4`-<|C^ z7!Hb!oCcqOzj`vY3pW)+O0&fb0xaE*uTRg!lx6vSKQo?uk@R{SUdk|n5@u0EhOq|) z2P>()xTIi64~>T7Wx}T&yo)h0FYhE?XG{_=(Hhqfbc8U9Fyh2EpfXBgm~kx{L5lY} z&p;QV^?HFt1+m;L4mp*^;{Bzh7Fd*(+-`R@yRi-sLypYJ1LF?LoObVqg5hNNL`0XY z;||uK_NFZFB-X2{3NM?XFaZl$k&f@Cx;{4 zMp7X?BG?--VP*4^MKxz;rY3FS=pbx&A}k<=^LoRis9KBV~MCN=F|j0^vtLx32WwGG9x1$qY~slG$ygeu-TGLnt(O< zK$ITDQV)U%MbNW&^?`U(!ADQUQ}H5rP}HLyoY~zrZNbOC%=|Muv){~qGYJ4LcL!dOL(5XoL=9Es$a7=T z!BXQdFH0EJLMb`cBKWL>cPT0w9AmtVQEt3xp|sT~G=z|Xj2IMT;2}#zm~7ihB296P ze4r1Jm3o0`3DHsw`<+T-u_H}29+>7zJv}{EcDo8&QIx22>I35rNq_Zf%B#;|nbbc|+HYiiyK3 z=Er#yw9_&nDqlQGnN~SHT&|-bTU`;+v<%w}*sJgQSa|mIofk&tUtBplaC~X&;?Rw6 TT^m02&XgkuDBb_BIAi_>ek8xTDR?rz>@+v;2a4P~VDep?URv$G7gZa`#gc_BtGQ zHesK;TP)Pn_qBM#z=*SaU~i+ zi05JEKs$WZey#}x{&WHRtxiLMBPBKHnvk8Eo14w*+CG3EX)-4dj94he;Js@KJgWP+ zH!N8PEUcd3yRtl!SZ_=fUN4105;_vdAVno?#MP+BC^Lg~?POa~xF+7) zi4Sb0U9!BK1)1Aua!t}&XbrUR?w$7)dT@Zv`4pAO@e6&>WNG$ zZ%b>j(Uc)9t%Xs{rSfBGL(eDeNfFoeg26G**%Nw}L>y#ghge}>s=VMmR?|%IoveOW zAE&sR9TvY9f&ZU$Va=*oW6p%B>*AEG#dQ~@#)=kuOujzNh^z_me*xW;$7|7jnJt7?66F@#*0SGaJmjU4>@dF<_sQQzH(}&;oPy=my<`gN9VVE?7I51aoyLJ Psf_Oc%bou^mzaM66y^Oj literal 0 HcmV?d00001 diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf deleted file mode 100644 index b48213c..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk new file mode 100644 index 0000000000000000000000000000000000000000..6184e183e84906986b434e9563e5e0e645f450ec GIT binary patch literal 1303 zcma)5OK1~O6g~YcS~pI$Nd16uVe!Mnrf3aCt?lGTOq(>(Qe_rRlUPGKnKqNODc0Zz z5xWp;i-;l=LD$kv7k&_TD)=b~;;M)sxKPxs3(uQO)3j*q3-{go^6s2--hKB4094dh zat5B6Gdy=t0~_P%o3`$cXXh&x{oLo`dbXH{3d=9LxypKM4=n`W+P=0XrF$Q0u(zq{ z?q=*$TGhpzG9Q4$TxtIE(-PeIY$ggPsjKn*X}?#R6#wsG#*=SCUSrvpB6K5}> z94eu|D~KI|KKoPDcya}LjOZGFI?^WMCiE{ujGPF$!#xFu-Bqr6zJz+?mq4vqfodSf z`x1&rsMsKShUc);fGv!hAsS35v!U~3q&py!K?!yNJtgH!9Jfu5bt^bDj1hSlgRF{1 zt}N?fa28w%a*30rahhLkqfJ;|^o3ZEy>S$H8@a~-tBzmx!&0bZDUFMF+O3>xz z)ER0IyH)mr8rNf~L`F_((Sd}aT6I^A+)yGjk~H*8+#Xl8F=Kd8))LvMk<~K^drVJL zP7Af!q5frFeitfF^RFtZJn`C2?_rO(l5OcNsaX3vP_0f@SLs64tn^Y3t)_byi_7;i zBW}PE2}M3l^m-l#oXVr$8>bp?X|F}`#pt8)MrL}&f@3siL=A_t#c{An)>3Ei@bZ;Y kQ&s1V&OE<#WXJJ^t?!$!{it65rEVhSIly$s|MmsuAMWS{kN^Mx literal 0 HcmV?d00001 diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf deleted file mode 100644 index 905e977..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk new file mode 100644 index 0000000000000000000000000000000000000000..0727bd1472ea64502a03bf7ef5bf94d736aa5c11 GIT binary patch literal 1283 zcma)5OH30{6g_+vuyRmh@RNuO@t3KT_-PsyVLELy0#jzx5Hkx(Td_9NX=Yj|5gR{2 z6Bb6qg^9rs6WwauqK3Fr6F}_wK z*@S&k(5U28`5yS)ovB>BB|t9b?KlEdx*p%24Ml`0e!qt@o_=L|ZB<|LQbrMZ*of1* z895J2RbE^c(5-d#NzoF)4|sT&BE~4k7;k-qi5JPLM>jNdp$`(GgsnrQm5LCp%d1|* zD6Y{SXhC?XUSM26B%Q~8uTp>XNKsV+<6Nn+vC+-0t^-7nCg#KggC0sjxqm}INR^Ju z8ncdiSe?q762}C! z68gImM;QI_57u~U6?%fuO?oV3UBrzTT7jsajLF)mlHV7q=De_kdh;ivHmpK*P@wx_ z#v{boA#|J0;Ufb!;~YA`4aUq`p_3-MNEIZt4|F@_?%-I3`CYe$vZ@yum347Ql^o$) zbl+oCgHM#j8N%O6cGdIzG4Dk(9us6uWOZcP?i~*V@}MSmd(<|I86NZGWi7tDNKpDWW?6N=9@9Tf4uqfd z1f;oN8>ZWCo3B}L9)o0b3$p{fp)p!;;*LLX(0{01G&5)MXyV%G>ADNYW}jan?3WxW$YBy0 zL@92>gu_z9FD^;w)p`cxP@eEZ4)0=6O!FP{HV0XFj;wt3LPHM*AR|O<9WIJg_$e+% z{fN-LMrWW2fl@u+jD%ov68oJ>gQ252HSC+=O7->ic6NCkAc!O>Cmk4ZDB19X>k|B` zd_vJ!b;x0Lh2P})j#;nlF1%ciLL4SyNFhO;a1qy_evHw{qP!+jTkLRGcsVa2W*UZ> zws53Gf0ySQra%6{J)Wze$B1sw$3)sDZpG+2qJAo(Xs7dDx4&HTa*2B5CsXSxs1~yH zzF6@TDYl57q33Xu0oyT*4lGc_%9^3;AwThDbpYrt>0YZ)uq)iIm_&=H<2I9@AdNFN z88}aVCmE-+caibRlQodlkZEmu9!lgvLu=2ZC{B~2yJXd`(AFR6(G)@NT*S>-Dv@y| zjp$gy5>9`Gm7PdrrjnMKiM!*%vc}`?n3*OG?evgth(GM>8!brw-&(|Xl@Ax7QSBt| z+FwH8|HoN411UBbCP711J_;KtY^C1O*cZ6LUOl3JnjH!}>GR2pzqZV`+%jHsZ+VT7 zQNs&Ee4sU*wW5~SchGyt<1$j?cy#6J@O;g=?!xoS$9A7w*|FGh?R)K}&keJwzyV74 J{_CEZzX5d9_Cx>x literal 0 HcmV?d00001 From 52ae1d5c22483f33b10112c0aa6d5a227959d64d Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 13:37:14 -0400 Subject: [PATCH 16/43] Reverting links Reverting links --- .../mapr52-hive/files/socks-proxy.sh.lnk | Bin 1097 -> 0 bytes .../mapr52-hive/files/supervisord.conf.lnk | Bin 1109 -> 0 bytes .../files/supervisord.d/bootstrap.conf.lnk | Bin 1273 -> 0 bytes .../files/supervisord.d/mysql-metastore.conf.lnk | Bin 1303 -> 0 bytes .../files/supervisord.d/socks-proxy.conf.lnk | Bin 1283 -> 0 bytes .../files/supervisord.d/sshd.conf.lnk | Bin 1246 -> 0 bytes 6 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk delete mode 100644 teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf.lnk diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk b/teradatalabs/mapr52-hive/files/socks-proxy.sh.lnk deleted file mode 100644 index 910c9ac2fc3bff0af26802936fff249435a4dbee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1097 zcma)4OK1~O6g~Ycl9kgMq<)}vVe!Mnr1%NJwx(%5jA@h3xQOx=jcI8^m`=#FHpLqJ zAVL>vsS81bAn01$`a#^O;HR64tKvp*p{QFIo|jBxQbc{>zI)%i`_4T%_a*?ecv`sw z&#F73x8Xt$Y!sdhe9%R`IfH(adMgYMLN@@>!OU{1OF-h~4`-<|C^ z7!Hb!oCcqOzj`vY3pW)+O0&fb0xaE*uTRg!lx6vSKQo?uk@R{SUdk|n5@u0EhOq|) z2P>()xTIi64~>T7Wx}T&yo)h0FYhE?XG{_=(Hhqfbc8U9Fyh2EpfXBgm~kx{L5lY} z&p;QV^?HFt1+m;L4mp*^;{Bzh7Fd*(+-`R@yRi-sLypYJ1LF?LoObVqg5hNNL`0XY z;||uK_NFZFB-X2{3NM?XFaZl$k&f@Cx;{4 zMp7X?BG?--VP*4^MKxz;rY3FS=pbx&A}k<=^LoRis9KBV~MCN=F|j0^vtLx32WwGG9x1$qY~slG$ygeu-TGLnt(O< zK$ITDQV)U%MbNW&^?`U(!ADQUQ}H5rP}HLyoY~zrZNbOC%=|Muv){~qGYJ4LcL!dOL(5XoL=9Es$a7=T z!BXQdFH0EJLMb`cBKWL>cPT0w9AmtVQEt3xp|sT~G=z|Xj2IMT;2}#zm~7ihB296P ze4r1Jm3o0`3DHsw`<+T-u_H}29+>7zJv}{EcDo8&QIx22>I35rNq_Zf%B#;|nbbc|+HYiiyK3 z=Er#yw9_&nDqlQGnN~SHT&|-bTU`;+v<%w}*sJgQSa|mIofk&tUtBplaC~X&;?Rw6 TT^m02&XgkuDBb_BIAi_>ek8xTDR?rz>@+v;2a4P~VDep?URv$G7gZa`#gc_BtGQ zHesK;TP)Pn_qBM#z=*SaU~i+ zi05JEKs$WZey#}x{&WHRtxiLMBPBKHnvk8Eo14w*+CG3EX)-4dj94he;Js@KJgWP+ zH!N8PEUcd3yRtl!SZ_=fUN4105;_vdAVno?#MP+BC^Lg~?POa~xF+7) zi4Sb0U9!BK1)1Aua!t}&XbrUR?w$7)dT@Zv`4pAO@e6&>WNG$ zZ%b>j(Uc)9t%Xs{rSfBGL(eDeNfFoeg26G**%Nw}L>y#ghge}>s=VMmR?|%IoveOW zAE&sR9TvY9f&ZU$Va=*oW6p%B>*AEG#dQ~@#)=kuOujzNh^z_me*xW;$7|7jnJt7?66F@#*0SGaJmjU4>@dF<_sQQzH(}&;oPy=my<`gN9VVE?7I51aoyLJ Psf_Oc%bou^mzaM66y^Oj diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf.lnk deleted file mode 100644 index 6184e183e84906986b434e9563e5e0e645f450ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1303 zcma)5OK1~O6g~YcS~pI$Nd16uVe!Mnrf3aCt?lGTOq(>(Qe_rRlUPGKnKqNODc0Zz z5xWp;i-;l=LD$kv7k&_TD)=b~;;M)sxKPxs3(uQO)3j*q3-{go^6s2--hKB4094dh zat5B6Gdy=t0~_P%o3`$cXXh&x{oLo`dbXH{3d=9LxypKM4=n`W+P=0XrF$Q0u(zq{ z?q=*$TGhpzG9Q4$TxtIE(-PeIY$ggPsjKn*X}?#R6#wsG#*=SCUSrvpB6K5}> z94eu|D~KI|KKoPDcya}LjOZGFI?^WMCiE{ujGPF$!#xFu-Bqr6zJz+?mq4vqfodSf z`x1&rsMsKShUc);fGv!hAsS35v!U~3q&py!K?!yNJtgH!9Jfu5bt^bDj1hSlgRF{1 zt}N?fa28w%a*30rahhLkqfJ;|^o3ZEy>S$H8@a~-tBzmx!&0bZDUFMF+O3>xz z)ER0IyH)mr8rNf~L`F_((Sd}aT6I^A+)yGjk~H*8+#Xl8F=Kd8))LvMk<~K^drVJL zP7Af!q5frFeitfF^RFtZJn`C2?_rO(l5OcNsaX3vP_0f@SLs64tn^Y3t)_byi_7;i zBW}PE2}M3l^m-l#oXVr$8>bp?X|F}`#pt8)MrL}&f@3siL=A_t#c{An)>3Ei@bZ;Y kQ&s1V&OE<#WXJJ^t?!$!{it65rEVhSIly$s|MmsuAMWS{kN^Mx diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf.lnk deleted file mode 100644 index 0727bd1472ea64502a03bf7ef5bf94d736aa5c11..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1283 zcma)5OH30{6g_+vuyRmh@RNuO@t3KT_-PsyVLELy0#jzx5Hkx(Td_9NX=Yj|5gR{2 z6Bb6qg^9rs6WwauqK3Fr6F}_wK z*@S&k(5U28`5yS)ovB>BB|t9b?KlEdx*p%24Ml`0e!qt@o_=L|ZB<|LQbrMZ*of1* z895J2RbE^c(5-d#NzoF)4|sT&BE~4k7;k-qi5JPLM>jNdp$`(GgsnrQm5LCp%d1|* zD6Y{SXhC?XUSM26B%Q~8uTp>XNKsV+<6Nn+vC+-0t^-7nCg#KggC0sjxqm}INR^Ju z8ncdiSe?q762}C! z68gImM;QI_57u~U6?%fuO?oV3UBrzTT7jsajLF)mlHV7q=De_kdh;ivHmpK*P@wx_ z#v{boA#|J0;Ufb!;~YA`4aUq`p_3-MNEIZt4|F@_?%-I3`CYe$vZ@yum347Ql^o$) zbl+oCgHM#j8N%O6cGdIzG4Dk(9us6uWOZcP?i~*V@}MSmd(<|I86NZGWi7tDNKpDWW?6N=9@9Tf4uqfd z1f;oN8>ZWCo3B}L9)o0b3$p{fp)p!;;*LLX(0{01G&5)MXyV%G>ADNYW}jan?3WxW$YBy0 zL@92>gu_z9FD^;w)p`cxP@eEZ4)0=6O!FP{HV0XFj;wt3LPHM*AR|O<9WIJg_$e+% z{fN-LMrWW2fl@u+jD%ov68oJ>gQ252HSC+=O7->ic6NCkAc!O>Cmk4ZDB19X>k|B` zd_vJ!b;x0Lh2P})j#;nlF1%ciLL4SyNFhO;a1qy_evHw{qP!+jTkLRGcsVa2W*UZ> zws53Gf0ySQra%6{J)Wze$B1sw$3)sDZpG+2qJAo(Xs7dDx4&HTa*2B5CsXSxs1~yH zzF6@TDYl57q33Xu0oyT*4lGc_%9^3;AwThDbpYrt>0YZ)uq)iIm_&=H<2I9@AdNFN z88}aVCmE-+caibRlQodlkZEmu9!lgvLu=2ZC{B~2yJXd`(AFR6(G)@NT*S>-Dv@y| zjp$gy5>9`Gm7PdrrjnMKiM!*%vc}`?n3*OG?evgth(GM>8!brw-&(|Xl@Ax7QSBt| z+FwH8|HoN411UBbCP711J_;KtY^C1O*cZ6LUOl3JnjH!}>GR2pzqZV`+%jHsZ+VT7 zQNs&Ee4sU*wW5~SchGyt<1$j?cy#6J@O;g=?!xoS$9A7w*|FGh?R)K}&keJwzyV74 J{_CEZzX5d9_Cx>x From b4edeb907d295b189ff9f6d0ad0111f3ce88d30f Mon Sep 17 00:00:00 2001 From: Anton Petrov Date: Thu, 23 Mar 2017 14:16:42 -0400 Subject: [PATCH 17/43] Add symlinks to mapr52-hive --- teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 + teradatalabs/mapr52-hive/files/supervisord.conf | 1 + teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf | 1 + .../mapr52-hive/files/supervisord.d/mysql-metastore.conf | 1 + teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf | 1 + teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf | 1 + 6 files changed, 6 insertions(+) create mode 120000 teradatalabs/mapr52-hive/files/socks-proxy.sh create mode 120000 teradatalabs/mapr52-hive/files/supervisord.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh new file mode 120000 index 0000000..852d770 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/socks-proxy.sh @@ -0,0 +1 @@ +../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf new file mode 120000 index 0000000..c821a2d --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.conf @@ -0,0 +1 @@ +../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf new file mode 120000 index 0000000..948ad9f --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf @@ -0,0 +1 @@ +../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf new file mode 120000 index 0000000..e686e37 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf @@ -0,0 +1 @@ +../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf new file mode 120000 index 0000000..fe5c063 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf @@ -0,0 +1 @@ +../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf new file mode 120000 index 0000000..c257c15 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf @@ -0,0 +1 @@ +../../../commons/supervisord.d/sshd.conf \ No newline at end of file From 3e15ca4cd6610b8c19cc03d0edca1fff89208dd5 Mon Sep 17 00:00:00 2001 From: Anton Petrov Date: Thu, 23 Mar 2017 15:26:28 -0400 Subject: [PATCH 18/43] Fix links inside supervisord.d directory --- teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf | 2 +- .../mapr52-hive/files/supervisord.d/mysql-metastore.conf | 2 +- teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf | 2 +- teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf index 948ad9f..16a8759 120000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf +++ b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf @@ -1 +1 @@ -../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file +../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf index e686e37..b48213c 120000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf +++ b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf @@ -1 +1 @@ -../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file +../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf index fe5c063..905e977 120000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf +++ b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf @@ -1 +1 @@ -../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file +../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf index c257c15..31d04a8 120000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf +++ b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf @@ -1 +1 @@ -../../../commons/supervisord.d/sshd.conf \ No newline at end of file +../../../../commons/supervisord.d/sshd.conf \ No newline at end of file From b7a3b5851cd2a9cc5af0a46c11dd93b770963b83 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 15:34:41 -0400 Subject: [PATCH 19/43] Modification to include env.sh changes Modification to include env.sh changes in Kerberized MapR cluster --- teradatalabs/mapr52-hive-kerberized/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 5ffb495..76a1a94 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -32,7 +32,6 @@ ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl ADD files/jceJars/local_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/local_policy.jar ADD files/jceJars/US_export_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/US_export_policy.jar -ADD files/conf/env.sh /opt/mapr/conf/env.sh # ENABLE HIVE SECURITY ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml @@ -66,6 +65,10 @@ RUN /usr/sbin/kdb5_util create -s -P password \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" \ && chmod 644 /etc/presto/conf/*.keytab + && cat /opt/mapr/conf/env.sh | sed -e '0,/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > env_new.sh \ + && cat env_new.sh | sed -e '0,/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > /opt/mapr/conf/env.sh \ + && rm -rf env_new.sh + # CREATE SSL KEYSTORE RUN keytool -genkeypair \ From 56c37e9fe61c3e839614feee47802be50d407d67 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 15:57:23 -0400 Subject: [PATCH 20/43] Adding commands to start httpfs services Adding commands to start httpfs services --- teradatalabs/mapr52-hive-kerberized/Dockerfile | 2 +- teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh | 4 ++++ teradatalabs/mapr52-hive/files/bootstrap.sh | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 76a1a94..1c0278d 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -64,7 +64,7 @@ RUN /usr/sbin/kdb5_util create -s -P password \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-server.keytab presto-server/presto-master.docker.cluster" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" \ - && chmod 644 /etc/presto/conf/*.keytab + && chmod 644 /etc/presto/conf/*.keytab \ && cat /opt/mapr/conf/env.sh | sed -e '0,/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > env_new.sh \ && cat env_new.sh | sed -e '0,/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > /opt/mapr/conf/env.sh \ && rm -rf env_new.sh diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index 88cc860..bb22412 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -29,6 +29,10 @@ service mapr-warden start # WAIT FOR WARDEN TO START ALL THE SERVICES sh /root/wardenTracker.sh +# START HTTPFS SERVICES +maprcli node services -name httpfs -action start -nodes $(hostname) +chown -R mapr /opt/mapr/httpfs + # CREATE KERBEROS TICKET kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 09bfa3f..b600d0a 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -25,6 +25,10 @@ service mapr-warden start # WAIT FOR WARDEN TO START ALL THE SERVICES sh /root/wardenTracker.sh +# START HTTPFS SERVICES +maprcli node services -name httpfs -action start -nodes $(hostname) +chown -R mapr /opt/mapr/httpfs + # RUN HDFS COMMANDS hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive From 875f70d6fa797a12badf255268b109ad48568b2b Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 16:25:25 -0400 Subject: [PATCH 21/43] Adding privileged in docker-compose for MapR Adding privileged in docker-compose for MapR --- test/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 2e04ebe..1908301 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -4,6 +4,7 @@ services: hadoop-master: image: ${TESTED_IMAGE} hostname: hadoop-master + privileged: true ports: - '1180' From 86ae2031ea379299809380788023e1b45e4c81b1 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 23 Mar 2017 17:42:12 -0400 Subject: [PATCH 22/43] Changing Test container to accomodate mapr Changing Test container to accomodate mapr --- .../mapr52-hive/files/wait_for_mapr_hive.sh | 15 ------------- test/files/expose_mapr_hive.sh | 21 ++++--------------- test/image_tests/image_tests.bats.sh | 6 ++++-- 3 files changed, 8 insertions(+), 34 deletions(-) delete mode 100644 teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh diff --git a/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh b/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh deleted file mode 100644 index eef4670..0000000 --- a/teradatalabs/mapr52-hive/files/wait_for_mapr_hive.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -nohup hadoop fs -ls /user/hive/warehouse > hivecliReady.log 2>&1 & -wait -hivecliReady=$(cat hivecliReady.log | grep 'No such file or directory' | wc -l) - -# WAIT FOR hive directories to be created -while [ $hivecliReady == 1 ] -do - nohup hadoop fs -ls /user/hive/warehouse > hivecliReady.log 2>&1 & - wait - hivecliReady=$(cat hivecliReady.log | grep 'No such file or directory' | wc -l -) -done -rm -rf hivecliReady.log diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh index 264c6e3..eac8f42 100644 --- a/test/files/expose_mapr_hive.sh +++ b/test/files/expose_mapr_hive.sh @@ -11,23 +11,10 @@ do fi done -hiveportReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) - -# WAIT FOR HIVE PORT TO OPEN -while [ $hiveportReady -ne 1 ] +hivecliReady=1 +while [ $hivecliReady -ne 0 ] do sleep 5s - hiveportReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'netstat -tuplen' | grep 10000 | wc -l) + ssh -o StrictHostKeyChecking=no root@hadoop-master 'hadoop fs -ls /user/hive/warehouse' + hivecliReady=$? done - -maprReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'ps -ef | grep wardenTracker.sh' | wc -l) - -# WAIT FOR WARDEN TO START ALL SERVICES -while [ $maprReady -ne 2 ] -do - maprReady=$(ssh -o StrictHostKeyChecking=no root@hadoop-master 'ps -ef | grep wardenTracker.sh' | wc -l) -done - -# WAIT FOR HIVE DIRECTORIES TO BE CREATED -ssh -o StrictHostKeyChecking=no root@hadoop-master './wait_for_mapr_hive.sh' - diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index a67d573..e4eccd6 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +user=hdfs skip_if_needed() { SHOULD_RUN=true # Can't rely on exit codes here, as BATS will fail the test if any of the statements @@ -24,6 +25,7 @@ assert_output_contains() { function wait_for_mapr { skip_if_needed + user=hive assert_run sh /root/expose_mapr_hive.sh } @@ -35,12 +37,12 @@ function exposes_hive { function allows_creating_a_table_in_hive { skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n $user -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n $user -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From d4223b447d4f28a1c3707f3a110128fd10d00c98 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Fri, 24 Mar 2017 13:20:18 -0400 Subject: [PATCH 23/43] modification of tests to accomadate hive user modification of tests to accomadate hive user --- .../mapr52-hive-kerberized/capabilities.txt | 4 ---- teradatalabs/mapr52-hive/capabilities.txt | 9 ++++---- test/image_tests/image_tests.bats.sh | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 16 deletions(-) delete mode 100644 teradatalabs/mapr52-hive-kerberized/capabilities.txt diff --git a/teradatalabs/mapr52-hive-kerberized/capabilities.txt b/teradatalabs/mapr52-hive-kerberized/capabilities.txt deleted file mode 100644 index 6892a21..0000000 --- a/teradatalabs/mapr52-hive-kerberized/capabilities.txt +++ /dev/null @@ -1,4 +0,0 @@ -exposes_hive -allows_creating_a_table_in_hive -allows_selecting_from_the_table -exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt index ba21a5b..2518bb1 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -1,5 +1,4 @@ -wait_for_mapr -exposes_hive -allows_creating_a_table_in_hive -allows_selecting_from_the_table -exposes_socks_proxy +exposes_mapr +allows_creating_a_table_in_hive_mapr +allows_selecting_from_the_table_mapr +exposes_socks_proxy \ No newline at end of file diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index e4eccd6..361574e 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -user=hdfs skip_if_needed() { SHOULD_RUN=true # Can't rely on exit codes here, as BATS will fail the test if any of the statements @@ -10,7 +9,6 @@ skip_if_needed() { skip fi } - assert_run() { run "$@" echo "Output of [$*]:" @@ -23,9 +21,8 @@ assert_output_contains() { printf '%s\n' "${lines[@]}" | grep -q $1 } -function wait_for_mapr { +function exposes_mapr { skip_if_needed - user=hive assert_run sh /root/expose_mapr_hive.sh } @@ -34,15 +31,26 @@ function exposes_hive { assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } +function allows_creating_a_table_in_hive_mapr { + skip_if_needed + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' +} + +function allows_selecting_from_the_table_mapr { + skip_if_needed + assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_output_contains 'test.id' + assert_output_contains '42' +} + function allows_creating_a_table_in_hive { - skip_if_needed - assert_run beeline -n $user -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n $user -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From 3ebb7482dc3926081c392358124ca7cc653cf87b Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Fri, 24 Mar 2017 16:27:56 -0400 Subject: [PATCH 24/43] Fixing kerberos container Fixing kerberos container --- teradatalabs/mapr52-hive-kerberized/Dockerfile | 11 +++++------ .../files/kerberos/kadm5.acl | 1 + .../files/kerberos/kdc.conf | 12 ++++++++++++ .../files/kerberos/krb5.conf | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 1c0278d..1a98fda 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -38,8 +38,8 @@ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml # CREATE KERBEROS DATABASE RUN /usr/sbin/kdb5_util create -s -P password \ - && usermod -g root hdfs - && usermod -g mapr hdfs + && usermod -g root hdfs \ + && usermod -g mapr hdfs \ # CREATE MAPR AND HIVE PRINCIPALS AND KEYTABS && /usr/sbin/kadmin.local -q "addprinc -randkey mapr/mycluster@LABS.TERADATA.COM" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM" \ @@ -65,11 +65,10 @@ RUN /usr/sbin/kdb5_util create -s -P password \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" \ && chmod 644 /etc/presto/conf/*.keytab \ - && cat /opt/mapr/conf/env.sh | sed -e '0,/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > env_new.sh \ - && cat env_new.sh | sed -e '0,/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/ s/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=simple"/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > /opt/mapr/conf/env.sh \ + && cat /opt/mapr/conf/env.sh | sed -e '0,/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=maprsasl_keytab"/ s/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=maprsasl_keytab"/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > env_new.sh \ + && cat env_new.sh | sed -e '0,/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=maprsasl"/ s/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=maprsasl"/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > /opt/mapr/conf/env.sh \ && rm -rf env_new.sh - # CREATE SSL KEYSTORE RUN keytool -genkeypair \ -alias presto \ @@ -84,4 +83,4 @@ RUN chmod 644 /etc/presto/conf/keystore.jks EXPOSE 88 EXPOSE 749 -CMD /root/startup.sh \ No newline at end of file +CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl new file mode 100644 index 0000000..0530526 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl @@ -0,0 +1 @@ +*/admin@LABS.TERADATA.COM * diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf new file mode 100644 index 0000000..c864897 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf @@ -0,0 +1,12 @@ +[kdcdefaults] + kdc_ports = 88 + kdc_tcp_ports = 88 + +[realms] + LABS.TERADATA.COM = { + #master_key_type = aes256-cts + acl_file = /var/kerberos/krb5kdc/kadm5.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab + supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + } diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf b/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf new file mode 100644 index 0000000..344e348 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf @@ -0,0 +1,18 @@ +[logging] +default = FILE:/var/log/krb5libs.log +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmind.log + +[libdefaults] +default_realm = LABS.TERADATA.COM +dns_lookup_realm = false +dns_lookup_kdc = false +ticket_lifetime = 24h +renew_lifetime = 7d +forwardable = true + +[realms] +LABS.TERADATA.COM = { +kdc = hadoop-master +admin_server = hadoop-master +} From 1f5f062165b885f3d34baf384eeabca6b8d1bc90 Mon Sep 17 00:00:00 2001 From: Anton Petrov Date: Fri, 24 Mar 2017 16:22:46 -0400 Subject: [PATCH 25/43] Start sshd and the socks-proxy from bootstrap.sh --- teradatalabs/mapr52-hive/files/bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index b600d0a..5b7b3e2 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -1,7 +1,8 @@ #!/bin/sh -# START SSH SERVICE -service sshd start +# START SSHD AND THE SOCKS PROXY FOR THE HIVE METASTORE +supervisorctl start sshd +supervisorctl start socks-proxy # CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart From 239e58767a6380cec40a953b6c55ff472ef901a3 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Mon, 27 Mar 2017 15:37:38 -0400 Subject: [PATCH 26/43] Changes for Debugging Bootsrap to know where its stuck Debugging Bootsrap to know where its stuck --- teradatalabs/mapr52-base/Dockerfile | 2 +- teradatalabs/mapr52-hive/files/bootstrap.sh | 11 ++++++++--- test/files/expose_mapr_hive.sh | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 67ea40f..d55bd9c 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -33,7 +33,7 @@ RUN yum update -y \ # INSTALL MapR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ - && yum install -y mapr-webserver mapr-gateway \ + && yum install -y mapr-webserver mapr-gateway mapr-httpfs \ # CONFIGURE ZOOKEEPER'S DATA DIRECTORY && rm -rf /opt/mapr/zkdata \ diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 5b7b3e2..5adbcad 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -7,25 +7,30 @@ supervisorctl start socks-proxy # CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart +date + # SETUP FLAT FILE /home/mapr/storagefile dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 +date + # SETUP DISK FOR MAPR BY RUNNING disksetup /opt/mapr/server/disksetup -M -F /root/disk.txt # CREATE HIVE PROXY USERS chmod 755 /opt/mapr/conf/proxy +date # START SERVICES service mapr-zookeeper start service mapr-warden start - +date # CONFIGURE HIVE /opt/mapr/server/configure.sh -R - +date # WAIT FOR WARDEN TO START ALL THE SERVICES sh /root/wardenTracker.sh - +date # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) chown -R mapr /opt/mapr/httpfs diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh index eac8f42..b5bf405 100644 --- a/test/files/expose_mapr_hive.sh +++ b/test/files/expose_mapr_hive.sh @@ -16,5 +16,8 @@ while [ $hivecliReady -ne 0 ] do sleep 5s ssh -o StrictHostKeyChecking=no root@hadoop-master 'hadoop fs -ls /user/hive/warehouse' + ssh -o StrictHostKeyChecking=no root@hadoop-master 'cat /var/log/bootstrap.log' > /root/bootstrap.log + ssh -o StrictHostKeyChecking=no root@sdl03545.labs.teradata.com 'ls -ltr /root/' + scp /root/bootstrap.log root@sdl03545.labs.teradata.com:/root/ hivecliReady=$? done From 2fb8915e73032f9603ee79691e6399f12c3e031f Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Mon, 27 Mar 2017 16:36:35 -0400 Subject: [PATCH 27/43] Reverting changes for debugging bootstrap Reverting changes for debugging bootstrap --- teradatalabs/mapr52-hive/files/bootstrap.sh | 11 +++-------- test/files/expose_mapr_hive.sh | 3 --- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 5adbcad..5b7b3e2 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -7,30 +7,25 @@ supervisorctl start socks-proxy # CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -date - # SETUP FLAT FILE /home/mapr/storagefile dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 -date - # SETUP DISK FOR MAPR BY RUNNING disksetup /opt/mapr/server/disksetup -M -F /root/disk.txt # CREATE HIVE PROXY USERS chmod 755 /opt/mapr/conf/proxy -date # START SERVICES service mapr-zookeeper start service mapr-warden start -date + # CONFIGURE HIVE /opt/mapr/server/configure.sh -R -date + # WAIT FOR WARDEN TO START ALL THE SERVICES sh /root/wardenTracker.sh -date + # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) chown -R mapr /opt/mapr/httpfs diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh index b5bf405..eac8f42 100644 --- a/test/files/expose_mapr_hive.sh +++ b/test/files/expose_mapr_hive.sh @@ -16,8 +16,5 @@ while [ $hivecliReady -ne 0 ] do sleep 5s ssh -o StrictHostKeyChecking=no root@hadoop-master 'hadoop fs -ls /user/hive/warehouse' - ssh -o StrictHostKeyChecking=no root@hadoop-master 'cat /var/log/bootstrap.log' > /root/bootstrap.log - ssh -o StrictHostKeyChecking=no root@sdl03545.labs.teradata.com 'ls -ltr /root/' - scp /root/bootstrap.log root@sdl03545.labs.teradata.com:/root/ hivecliReady=$? done From 54221d4c51380fe5b843d1cb6a1320b9f1b2e45e Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 28 Mar 2017 10:38:08 -0400 Subject: [PATCH 28/43] Moving partition creation to mapr-base Moving partition creation to mapr-base and changes in capabilities --- teradatalabs/mapr52-base/Dockerfile | 8 +++++--- teradatalabs/mapr52-hive/capabilities.txt | 4 ++-- teradatalabs/mapr52-hive/files/bootstrap.sh | 3 --- test/image_tests/image_tests.bats.sh | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index d55bd9c..8edb796 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -40,8 +40,6 @@ RUN yum update -y \ && mkdir /opt/mapr/zkdata \ && chmod 777 /opt/mapr/zkdata \ && mkdir -p /mapr \ -# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK - && echo "/home/mapr/storagefile" > /root/disk.txt \ # INSTALL PYTHON AND SUPERVISORD && yum install -y python-setuptools \ @@ -68,4 +66,8 @@ RUN yum update -y \ && echo "cd /home/hdfs" >> /home/hdfs/.bashrc \ && chown mapr:mapr /home/mapr \ && chown hive:hive /home/hive \ - && chown hdfs:hdfs /home/hdfs + && chown hdfs:hdfs /home/hdfs \ + # SETUP FLAT FILE /home/mapr/storagefile + && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ + # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK + && echo "/home/mapr/storagefile" > /root/disk.txt diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt index 2518bb1..8fd71e0 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -1,4 +1,4 @@ exposes_mapr -allows_creating_a_table_in_hive_mapr -allows_selecting_from_the_table_mapr +allows_creating_a_table_in_mapr_hive +allows_selecting_from_the_table_mapr_hive exposes_socks_proxy \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 5b7b3e2..3da3253 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -7,9 +7,6 @@ supervisorctl start socks-proxy # CONFIGURE MapR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -# SETUP FLAT FILE /home/mapr/storagefile -dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 - # SETUP DISK FOR MAPR BY RUNNING disksetup /opt/mapr/server/disksetup -M -F /root/disk.txt diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 361574e..ca92762 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -31,12 +31,12 @@ function exposes_hive { assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } -function allows_creating_a_table_in_hive_mapr { +function allows_creating_a_table_in_mapr_hive { skip_if_needed assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } -function allows_selecting_from_the_table_mapr { +function allows_selecting_from_the_table_mapr_hive { skip_if_needed assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' From 73461bb29e5db5d28b4fd76c7226a41c8b687dd2 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 28 Mar 2017 12:39:01 -0400 Subject: [PATCH 29/43] decreasing allocation of space to 3 GB decreasing allocation of space to 3 GB --- teradatalabs/mapr52-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 8edb796..aa0152a 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -68,6 +68,6 @@ RUN yum update -y \ && chown hive:hive /home/hive \ && chown hdfs:hdfs /home/hdfs \ # SETUP FLAT FILE /home/mapr/storagefile - && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ + && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=3 \ # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK && echo "/home/mapr/storagefile" > /root/disk.txt From 7cd2813c9236eb240c708b397225581a35aba1f1 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 28 Mar 2017 13:55:16 -0400 Subject: [PATCH 30/43] Increasing allocation of space back to 10 GB Increasing allocation of space back to 10 GB --- teradatalabs/mapr52-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index aa0152a..8edb796 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -68,6 +68,6 @@ RUN yum update -y \ && chown hive:hive /home/hive \ && chown hdfs:hdfs /home/hdfs \ # SETUP FLAT FILE /home/mapr/storagefile - && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=3 \ + && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK && echo "/home/mapr/storagefile" > /root/disk.txt From 904a566c66340558bba49abc6ec65c34592041f5 Mon Sep 17 00:00:00 2001 From: Anton Petrov Date: Tue, 28 Mar 2017 14:38:00 -0400 Subject: [PATCH 31/43] Pick user to execute Hadoop commands based on image name --- Makefile | 2 +- teradatalabs/mapr52-hive/capabilities.txt | 4 ++-- test/image_tests/image_tests.bats.sh | 27 +++++++++++------------ 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 1024abe..97bcaa0 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ $(IMAGE_DIRS): %: %/Dockerfile | check-links export TESTED_IMAGE=$* && \ cd test && \ docker-compose up -t 0 -d hadoop-master && \ - time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" --rm test-runner + time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" -e IMAGE=$* --rm test-runner # # Static pattern rule to pull docker images that are external dependencies of diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt index 8fd71e0..c1da436 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -1,4 +1,4 @@ exposes_mapr -allows_creating_a_table_in_mapr_hive -allows_selecting_from_the_table_mapr_hive +allows_creating_a_table_in_hive +allows_selecting_from_the_table exposes_socks_proxy \ No newline at end of file diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index ca92762..7c1abe1 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +get_user() { + skip_if_needed + if [ "${IMAGE}" == "teradatalabs/mapr52-hive" ] + then + echo "hive" + else + echo "hdfs" + fi +} + skip_if_needed() { SHOULD_RUN=true # Can't rely on exit codes here, as BATS will fail the test if any of the statements @@ -9,6 +19,7 @@ skip_if_needed() { skip fi } + assert_run() { run "$@" echo "Output of [$*]:" @@ -31,26 +42,14 @@ function exposes_hive { assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s } -function allows_creating_a_table_in_mapr_hive { - skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' -} - -function allows_selecting_from_the_table_mapr_hive { - skip_if_needed - assert_run beeline -n hive -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' - assert_output_contains 'test.id' - assert_output_contains '42' -} - function allows_creating_a_table_in_hive { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n $(get_user) -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n $(get_user) -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From 7259c4522afdaa5bc5523a1eeda6f256038aecde Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 28 Mar 2017 15:28:23 -0400 Subject: [PATCH 32/43] makefile changes to debug images size makefile changes to debug images size --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 97bcaa0..c94941d 100644 --- a/Makefile +++ b/Makefile @@ -154,6 +154,8 @@ $(IMAGE_DIRS): %: %/Dockerfile | check-links @echo export TESTED_IMAGE=$* && \ cd test && \ + docker ps && \ + docker images && \ docker-compose up -t 0 -d hadoop-master && \ time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" -e IMAGE=$* --rm test-runner From f5b8ae057087d6532bedea5312a3112022f5804e Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Tue, 28 Mar 2017 15:55:21 -0400 Subject: [PATCH 33/43] Changes in travis to increase wait time during build Changes in travis to increase wait time during build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e501d61..01180ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: make ${IMAGES[@]} +install: travis_wait 15 make ${IMAGES[@]} before_deploy: # decrypt the github deploy key From c0fd0ea3c67961d5288df8529758803026296371 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 29 Mar 2017 10:07:57 -0400 Subject: [PATCH 34/43] Reverting the debugging changes Reverting the debugging changes --- .travis.yml | 2 +- Makefile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01180ed..e501d61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: travis_wait 15 make ${IMAGES[@]} +install: make ${IMAGES[@]} before_deploy: # decrypt the github deploy key diff --git a/Makefile b/Makefile index c94941d..97bcaa0 100644 --- a/Makefile +++ b/Makefile @@ -154,8 +154,6 @@ $(IMAGE_DIRS): %: %/Dockerfile | check-links @echo export TESTED_IMAGE=$* && \ cd test && \ - docker ps && \ - docker images && \ docker-compose up -t 0 -d hadoop-master && \ time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" -e IMAGE=$* --rm test-runner From 0dfc67366490e7ab72a5895d7a6816c8854cb51d Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 29 Mar 2017 13:15:40 -0400 Subject: [PATCH 35/43] Adding httpfs to MapR-base Adding httpfs to MapR-base and moving storagefile partition creation to hive --- teradatalabs/mapr52-base/Dockerfile | 32 +++++++++------------ teradatalabs/mapr52-hive/Dockerfile | 8 ++++-- teradatalabs/mapr52-hive/files/bootstrap.sh | 3 +- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 8edb796..4c010d3 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -24,17 +24,26 @@ RUN yum update -y \ # INSTALL UTILITY SOFTWARE && yum install -y iputils vim openssh-server openssh-clients sudo \ - # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ - && service sshd start \ -# INSTALL MapR +# INSTALL MAPR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ && yum install -y mapr-webserver mapr-gateway mapr-httpfs \ +# ADD USERS AND CHANGE OWNERSHIPS + && adduser mapr \ + && adduser hive \ + && adduser hdfs \ + && touch /home/mapr /home/hive /home/hdfs \ + && echo "cd /home/mapr" >> /home/mapr/.bashrc \ + && echo "cd /home/hive" >> /home/hive/.bashrc \ + && echo "cd /home/hdfs" >> /home/hdfs/.bashrc \ + && chown -R mapr:mapr /home/mapr /opt/mapr/httpfs \ + && chown hive:hive /home/hive \ + && chown hdfs:hdfs /home/hdfs \ # CONFIGURE ZOOKEEPER'S DATA DIRECTORY && rm -rf /opt/mapr/zkdata \ && mkdir /opt/mapr/zkdata \ @@ -55,19 +64,4 @@ RUN yum update -y \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ - && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys \ - - && adduser mapr \ - && adduser hive \ - && adduser hdfs \ - && touch /home/mapr /home/hive /home/hdfs \ - && echo "cd /home/mapr" >> /home/mapr/.bashrc \ - && echo "cd /home/hive" >> /home/hive/.bashrc \ - && echo "cd /home/hdfs" >> /home/hdfs/.bashrc \ - && chown mapr:mapr /home/mapr \ - && chown hive:hive /home/hive \ - && chown hdfs:hdfs /home/hdfs \ - # SETUP FLAT FILE /home/mapr/storagefile - && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ - # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK - && echo "/home/mapr/storagefile" > /root/disk.txt + && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index 5f9cebb..f765e62 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -30,12 +30,16 @@ RUN chmod 777 /root/*.sh \ # INSTALL MYSQL SERVER && yum install -y mysql-server mysql-connector-java \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ - && /root/setup.sh \ # INSTALL HIVE && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + +# SETUP FLAT FILE /home/mapr/storagefile + && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ +# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK + && echo "/home/mapr/storagefile" > /root/disk.txt # HDFS PORTS EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index 3da3253..b087954 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -24,8 +24,7 @@ service mapr-warden start sh /root/wardenTracker.sh # START HTTPFS SERVICES -maprcli node services -name httpfs -action start -nodes $(hostname) -chown -R mapr /opt/mapr/httpfs +maprcli node services -name httpfs -action start -nodes $(hostname) # RUN HDFS COMMANDS hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive From e4c5f9144212bd0a8e607d8961c6e043488e367f Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 29 Mar 2017 13:26:33 -0400 Subject: [PATCH 36/43] Removing creation of partition from kerberized container Removing creation of partition from bootstrap of kerberized container and removing chown --- teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index bb22412..1c45c5d 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -3,9 +3,6 @@ # CONFIGURE MAPR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -# SETUP FLAT FILE /home/mapr/storagefile -dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 - # SETUP DISK FOR MAPR BY RUNNING disksetup /opt/mapr/server/disksetup -M -F /root/disk.txt @@ -31,7 +28,6 @@ sh /root/wardenTracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) -chown -R mapr /opt/mapr/httpfs # CREATE KERBEROS TICKET kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM From c01bcd5e7ca6e4c2b1eccc861364c6a0f12817b5 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Wed, 29 Mar 2017 13:29:09 -0400 Subject: [PATCH 37/43] Adding SSHD AND THE SOCKS PROXY for kerberized mapr Adding SSHD AND THE SOCKS PROXY for kerberized mapr --- teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index 1c45c5d..00a90a9 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -1,5 +1,10 @@ #!/bin/sh + +# START SSHD AND THE SOCKS PROXY FOR THE HIVE METASTORE +supervisorctl start sshd +supervisorctl start socks-proxy + # CONFIGURE MAPR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart From d15d552912ab28bcb076acab1cbb2fad0bfd7d22 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 10:39:33 -0400 Subject: [PATCH 38/43] Rearranging the code in MapR cluster to make it ready for review Rearranging the code in MapR cluster to make it ready for review --- teradatalabs/mapr52-base/Dockerfile | 6 +- .../mapr52-hive-kerberized/Dockerfile | 5 +- .../mapr52-hive-kerberized/files/bootstrap.sh | 2 +- .../files/conf/hive-site.xml | 94 +++++++++---------- teradatalabs/mapr52-hive/Dockerfile | 9 +- teradatalabs/mapr52-hive/files/bootstrap.sh | 4 +- 6 files changed, 59 insertions(+), 61 deletions(-) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 4c010d3..25edeac 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -15,7 +15,7 @@ FROM teradatalabs/centos6-java8-oracle MAINTAINER Teradata Docker Team -# ADD REPO FOR MapR +# ADD REPO FOR MAPR ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo COPY files/id_rsa.pub /root/ RUN yum update -y \ @@ -23,7 +23,7 @@ RUN yum update -y \ && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ # INSTALL UTILITY SOFTWARE - && yum install -y iputils vim openssh-server openssh-clients sudo \ + && yum install -y iputils vim openssh-server openssh-clients sudo lsof \ # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ @@ -62,6 +62,8 @@ RUN yum update -y \ # CLEANUP && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + +# GENERATE SSH KEYS && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 1a98fda..24956c2 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -15,7 +15,7 @@ FROM teradatalabs/mapr52-hive MAINTAINER Teradata Docker Team -# REMOVE THESE FILES IF IT IS ALREADY THERE +# REMOVE UNNECESSARY FILES RUN rm -rf /opt/mapr/conf/ssl_truststore \ && rm -rf /opt/mapr/conf/maprserverticket \ && rm -rf /opt/mapr/conf/cldb.key \ @@ -53,7 +53,8 @@ RUN /usr/sbin/kdb5_util create -s -P password \ # CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ && chmod 777 /root/bootstrap.sh \ - && chown mapr:mapr /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab \ + && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ + && chown hive:hive /opt/mapr/conf/hive.keytab \ && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ # CREATE PRESTO PRINCIPAL AND KEYTAB diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index 00a90a9..240ceab 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -17,7 +17,7 @@ chmod 755 /opt/mapr/conf/proxy # CONFIGURE HIVE /opt/mapr/server/configure.sh -R -# CONFIGURE KERBEROS +# ENABLE SECURITY IN MAPR /opt/mapr/server/configure.sh -secure -genkeys -C localhost -Z localhost -N mycluster -no-autostart # START KERBEROS SERVICES diff --git a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml index 2031be7..beeeebd 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml +++ b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml @@ -16,79 +16,79 @@ --> - + datanucleus.schema.autoCreateAll true creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once - + - - hive.server2.enable.doAs - true - Set this property to enable impersonation in Hive Server 2 - + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + - - hive.metastore.execute.setugi - true - Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. - + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + - + javax.jdo.option.ConnectionURL jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true JDBC connect string for a JDBC metastore - + - + javax.jdo.option.ConnectionDriverName com.mysql.jdbc.Driver Driver class name for a JDBC metastore - + - + javax.jdo.option.ConnectionUserName root username to use against metastore database - + - + javax.jdo.option.ConnectionPassword root password to use against metastore database - + - + hive.metastore.uris thrift://localhost:9083 - + - - hive.metastore.kerberos.keytab.file - /opt/mapr/conf/hive.keytab - The path to the Kerberos Keytab file containing the metastore thrift server's service principal. - - - hive.metastore.kerberos.principal - hive/mycluster@LABS.TERADATA.COM - The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct hostname. - - - hive.server2.authentication - KERBEROS - authenticationtype - - - hive.server2.authentication.kerberos.principal - hive/mycluster@LABS.TERADATA.COM - HiveServer2 principal. If _HOST is used as the FQDN portion, it will be replaced with the actual hostname of the running instance. - - - hive.server2.authentication.kerberos.keytab - /opt/mapr/conf/hive.keytab - Keytab file for HiveServer2 principal - + + hive.metastore.kerberos.keytab.file + /opt/mapr/conf/hive.keytab + The path to the Kerberos Keytab file containing the metastore thrift server's service principal. + + + hive.metastore.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct hostname. + + + hive.server2.authentication + KERBEROS + authenticationtype + + + hive.server2.authentication.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + HiveServer2 principal. If _HOST is used as the FQDN portion, it will be replaced with the actual hostname of the running instance. + + + hive.server2.authentication.kerberos.keytab + /opt/mapr/conf/hive.keytab + Keytab file for HiveServer2 principal + diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile index f765e62..9d3f68f 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -21,18 +21,13 @@ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml ADD files/supervisord.conf /etc/supervisord.conf COPY files/supervisord.d/* /etc/supervisord.d/ - RUN chmod 777 /root/*.sh \ -# INSTALL UTILITY SOFTWARE - && yum install -y openssh \ - && yum install -y lsof \ - + # INSTALL MYSQL SERVER && yum install -y mysql-server mysql-connector-java \ - && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ && /root/setup.sh \ -# INSTALL HIVE +# INSTALL MAPR HIVE && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh index b087954..384ad26 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -4,10 +4,10 @@ supervisorctl start sshd supervisorctl start socks-proxy -# CONFIGURE MapR +# CONFIGURE MAPR /opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart -# SETUP DISK FOR MAPR BY RUNNING disksetup +# SETUP DISK FOR MAPR BY RUNNING DISKSETUP /opt/mapr/server/disksetup -M -F /root/disk.txt # CREATE HIVE PROXY USERS From 6458491f81ea35fb7ae18d3a734c8fe514e8d699 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 16:15:09 -0400 Subject: [PATCH 39/43] Adding missing sshd service start in base --- teradatalabs/mapr52-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 25edeac..851955e 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -27,6 +27,7 @@ RUN yum update -y \ # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + && services sshd start \ # INSTALL MAPR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ From ad61f7a09247f3000f16b39eefc3bf4ee7464f31 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 6 Apr 2017 10:35:42 -0400 Subject: [PATCH 40/43] Adding Logic to wait for MapR ticket in Warden This changes will make maprcli wait for MapR ticket in wardenTracker --- .../mapr52-hive-kerberized/Dockerfile | 5 +- teradatalabs/mapr52-hive-kerberized/README.md | 6 +- .../mapr52-hive-kerberized/files/bootstrap.sh | 12 +--- .../files/wardenTracker.sh | 64 +++++++++++++++++++ 4 files changed, 71 insertions(+), 16 deletions(-) create mode 100644 teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 24956c2..0a8b477 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -20,13 +20,14 @@ RUN rm -rf /opt/mapr/conf/ssl_truststore \ && rm -rf /opt/mapr/conf/maprserverticket \ && rm -rf /opt/mapr/conf/cldb.key \ && rm -rf /opt/mapr/conf/ssl_keystore \ - && rm -rf /root/bootstrap.sh \ + && rm -rf /root/bootstrap.sh /root/wardenTracker.sh \ # INSTALL KERBEROS && yum install -y krb5-libs krb5-server krb5-workstation # ADD KERBEROS CONFIGURATION ADD files/bootstrap.sh /root/ +ADD files/wardenTracker.sh /root/ ADD files/kerberos/krb5.conf /etc/krb5.conf ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl @@ -52,7 +53,7 @@ RUN /usr/sbin/kdb5_util create -s -P password \ # CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ - && chmod 777 /root/bootstrap.sh \ + && chmod 777 /root/bootstrap.sh /root/wardenTracker.sh \ && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ && chown hive:hive /opt/mapr/conf/hive.keytab \ && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ diff --git a/teradatalabs/mapr52-hive-kerberized/README.md b/teradatalabs/mapr52-hive-kerberized/README.md index ebf092a..4949497 100644 --- a/teradatalabs/mapr52-hive-kerberized/README.md +++ b/teradatalabs/mapr52-hive-kerberized/README.md @@ -1,6 +1,6 @@ -# mapr52-hive +# mapr52-hive-kerberized -Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +Docker image with kerberos enabled for MapR. Please note that running services have lower memory heap size set. For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal for general correctness testing. @@ -8,7 +8,7 @@ for general correctness testing. ## Run ``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive-kerberized ``` ## Oracle license diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index 240ceab..9136588 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -34,16 +34,6 @@ sh /root/wardenTracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) -# CREATE KERBEROS TICKET -kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM - -# CREATE MAPR TICKET -maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM - # RUN HDFS COMMANDS -hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -mkdir /user /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive - -# REMOVE MAPR TICKET AND KERBEROS TICKET -kdestroy -rm -rf /tmp/* diff --git a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh new file mode 100644 index 0000000..802c8ef --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +hname=$(hostname) + +Services=0 +maprloginReady=1 +maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + +# WAIT FOR CLDB TO START +while [ $Services -ne 2 ] +do +if [ $maprcliReady == 1 ] +then + maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + Services=0 + # Get MapR ticket + if [ $maprloginReady -ne 0 ] + then + # CREATE KERBEROS TICKET + kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + + # CREATE MAPR TICKET + maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + maprloginReady=$? + fi +else + Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +fi +done + +# WAIT FOR NODEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR RESOURCEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVE METASTORE TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVESERVER2 TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR JOBHISTORYSERVER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done From 96627fc70d4147c3e005e5087e61d96f6cdf7e6b Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 20 Apr 2017 12:58:42 -0400 Subject: [PATCH 41/43] changes in travis changes in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e501d61..29b39c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: make ${IMAGES[@]} +install: travis_wait 48 make teradatalabs/mapr52-hive before_deploy: # decrypt the github deploy key From 6046f9f243d94c4898d642a47e1ea2b909b7caa5 Mon Sep 17 00:00:00 2001 From: AS186170 Date: Fri, 21 Apr 2017 15:42:36 +0000 Subject: [PATCH 42/43] Adding Review Changes for MapR --- .travis.yml | 3 +- .../{mapr52-base => mapr-base}/Dockerfile | 16 ++++- teradatalabs/mapr-base/README.md | 13 ++++ .../files/id_rsa.pub | 0 .../files/maprtech.repo | 0 teradatalabs/mapr-base/files/startup.sh | 1 + .../files/supervisord.conf | 0 .../files/supervisord.d/bootstrap.conf | 0 .../Dockerfile | 9 +-- teradatalabs/mapr-hive-kerberized/README.md | 23 +++++++ .../files/bootstrap.sh | 2 +- .../files/conf/hive-site.xml | 0 .../files/jceJars/US_export_policy.jar | Bin .../files/jceJars/local_policy.jar | Bin .../files/kerberos/kadm5.acl | 0 .../files/kerberos/kdc.conf | 0 .../files/kerberos/krb5.conf | 0 .../files/warden_tracker.sh | 37 ++++++++++ .../{mapr52-hive => mapr-hive}/Dockerfile | 17 +++-- teradatalabs/mapr-hive/README.md | 22 ++++++ .../capabilities.txt | 2 +- .../files/bootstrap.sh | 4 +- .../files/conf/core-site.xml | 0 .../files/conf/hive-site.xml | 0 .../{mapr52-hive => mapr-hive}/files/setup.sh | 0 .../files/socks-proxy.sh | 0 .../files/supervisord.d/mysql-metastore.conf | 0 .../files/supervisord.d/socks-proxy.conf | 0 .../files/supervisord.d/sshd.conf | 0 .../mapr-hive/files/warden_tracker.sh | 28 ++++++++ teradatalabs/mapr52-base/README.md | 9 --- teradatalabs/mapr52-hive-kerberized/README.md | 17 ----- .../files/wardenTracker.sh | 64 ------------------ teradatalabs/mapr52-hive/README.md | 17 ----- teradatalabs/mapr52-hive/files/startup.sh | 3 - .../mapr52-hive/files/wardenTracker.sh | 54 --------------- test/files/expose_mapr_hive.sh | 2 + test/image_tests/image_tests.bats.sh | 2 +- 38 files changed, 161 insertions(+), 184 deletions(-) rename teradatalabs/{mapr52-base => mapr-base}/Dockerfile (85%) create mode 100644 teradatalabs/mapr-base/README.md rename teradatalabs/{mapr52-base => mapr-base}/files/id_rsa.pub (100%) rename teradatalabs/{mapr52-base => mapr-base}/files/maprtech.repo (100%) create mode 120000 teradatalabs/mapr-base/files/startup.sh rename teradatalabs/{mapr52-hive => mapr-base}/files/supervisord.conf (100%) rename teradatalabs/{mapr52-hive => mapr-base}/files/supervisord.d/bootstrap.conf (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/Dockerfile (95%) create mode 100644 teradatalabs/mapr-hive-kerberized/README.md rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/bootstrap.sh (97%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/conf/hive-site.xml (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/jceJars/US_export_policy.jar (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/jceJars/local_policy.jar (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/kadm5.acl (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/kdc.conf (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/krb5.conf (100%) create mode 100644 teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh rename teradatalabs/{mapr52-hive => mapr-hive}/Dockerfile (75%) create mode 100644 teradatalabs/mapr-hive/README.md rename teradatalabs/{mapr52-hive => mapr-hive}/capabilities.txt (79%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/bootstrap.sh (88%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/conf/core-site.xml (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/conf/hive-site.xml (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/setup.sh (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/socks-proxy.sh (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/mysql-metastore.conf (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/socks-proxy.conf (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/sshd.conf (100%) create mode 100644 teradatalabs/mapr-hive/files/warden_tracker.sh delete mode 100644 teradatalabs/mapr52-base/README.md delete mode 100644 teradatalabs/mapr52-hive-kerberized/README.md delete mode 100644 teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh delete mode 100644 teradatalabs/mapr52-hive/README.md delete mode 100644 teradatalabs/mapr52-hive/files/startup.sh delete mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/.travis.yml b/.travis.yml index 29b39c7..468dac2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ dist: trusty env: matrix: - - IMAGES='teradatalabs/centos6-java8-oracle.dependants' - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv} ) @@ -22,7 +21,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: travis_wait 48 make teradatalabs/mapr52-hive +install: travis_wait 48 make teradatalabs/mapr-hive-kerberized before_deploy: # decrypt the github deploy key diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr-base/Dockerfile similarity index 85% rename from teradatalabs/mapr52-base/Dockerfile rename to teradatalabs/mapr-base/Dockerfile index 851955e..de73416 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr-base/Dockerfile @@ -15,19 +15,21 @@ FROM teradatalabs/centos6-java8-oracle MAINTAINER Teradata Docker Team -# ADD REPO FOR MAPR +# ADD MAPR REPO ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo COPY files/id_rsa.pub /root/ + RUN yum update -y \ + # ... GET MapRGPG KEY && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ # INSTALL UTILITY SOFTWARE - && yum install -y iputils vim openssh-server openssh-clients sudo lsof \ + && yum install -y iputils openssh-server openssh-clients sudo lsof \ # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ - && services sshd start \ + && service sshd start \ # INSTALL MAPR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ @@ -68,3 +70,11 @@ RUN yum update -y \ && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys + +# Copy supervisord startup script and base configs +COPY files/startup.sh /root/startup.sh +COPY files/supervisord.conf /etc/supervisord.conf +COPY files/supervisord.d/bootstrap.conf /etc/supervisord.d/bootstrap.conf + +# Add supervisord configs in child images +ONBUILD COPY files/supervisord.d/* /etc/supervisord.d/ diff --git a/teradatalabs/mapr-base/README.md b/teradatalabs/mapr-base/README.md new file mode 100644 index 0000000..d0cae4a --- /dev/null +++ b/teradatalabs/mapr-base/README.md @@ -0,0 +1,13 @@ +# mapr-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-base.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-base +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-base.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-base + +A Hadoop base image for creating images testing Presto, based on the MapR distro. + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) \ No newline at end of file diff --git a/teradatalabs/mapr52-base/files/id_rsa.pub b/teradatalabs/mapr-base/files/id_rsa.pub similarity index 100% rename from teradatalabs/mapr52-base/files/id_rsa.pub rename to teradatalabs/mapr-base/files/id_rsa.pub diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr-base/files/maprtech.repo similarity index 100% rename from teradatalabs/mapr52-base/files/maprtech.repo rename to teradatalabs/mapr-base/files/maprtech.repo diff --git a/teradatalabs/mapr-base/files/startup.sh b/teradatalabs/mapr-base/files/startup.sh new file mode 120000 index 0000000..9a9f191 --- /dev/null +++ b/teradatalabs/mapr-base/files/startup.sh @@ -0,0 +1 @@ +../../../commons/startup.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr-base/files/supervisord.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.conf rename to teradatalabs/mapr-base/files/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr-base/files/supervisord.d/bootstrap.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf rename to teradatalabs/mapr-base/files/supervisord.d/bootstrap.conf diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr-hive-kerberized/Dockerfile similarity index 95% rename from teradatalabs/mapr52-hive-kerberized/Dockerfile rename to teradatalabs/mapr-hive-kerberized/Dockerfile index 0a8b477..fc2fa5e 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr-hive-kerberized/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM teradatalabs/mapr52-hive +FROM teradatalabs/mapr-hive MAINTAINER Teradata Docker Team # REMOVE UNNECESSARY FILES @@ -20,14 +20,14 @@ RUN rm -rf /opt/mapr/conf/ssl_truststore \ && rm -rf /opt/mapr/conf/maprserverticket \ && rm -rf /opt/mapr/conf/cldb.key \ && rm -rf /opt/mapr/conf/ssl_keystore \ - && rm -rf /root/bootstrap.sh /root/wardenTracker.sh \ + && rm -rf /root/bootstrap.sh /root/warden_tracker.sh \ # INSTALL KERBEROS && yum install -y krb5-libs krb5-server krb5-workstation # ADD KERBEROS CONFIGURATION ADD files/bootstrap.sh /root/ -ADD files/wardenTracker.sh /root/ +ADD files/warden_tracker.sh /root/ ADD files/kerberos/krb5.conf /etc/krb5.conf ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl @@ -41,6 +41,7 @@ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml RUN /usr/sbin/kdb5_util create -s -P password \ && usermod -g root hdfs \ && usermod -g mapr hdfs \ + # CREATE MAPR AND HIVE PRINCIPALS AND KEYTABS && /usr/sbin/kadmin.local -q "addprinc -randkey mapr/mycluster@LABS.TERADATA.COM" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM" \ @@ -53,7 +54,7 @@ RUN /usr/sbin/kdb5_util create -s -P password \ # CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ - && chmod 777 /root/bootstrap.sh /root/wardenTracker.sh \ + && chmod 777 /root/bootstrap.sh /root/warden_tracker.sh \ && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ && chown hive:hive /opt/mapr/conf/hive.keytab \ && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ diff --git a/teradatalabs/mapr-hive-kerberized/README.md b/teradatalabs/mapr-hive-kerberized/README.md new file mode 100644 index 0000000..d38d024 --- /dev/null +++ b/teradatalabs/mapr-hive-kerberized/README.md @@ -0,0 +1,23 @@ +# mapr-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-hive-kerberized.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-hive-kerberized +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-hive-kerberized.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-hive-kerberized + + +Docker image with MapR FS, YARN and HIVE installed in a kerberized environment. Please note that running services have lower memory heap size set. +For more details please check hadoop-env.sh(configuration) file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr-hive-kerberized +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr-hive-kerberized/files/bootstrap.sh similarity index 97% rename from teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh rename to teradatalabs/mapr-hive-kerberized/files/bootstrap.sh index 9136588..5553f30 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr-hive-kerberized/files/bootstrap.sh @@ -29,7 +29,7 @@ service mapr-zookeeper start service mapr-warden start # WAIT FOR WARDEN TO START ALL THE SERVICES -sh /root/wardenTracker.sh +/root/warden_tracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) diff --git a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml b/teradatalabs/mapr-hive-kerberized/files/conf/hive-site.xml similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml rename to teradatalabs/mapr-hive-kerberized/files/conf/hive-site.xml diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar b/teradatalabs/mapr-hive-kerberized/files/jceJars/US_export_policy.jar similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar rename to teradatalabs/mapr-hive-kerberized/files/jceJars/US_export_policy.jar diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar b/teradatalabs/mapr-hive-kerberized/files/jceJars/local_policy.jar similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar rename to teradatalabs/mapr-hive-kerberized/files/jceJars/local_policy.jar diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl b/teradatalabs/mapr-hive-kerberized/files/kerberos/kadm5.acl similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl rename to teradatalabs/mapr-hive-kerberized/files/kerberos/kadm5.acl diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf b/teradatalabs/mapr-hive-kerberized/files/kerberos/kdc.conf similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf rename to teradatalabs/mapr-hive-kerberized/files/kerberos/kdc.conf diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf b/teradatalabs/mapr-hive-kerberized/files/kerberos/krb5.conf similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf rename to teradatalabs/mapr-hive-kerberized/files/kerberos/krb5.conf diff --git a/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh b/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh new file mode 100644 index 0000000..78e1ef2 --- /dev/null +++ b/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +HOSTNAME=$(hostname) +MAPR_LOGIN_READY=1 +MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + +# CHECK RUNNING STATUS OF GIVEN SERVICES USING MAPRCLI +function exposes_mapr_services { + SERVICES=0 + while [ $SERVICES -ne 2 ] + do + SERVICES=$(maprcli service list -node $HOSTNAME | grep $1 | awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) + done +} + +# WAIT FOR MAPRCLI +while [ $MAPR_CLI_READY == 1 ] +do + MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + if [ $MAPR_LOGIN_READY -ne 0 ] + then + # CREATE KERBEROS TICKET + kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + + # CREATE MAPR TICKET + maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + MAPR_LOGIN_READY=$? + fi +done + +# WAIT FOR MAPR SERVICES TO START +exposes_mapr_services CLDB +exposes_mapr_services NodeManager +exposes_mapr_services ResourceManager +exposes_mapr_services HiveMetastore +exposes_mapr_services HiveServer2 +exposes_mapr_services JobHistoryServer diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr-hive/Dockerfile similarity index 75% rename from teradatalabs/mapr52-hive/Dockerfile rename to teradatalabs/mapr-hive/Dockerfile index 9d3f68f..b29a83d 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr-hive/Dockerfile @@ -12,16 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM teradatalabs/mapr52-base +FROM teradatalabs/mapr-base MAINTAINER Teradata Docker Team # ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY -ADD files/*.sh /root/ +ADD files/setup.sh /root/ +ADD files/warden_tracker.sh /root/ +ADD files/bootstrap.sh /root/ +ADD files/socks-proxy.sh /root/ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml -ADD files/supervisord.conf /etc/supervisord.conf -COPY files/supervisord.d/* /etc/supervisord.d/ -RUN chmod 777 /root/*.sh \ +COPY files/supervisord.d/mysql-metastore.conf /etc/supervisord.d/ +COPY files/supervisord.d/socks-proxy.conf /etc/supervisord.d/ +COPY files/supervisord.d/sshd.conf /etc/supervisord.d/ + +RUN chmod 777 /root/setup.sh /root/warden_tracker.sh /root/bootstrap.sh /root/socks-proxy.sh /root/startup.sh \ # INSTALL MYSQL SERVER && yum install -y mysql-server mysql-connector-java \ @@ -32,7 +37,7 @@ RUN chmod 777 /root/*.sh \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # SETUP FLAT FILE /home/mapr/storagefile - && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ + && dd if=/dev/zero of=/home/mapr/storagefile bs=250M count=30 \ # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK && echo "/home/mapr/storagefile" > /root/disk.txt diff --git a/teradatalabs/mapr-hive/README.md b/teradatalabs/mapr-hive/README.md new file mode 100644 index 0000000..f82e4da --- /dev/null +++ b/teradatalabs/mapr-hive/README.md @@ -0,0 +1,22 @@ +# mapr-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-hive.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-hive +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-hive.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-hive + +Docker image with MapR FS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check hadoop-env.sh(configuration) file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr-hive/capabilities.txt similarity index 79% rename from teradatalabs/mapr52-hive/capabilities.txt rename to teradatalabs/mapr-hive/capabilities.txt index c1da436..9d44a71 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr-hive/capabilities.txt @@ -1,4 +1,4 @@ exposes_mapr allows_creating_a_table_in_hive allows_selecting_from_the_table -exposes_socks_proxy \ No newline at end of file +exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr-hive/files/bootstrap.sh similarity index 88% rename from teradatalabs/mapr52-hive/files/bootstrap.sh rename to teradatalabs/mapr-hive/files/bootstrap.sh index 384ad26..b460bd7 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr-hive/files/bootstrap.sh @@ -5,7 +5,7 @@ supervisorctl start sshd supervisorctl start socks-proxy # CONFIGURE MAPR -/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart # SETUP DISK FOR MAPR BY RUNNING DISKSETUP /opt/mapr/server/disksetup -M -F /root/disk.txt @@ -21,7 +21,7 @@ service mapr-warden start /opt/mapr/server/configure.sh -R # WAIT FOR WARDEN TO START ALL THE SERVICES -sh /root/wardenTracker.sh +/root/warden_tracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr-hive/files/conf/core-site.xml similarity index 100% rename from teradatalabs/mapr52-hive/files/conf/core-site.xml rename to teradatalabs/mapr-hive/files/conf/core-site.xml diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr-hive/files/conf/hive-site.xml similarity index 100% rename from teradatalabs/mapr52-hive/files/conf/hive-site.xml rename to teradatalabs/mapr-hive/files/conf/hive-site.xml diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr-hive/files/setup.sh similarity index 100% rename from teradatalabs/mapr52-hive/files/setup.sh rename to teradatalabs/mapr-hive/files/setup.sh diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr-hive/files/socks-proxy.sh similarity index 100% rename from teradatalabs/mapr52-hive/files/socks-proxy.sh rename to teradatalabs/mapr-hive/files/socks-proxy.sh diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr-hive/files/supervisord.d/mysql-metastore.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf rename to teradatalabs/mapr-hive/files/supervisord.d/mysql-metastore.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr-hive/files/supervisord.d/socks-proxy.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf rename to teradatalabs/mapr-hive/files/supervisord.d/socks-proxy.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf b/teradatalabs/mapr-hive/files/supervisord.d/sshd.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf rename to teradatalabs/mapr-hive/files/supervisord.d/sshd.conf diff --git a/teradatalabs/mapr-hive/files/warden_tracker.sh b/teradatalabs/mapr-hive/files/warden_tracker.sh new file mode 100644 index 0000000..4cab14b --- /dev/null +++ b/teradatalabs/mapr-hive/files/warden_tracker.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +HOSTNAME=$(hostname) +MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + +# CHECK RUNNING STATUS OF GIVEN SERVICES USING MAPRCLI +function exposes_mapr_services { + SERVICES=0 + while [ $SERVICES -ne 2 ] + do + SERVICES=$(maprcli service list -node $HOSTNAME | grep $1 | awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) + done +} + +# WAIT FOR MAPRCLI +while [ $MAPR_CLI_READY == 1 ] +do + MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) +done + + +# WAIT FOR MAPR SERVICES TO START +exposes_mapr_services CLDB +exposes_mapr_services NodeManager +exposes_mapr_services ResourceManager +exposes_mapr_services HiveMetastore +exposes_mapr_services HiveServer2 +exposes_mapr_services JobHistoryServer \ No newline at end of file diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md deleted file mode 100644 index e9d1cd8..0000000 --- a/teradatalabs/mapr52-base/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# mapr52-base - - -Docker image with Hive installed from MapR repositories. - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/README.md b/teradatalabs/mapr52-hive-kerberized/README.md deleted file mode 100644 index 4949497..0000000 --- a/teradatalabs/mapr52-hive-kerberized/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# mapr52-hive-kerberized - -Docker image with kerberos enabled for MapR. Please note that running services have lower memory heap size set. -For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. -If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal -for general correctness testing. - -## Run - -``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive-kerberized -``` - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh deleted file mode 100644 index 802c8ef..0000000 --- a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -hname=$(hostname) - -Services=0 -maprloginReady=1 -maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - -# WAIT FOR CLDB TO START -while [ $Services -ne 2 ] -do -if [ $maprcliReady == 1 ] -then - maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - Services=0 - # Get MapR ticket - if [ $maprloginReady -ne 0 ] - then - # CREATE KERBEROS TICKET - kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM - - # CREATE MAPR TICKET - maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM - maprloginReady=$? - fi -else - Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -fi -done - -# WAIT FOR NODEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR RESOURCEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVE METASTORE TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVESERVER2 TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR JOBHISTORYSERVER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md deleted file mode 100644 index ebf092a..0000000 --- a/teradatalabs/mapr52-hive/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# mapr52-hive - -Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. -For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. -If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal -for general correctness testing. - -## Run - -``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive -``` - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh deleted file mode 100644 index afd8798..0000000 --- a/teradatalabs/mapr52-hive/files/startup.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh deleted file mode 100644 index 5124a4d..0000000 --- a/teradatalabs/mapr52-hive/files/wardenTracker.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -hname=$(hostname) - -Services=0 - -maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - -# WAIT FOR CLDB TO START -while [ $Services -ne 2 ] -do -if [ $maprcliReady == 1 ] -then - maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - Services=0 -else - Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -fi -done - -# WAIT FOR NODEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR RESOURCEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVE METASTORE TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVESERVER2 TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR JOBHISTORYSERVER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh index eac8f42..5fbf875 100644 --- a/test/files/expose_mapr_hive.sh +++ b/test/files/expose_mapr_hive.sh @@ -1,5 +1,6 @@ #!/bin/bash +#Wait for hadoop-master to be available conn=0 while [ $conn -ne 1 ] do @@ -11,6 +12,7 @@ do fi done +# Wait for hive and warehouse directories in HDFS hivecliReady=1 while [ $hivecliReady -ne 0 ] do diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 7c1abe1..aee8bb4 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -2,7 +2,7 @@ get_user() { skip_if_needed - if [ "${IMAGE}" == "teradatalabs/mapr52-hive" ] + if [ "${IMAGE}" == "teradatalabs/mapr-hive" ] then echo "hive" else From ec747149788627d960ebaf0f9362a0f21c0e6a75 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Mon, 24 Apr 2017 13:07:11 -0400 Subject: [PATCH 43/43] Travis yml changes Travis yml changes --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 468dac2..902f0c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ dist: trusty env: matrix: - - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv} ) # Prevent duplicate builds on tag pushes.