[DLAB-656] Prepare docker images for MongoDB/UI/Billing services

infrastructure-provisioning/src/general/lib/aws/actions_lib.py

@@ -436,6 +436,14 @@ def create_instance(definitions, instance_tag, primary_disk_size=12):
         elif definitions.instance_class == 'ssn':
             get_iam_profile(definitions.iam_profile)
             instances = ec2.create_instances(ImageId=definitions.ami_id, MinCount=1, MaxCount=1,
+                                             BlockDeviceMappings=[
+                                                 {
+                                                     "DeviceName": "/dev/sda1",
+                                                     "Ebs":
+                                                         {
+                                                             "VolumeSize": int(primary_disk_size)
+                                                         }
+                                                 }],
                                              KeyName=definitions.key_name,
                                              SecurityGroupIds=security_groups_ids,
                                              InstanceType=definitions.instance_type,

infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py

@@ -175,7 +175,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
              locale, region_info, ldap_login, tenant_id,
              application_id, hostname, data_lake_name, subscription_id,
              validate_permission_scope, dlab_id, usage_date, product,
-             usage_type, usage, cost, resource_id, tags, report_path=''):
+             usage_type, usage, cost, resource_id, tags, report_path='', k8s=False):
     try:
         if not exists(os.environ['ssn_dlab_path'] + 'tmp/ss_started'):
             java_path = sudo("update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'")
@@ -190,17 +190,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
             sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
             put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
             sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
-            with open('/root/templates/supervisor_svc.conf', 'r') as f:
-                text = f.read()
-            text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
-            with open('/root/templates/supervisor_svc.conf', 'w') as f:
-                f.write(text)
-            put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf')
-            sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
-            sudo('cp ' + os.environ['ssn_dlab_path'] +
-                 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf')
-            sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.conf {}'.format(supervisor_conf))
-            sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf))
             try:
                 sudo('mkdir -p /var/log/application')
                 run('mkdir -p /tmp/yml_tmp/')
@@ -292,16 +281,47 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
             except:
                 append_result("Unable to generate cert and copy to java keystore")
                 sys.exit(1)
-            sudo('service supervisor start')
-            sudo('service nginx restart')
-            sudo('service supervisor restart')
-            sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started')
+            if k8s:
+                try:
+                    dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path)
+                    k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir)
+                    sudo("mkdir {}".format(k8s_dir))
+                    sudo("cp -r {0}conf/ {1}".format(dlab_path))
+                    sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir))
+                    sudo("cd {}; docker build --file ssn/files/os/webui_Dockerfile -t docker.dlab-ui ."
+                            .format(dlab_source_dir))
+                    sudo("cd {}; docker build --file ssn/files/os/billing_Dockerfile -t docker.dlab-billing ."
+                            .format(dlab_source_dir))
+                    sudo("rm -rf {}".format(k8s_dir))
+                except Exception as err:
+                    traceback.print_exc()
+                    print('Failed to build images for services: ', str(err))
+                    sys.exit(1)
+            else:
+                with open('/root/templates/supervisor_svc.conf', 'r') as f:
+                    text = f.read()
+                text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
+                with open('/root/templates/supervisor_svc.conf', 'w') as f:
+                    f.write(text)
+                put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf')
+                sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
+                sudo('cp ' + os.environ['ssn_dlab_path'] +
+                     'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf')
+                sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.conf {}'.format(supervisor_conf))
+                sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf))
+                sudo('service supervisor start')
+                sudo('service nginx restart')
+                sudo('service supervisor restart')
+                sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started')
     except Exception as err:
         traceback.print_exc()
         print('Failed to start Self-service: ', str(err))
         sys.exit(1)
 
-
 def install_build_dep():
     try:
         if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_dlab_path'])):

infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py

@@ -216,17 +216,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
             sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
             put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
             sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
-            with open('/root/templates/supervisor_svc.conf', 'r') as f:
-                text = f.read()
-            text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
-            with open('/root/templates/supervisor_svc.ini', 'w') as f:
-                f.write(text)
-            put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini')
-            sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/')
-            sudo('cp ' + os.environ['ssn_dlab_path'] +
-                 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf')
-            sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.ini {}'.format(supervisor_conf))
-            sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf))
             try:
                 sudo('mkdir -p /var/log/application')
                 run('mkdir -p /tmp/yml_tmp/')
@@ -323,9 +312,41 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
             except:
                 append_result("Unable to generate cert and copy to java keystore")
                 sys.exit(1)
-            sudo('systemctl restart supervisord')
-            sudo('service nginx restart')
-            sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started')
+            if kuber_enabled:
+                try:
+                    dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path)
+                    k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir)
+                    sudo("mkdir {}".format(k8s_dir))
+                    sudo("cp -r {0}conf/ {1}".format(dlab_path))
+                    sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir))
+                    sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir))
+                    sudo("cd {}; docker build --file ssn/files/os/webui_Dockerfile -t docker.dlab-ui ."
+                            .format(dlab_source_dir))
+                    sudo("cd {}; docker build --file ssn/files/os/billing_Dockerfile -t docker.dlab-billing ."
+                            .format(dlab_source_dir))
+                    sudo("rm -rf {}".format(k8s_dir))
+                except Exception as err:
+                    traceback.print_exc()
+                    print('Failed to build images for services: ', str(err))
+                    sys.exit(1)
+            else:
+                with open('/root/templates/supervisor_svc.conf', 'r') as f:
+                    text = f.read()
+                text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
+                with open('/root/templates/supervisor_svc.ini', 'w') as f:
+                    f.write(text)
+                put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini')
+                sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/')
+                sudo('cp ' + os.environ['ssn_dlab_path'] +
+                     'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf')
+                sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.ini {}'.format(supervisor_conf))
+                sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf))
+                sudo('systemctl restart supervisord')
+                sudo('service nginx restart')
+                sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started')
     except Exception as err:
         traceback.print_exc()
         print('Failed to start Self-service: ', str(err))

infrastructure-provisioning/src/ssn/files/os/billing_Dockerfile

@@ -0,0 +1,40 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM ubuntu:16.04
+
+
+USER root
+RUN  apt-get update && \
+     apt-get -y upgrade && \
+     apt-get -y install software-properties-common && \
+     apt-get -y install curl tar sudo
+
+
+# java
+RUN apt-get install -y default-jre && \
+    apt-get install -y default-jdk && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY tmp-kuber/* /root/
+COPY tmp-kuber/conf/* /root/
+
+CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/billing-aws-2.1.jar --conf /root/billing.yml

infrastructure-provisioning/src/ssn/files/os/create_db.js

@@ -0,0 +1,8 @@
+use dlabdb
+db.createUser(
+    {
+      user: "admin",
+      pwd: "PASSWORD",
+      roles: [{'role':'userAdminAnyDatabase','db':'admin'}]
+    }
+);

infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile

@@ -0,0 +1,35 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM mongo:4.0-xenial
+
+ARG CLOUD_PROVIDER
+
+COPY ssn/files/os/create_db.js /root/
+COPY ssn/files/os/mongodb.conf /root/
+COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/
+RUN mv /root/mongodb.conf /etc/mongodb.conf
+
+RUN mkdir -p /data/logs \
+    && chown -R mongodb:mongodb /data/logs
+
+CMD ["mongod", "--config", "/etc/mongodb.conf"]
+

infrastructure-provisioning/src/ssn/files/os/mongodb.conf

@@ -0,0 +1,42 @@
+# mongod.conf
+
+# for documentation of all options, see:
+#   http://docs.mongodb.org/manual/reference/configuration-options/
+
+# Where and how to store data.
+storage:
+  dbPath: /data/db2
+  journal:
+    enabled: true
+#  engine:
+#  mmapv1:
+#  wiredTiger:
+
+# where to write logging data.
+systemLog:
+  destination: file
+  logAppend: true
+  path: /data/logs/mongod.log
+
+# network interfaces
+net:
+  port: 27017
+  bindIp: 0.0.0.0
+
+
+#processManagement:
+
+#security:
+#  authorization: enabled
+
+#operationProfiling:
+
+#replication:
+
+#sharding:
+
+## Enterprise-Only Options:
+
+#auditLog:
+
+#snmp:

infrastructure-provisioning/src/ssn/files/os/webui_Dockerfile

@@ -0,0 +1,40 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM ubuntu:16.04
+
+
+USER root
+RUN  apt-get update && \
+     apt-get -y upgrade && \
+     apt-get -y install software-properties-common && \
+     apt-get -y install curl tar sudo
+
+
+# java
+RUN apt-get install -y default-jre && \
+    apt-get install -y default-jdk && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY tmp-kuber/* /root/
+COPY tmp-kuber/conf/* /root/
+
+CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/self-service-2.1.jar server /root/self-service.yml

infrastructure-provisioning/src/ssn/scripts/configure_mongo.py

@@ -22,6 +22,7 @@
 # ******************************************************************************
 
 from pymongo import MongoClient
+from fabric.api import *
 import yaml, json, sys
 import subprocess
 import time
@@ -34,6 +35,8 @@
 parser = argparse.ArgumentParser()
 parser.add_argument('--dlab_path', type=str, default='')
 parser.add_argument('--mongo_parameters', type=str, default='')
+parser.add_argument('--cloud_provider', type=str, default='')
+parser.add_argument('--k8s', type=str, default=False)
 args = parser.parse_args()
 
 
@@ -78,24 +81,32 @@ def read_yml_conf(path, section, param):
     # Setting up admin's password and enabling security
     client = MongoClient(mongo_ip + ':' + str(mongo_port))
     pass_upd = True
-    try:
-        command = ['service', 'mongod', 'start']
-        subprocess.call(command, shell=False)
-        time.sleep(5)
-        client.dlabdb.add_user('admin', mongo_passwd, roles=[{'role':'userAdminAnyDatabase','db':'admin'}])
-        client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"])
-        set_mongo_parameters(client, mongo_parameters)
-        with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data:
-            json_data = json.load(data)
-        for i in json_data:
-            client.dlabdb.roles.insert_one(i)
-        client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200)
-        if add_2_yml_config(path,'security','authorization','enabled'):
-            command = ['service', 'mongod', 'restart']
+    if args.k8s:
+        local("cd {}sources/infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ."
+             .format(args.dlab_path, args.cloud_provider))
+        local("mkdir -p /opt/mongo_vol")
+        local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db -d docker.dlab-mongo --smallfiles")
+        local("sleep 10; docker exec dlab-mongo sh -c 'mongo < /root/create_db.js'")
+        local("docker exec dlab-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'")
+    else:
+        try:
+            command = ['service', 'mongod', 'start']
             subprocess.call(command, shell=False)
-    except:
-        print("Looks like MongoDB have already been secured")
-        pass_upd = False
+            time.sleep(5)
+            client.dlabdb.add_user('admin', mongo_passwd, roles=[{'role':'userAdminAnyDatabase','db':'admin'}])
+            client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"])
+            set_mongo_parameters(client, mongo_parameters)
+            with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data:
+                json_data = json.load(data)
+            for i in json_data:
+                client.dlabdb.roles.insert_one(i)
+            client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200)
+            if add_2_yml_config(path,'security','authorization','enabled'):
+                command = ['service', 'mongod', 'restart']
+                subprocess.call(command, shell=False)
+        except:
+            print("Looks like MongoDB have already been secured")
+            pass_upd = False
 
     # Generating output config
     add_2_yml_config(outfile, 'network', 'ip', mongo_ip)