From 1090d6026ce61f736b00d8d2a89b65de8d1f7a86 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 23 May 2019 11:57:17 +0300 Subject: [PATCH 01/21] POC for webui image --- .../src/general/files/aws/webui_Dockerfile | 39 +++++++++++++++++++ .../src/general/lib/os/debian/ssn_lib.py | 14 +++++++ .../src/ssn/scripts/configure_ui.py | 4 ++ 3 files changed, 57 insertions(+) create mode 100644 infrastructure-provisioning/src/general/files/aws/webui_Dockerfile diff --git a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile new file mode 100644 index 0000000000..35911a2378 --- /dev/null +++ b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile @@ -0,0 +1,39 @@ +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + +FROM ubuntu:16.04 + + +USER root +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install software-properties-common && \ + apt-get -y install curl tar sudo + + +# java +RUN apt-get install -y default-jre && \ + apt-get install -y default-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY conf/* /root/ + +CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/self-service-2.1.jar server /root/self-service.yml \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index 2b2c075272..36f12b0602 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -302,6 +302,20 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, sys.exit(1) +def create_kuber_images(): + try: + sudo("mkdir /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cp -r /opt/dlab/conf/ /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cp -r /opt/dlab/webapp/self-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cp -r /opt/dlab/webapp/security-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cp -r /opt/dlab/webapp/provisioning-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cp -r /opt/dlab/webapp/billing/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") + sudo("cd /opt/dlab/sources/infrastructure-provisioning/src/; docker build --file general/files/aws/webui_Dockerfile --no-cache -t docker.dlab-ui .") + except Exception as err: + traceback.print_exc() + print('Failed to build images for Kubernetes', str(err)) + sys.exit(1) + def install_build_dep(): try: if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_dlab_path'])): diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py index 364a45ff5f..87fda845ae 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py @@ -240,3 +240,7 @@ def build_ui(): args.hostname, args.datalake_store_name, args.subscription_id, args.validate_permission_scope, args.dlab_id, args.usage_date, args.product, args.usage_type, args.usage, args.cost, args.resource_id, args.tags) + + #raw draft for kuber + print("Creating Kuber images") + create_kuber_images() \ No newline at end of file From ff128487489e74f5d531962038f53f97ea449e5d Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 23 May 2019 14:38:24 +0300 Subject: [PATCH 02/21] increase size of storage device for aws(alrd in dev) --- .../src/general/lib/aws/actions_lib.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py index 9fe44825fa..d8e00f3bec 100644 --- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py @@ -436,6 +436,14 @@ def create_instance(definitions, instance_tag, primary_disk_size=12): elif definitions.instance_class == 'ssn': get_iam_profile(definitions.iam_profile) instances = ec2.create_instances(ImageId=definitions.ami_id, MinCount=1, MaxCount=1, + BlockDeviceMappings=[ + { + "DeviceName": "/dev/sda1", + "Ebs": + { + "VolumeSize": int(primary_disk_size) + } + }], KeyName=definitions.key_name, SecurityGroupIds=security_groups_ids, InstanceType=definitions.instance_type, From 202cd68dfb1006fd057e05308caf255ba7118f37 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 23 May 2019 15:05:39 +0300 Subject: [PATCH 03/21] typo fix --- .../src/general/files/aws/webui_Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile index 35911a2378..1ec872145c 100644 --- a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile +++ b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile @@ -34,6 +34,6 @@ RUN apt-get install -y default-jre && \ apt-get install -y default-jdk && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY conf/* /root/ +COPY tmp-kuber/* /root/ CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/self-service-2.1.jar server /root/self-service.yml \ No newline at end of file From f166b16d0b0855a8819e537b32af1ac4d1432747 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Mon, 27 May 2019 16:31:33 +0300 Subject: [PATCH 04/21] [DLAB-656] self-service and billing separate images --- .../src/general/files/aws/billing_Dockerfile | 40 +++++++++++ .../src/general/files/aws/webui_Dockerfile | 1 + .../src/general/lib/os/debian/ssn_lib.py | 72 +++++++++++++------ .../src/ssn/scripts/configure_ui.py | 6 +- 4 files changed, 91 insertions(+), 28 deletions(-) create mode 100644 infrastructure-provisioning/src/general/files/aws/billing_Dockerfile diff --git a/infrastructure-provisioning/src/general/files/aws/billing_Dockerfile b/infrastructure-provisioning/src/general/files/aws/billing_Dockerfile new file mode 100644 index 0000000000..c0952c40ed --- /dev/null +++ b/infrastructure-provisioning/src/general/files/aws/billing_Dockerfile @@ -0,0 +1,40 @@ +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + +FROM ubuntu:16.04 + + +USER root +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install software-properties-common && \ + apt-get -y install curl tar sudo + + +# java +RUN apt-get install -y default-jre && \ + apt-get install -y default-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY tmp-kuber/* /root/ +COPY tmp-kuber/conf/* /root/ + +CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/billing-aws-2.1.jar --conf /root/billing.yml \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile index 1ec872145c..78e66808ce 100644 --- a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile +++ b/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile @@ -35,5 +35,6 @@ RUN apt-get install -y default-jre && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY tmp-kuber/* /root/ +COPY tmp-kuber/conf/* /root/ CMD sudo java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 -DDLAB_CONF_DIR=/root/ /root/self-service-2.1.jar server /root/self-service.yml \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index 36f12b0602..eac937e0ec 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -190,17 +190,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/') put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf') sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/') - with open('/root/templates/supervisor_svc.conf', 'r') as f: - text = f.read() - text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user) - with open('/root/templates/supervisor_svc.conf', 'w') as f: - f.write(text) - put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf') - sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/') - sudo('cp ' + os.environ['ssn_dlab_path'] + - 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf') - sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.conf {}'.format(supervisor_conf)) - sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf)) try: sudo('mkdir -p /var/log/application') run('mkdir -p /tmp/yml_tmp/') @@ -292,25 +281,62 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, except: append_result("Unable to generate cert and copy to java keystore") sys.exit(1) - sudo('service supervisor start') - sudo('service nginx restart') - sudo('service supervisor restart') - sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started') + if kuber_enabled: + try: + dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path) + k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir) + sudo("mkdir {}".format(k8s_dir)) + sudo("cp -r {0}conf/ {1}".format(dlab_path)) + sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) + sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui ." + .format(dlab_source_dir)) + sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." + .format(dlab_source_dir)) + except Exception as err: + traceback.print_exc() + print('Failed to build images for services: ', str(err)) + sys.exit(1) + else: + with open('/root/templates/supervisor_svc.conf', 'r') as f: + text = f.read() + text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user) + with open('/root/templates/supervisor_svc.conf', 'w') as f: + f.write(text) + put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf') + sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/') + sudo('cp ' + os.environ['ssn_dlab_path'] + + 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf') + sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.conf {}'.format(supervisor_conf)) + sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf)) + sudo('service supervisor start') + sudo('service nginx restart') + sudo('service supervisor restart') + sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started') except Exception as err: traceback.print_exc() print('Failed to start Self-service: ', str(err)) sys.exit(1) -def create_kuber_images(): +def create_kuber_images(dlab_path): try: - sudo("mkdir /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cp -r /opt/dlab/conf/ /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cp -r /opt/dlab/webapp/self-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cp -r /opt/dlab/webapp/security-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cp -r /opt/dlab/webapp/provisioning-service/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cp -r /opt/dlab/webapp/billing/lib/*.jar /opt/dlab/sources/infrastructure-provisioning/src/tmp-kuber/") - sudo("cd /opt/dlab/sources/infrastructure-provisioning/src/; docker build --file general/files/aws/webui_Dockerfile --no-cache -t docker.dlab-ui .") + dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path) + k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir) + sudo("mkdir {}".format(k8s_dir)) + sudo("cp -r {0}conf/ {1}".format(dlab_path)) + sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) + sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui .".format(dlab_source_dir)) + sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." + .format(dlab_source_dir)) + except Exception as err: traceback.print_exc() print('Failed to build images for Kubernetes', str(err)) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py index 87fda845ae..daafc19c2b 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py @@ -239,8 +239,4 @@ def build_ui(): args.region_info, args.ldap_login, args.tenant_id, args.application_id, args.hostname, args.datalake_store_name, args.subscription_id, args.validate_permission_scope, args.dlab_id, args.usage_date, args.product, args.usage_type, - args.usage, args.cost, args.resource_id, args.tags) - - #raw draft for kuber - print("Creating Kuber images") - create_kuber_images() \ No newline at end of file + args.usage, args.cost, args.resource_id, args.tags) \ No newline at end of file From 9e973471b9ef7fe4cf69bd8a7db9b0120fd00b05 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Mon, 27 May 2019 17:55:53 +0300 Subject: [PATCH 05/21] fix+redhat version --- .../src/general/lib/os/debian/common_lib.py | 2 +- .../src/general/lib/os/debian/ssn_lib.py | 22 +-------- .../src/general/lib/os/redhat/ssn_lib.py | 49 +++++++++++++------ 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py index 2cc64a8624..b4a5ca588d 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py @@ -64,7 +64,7 @@ def change_pkg_repos(): sudo('touch /tmp/pkg_china_ensured') -def find_java_path_remote(): +def update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'(): java_path = sudo("sh -c \"update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'\"") return java_path diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index eac937e0ec..eef8b25906 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -296,6 +296,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, .format(dlab_source_dir)) sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." .format(dlab_source_dir)) + sudo("rm -rf {}".format(k8s_dir)) except Exception as err: traceback.print_exc() print('Failed to build images for services: ', str(err)) @@ -321,27 +322,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, print('Failed to start Self-service: ', str(err)) sys.exit(1) - -def create_kuber_images(dlab_path): - try: - dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path) - k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir) - sudo("mkdir {}".format(k8s_dir)) - sudo("cp -r {0}conf/ {1}".format(dlab_path)) - sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) - sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) - sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) - sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) - sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) - sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui .".format(dlab_source_dir)) - sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." - .format(dlab_source_dir)) - - except Exception as err: - traceback.print_exc() - print('Failed to build images for Kubernetes', str(err)) - sys.exit(1) - def install_build_dep(): try: if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_dlab_path'])): diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py index 164dfc5a6c..502e8c8a58 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py @@ -216,17 +216,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/') put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf') sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/') - with open('/root/templates/supervisor_svc.conf', 'r') as f: - text = f.read() - text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user) - with open('/root/templates/supervisor_svc.ini', 'w') as f: - f.write(text) - put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini') - sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/') - sudo('cp ' + os.environ['ssn_dlab_path'] + - 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf') - sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.ini {}'.format(supervisor_conf)) - sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf)) try: sudo('mkdir -p /var/log/application') run('mkdir -p /tmp/yml_tmp/') @@ -323,9 +312,41 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, except: append_result("Unable to generate cert and copy to java keystore") sys.exit(1) - sudo('systemctl restart supervisord') - sudo('service nginx restart') - sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started') + if kuber_enabled: + try: + dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path) + k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir) + sudo("mkdir {}".format(k8s_dir)) + sudo("cp -r {0}conf/ {1}".format(dlab_path)) + sudo("cp -r {0}webapp/self-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/security-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) + sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) + sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui ." + .format(dlab_source_dir)) + sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." + .format(dlab_source_dir)) + sudo("rm -rf {}".format(k8s_dir)) + except Exception as err: + traceback.print_exc() + print('Failed to build images for services: ', str(err)) + sys.exit(1) + else: + with open('/root/templates/supervisor_svc.conf', 'r') as f: + text = f.read() + text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user) + with open('/root/templates/supervisor_svc.ini', 'w') as f: + f.write(text) + put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini') + sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/') + sudo('cp ' + os.environ['ssn_dlab_path'] + + 'tmp/proxy_location_webapp_template.conf /etc/nginx/locations/proxy_location_webapp.conf') + sudo('cp ' + os.environ['ssn_dlab_path'] + 'tmp/supervisor_svc.ini {}'.format(supervisor_conf)) + sudo('sed -i \'s=WEB_APP_DIR={}=\' {}'.format(web_path, supervisor_conf)) + sudo('systemctl restart supervisord') + sudo('service nginx restart') + sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/ss_started') except Exception as err: traceback.print_exc() print('Failed to start Self-service: ', str(err)) From 577b815147be5c5e7c034c2f95793bf04cc841b9 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Wed, 5 Jun 2019 11:14:50 +0300 Subject: [PATCH 06/21] fix+redhat version --- .../src/general/lib/os/debian/ssn_lib.py | 4 +- .../src/general/lib/os/redhat/ssn_lib.py | 4 +- .../aws => ssn/files/os}/billing_Dockerfile | 0 .../src/ssn/files/os/create_db.js | 8 ++++ .../src/ssn/files/os/mongo_Dockerfile | 36 +++++++++++++++ .../src/ssn/files/os/mongodb.conf | 42 +++++++++++++++++ .../aws => ssn/files/os}/webui_Dockerfile | 0 .../src/ssn/scripts/configure_mongo.py | 45 ++++++++++++------- .../src/ssn/scripts/configure_ui.py | 4 +- 9 files changed, 120 insertions(+), 23 deletions(-) rename infrastructure-provisioning/src/{general/files/aws => ssn/files/os}/billing_Dockerfile (100%) create mode 100644 infrastructure-provisioning/src/ssn/files/os/create_db.js create mode 100644 infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile create mode 100644 infrastructure-provisioning/src/ssn/files/os/mongodb.conf rename infrastructure-provisioning/src/{general/files/aws => ssn/files/os}/webui_Dockerfile (100%) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index eef8b25906..9079c99e72 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -292,9 +292,9 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) - sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui ." + sudo("cd {}; docker build --file ssn/files/os/webui_Dockerfile -t docker.dlab-ui ." .format(dlab_source_dir)) - sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." + sudo("cd {}; docker build --file ssn/files/os/billing_Dockerfile -t docker.dlab-billing ." .format(dlab_source_dir)) sudo("rm -rf {}".format(k8s_dir)) except Exception as err: diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py index 502e8c8a58..9d22c73d9f 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py @@ -323,9 +323,9 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, sudo("cp -r {0}webapp/provisioning-service/lib/*.jar {1}".format(dlab_path, k8s_dir)) sudo("cp -r {0}webapp/billing/lib/*.jar {1}".format(dlab_path, k8s_dir)) sudo('sed -i "s|ssn.yml|/root/ssn.yml|g" {}self-service.yml'.format(k8s_dir)) - sudo("cd {}; docker build --file general/files/aws/webui_Dockerfile -t docker.dlab-ui ." + sudo("cd {}; docker build --file ssn/files/os/webui_Dockerfile -t docker.dlab-ui ." .format(dlab_source_dir)) - sudo("cd {}; docker build --file general/files/aws/billing_Dockerfile -t docker.dlab-billing ." + sudo("cd {}; docker build --file ssn/files/os/billing_Dockerfile -t docker.dlab-billing ." .format(dlab_source_dir)) sudo("rm -rf {}".format(k8s_dir)) except Exception as err: diff --git a/infrastructure-provisioning/src/general/files/aws/billing_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/billing_Dockerfile similarity index 100% rename from infrastructure-provisioning/src/general/files/aws/billing_Dockerfile rename to infrastructure-provisioning/src/ssn/files/os/billing_Dockerfile diff --git a/infrastructure-provisioning/src/ssn/files/os/create_db.js b/infrastructure-provisioning/src/ssn/files/os/create_db.js new file mode 100644 index 0000000000..96163d2f94 --- /dev/null +++ b/infrastructure-provisioning/src/ssn/files/os/create_db.js @@ -0,0 +1,8 @@ +use dlabdb +db.createUser( + { + user: "admin", + pwd: "PASSWORD", + roles: [{'role':'userAdminAnyDatabase','db':'admin'}] + } +); diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile new file mode 100644 index 0000000000..39ab400493 --- /dev/null +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -0,0 +1,36 @@ +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + +FROM mongo:4.0-xenial + +ARG CLOUD_PROVIDER + +COPY create_db.js /root/ +COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ +RUN mv /root/mongodb.conf /etc/mongodb.conf + +RUN mkdir -p /data/db2 \ + && mkdir -p /data/logs \ + && chown -R mongodb:mongodb /data/db2 && \ + chown -R mongodb:mongodb /data/logs + +CMD ["mongod", "--config", "/etc/mongodb.conf"] + diff --git a/infrastructure-provisioning/src/ssn/files/os/mongodb.conf b/infrastructure-provisioning/src/ssn/files/os/mongodb.conf new file mode 100644 index 0000000000..28d182bb5c --- /dev/null +++ b/infrastructure-provisioning/src/ssn/files/os/mongodb.conf @@ -0,0 +1,42 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# Where and how to store data. +storage: + dbPath: /data/db2 + journal: + enabled: true +# engine: +# mmapv1: +# wiredTiger: + +# where to write logging data. +systemLog: + destination: file + logAppend: true + path: /data/logs/mongod.log + +# network interfaces +net: + port: 27017 + bindIp: 0.0.0.0 + + +#processManagement: + +#security: +# authorization: enabled + +#operationProfiling: + +#replication: + +#sharding: + +## Enterprise-Only Options: + +#auditLog: + +#snmp: diff --git a/infrastructure-provisioning/src/general/files/aws/webui_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/webui_Dockerfile similarity index 100% rename from infrastructure-provisioning/src/general/files/aws/webui_Dockerfile rename to infrastructure-provisioning/src/ssn/files/os/webui_Dockerfile diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index a7a5fc8811..b4f26d6601 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -22,6 +22,7 @@ # ****************************************************************************** from pymongo import MongoClient +from fabric.api import * import yaml, json, sys import subprocess import time @@ -34,6 +35,8 @@ parser = argparse.ArgumentParser() parser.add_argument('--dlab_path', type=str, default='') parser.add_argument('--mongo_parameters', type=str, default='') +parser.add_argument('--cloud_provider', type=str, default='') +parser.add_argument('--kuber', type=str, default=False) args = parser.parse_args() @@ -78,24 +81,32 @@ def read_yml_conf(path, section, param): # Setting up admin's password and enabling security client = MongoClient(mongo_ip + ':' + str(mongo_port)) pass_upd = True - try: - command = ['service', 'mongod', 'start'] - subprocess.call(command, shell=False) - time.sleep(5) - client.dlabdb.add_user('admin', mongo_passwd, roles=[{'role':'userAdminAnyDatabase','db':'admin'}]) - client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"]) - set_mongo_parameters(client, mongo_parameters) - with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data: - json_data = json.load(data) - for i in json_data: - client.dlabdb.roles.insert_one(i) - client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200) - if add_2_yml_config(path,'security','authorization','enabled'): - command = ['service', 'mongod', 'restart'] + if kuber: + local("cd {}; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." + .format(args.dlab_path, args.cloud_provider)) + local("mkdir -p /opt/mongo-vol") + local("docker run --name dlab-mongo -p 27017:27017 -v -d docker.dlab-mongo") + local("docker exec some-mongo sh -c 'mongo < /root/create_db.js'") + local("docker exec some-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") + else: + try: + command = ['service', 'mongod', 'start'] subprocess.call(command, shell=False) - except: - print("Looks like MongoDB have already been secured") - pass_upd = False + time.sleep(5) + client.dlabdb.add_user('admin', mongo_passwd, roles=[{'role':'userAdminAnyDatabase','db':'admin'}]) + client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"]) + set_mongo_parameters(client, mongo_parameters) + with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data: + json_data = json.load(data) + for i in json_data: + client.dlabdb.roles.insert_one(i) + client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200) + if add_2_yml_config(path,'security','authorization','enabled'): + command = ['service', 'mongod', 'restart'] + subprocess.call(command, shell=False) + except: + print("Looks like MongoDB have already been secured") + pass_upd = False # Generating output config add_2_yml_config(outfile, 'network', 'ip', mongo_ip) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py index daafc19c2b..196e44f15a 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py @@ -122,8 +122,8 @@ def configure_mongo(mongo_passwd): env.host_string)) sudo('mv /tmp/mongo_roles.json ' + args.dlab_path + 'tmp/') mongo_parameters = json.loads(args.mongo_parameters) - sudo("python " + args.dlab_path + "tmp/configure_mongo.py --dlab_path {} --mongo_parameters '{}'".format( - args.dlab_path, json.dumps(mongo_parameters))) + sudo("python " + args.dlab_path + "tmp/configure_mongo.py --dlab_path {} --mongo_parameters '{}' --cloud_provider {}".format( + args.dlab_path, json.dumps(mongo_parameters), args.cloud_provider)) except Exception as err: traceback.print_exc() print('Failed to configure MongoDB: ', str(err)) From e647dbe8006c44f048be9515cce0dd4ed555c217 Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 11:52:17 +0300 Subject: [PATCH 07/21] Update configure_mongo.py --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index b4f26d6601..d64fbeeb2c 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -36,7 +36,7 @@ parser.add_argument('--dlab_path', type=str, default='') parser.add_argument('--mongo_parameters', type=str, default='') parser.add_argument('--cloud_provider', type=str, default='') -parser.add_argument('--kuber', type=str, default=False) +parser.add_argument('--kuber', type=str, default=True) args = parser.parse_args() From 0f68cfa53ea3c0bfb835e1855e2e7a4ae60130c7 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Wed, 5 Jun 2019 12:55:48 +0300 Subject: [PATCH 08/21] fix --- .../src/general/lib/os/debian/common_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py index b4a5ca588d..2cc64a8624 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py @@ -64,7 +64,7 @@ def change_pkg_repos(): sudo('touch /tmp/pkg_china_ensured') -def update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'(): +def find_java_path_remote(): java_path = sudo("sh -c \"update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'\"") return java_path From 3642dda8159eb75dbb8a9b68e8ce74f0701a6e3a Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 13:16:25 +0300 Subject: [PATCH 09/21] Update configure_mongo.py --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index d64fbeeb2c..4115798d24 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -81,7 +81,7 @@ def read_yml_conf(path, section, param): # Setting up admin's password and enabling security client = MongoClient(mongo_ip + ':' + str(mongo_port)) pass_upd = True - if kuber: + if args.kuber: local("cd {}; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo-vol") From 014ae9b0ff34f0b0dc169e28dbf9f376c836d78e Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 13:53:26 +0300 Subject: [PATCH 10/21] Update configure_mongo.py --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 4115798d24..57c556b298 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -82,7 +82,7 @@ def read_yml_conf(path, section, param): client = MongoClient(mongo_ip + ':' + str(mongo_port)) pass_upd = True if args.kuber: - local("cd {}; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." + local("cd {}infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo-vol") local("docker run --name dlab-mongo -p 27017:27017 -v -d docker.dlab-mongo") From 380136c287c7e3e58d77a091dd17d06f6f6f8795 Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 14:15:33 +0300 Subject: [PATCH 11/21] Update configure_mongo.py --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 57c556b298..469c0b30be 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -82,7 +82,7 @@ def read_yml_conf(path, section, param): client = MongoClient(mongo_ip + ':' + str(mongo_port)) pass_upd = True if args.kuber: - local("cd {}infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." + local("cd {}sources/infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo-vol") local("docker run --name dlab-mongo -p 27017:27017 -v -d docker.dlab-mongo") From 8e5d926294008146dce6615b1d908c61575ba241 Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 14:39:22 +0300 Subject: [PATCH 12/21] Update mongo_Dockerfile --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index 39ab400493..62dacf043f 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -23,7 +23,7 @@ FROM mongo:4.0-xenial ARG CLOUD_PROVIDER -COPY create_db.js /root/ +COPY ssn/files/os/create_db.js /root/ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf From 7a7d91c6c39dfcd21ff7004421621c0a750a1ee1 Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 15:11:29 +0300 Subject: [PATCH 13/21] Update mongo_Dockerfile --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index 62dacf043f..994adfdd84 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -24,6 +24,7 @@ FROM mongo:4.0-xenial ARG CLOUD_PROVIDER COPY ssn/files/os/create_db.js /root/ +COPY ssn/files/os/mongodb.conf /root/ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf From 340e5ae7dd422625d1f5b50f5854f3b39d897d5d Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 15:32:00 +0300 Subject: [PATCH 14/21] Update configure_mongo.py --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 469c0b30be..1fbf3fb6a8 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -85,7 +85,7 @@ def read_yml_conf(path, section, param): local("cd {}sources/infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo-vol") - local("docker run --name dlab-mongo -p 27017:27017 -v -d docker.dlab-mongo") + local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db2 -d docker.dlab-mongo") local("docker exec some-mongo sh -c 'mongo < /root/create_db.js'") local("docker exec some-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") else: From cf69f39d26553cf28e7479a09853be9c8930b47b Mon Sep 17 00:00:00 2001 From: "i.oleksandr" Date: Wed, 5 Jun 2019 15:46:15 +0300 Subject: [PATCH 15/21] Update configure_mongo.py --- .../src/ssn/scripts/configure_mongo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 1fbf3fb6a8..9ab678a49d 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -86,8 +86,8 @@ def read_yml_conf(path, section, param): .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo-vol") local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db2 -d docker.dlab-mongo") - local("docker exec some-mongo sh -c 'mongo < /root/create_db.js'") - local("docker exec some-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") + local("docker exec dlab-mongo sh -c 'mongo < /root/create_db.js'") + local("docker exec dlab-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") else: try: command = ['service', 'mongod', 'start'] From 796e2953448c239ae2f4b7197d9dfb496be58525 Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Wed, 5 Jun 2019 17:33:20 +0300 Subject: [PATCH 16/21] fixes --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 4 +--- .../src/ssn/scripts/configure_mongo.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index 994adfdd84..e254491503 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -28,9 +28,7 @@ COPY ssn/files/os/mongodb.conf /root/ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf -RUN mkdir -p /data/db2 \ - && mkdir -p /data/logs \ - && chown -R mongodb:mongodb /data/db2 && \ +RUN && chown -R mongodb:mongodb /data/db2 && \ chown -R mongodb:mongodb /data/logs CMD ["mongod", "--config", "/etc/mongodb.conf"] diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 9ab678a49d..ee792d8bf4 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -84,8 +84,8 @@ def read_yml_conf(path, section, param): if args.kuber: local("cd {}sources/infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) - local("mkdir -p /opt/mongo-vol") - local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db2 -d docker.dlab-mongo") + local("mkdir -p /opt/mongo_vol") + local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db -d docker.dlab-mongo --smallfiles") local("docker exec dlab-mongo sh -c 'mongo < /root/create_db.js'") local("docker exec dlab-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") else: From 690efb0e5a96b58fdd12cacfa5ea57a4f607865b Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Wed, 5 Jun 2019 17:51:43 +0300 Subject: [PATCH 17/21] fixes --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index e254491503..a558c77111 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -28,7 +28,7 @@ COPY ssn/files/os/mongodb.conf /root/ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf -RUN && chown -R mongodb:mongodb /data/db2 && \ +RUN chown -R mongodb:mongodb /data/db2 && \ chown -R mongodb:mongodb /data/logs CMD ["mongod", "--config", "/etc/mongodb.conf"] From be9706770f35afbbc92cbdb591f38f0c37c3ffae Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Wed, 5 Jun 2019 18:27:41 +0300 Subject: [PATCH 18/21] fixes --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index a558c77111..e0d7ff2b3b 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -28,7 +28,7 @@ COPY ssn/files/os/mongodb.conf /root/ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf -RUN chown -R mongodb:mongodb /data/db2 && \ +RUN mkdir -p /data/logs \ chown -R mongodb:mongodb /data/logs CMD ["mongod", "--config", "/etc/mongodb.conf"] From d8a683b133de2e4b3866d862246c0ba3f64fa16a Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 6 Jun 2019 11:21:51 +0300 Subject: [PATCH 19/21] fixes --- infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile index e0d7ff2b3b..9c2a6b5724 100644 --- a/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile +++ b/infrastructure-provisioning/src/ssn/files/os/mongo_Dockerfile @@ -29,7 +29,7 @@ COPY ssn/files/${CLOUD_PROVIDER}/mongo_roles.json /root/ RUN mv /root/mongodb.conf /etc/mongodb.conf RUN mkdir -p /data/logs \ - chown -R mongodb:mongodb /data/logs + && chown -R mongodb:mongodb /data/logs CMD ["mongod", "--config", "/etc/mongodb.conf"] From aa3537275c59f46a8221c3a457c5c8c8adf5e17a Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 6 Jun 2019 11:54:21 +0300 Subject: [PATCH 20/21] fixes --- infrastructure-provisioning/src/ssn/scripts/configure_mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index ee792d8bf4..1a0cbabddc 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -86,7 +86,7 @@ def read_yml_conf(path, section, param): .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo_vol") local("docker run --name dlab-mongo -p 27017:27017 -v /opt/mongo_vol:/data/db -d docker.dlab-mongo --smallfiles") - local("docker exec dlab-mongo sh -c 'mongo < /root/create_db.js'") + local("sleep 10; docker exec dlab-mongo sh -c 'mongo < /root/create_db.js'") local("docker exec dlab-mongo sh -c 'mongoimport --jsonArray --db dlabdb --collection roles --file /root/mongo_roles.json'") else: try: From 205c3bc5c6ae2595043d0d6096bc343e023d9b5f Mon Sep 17 00:00:00 2001 From: Oleksandr_Isniuk Date: Thu, 6 Jun 2019 16:23:52 +0300 Subject: [PATCH 21/21] [DLAB-656] Prepare docker images for MongoDB/UI/Billing services --- .../src/general/lib/os/debian/ssn_lib.py | 4 ++-- .../src/ssn/scripts/configure_mongo.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index 9079c99e72..84c6a9a9ce 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -175,7 +175,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, locale, region_info, ldap_login, tenant_id, application_id, hostname, data_lake_name, subscription_id, validate_permission_scope, dlab_id, usage_date, product, - usage_type, usage, cost, resource_id, tags, report_path=''): + usage_type, usage, cost, resource_id, tags, report_path='', k8s=False): try: if not exists(os.environ['ssn_dlab_path'] + 'tmp/ss_started'): java_path = sudo("update-alternatives --query java | grep 'Value: ' | grep -o '/.*/jre'") @@ -281,7 +281,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path, except: append_result("Unable to generate cert and copy to java keystore") sys.exit(1) - if kuber_enabled: + if k8s: try: dlab_source_dir = '{}sources/infrastructure-provisioning/src/'.format(dlab_path) k8s_dir = '{}tmp-kuber/'.format(dlab_source_dir) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index 1a0cbabddc..301b228a91 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -36,7 +36,7 @@ parser.add_argument('--dlab_path', type=str, default='') parser.add_argument('--mongo_parameters', type=str, default='') parser.add_argument('--cloud_provider', type=str, default='') -parser.add_argument('--kuber', type=str, default=True) +parser.add_argument('--k8s', type=str, default=False) args = parser.parse_args() @@ -81,7 +81,7 @@ def read_yml_conf(path, section, param): # Setting up admin's password and enabling security client = MongoClient(mongo_ip + ':' + str(mongo_port)) pass_upd = True - if args.kuber: + if args.k8s: local("cd {}sources/infrastructure-provisioning/src/; docker build --build-arg CLOUD_PROVIDER={} --file ssn/files/os/mongo_Dockerfile -t docker.dlab-mongo ." .format(args.dlab_path, args.cloud_provider)) local("mkdir -p /opt/mongo_vol")