Kubernetes

apluslms_roman/backends/__init__.py

@@ -1,11 +1,14 @@
+import logging
 from collections import namedtuple
 from collections.abc import Mapping
 
+from apluslms_roman.utils.path_mapping import get_host_path
 from ..observer import BuildObserver
 
 
 BACKENDS = {
     'docker': 'apluslms_roman.backends.docker.DockerBackend',
+    'kubernetes': 'apluslms_roman.backends.kubernetes.KubernetesBackend',
 }
 
 
@@ -15,6 +18,9 @@
 ])
 
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 def clean_image_name(image):
     if ':' not in image:
         image += ':latest'
@@ -107,3 +113,9 @@ def verify(self):
 
     def version_info(self):
         pass
+
+    def remap_path(self, path):
+        map_ = self.environment.environ.get('directory_map', {})
+        logger.debug("get mapping from environment:{}".format(map_))
+        map_ = dict(map_) if len(map_) == 0 else map_
+        return get_host_path(path, map_)

apluslms_roman/backends/docker.py

@@ -1,15 +1,19 @@
+import os
+import logging
 import docker
 from os.path import join
 
 from apluslms_yamlidator.utils.decorator import cached_property
+from docker import DockerClient
 
 from ..utils.translation import _
 from . import (
     Backend,
     BuildResult,
 )
 
-
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
 Mount = docker.types.Mount
 
 
@@ -22,14 +26,28 @@ class DockerBackend(Backend):
     @cached_property
     def _client(self):
         env = self.environment.environ
-        kwargs = {}
-        version = env.get('DOCKER_VERSION', None)
-        if version:
-            kwargs['version'] = version
-        timeout = env.get('DOCKER_TIMEOUT', None)
-        if timeout:
-            kwargs['timeout'] = timeout
-        return docker.from_env(environment=env, **kwargs)
+        params = {
+            'base_url': env.get('host'),
+            'version': env.get('version'),
+        }
+        if 'timeout' in env:
+            params['timeout'] = env['timeout']
+
+        # false values: 0, false, '', None
+        # true values: 1, true, "yes", unset
+        tls_verify = bool(env.get('tls_verify', False))
+        cert_path = env.get('cert_path') or None
+        if tls_verify or cert_path:
+            if not cert_path:
+                cert_path = os.path.join(os.path.expanduser('~'), '.docker')
+            params['tls'] = docker.tls.TLSConfig(
+                client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')),
+                ca_cert=os.path.join(cert_path, 'ca.pem'),
+                verify=tls_verify,
+                ssl_version=env.get('tls_ssl_version'),
+                assert_hostname=tls_verify and env.get('tls_assert_hostname'),
+            )
+        return DockerClient(**params)
 
     def _run_opts(self, task, step):
         env = self.environment
@@ -40,17 +58,18 @@ def _run_opts(self, task, step):
             environment=step.env,
             user='{}:{}'.format(env.uid, env.gid),
         )
-
+        path = self.remap_path(task.path)
+        logger.debug("Final path is:{}".format(path))
         # mounts and workdir
         if step.mnt:
-            opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)]
+            opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)]
             opts['working_dir'] = step.mnt
         else:
             wpath = self.WORK_PATH
             opts['mounts'] = [
                 Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE),
-                Mount(join(wpath, 'src'), task.path, type='bind', read_only=True),
-                Mount(join(wpath, 'build'), join(task.path, '_build'), type='bind', read_only=False),
+                Mount(join(wpath, 'src'), path, type='bind', read_only=True),
+                Mount(join(wpath, 'build'), join(path, '_build'), type='bind', read_only=False),
             ]
             opts['working_dir'] = wpath
 

apluslms_roman/backends/kubernetes.py

@@ -0,0 +1,135 @@
+import logging
+
+from os.path import join
+from apluslms_yamlidator.utils.decorator import cached_property
+from kubernetes import client, config, watch
+from apluslms_roman.utils.kubernetes import create_pod
+from apluslms_roman.backends import BuildTask
+from apluslms_roman.observer import BuildObserver
+from . import (
+    Backend,
+    BuildResult,
+)
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+class KubernetesBackend(Backend):
+    """
+    Run each step as a Kubernetes Deployment
+    Mounting: using mounting in deployment, mapping is same as shepherd
+    """
+    name = 'kubernetes'
+
+    @cached_property
+    def _client(self):
+        # Load kubernetes config from from $Home/.kube/config
+        config.load_kube_config()
+        api = client.CoreV1Api()
+        return api
+
+    def _run_opts(self, task, step):
+        """
+        Define the Pod model
+        """
+        env = self.environment
+        opts = dict(
+            image=step.img,
+            command=step.cmd,
+            environment=step.env,
+            namespace=env.environ['namespace'],
+            name=step.img.split(':')[0].replace('/', '-')
+        )
+        if step.mnt:
+            opts['volumes'] = [
+                client.V1Volume(
+                    name='build-path',
+                    host_path=client.V1HostPathVolumeSource(path="/build-source")
+                )
+            ]
+            opts['mounts'] = [
+                client.V1VolumeMount(
+                    mount_path=step.mnt,
+                    name='build-path'
+                )
+            ]
+            opts['working_dir'] = step.mnt
+        else:
+            wpath = self.WORK_PATH
+
+            opts['volumes'] = [
+                client.V1Volume(
+                    name='cache',
+                    empty_dir=client.V1EmptyDirVolumeSource(size_limit=self.WORK_SIZE, medium='Memory')
+                ),
+                client.V1Volume(
+                    name='source',
+                    host_path=client.V1HostPathVolumeSource(path=join(wpath, 'src'))
+                ),
+                client.V1Volume(
+                    name='build',
+                    host_path=client.V1HostPathVolumeSource(path=join(wpath, 'build'))
+                )
+               ]
+            opts['mounts'] = [
+                client.V1VolumeMount(
+                    mount_path=wpath,
+                    name='cache',
+                    read_only=False
+                ),
+                client.V1VolumeMount(
+                    mount_path=join(wpath, 'src'),
+                    name='source',
+                    read_only=True
+                ),
+                client.V1VolumeMount(
+                    mount_path=join(wpath, 'build'),
+                    name='build',
+                    read_only=False
+                )
+            ]
+        return opts
+
+    def prepare(self, task: BuildTask, observer: BuildObserver):
+        pass
+
+    def build(self, task: BuildTask, observer: BuildObserver):
+        api_client = self._client
+        for step in task.steps:
+            observer.start_step(step)
+            opts = self._run_opts(task, step)
+            observer.manager_msg(step, "Running deployment with image {}:".format(opts['image']))
+            name = opts['name']
+            try:
+                create_resp = create_pod(**opts)
+                print(create_resp)
+                name = create_resp.metadata.name
+                # Waiting pod finished
+                while True:
+                    resp = api_client.read_namespaced_pod(name=name, namespace=opts['namespace'])
+                    if resp.status.phase != "Pending":
+                        break
+                for line in api_client.read_namespaced_pod_log(
+                        name=name,
+                        namespace=opts['namespace'],
+                        follow=True,
+                        _preload_content=False).stream():
+                    observer.container_msg(step, line.decode('utf-8'))
+            except client.rest.ApiException as e:
+                logger.warning('Error when create Pod: %s.\n' % e)
+                return BuildResult(1, e, step)
+            finally:
+                api_client.delete_namespaced_pod(
+                    name=name,
+                    namespace=opts['namespace'],
+                )
+                observer.end_step(step)
+            return BuildResult()
+
+    def verify(self):
+        try:
+            api_client = self._client
+            api_client.list_component_status()
+        except Exception as e:
+            return "{}: {}".format(e.__class__.__name__, e)

apluslms_roman/builder.py

@@ -1,14 +1,20 @@
-from os import environ, getuid, getegid
-from os.path import isdir, join
+import logging
+from os import getuid, getegid
+from os.path import isdir
 
-from apluslms_yamlidator.utils.decorator import cached_property
 from apluslms_yamlidator.utils.collections import OrderedDict
+from apluslms_yamlidator.utils.decorator import cached_property
 
+from apluslms_roman.utils.path_mapping import load_from_env
 from .backends import BACKENDS, BuildTask, BuildStep, Environment
 from .observer import StreamObserver
 from .utils.importing import import_string
 from .utils.translation import _
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
 class Builder:
     def __init__(self, engine, config, observer=None):
         if not isdir(config.dir):
@@ -18,7 +24,6 @@ def __init__(self, engine, config, observer=None):
         self._engine = engine
         self._observer = observer or StreamObserver()
 
-
     def get_steps(self, refs: list = None):
         steps = [BuildStep.from_config(i, step) for i, step in enumerate(self.config.steps)]
         if refs:
@@ -60,11 +65,13 @@ def __init__(self, backend_class=None, settings=None):
 
         name = getattr(backend_class, 'name', None) or backend_class.__name__.lower()
         env_prefix = name.upper() + '_'
-        env = {k: v for k, v in environ.items() if k.startswith(env_prefix)}
+        env = load_from_env(env_prefix, '.')
+        logger.debug("env without reading global config:{}".format(env))
         if settings:
             for k, v in settings.get(name, {}).items():
                 if v is not None and v != '':
-                    env[env_prefix + k.replace('-', '_').upper()] = v
+                    env[k] = v
+        logger.debug("env after read from global config:{}".format(env))
         self._environment = Environment(getuid(), getegid(), env)
 
     @cached_property
@@ -78,4 +85,4 @@ def version_info(self):
         return self.backend.version_info()
 
     def create_builder(self, *args, **kwargs):
-        return Builder(self, *args, **kwargs)
+        return Builder(self, *args, **kwargs)

apluslms_roman/cli.py

@@ -294,11 +294,8 @@ def main():
     exit(context.run())
 
 
-
 ## Actions
-
 # action utils
-
 def get_engine(context):
     try:
         return Engine(settings=context.settings)
@@ -341,7 +338,6 @@ def build_action(context):
     config = get_config(context)
     engine = get_engine(context)
     builder = engine.create_builder(config)
-
     if context.args.list_steps:
         steps = builder.get_steps()
         num_len = max(2, len(str(len(steps)-1)))
@@ -436,5 +432,6 @@ def backend_test_action(context, verbose=False):
         print(engine.version_info())
     return 0
 
+
 if __name__ == '__main__':
     main()

apluslms_roman/schemas/roman_settings-v1.0.yaml

@@ -9,6 +9,7 @@ allOf:
 optional:
   - backend
   - docker
+  - kubernetes
 
 additionalProperties: false
 properties:
@@ -24,6 +25,10 @@ properties:
     type: object
     additionalProperties: false
     properties:
+      directory_map:
+        title: docker conatiner-host machine path mapping
+        description: The dictornary mapping between docker and it's host
+        type: object
       host:
         title: docker host
         description: the URL to the Docker host
@@ -46,3 +51,13 @@ properties:
         type: integer
         minimum: 0
         exclusiveMinimum: true
+  kubernetes:
+    title: kubernetes backend options
+    description: options for Kubernetes  backend
+    type: object
+    additionalProperties: false
+    properties:
+      namespace:
+        title: default pod namespace
+        description: default namesapce for generated pod which runs build container.
+        type: string