diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index e797bc1..c3cc5b8 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -1,8 +1,11 @@ +import logging from collections import namedtuple from collections.abc import Mapping +from pathlib import PurePosixPath from ..observer import BuildObserver +logger = logging.getLogger(__name__) BACKENDS = { 'docker': 'apluslms_roman.backends.docker.DockerBackend', @@ -112,3 +115,18 @@ def verify(self): def version_info(self): pass + + def get_host_path(self, original): + mapping = self.environment.environ.get('directory_map', {}) + if not mapping: + return original + logger.debug("Get mapping from environment:%s", mapping) + path = PurePosixPath(original) + for container, host in mapping.items(): + try: + logger.debug("Mapping:%s:%s", container, host) + return str(PurePosixPath(host, path.relative_to(container))) + except ValueError: + logger.error("Error when composing new path!") + continue + return ValueError("Unable to map path '%s' to any backend host path" % (path)) diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index 885ddd6..efc4c77 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -1,6 +1,7 @@ -import docker -from os.path import join +import logging +from os.path import join, expanduser +import docker from apluslms_yamlidator.utils.decorator import cached_property from ..utils.translation import _ @@ -12,6 +13,8 @@ Mount = docker.types.Mount +logger = logging.getLogger(__name__) + class DockerBackend(Backend): name = 'docker' @@ -22,14 +25,28 @@ class DockerBackend(Backend): @cached_property def _client(self): env = self.environment.environ - kwargs = {} - version = env.get('DOCKER_VERSION', None) - if version: - kwargs['version'] = version - timeout = env.get('DOCKER_TIMEOUT', None) - if timeout: - kwargs['timeout'] = timeout - return docker.from_env(environment=env, **kwargs) + params = { + 'base_url': env.get('host'), + 'version': env.get('version'), + } + if 'timeout' in env: + params['timeout'] = env['timeout'] + + # false values: 0, false, '', unset + # true values: 1, true, "yes" + tls_verify = bool(env.get('tls_verify', False)) + cert_path = env.get('cert_path') or None + if tls_verify or cert_path: + if not cert_path: + cert_path = join(expanduser('~'), '.docker') + params['tls'] = docker.tls.TLSConfig( + client_cert=(join(cert_path, 'cert.pem'), join(cert_path, 'key.pem')), + ca_cert=join(cert_path, 'ca.pem'), + verify=tls_verify, + ssl_version=env.get('tls_ssl_version'), + assert_hostname=tls_verify and env.get('tls_assert_hostname'), + ) + return docker.DockerClient(**params) def _run_opts(self, task, step): env = self.environment @@ -41,16 +58,19 @@ def _run_opts(self, task, step): user='{}:{}'.format(env.uid, env.gid), ) + path = self.get_host_path(task.path) + + logger.debug("Final path is:%s", path) # mounts and workdir if step.mnt: - opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)] + opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)] opts['working_dir'] = step.mnt else: wpath = self.WORK_PATH opts['mounts'] = [ Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE), - Mount(join(wpath, 'src'), task.path, type='bind', read_only=True), - Mount(join(wpath, 'build'), join(task.path, '_build'), type='bind', read_only=False), + Mount(join(wpath, 'src'), path, type='bind', read_only=True), + Mount(join(wpath, 'build'), join(path, '_build'), type='bind', read_only=False), ] opts['working_dir'] = wpath diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 89ee2ea..6e4d79e 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -9,6 +9,7 @@ from .utils.importing import import_string from .utils.translation import _ + class Builder: def __init__(self, engine, config, observer=None): if not isdir(config.dir): @@ -21,7 +22,7 @@ def __init__(self, engine, config, observer=None): def get_steps(self, refs: list = None): steps = [BuildStep.from_config(i, step) - for i, step in enumerate(self.config.steps)] + for i, step in enumerate(self.config.steps)] if refs: name_dict = {step.name: step for step in steps} refs = [int(ref) if ref.isdigit() else ref.lower() for ref in refs] @@ -64,11 +65,9 @@ def __init__(self, backend_class=None, settings=None): name = getattr(backend_class, 'name', None) or backend_class.__name__.lower() env_prefix = name.upper() + '_' - env = {k: v for k, v in environ.items() if k.startswith(env_prefix)} + env = {key: value for key, value in environ.items() if key.startswith(env_prefix)} if settings: - for k, v in settings.get(name, {}).items(): - if v is not None and v != '': - env[env_prefix + k.replace('-', '_').upper()] = v + env.update(settings.get(name, {})) self._environment = Environment(getuid(), getegid(), env) @cached_property diff --git a/apluslms_roman/schemas/roman_settings-v1.0.yaml b/apluslms_roman/schemas/roman_settings-v1.0.yaml index a972077..22d9386 100644 --- a/apluslms_roman/schemas/roman_settings-v1.0.yaml +++ b/apluslms_roman/schemas/roman_settings-v1.0.yaml @@ -24,6 +24,10 @@ properties: type: object additionalProperties: false properties: + directory_map: + title: docker container-host machine path mapping + description: The dictornary mapping between docker and it's host + type: object host: title: docker host description: the URL to the Docker host diff --git a/apluslms_roman/utils/path_mapping.py b/apluslms_roman/utils/path_mapping.py new file mode 100644 index 0000000..d3740d4 --- /dev/null +++ b/apluslms_roman/utils/path_mapping.py @@ -0,0 +1,25 @@ +import json +import logging +import re +from os import environ +from apluslms_yamlidator.document import find_ml + +logger = logging.getLogger(__name__) +json_re = re.compile(r'^(?:["[{]|(?:-?[1-9]\d*(?:\.\d+)?|null|true|false)$)') + + +def nest_dict(flat_dict): + nested = {} + for keys, value in flat_dict.items(): + dict_, key = find_ml(nested, keys, create_dicts=True) + dict_[key] = value + return nested + + +def load_from_env(env_prefix=None, decode_json=True): + if decode_json: + decode = lambda s: json.loads(s) if json_re.match(s) is not None else s + else: + decode = lambda s: s + env = {key[len(env_prefix):].lower(): decode(value) for key, value in environ.items() if key.startswith(env_prefix)} + return nest_dict(env) diff --git a/tests/utils/test_path_mapping.py b/tests/utils/test_path_mapping.py new file mode 100644 index 0000000..9715d7c --- /dev/null +++ b/tests/utils/test_path_mapping.py @@ -0,0 +1,64 @@ +import os +import unittest +from json import loads + +from apluslms_roman.utils.path_mapping import json_re, load_from_env + +test_case_loadable = ( + 'true', + 'false', + 'null', + '123', + '-123', + '3.14', + '-3.14', + '{"foo": "bar"}', + '[1, 2, 3]', + '"foo bar"' +) + +test_case_not_loadable = ( + "/foobar.py", + "text", + "yes", + "0123123", +) + + +class TestJsonLoadable(unittest.TestCase): + + def test_loadable_not_raise(self): + for case in test_case_loadable: + with self.subTest(non_json=case): + loads(case) + + def test_not_loadable_raise(self): + for case in test_case_not_loadable: + with self.subTest(non_json=case): + with self.assertRaises(ValueError, msg="Testing:{}".format(case)): + loads(case) + + +class TestJsonRegex(unittest.TestCase): + + def test_loadable_match(self): + for case in test_case_loadable: + with self.subTest(non_json=case): + self.assertTrue(json_re.match(case)is not None, msg="Testing:{}".format(case)) + + def test_not_loadable_not_match(self): + for case in test_case_not_loadable: + with self.subTest(non_json=case): + self.assertFalse(json_re.match(case) is not None, msg="Testing:{}".format(case)) + + +class TestLoadFromEnv(unittest.TestCase): + + def test_with_decode_json(self): + os.environ['DOCKER.FOO.BAR'] = '123' + self.assertEqual({'foo': {'bar': '123'}}, load_from_env('DOCKER.', False)) + + def test_without_decode_json(self): + os.environ['DOCKER.FOO.BAR'] = '123' + self.assertEqual({'foo': {'bar': 123}}, load_from_env('DOCKER.', True)) +