diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index e7b5ff1..568fadc 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -86,7 +86,7 @@ def __str__(self): return "Build failed on step {}: {}".format(self.step, error) -Environment = namedtuple('Environment', [ +BackendContext = namedtuple('BackendContext', [ 'uid', 'gid', 'environ', @@ -98,8 +98,8 @@ class Backend: WORK_PATH = '/work' LABEL_PREFIX = 'io.github.apluslm.roman' - def __init__(self, environment: Environment): - self.environment = environment + def __init__(self, context: BackendContext): + self.context = context def prepare(self, task: BuildTask, observer: BuildObserver): raise NotImplementedError diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index b278c10..9847751 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -40,7 +40,7 @@ class DockerBackend(Backend): @cached_property def _client(self): - env = self.environment.environ + env = self.context.environ kwargs = {} version = env.get('DOCKER_VERSION', None) if version: @@ -51,7 +51,7 @@ def _client(self): return docker.from_env(environment=env, **kwargs) def _run_opts(self, task, step): - env = self.environment + env = self.context now = datetime.now() expire = now + timedelta(days=1) diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 173a3ab..bdf8ede 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -5,7 +5,7 @@ from apluslms_yamlidator.utils.decorator import cached_property from apluslms_yamlidator.utils.collections import OrderedDict -from .backends import BACKENDS, BuildTask, BuildStep, Environment +from .backends import BACKENDS, BackendContext, BuildTask, BuildStep from .observer import StreamObserver from .utils.importing import import_string from .utils.translation import _ @@ -55,31 +55,54 @@ def build(self, step_refs: list = None, clean_build=False): return result +class BackendError(Exception): + + def __init__(self, backend): + super().__init__() + self.backend = backend + + class Engine: def __init__(self, backend_class=None, settings=None): + backend_name = None if backend_class is None: if settings and 'backend' in settings: backend_class = settings['backend'] + if 'backends' in settings and backend_class in settings['backends']: + backend_name = backend_class + if 'type' in settings['backends'][backend_name]: + backend_class = settings['backends'][backend_name]['type'] else: from .backends.docker import DockerBackend as backend_class if isinstance(backend_class, str): if '.' not in backend_class: backend_class = BACKENDS.get(backend_class, backend_class) - backend_class = import_string(backend_class) + try: + backend_class = import_string(backend_class) + except ImportError: + raise BackendError(backend_class) self._backend_class = backend_class name = getattr(backend_class, 'name', None) or backend_class.__name__.lower() - env_prefix = name.upper() + '_' - env = {k: v for k, v in environ.items() if k.startswith(env_prefix)} + prefix = name.upper() + '_' + options = {} + # config + if backend_name is not None: + options.update({prefix + k.replace('-', '_').upper(): v + for k, v in settings['backends'][backend_name].items() + if k != 'type'}) + # environment + options.update({k: v for k, v in environ.items() if k.startswith(prefix)}) + # command line if settings: - for k, v in settings.get(name, {}).items(): - if v is not None and v != '': - env[env_prefix + k.replace('-', '_').upper()] = v - self._environment = Environment(getuid(), getegid(), env) + options.update({prefix + k.replace('-', '_').upper(): v + for k, v in settings.get(name, {}).items()}) + + self._backend_context = BackendContext(getuid(), getegid(), options) @cached_property def backend(self): - return self._backend_class(self._environment) + return self._backend_class(self._backend_context) def verify(self): return self.backend.verify() diff --git a/apluslms_roman/cli.py b/apluslms_roman/cli.py index 9131590..82650a9 100644 --- a/apluslms_roman/cli.py +++ b/apluslms_roman/cli.py @@ -13,7 +13,7 @@ from apluslms_yamlidator.validator import ValidationError, render_error from . import __version__ -from .builder import Engine +from .builder import BackendError, Engine from .configuration import ProjectConfig, ProjectConfigError from .settings import GlobalSettings from .utils.env import EnvDict, EnvError @@ -397,9 +397,8 @@ def main(*, args=None): def get_engine(context): try: return Engine(settings=context.settings) - except ImportError: - exit(1, _("ERROR: Unable to find backend '{}'.").format( - context.settings.get('backend', 'docker'))) + except BackendError as err: + exit(1, _("ERROR: Unable to find backend '{}'.").format(err.backend)) def get_config(context): diff --git a/apluslms_roman/schemas/roman_settings-v1.0.yaml b/apluslms_roman/schemas/roman_settings-v1.0.yaml index 4d82e5e..f44c8f6 100644 --- a/apluslms_roman/schemas/roman_settings-v1.0.yaml +++ b/apluslms_roman/schemas/roman_settings-v1.0.yaml @@ -8,18 +8,9 @@ allOf: optional: - backend - - docker + - backends -additionalProperties: false -properties: - version: {} - environment: - $ref: "roman_environment-v1.0#/properties/environment" - backend: - title: backend driver - description: the container backend driver class - type: string - default: docker +definitions: docker: title: docker backend options description: options for Docker container executor backend @@ -47,3 +38,38 @@ properties: description: default timeout for API calls type: integer exclusiveMinimum: 0 + type: + type: string + backend: + required: + - type + properties: + type: + type: string + if: + properties: + type: + const: docker + then: + $ref: "#/definitions/docker" + else: + type: object + +additionalProperties: false +properties: + version: {} + environment: + $ref: "roman_environment-v1.0#/properties/environment" + backend: + title: backend driver + description: name of the container backend driver class + type: string + default: docker + backends: + type: object + properties: + docker: + $ref: "#/definitions/docker" + patternProperties: + "^(?!docker$)": + $ref: "#/definitions/backend" diff --git a/tests/schemas/__init__.py b/tests/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/schemas/test_project_conf.py b/tests/schemas/test_project_conf.py new file mode 100644 index 0000000..c873f9c --- /dev/null +++ b/tests/schemas/test_project_conf.py @@ -0,0 +1,36 @@ +from ..test_cli import CliTestCase + +class TestBackendValidation(CliTestCase): + + def test_basicBackend_shouldValidate(self): + r = self.command_test('config -g', settings={ + 'version': '1.0', + 'backends': { + 'docker1': {'type': 'docker', 'timeout': 50}, + 'docker2': {'type': 'docker', 'timeout': 100} + } + }) + + def test_dockerBackendWithNoType_shouldValidate(self): + r = self.command_test('config -g', settings={ + 'version': '1.0', + 'backends': { + 'docker': {'timeout': 50} + } + }) + + def test_backendWithUnfamiliarType_shouldValidate(self): + r = self.command_test('config -g', settings={ + 'version': '1.0', + 'backends': { + 'backend1': {'type': 'unkown', 'asdf': True} + } + }) + + def test_backendWithNoType_shouldFail(self): + r = self.command_test('config -g', settings={ + 'version': '1.0', + 'backends': { + 'backend1': {'test': True} + } + }, exit_code=1) diff --git a/tests/test_cli.py b/tests/test_cli.py index 190d977..e0d2c67 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -360,30 +360,19 @@ class TestConfigSetAction(CliTestCase): SETTINGS = "version: '1'" def test_normal(self): - r = self.command_test("config -g set docker.timeout=100", settings=self.SETTINGS) + r = self.command_test("config -g set backend=docker1", settings=self.SETTINGS) self.assertEqual(r.out.strip(), "File successfully edited.") data = r.files[SETTINGS_FN].get_written_yaml() - self.assertIn('docker', data) - self.assertIn('timeout', data['docker']) - self.assertEqual(data['docker']['timeout'], 100) + self.assertIn('backend', data) + self.assertEqual(data['backend'], 'docker1') def test_withNewFile_shouldCreateFile(self): - r = self.command_test("config -g set docker.timeout=100") + r = self.command_test("config -g set backend=docker1") self.assertEqual(r.out.strip(), "File created.") self.assertIn(SETTINGS_FN, r.files) data = r.files[SETTINGS_FN].get_written_yaml() - self.assertIn('docker', data) - self.assertIn('timeout', data['docker']) - self.assertEqual(data['docker']['timeout'], 100) - - def test_withWrongType_shouldError(self): - r = self.command_test("config -g set docker.tls_verify=2", exit_code=1) - self.assertEqual( - "docker.tls_verify should be of type 'boolean', but was 'str'.", - r.err.strip()) - r = self.command_test("config -g set docker.timeout=a", exit_code=1) - self.assertEqual( - "docker.timeout should be of type 'integer', but was 'str'.", r.err.strip()) + self.assertIn('backend', data) + self.assertEqual(data['backend'], 'docker1') def test_withLocalAndGlobalSeletcted_shouldError(self): r = self.command_test("config -g -p set a=b", exit_code=1) @@ -398,16 +387,15 @@ def test_withWrongCommandSyntax_shouldError(self): class TestConfigRemoveAction(CliTestCase): SETTINGS = { 'version': '1.0', - 'docker': {'tls_verify': True} + 'backend': 'docker1' } def test_normal(self): - r = self.command_test("config -g remove docker.tls_verify", - settings=self.SETTINGS) + r = self.command_test("config -g remove backend", settings=self.SETTINGS) self.assertEqual("File successfully edited.", r.out.strip()) def test_withEmptyFile_shouldInformAboutEmptyFileAndNotError(self): - r = self.command_test("config -g remove docker.test") + r = self.command_test("config -g remove backend") self.assertEqual( "Cannot delete from config because config file doesn't exist.", r.out.strip())