diff --git a/README.rst b/README.rst index 033d182..d1981e2 100644 --- a/README.rst +++ b/README.rst @@ -29,4 +29,18 @@ Proof of concept. Features -------- +* starts temporary mongo containers + * py.test integration +* executes every test against multiple versions of mongo +* resets the temporary mongo container before every test +* restores archived mongo dumps before every test +* support for single member replica sets +* ... + + +Usage +----- + +Dockerdb uses `mongorestore --archive` to restore mongo dumps. +To create an archived dump, run `mongodump --archive > ./dump` diff --git a/dockerdb/mongo_pytest.py b/dockerdb/mongo_pytest.py index 6070729..537b773 100644 --- a/dockerdb/mongo_pytest.py +++ b/dockerdb/mongo_pytest.py @@ -21,15 +21,16 @@ def insert_data(client, data): def mongorestore(service, restore): - dst = os.path.join(service.share, 'dump') - if os.path.exists(dst): - shutil.rmtree(dst) - shutil.copytree(restore, dst) - command = ['mongorestore', dst] - exit_code, output = service.container.exec_run(command) + command = ['mongorestore', "--archive"] + + with open(restore, 'rb') as restore_file: + exit_code, output = service.exec_run(command, restore_file) if exit_code != 0: - LOG.error(output.decode('utf-8')) + if isinstance(output, bytes): + output = output.decode('utf-8', errors='ignore') + + LOG.error(output) raise subprocess.CalledProcessError(exit_code, command, output) diff --git a/dockerdb/service.py b/dockerdb/service.py index ec2d98c..88d1e1c 100644 --- a/dockerdb/service.py +++ b/dockerdb/service.py @@ -1,10 +1,13 @@ import os import time +import sys import shutil import tempfile import weakref import atexit import functools +import select +import socket import docker @@ -53,6 +56,65 @@ def inspect(self): """get docker inspect data for container""" return self.client.api.inspect_container(self.container.id) + def exec_run(self, command, input_file): + """Execute a command and pipe data into it """ + exec_info = self.client.api.exec_create( + self.container.name, command, stdin=True) + + exec_id = exec_info['Id'] + + sock = self.client.api.exec_start(exec_id, socket=True) + + if hasattr(sock, "_sock"): + # On python 3 docker-py returns a socket.SocketIO + # Wrapper doesn't give access to all needed methods + # So we extract the low level socket instance + sock = sock._sock + + sock.setblocking(False) + output = b'' + + while True: + r_list, w_list, _ = select.select([sock], [sock], []) + + if w_list: + file_data = input_file.read(4096) + + if file_data == b'': + break + else: + sock.send(file_data) + + if r_list: + socket_output = sock.recv(4096) + if socket_output == b'': + break + else: + output += socket_output + + # TODO Find a way to avoid hardcoded sleep limits for python 2 + # TODO Solutions is still unknown, see: STUD-583 + python_version = sys.version_info[0] + if python_version < 3: + time.sleep(1) + + sock.shutdown(socket.SHUT_WR) + + if python_version < 3: + time.sleep(0.2) + + sock.setblocking(True) + output += sock.recv(4096 * 100) + sock.close() + + while True: + exec_info = self.client.api.exec_inspect(exec_id) + exit_code = exec_info['ExitCode'] + if exit_code is not None: + break + + return exit_code, output + def ip_address(self): network_settings = self.inspect()['NetworkSettings'] ip_address = network_settings['IPAddress'] diff --git a/tests/dump b/tests/dump new file mode 100644 index 0000000..e490a8a Binary files /dev/null and b/tests/dump differ diff --git a/tests/dump/test/user.bson b/tests/dump/test/user.bson deleted file mode 100644 index d016e6c..0000000 Binary files a/tests/dump/test/user.bson and /dev/null differ diff --git a/tests/dump/test/user.metadata.json b/tests/dump/test/user.metadata.json deleted file mode 100644 index 1a37086..0000000 --- a/tests/dump/test/user.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"options":{},"indexes":[{"v":2,"key":{"_id":1},"name":"_id_","ns":"test.user"}]} \ No newline at end of file diff --git a/tests/error_dump b/tests/error_dump new file mode 100644 index 0000000..db69414 Binary files /dev/null and b/tests/error_dump differ diff --git a/tests/error_dump/test/user.bson b/tests/error_dump/test/user.bson deleted file mode 100644 index 6570cc0..0000000 Binary files a/tests/error_dump/test/user.bson and /dev/null differ diff --git a/tests/error_dump/test/user.metadata.json b/tests/error_dump/test/user.metadata.json deleted file mode 100644 index 1a37086..0000000 --- a/tests/error_dump/test/user.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"options":{},"indexes":[{"v":2,"key":{"_id":1},"name":"_id_","ns":"test.user"}]} \ No newline at end of file diff --git a/tests/test_usage_mongo.py b/tests/test_usage_mongo.py index 6c9805e..39ef9bd 100644 --- a/tests/test_usage_mongo.py +++ b/tests/test_usage_mongo.py @@ -7,7 +7,7 @@ BASE_PATH = os.path.dirname(__file__) DUMP_PATH = os.path.join(BASE_PATH, 'dump') -BROKEN_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump') +ERROR_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump') DATA = { 'dbname': { @@ -27,14 +27,8 @@ def test_package_consistent(): # ensure restore path does actually exist - assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.bson')) - assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.metadata.json')) - - dump_data_path = os.path.join(BROKEN_DUMP_PATH, 'test', 'user.bson') - dump_metadata_path = os.path.join( - BROKEN_DUMP_PATH, 'test', 'user.metadata.json') - assert os.path.exists(dump_data_path) - assert os.path.exists(dump_metadata_path) + assert os.path.exists(DUMP_PATH) + assert os.path.exists(ERROR_DUMP_PATH) def test_mongo_1(mongo): @@ -65,9 +59,9 @@ def test_mongo_restore(mongo2): # Check that loading a broken dump throws an error with pytest.raises(subprocess.CalledProcessError) as exc_info: - dockerdb.mongo_pytest.mongorestore(mongo2, BROKEN_DUMP_PATH) + dockerdb.mongo_pytest.mongorestore(mongo2, ERROR_DUMP_PATH) exception = exc_info.value assert exception.cmd[0] == 'mongorestore' assert exception.returncode == 1 - assert 'unexpected EOF' in exception.output.decode('utf-8') + assert 'unexpected EOF' in exception.output