From 649b2a0386e86797ef533c172675168f7f072bc7 Mon Sep 17 00:00:00 2001 From: Fabian Elsner Date: Thu, 12 Mar 2020 16:34:06 +0100 Subject: [PATCH 1/5] Safely restore mongo dumps without relying on volume mapping, added readme --- README.rst | 10 ++++++ dockerdb/mongo_pytest.py | 10 +++--- dockerdb/service.py | 44 +++++++++++++++++++++++ tests/dump | Bin 0 -> 519 bytes tests/dump/test/user.bson | Bin 65 -> 0 bytes tests/dump/test/user.metadata.json | 1 - tests/error_dump | Bin 0 -> 70 bytes tests/error_dump/test/user.bson | Bin 52 -> 0 bytes tests/error_dump/test/user.metadata.json | 1 - tests/test_usage_mongo.py | 14 +++----- 10 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 tests/dump delete mode 100644 tests/dump/test/user.bson delete mode 100644 tests/dump/test/user.metadata.json create mode 100644 tests/error_dump delete mode 100644 tests/error_dump/test/user.bson delete mode 100644 tests/error_dump/test/user.metadata.json diff --git a/README.rst b/README.rst index 033d182..6c216d7 100644 --- a/README.rst +++ b/README.rst @@ -30,3 +30,13 @@ Features -------- * py.test integration +* reset your mongo instance before every test +* restore archived mongo dumps before every test +* ... + + +Usage +----- + +Dockerdb uses `mongorestore --archive` to restore mongo dumps. +To create an archived dump, run `mongodump --archive > ./dump` diff --git a/dockerdb/mongo_pytest.py b/dockerdb/mongo_pytest.py index 6070729..6307e37 100644 --- a/dockerdb/mongo_pytest.py +++ b/dockerdb/mongo_pytest.py @@ -21,12 +21,10 @@ def insert_data(client, data): def mongorestore(service, restore): - dst = os.path.join(service.share, 'dump') - if os.path.exists(dst): - shutil.rmtree(dst) - shutil.copytree(restore, dst) - command = ['mongorestore', dst] - exit_code, output = service.container.exec_run(command) + command = ['mongorestore', "--archive"] + + with open(restore, 'rb') as restore_file: + exit_code, output = service.exec_run(command, restore_file) if exit_code != 0: LOG.error(output.decode('utf-8')) diff --git a/dockerdb/service.py b/dockerdb/service.py index ec2d98c..16d0157 100644 --- a/dockerdb/service.py +++ b/dockerdb/service.py @@ -5,6 +5,8 @@ import weakref import atexit import functools +import select +import socket import docker @@ -53,6 +55,48 @@ def inspect(self): """get docker inspect data for container""" return self.client.api.inspect_container(self.container.id) + def exec_run(self, command, input_file=None): + """Execute a command and pipe data into it """ + exec_info = self.client.api.exec_create(self.container.name, command, stdin=True) + exec_id = exec_info['Id'] + + sock = self.client.api.exec_start(exec_id, socket=True) + + # Wrapper doesn't give access to all needed methods + sock = sock._sock + sock.setblocking(False) + output = b'' + + while True: + r_list, w_list, _ = select.select([sock], [sock], []) + + if w_list and input_file: + file_data = input_file.read(4096) + if file_data == b'': + break + else: + sock.send(file_data) + + if r_list: + socket_output = sock.recv(4096) + if socket_output == b'': + break + else: + output += socket_output + + sock.shutdown(socket.SHUT_WR) + sock.setblocking(True) + output += sock.recv(4096) + sock.close() + + while True: + exec_info = self.client.api.exec_inspect(exec_id) + exit_code = exec_info['ExitCode'] + if exit_code is not None: + break + + return exit_code, output + def ip_address(self): network_settings = self.inspect()['NetworkSettings'] ip_address = network_settings['IPAddress'] diff --git a/tests/dump b/tests/dump new file mode 100644 index 0000000000000000000000000000000000000000..e490a8ab230fa8fc0dcc4a892dc49dee642cac03 GIT binary patch literal 519 zcmb7AJxjzu5FLYGjd0l550=Mqm*gauP7W185VR3QLoS(MFxecto73}1VWXAU*l2P8 zO0W^cAEHiH)WSmCVupF|&CJ`$U+*so02)ytqEbo{I*5uaqe!QPP%z7!H6p3FGDW-9 z0ZWl&Ecn9bS>(3tRvRo`6xlbvz_|P*;JF?Y4`7b7CZ)mpS!2MGhk})p4&zXVFlF5Y zi=nZB{-kPRDq=b(h5q3LM-1VrDDwfyZK2Ue8(DHnnI8GiLqm*7Wh}?q$cA{{Z^?mFNHf literal 0 HcmV?d00001 diff --git a/tests/dump/test/user.bson b/tests/dump/test/user.bson deleted file mode 100644 index d016e6c11b8751cec928c4ea65b0d6d52634c57c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65 zcmZ={U|?X6&rD&6_?Ma?ej_-peO7NOQ)zK(5d#}gDlsKDGmn9(AhEc(JijP~ffp!N Sl$w~6Q(2svT$EbEzyJV-w-c-Y diff --git a/tests/dump/test/user.metadata.json b/tests/dump/test/user.metadata.json deleted file mode 100644 index 1a37086..0000000 --- a/tests/dump/test/user.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"options":{},"indexes":[{"v":2,"key":{"_id":1},"name":"_id_","ns":"test.user"}]} \ No newline at end of file diff --git a/tests/error_dump b/tests/error_dump new file mode 100644 index 0000000000000000000000000000000000000000..db69414fdd52cdc0cf1285d2cd7f8a88f563bbfc GIT binary patch literal 70 zcmd0OG_x_4fq_9FIX^GCw5TXGuY`dW$S*BUEn?ts^><@n6mkx7W_Z7UuR9Q5Z37ZO K<{9 diff --git a/tests/error_dump/test/user.metadata.json b/tests/error_dump/test/user.metadata.json deleted file mode 100644 index 1a37086..0000000 --- a/tests/error_dump/test/user.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"options":{},"indexes":[{"v":2,"key":{"_id":1},"name":"_id_","ns":"test.user"}]} \ No newline at end of file diff --git a/tests/test_usage_mongo.py b/tests/test_usage_mongo.py index 6c9805e..170bb3e 100644 --- a/tests/test_usage_mongo.py +++ b/tests/test_usage_mongo.py @@ -7,7 +7,7 @@ BASE_PATH = os.path.dirname(__file__) DUMP_PATH = os.path.join(BASE_PATH, 'dump') -BROKEN_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump') +ERROR_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump') DATA = { 'dbname': { @@ -27,14 +27,8 @@ def test_package_consistent(): # ensure restore path does actually exist - assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.bson')) - assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.metadata.json')) - - dump_data_path = os.path.join(BROKEN_DUMP_PATH, 'test', 'user.bson') - dump_metadata_path = os.path.join( - BROKEN_DUMP_PATH, 'test', 'user.metadata.json') - assert os.path.exists(dump_data_path) - assert os.path.exists(dump_metadata_path) + assert os.path.exists(DUMP_PATH) + assert os.path.exists(ERROR_DUMP_PATH) def test_mongo_1(mongo): @@ -65,7 +59,7 @@ def test_mongo_restore(mongo2): # Check that loading a broken dump throws an error with pytest.raises(subprocess.CalledProcessError) as exc_info: - dockerdb.mongo_pytest.mongorestore(mongo2, BROKEN_DUMP_PATH) + dockerdb.mongo_pytest.mongorestore(mongo2, ERROR_DUMP_PATH) exception = exc_info.value assert exception.cmd[0] == 'mongorestore' From 286bca09b5a8750529ea0c7c795d91aed459f5f2 Mon Sep 17 00:00:00 2001 From: Fabian Elsner Date: Thu, 12 Mar 2020 16:46:49 +0100 Subject: [PATCH 2/5] Updated features in readme --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6c216d7..d1981e2 100644 --- a/README.rst +++ b/README.rst @@ -29,9 +29,13 @@ Proof of concept. Features -------- +* starts temporary mongo containers + * py.test integration -* reset your mongo instance before every test -* restore archived mongo dumps before every test +* executes every test against multiple versions of mongo +* resets the temporary mongo container before every test +* restores archived mongo dumps before every test +* support for single member replica sets * ... From b8bdf1a595e9677a7d867abcc0d7f108e601de83 Mon Sep 17 00:00:00 2001 From: Fabian Elsner Date: Thu, 12 Mar 2020 16:48:01 +0100 Subject: [PATCH 3/5] Input file should not be optional unless we actually test that case --- dockerdb/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerdb/service.py b/dockerdb/service.py index 16d0157..fdf4711 100644 --- a/dockerdb/service.py +++ b/dockerdb/service.py @@ -55,7 +55,7 @@ def inspect(self): """get docker inspect data for container""" return self.client.api.inspect_container(self.container.id) - def exec_run(self, command, input_file=None): + def exec_run(self, command, input_file): """Execute a command and pipe data into it """ exec_info = self.client.api.exec_create(self.container.name, command, stdin=True) exec_id = exec_info['Id'] @@ -70,7 +70,7 @@ def exec_run(self, command, input_file=None): while True: r_list, w_list, _ = select.select([sock], [sock], []) - if w_list and input_file: + if w_list: file_data = input_file.read(4096) if file_data == b'': break From 8038fe1ad934f92e7e561e6db3a1e33dc9681748 Mon Sep 17 00:00:00 2001 From: Fabian Elsner Date: Thu, 12 Mar 2020 17:08:01 +0100 Subject: [PATCH 4/5] Make code python2 safe --- dockerdb/service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dockerdb/service.py b/dockerdb/service.py index fdf4711..78de29c 100644 --- a/dockerdb/service.py +++ b/dockerdb/service.py @@ -62,8 +62,12 @@ def exec_run(self, command, input_file): sock = self.client.api.exec_start(exec_id, socket=True) - # Wrapper doesn't give access to all needed methods - sock = sock._sock + if hasattr(sock, "_sock"): + # On python 3 docker-py returns a socket.SocketIO + # Wrapper doesn't give access to all needed methods + # So we extract the low level socket instance + sock = sock._sock + sock.setblocking(False) output = b'' From 9a81d6ab618317be97e932dfa09e4cc42811bfe2 Mon Sep 17 00:00:00 2001 From: Fabian Elsner Date: Mon, 16 Mar 2020 08:23:51 +0100 Subject: [PATCH 5/5] Hacky fixed timing problem and exceptions in python2.7 --- dockerdb/mongo_pytest.py | 5 ++++- dockerdb/service.py | 18 ++++++++++++++++-- tests/test_usage_mongo.py | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dockerdb/mongo_pytest.py b/dockerdb/mongo_pytest.py index 6307e37..537b773 100644 --- a/dockerdb/mongo_pytest.py +++ b/dockerdb/mongo_pytest.py @@ -27,7 +27,10 @@ def mongorestore(service, restore): exit_code, output = service.exec_run(command, restore_file) if exit_code != 0: - LOG.error(output.decode('utf-8')) + if isinstance(output, bytes): + output = output.decode('utf-8', errors='ignore') + + LOG.error(output) raise subprocess.CalledProcessError(exit_code, command, output) diff --git a/dockerdb/service.py b/dockerdb/service.py index 78de29c..88d1e1c 100644 --- a/dockerdb/service.py +++ b/dockerdb/service.py @@ -1,5 +1,6 @@ import os import time +import sys import shutil import tempfile import weakref @@ -57,7 +58,9 @@ def inspect(self): def exec_run(self, command, input_file): """Execute a command and pipe data into it """ - exec_info = self.client.api.exec_create(self.container.name, command, stdin=True) + exec_info = self.client.api.exec_create( + self.container.name, command, stdin=True) + exec_id = exec_info['Id'] sock = self.client.api.exec_start(exec_id, socket=True) @@ -76,6 +79,7 @@ def exec_run(self, command, input_file): if w_list: file_data = input_file.read(4096) + if file_data == b'': break else: @@ -88,9 +92,19 @@ def exec_run(self, command, input_file): else: output += socket_output + # TODO Find a way to avoid hardcoded sleep limits for python 2 + # TODO Solutions is still unknown, see: STUD-583 + python_version = sys.version_info[0] + if python_version < 3: + time.sleep(1) + sock.shutdown(socket.SHUT_WR) + + if python_version < 3: + time.sleep(0.2) + sock.setblocking(True) - output += sock.recv(4096) + output += sock.recv(4096 * 100) sock.close() while True: diff --git a/tests/test_usage_mongo.py b/tests/test_usage_mongo.py index 170bb3e..39ef9bd 100644 --- a/tests/test_usage_mongo.py +++ b/tests/test_usage_mongo.py @@ -64,4 +64,4 @@ def test_mongo_restore(mongo2): exception = exc_info.value assert exception.cmd[0] == 'mongorestore' assert exception.returncode == 1 - assert 'unexpected EOF' in exception.output.decode('utf-8') + assert 'unexpected EOF' in exception.output