WIP: Restore without volume

README.rst

@@ -29,4 +29,18 @@ Proof of concept.
 Features
 --------
 
+* starts temporary mongo containers
+
 * py.test integration
+* executes every test against multiple versions of mongo
+* resets the temporary mongo container before every test
+* restores archived mongo dumps before every test
+* support for single member replica sets
+* ...
+
+
+Usage
+-----
+
+Dockerdb uses `mongorestore --archive` to restore mongo dumps.
+To create an archived dump, run `mongodump --archive > ./dump`

dockerdb/mongo_pytest.py

@@ -21,15 +21,16 @@ def insert_data(client, data):
 
 
 def mongorestore(service, restore):
-    dst = os.path.join(service.share, 'dump')
-    if os.path.exists(dst):
-        shutil.rmtree(dst)
-    shutil.copytree(restore, dst)
-    command = ['mongorestore', dst]
-    exit_code, output = service.container.exec_run(command)
+    command = ['mongorestore', "--archive"]
+
+    with open(restore, 'rb') as restore_file:
+        exit_code, output = service.exec_run(command, restore_file)
 
     if exit_code != 0:
-        LOG.error(output.decode('utf-8'))
+        if isinstance(output, bytes):
+            output = output.decode('utf-8', errors='ignore')
+
+        LOG.error(output)
 
         raise subprocess.CalledProcessError(exit_code, command, output)
 

dockerdb/service.py

@@ -1,10 +1,13 @@
 import os
 import time
+import sys
 import shutil
 import tempfile
 import weakref
 import atexit
 import functools
+import select
+import socket
 
 import docker
 
@@ -53,6 +56,65 @@ def inspect(self):
         """get docker inspect data for container"""
         return self.client.api.inspect_container(self.container.id)
 
+    def exec_run(self, command, input_file):
+        """Execute a command and pipe data into it """
+        exec_info = self.client.api.exec_create(
+            self.container.name, command, stdin=True)
+
+        exec_id = exec_info['Id']
+
+        sock = self.client.api.exec_start(exec_id, socket=True)
+
+        if hasattr(sock, "_sock"):
+            # On python 3 docker-py returns a socket.SocketIO
+            # Wrapper doesn't give access to all needed methods
+            # So we extract the low level socket instance
+            sock = sock._sock
+
+        sock.setblocking(False)
+        output = b''
+
+        while True:
+            r_list, w_list, _ = select.select([sock], [sock], [])
+
+            if w_list:
+                file_data = input_file.read(4096)
+
+                if file_data == b'':
+                    break
+                else:
+                    sock.send(file_data)
+
+            if r_list:
+                socket_output = sock.recv(4096)
+                if socket_output == b'':
+                    break
+                else:
+                    output += socket_output
+
+        # TODO Find a way to avoid hardcoded sleep limits for python 2
+        # TODO Solutions is still unknown, see: STUD-583
+        python_version = sys.version_info[0]
+        if python_version < 3:
+            time.sleep(1)
+
+        sock.shutdown(socket.SHUT_WR)
+
+        if python_version < 3:
+            time.sleep(0.2)
+
+        sock.setblocking(True)
+        output += sock.recv(4096 * 100)
+        sock.close()
+
+        while True:
+            exec_info = self.client.api.exec_inspect(exec_id)
+            exit_code = exec_info['ExitCode']
+            if exit_code is not None:
+                break
+
+        return exit_code, output
+
     def ip_address(self):
         network_settings = self.inspect()['NetworkSettings']
         ip_address = network_settings['IPAddress']

tests/test_usage_mongo.py

@@ -7,7 +7,7 @@
 
 BASE_PATH = os.path.dirname(__file__)
 DUMP_PATH = os.path.join(BASE_PATH, 'dump')
-BROKEN_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump')
+ERROR_DUMP_PATH = os.path.join(BASE_PATH, 'error_dump')
 
 DATA = {
     'dbname': {
@@ -27,14 +27,8 @@
 
 def test_package_consistent():
     # ensure restore path does actually exist
-    assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.bson'))
-    assert os.path.exists(os.path.join(DUMP_PATH, 'test', 'user.metadata.json'))
-
-    dump_data_path = os.path.join(BROKEN_DUMP_PATH, 'test', 'user.bson')
-    dump_metadata_path = os.path.join(
-        BROKEN_DUMP_PATH, 'test', 'user.metadata.json')
-    assert os.path.exists(dump_data_path)
-    assert os.path.exists(dump_metadata_path)
+    assert os.path.exists(DUMP_PATH)
+    assert os.path.exists(ERROR_DUMP_PATH)
 
 
 def test_mongo_1(mongo):
@@ -65,9 +59,9 @@ def test_mongo_restore(mongo2):
 
     # Check that loading a broken dump throws an error
     with pytest.raises(subprocess.CalledProcessError) as exc_info:
-        dockerdb.mongo_pytest.mongorestore(mongo2, BROKEN_DUMP_PATH)
+        dockerdb.mongo_pytest.mongorestore(mongo2, ERROR_DUMP_PATH)
 
     exception = exc_info.value
     assert exception.cmd[0] == 'mongorestore'
     assert exception.returncode == 1
-    assert 'unexpected EOF' in exception.output.decode('utf-8')
+    assert 'unexpected EOF' in exception.output