From 4af61d31a4ee4feb7b8ab87537bcb7ee3ee11051 Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Thu, 22 Oct 2020 12:44:48 -0400 Subject: [PATCH 1/2] Add contrib/snapshot_leveldb.py A copy of snapshot_lmdb with adapations to leveldb. The main difference is that it uses shutil.copytree() to copy the whole LevelDB directory. This script is only safe when the dredis server is stopped to guarantee that all LevelDB in-memory operations made it to the filesystem! --- dredis/contrib/snapshot_leveldb.py | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 dredis/contrib/snapshot_leveldb.py diff --git a/dredis/contrib/snapshot_leveldb.py b/dredis/contrib/snapshot_leveldb.py new file mode 100644 index 0000000..d6f5052 --- /dev/null +++ b/dredis/contrib/snapshot_leveldb.py @@ -0,0 +1,65 @@ +import argparse +import datetime +import json +import logging +import shutil +import sys + +from dredis import db +from dredis.keyspace import Keyspace + +logger = logging.getLogger(__name__) +BACKEND = 'leveldb' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--dir', help='dredis data directory', required=True) + parser.add_argument('--output', choices=['leveldb', 'rdb'], help='output file type', required=True) + parser.add_argument('--backend-option', action='append', + help='database backend options (e.g., --backend-option map_size=BYTES)') + args = parser.parse_args() + + db_backend_options = {} + if args.backend_option: + for option in args.backend_option: + if '=' not in option: + logger.error('Expected `key=value` pairs for --backend-option parameter') + sys.exit(1) + key, value = map(str.strip, option.split('=')) + db_backend_options[key] = json.loads(value) + + logger.info("Copying LevelDB files...") + output_dir = copy_dirs(args.dir, db_backend_options) + + if args.output == 'rdb': + logger.info("Saving RDB file...") + save_rdb(output_dir, db_backend_options) + shutil.rmtree(output_dir) + + logger.info("Done!") + + +def save_rdb(output_dir, db_backend_options): + db.DB_MANAGER.setup_dbs(output_dir, BACKEND, db_backend_options) + keyspace = Keyspace() + keyspace.save() + + +def copy_dirs(input_basedir, db_backend_options): + output_basedir = datetime.datetime.utcnow().strftime('leveldb-backup_%Y-%m-%dT%H:%M:%S') + shutil.copytree(input_basedir, output_basedir) + return output_basedir + + +def setup_logging(level): + logger.setLevel(level) + formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s') + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + logger.addHandler(handler) + + +if __name__ == '__main__': + setup_logging(logging.INFO) + main() From 0bfecee1943b0493f5b95acbf08457f12b70b13d Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Mon, 26 Oct 2020 10:12:06 -0400 Subject: [PATCH 2/2] Add leveldb to Travis CI The CI started complaining about missing leveldb/db.h when installing plyvel. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c9a4fbd..0b649de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ python: - "2.7" services: - redis-server +before_install: + - sudo apt-get update -q + - sudo apt-get install -y libleveldb-dev install: make setup env: - TEST_SUITE=fulltests BACKEND="--backend memory"