diff --git a/.gitignore b/.gitignore index c9b568f..087e8f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc *.swp +/.tox/ diff --git a/.pep8 b/.pep8 new file mode 100644 index 0000000..c1b094e --- /dev/null +++ b/.pep8 @@ -0,0 +1,2 @@ +[pep8] +max-line-length = 99 diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..4f2c1d1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.6.6 diff --git a/dbclean/ChangeLog b/dbclean/ChangeLog new file mode 100644 index 0000000..25d7878 --- /dev/null +++ b/dbclean/ChangeLog @@ -0,0 +1,9 @@ +2016-02-14: +* Fix --version parameter. +* Use a text-width of 99 chars. + +2013-07-21: +* Massive pep8 cleanup +* Update documentation (remove old --section parameter) +* Add this ChangeLog +* Add the "last" option, dbclean will always keep the given last backups. diff --git a/dbclean/README b/dbclean/README index 443ff84..131645d 100644 --- a/dbclean/README +++ b/dbclean/README @@ -8,7 +8,9 @@ are matched to those of dbdump, so they work together. === Installation === Simply check out the git-repository: - git clone http://git.git.fsinf.at/fsinf/db-backup.git + + git clone https://github.com/mathiasertl/db-backup.git + If you don't want to specify the full path, you can of course copy dbclean.py somewhere in your path (/usr/local/bin is usually good). @@ -22,5 +24,5 @@ A sample configuration file is included with the source code, see: dbclean.conf.example After that, you can start the script with - dbclean.py --section=example + dbclean.py example where example is the section in your config-file. diff --git a/dbclean/dbclean.conf.example b/dbclean/dbclean.conf.example index ad18a99..2cc52d8 100644 --- a/dbclean/dbclean.conf.example +++ b/dbclean/dbclean.conf.example @@ -4,6 +4,8 @@ # sections. #hourly = 72 #daily = 31 +# Always keep the last three backups +#last = 3 # # NOTE: You can also use the interpolation feature provided by the # ConfigParser python module. The following line is used in the diff --git a/dbclean/dbclean.py b/dbclean/dbclean.py index 6985414..1e1c6e7 100755 --- a/dbclean/dbclean.py +++ b/dbclean/dbclean.py @@ -1,61 +1,70 @@ #!/usr/bin/env python3 - -""" -This program is designed to clean files from a specified directory. -The program is designed to work together with dbdump.py, so it is -basically designed to clean out regular database dumps. The files -are kept at a certain granularity (so daily backups will be kept -for a month, monthly backups for a year, etc. -Please see the README file for how to use this script and supported -features. You might also try calling this program with '--help'. - -Copyright 2009 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os, time, sys, calendar, configparser, argparse - -config_file = ['/etc/dbclean/dbclean.conf', os.path.expanduser('~/.dbclean.conf')] - -parser = argparse.ArgumentParser(version="%prog 1.0", - description = """Cleanup regular database dumps created by dbdump. This script keeps backups - at given intervals for a given amount of time.""") -parser.add_argument('-c', '--config', type=str, dest='config', action='append', default=config_file, - help="""Additional config-files to use (default: %(default)s). Can be given multiple times - to name multiple config-files.""") -parser.add_argument('section', action='store', type=str, - help="Section in the config-file to use." ) +# +# This program is designed to clean files from a specified directory. The program is designed to +# work together with dbdump.py, so it is basically designed to clean out regular database dumps. +# The files are kept at a certain granularity (so daily backups will be kept for a month, monthly +# backups for a year, etc. Please see the README file for how to use this script and supported +# features. You might also try calling this program with '--help'. +# +# Copyright 2009 - 2019 Mathias Ertl +# +# This program is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import calendar +import configparser +import os +import sys +import time + + +def err(msg, *args): + print(msg % args, file=sys.stderr) + + +config_file = [ + '/etc/dbclean/dbclean.conf', + os.path.expanduser('~/.dbclean.conf') +] + +parser = argparse.ArgumentParser( + description="""Cleanup regular database dumps created by dbdump. This script keeps backups at +given intervals for a given amount of time.""") +parser.add_argument('--version', action='version', version='%(prog)s 1.1') +parser.add_argument( + '-c', '--config', type=str, dest='config', action='append', + default=config_file, help="""Additional config-files to use (default: %(default)s). Can be + given multiple times to name multiple config-files.""") +parser.add_argument('section', action='store', type=str, help="Section in the config-file to use.") args = parser.parse_args() -if args.section=='DEFAULT': +if args.section == 'DEFAULT': parser.error("--section must not be 'DEFAULT'.") -config = configparser.SafeConfigParser({ +config = configparser.ConfigParser({ 'format': '%%Y-%%m-%%d_%%H:%%M:%%S', 'hourly': '24', 'daily': '31', - 'monthly': '12', 'yearly': '3' + 'monthly': '12', 'yearly': '3', + 'last': '3', }) if not config.read(args.config): parser.error("No config-files could be read.") # check validity of config-file: if args.section not in config: - print("Error: %s: No section found with that name." % args.section, file=sys.stderr) + err("Error: %s: No section found with that name.", args.section) sys.exit(1) if 'datadir' not in config[args.section]: - print("Error: %s: Section does not contain option 'datadir'." % args.section, file=sys.stderr) + err("Error: %s: Section does not contain option 'datadir'.", args.section) sys.exit(1) # get directory containing backups: @@ -63,10 +72,10 @@ # check that given directory exists and is a directory: if not os.path.exists(datadir): - print("Error: %s: No such directory." % (datadir), file=sys.stderr) + err("Error: %s: No such directory.", datadir) sys.exit(1) elif not os.path.isdir(datadir): - print("Error: %s: Not a directory." % (datadir), file=sys.stderr) + err("Error: %s: Not a directory.", datadir) sys.exit(1) timeformat = config[args.section]['format'] @@ -74,106 +83,98 @@ daily = int(config[args.section]['daily']) monthly = int(config[args.section]['monthly']) yearly = int(config[args.section]['yearly']) +last = int(config[args.section]['last']) +now = time.time() + class backup(): files = [] - def __init__( self, time, base, file ): + def __init__(self, time, base, file): self.time = time self.base = base - self.files = [ file ] + self.files = [file] - def add( self, file ): - self.files.append( file ) + def add(self, file): + self.files.append(file) - def is_daily( self ): - if self.time[3] == 0: + def is_daily(self): + if self.time.tm_hour == 0: return True else: return False - def is_monthly( self ): - if self.is_daily() and self.time[2] == 1: + def is_monthly(self): + if self.is_daily() and self.time.tm_mday == 1: return True else: return False - def is_yearly( self ): - if self.is_monthly() and self.time[1] == 1: + def is_yearly(self): + if self.is_monthly() and self.time.tm_mon == 1: return True else: return False - def remove( self ): + def remove(self): for file in self.files: - os.remove( file ) + os.remove(file) - def __str__( self ): - return "%s in %s" %(self.files, self.base) + def __str__(self): + return "%s in %s" % (self.files, self.base) -now = time.time() # loop through each dir in datadir -for dir in os.listdir( datadir ): - if dir.startswith( '.' ): +for dir in os.listdir(datadir): + if dir.startswith('.'): # skip hidden directories continue + if dir == 'lost+found': + continue - fullpath = os.path.normpath( datadir + '/' + dir ) - if not os.path.isdir( fullpath ): - print( "Warning: %s: Not a directory." % (fullpath) ) + fullpath = os.path.normpath(datadir + '/' + dir) + if not os.path.isdir(fullpath): + print("Warning: %s: Not a directory." % fullpath) continue - os.chdir( fullpath ) + os.chdir(fullpath) backups = {} - files = os.listdir( '.' ) + files = os.listdir('.') files.sort() for file in files: - filestamp = file.split( '.' )[0] + filestamp = file.split('.')[0] timestamp = '' try: - timestamp = time.strptime( filestamp, timeformat ) + timestamp = time.strptime(filestamp, timeformat) except ValueError as e: - print( '%s: %s' %(file, e) ) + print('%s: %s' % (file, e)) + continue - if timestamp not in list( backups.keys() ): - backups[timestamp] = backup( timestamp, fullpath, file ) + if timestamp not in list(backups.keys()): + backups[timestamp] = backup(timestamp, fullpath, file) else: - backups[timestamp].add( file ) + backups[timestamp].add(file) + backup_items = sorted(backups.items(), key=lambda t: t[0]) + if last: # NOTE: if last == 0, the slice returns an empty list! + backup_items = backup_items[:-last] - for stamp in list(backups.keys()): + for stamp, bck in backup_items: bck = backups[stamp] - bck_seconds = calendar.timegm( stamp ) + bck_seconds = calendar.timegm(stamp) + + if bck_seconds > now - (hourly * 3600): + continue + + if bck.is_daily() and bck_seconds > now - (daily * 86400): + continue + + if bck.is_monthly() and bck_seconds > now - (monthly * 2678400): + continue - if bck_seconds > now - ( hourly * 3600 ): -# print ( "%s is hourly and will be kept" % ( time.asctime( stamp ) ) ) + if bck.is_yearly() and bck_seconds > now - (yearly * 31622400): continue -# else: -# print ("%s is hourly but to old." % ( time.asctime( stamp ) ) ) - - if bck.is_daily(): - if bck_seconds > now - ( daily * 86400 ): -# print( "%s is daily and will be kept." % ( time.asctime( stamp ) ) ) - continue -# else: -# print ("%s is daily but to old." % ( time.asctime( stamp ) ) ) - - if bck.is_monthly(): - if bck_seconds > now - ( monthly * 2678400 ): -# print( "%s is monthly and will be kept." % ( time.asctime( stamp ) ) ) - continue -# else: -# print ("%s is monthly but to old." % ( time.asctime( stamp ) ) ) - - if bck.is_yearly(): - if bck_seconds > now - ( yearly * 31622400 ): -# print( "%s is yearly and will be kept." % ( time.asctime( stamp ) ) ) - continue -# else: -# print ("%s is yearly but to old." % ( time.asctime( stamp ) ) ) -# print( "%s will be removed." % ( time.asctime( stamp ) ) ) bck.remove() diff --git a/dbdump/ChangeLog b/dbdump/ChangeLog new file mode 100644 index 0000000..6b06bc6 --- /dev/null +++ b/dbdump/ChangeLog @@ -0,0 +1,8 @@ +# 1.2 +* flake8/isort cleanup +* Add new options: ssh-timeout and ssh-options +* Switch to generating sha256 checksums + +2013-07-21: +* pep8 cleanup +* add this ChangeLog diff --git a/dbdump/README b/dbdump/README index b1188db..0f0e87a 100644 --- a/dbdump/README +++ b/dbdump/README @@ -13,7 +13,7 @@ to get it working. Simply check out the git-repository: - git clone https://git.fsinf.at/fsinf/db-backup.git + git clone https://github.com/mathiasertl/db-backup.git If you don't want to specify the full path, you can of course copy dbdump.py somewhere in your path (/usr/local/bin is usually good). Take care to copy diff --git a/dbdump/dbdump.conf.example b/dbdump/dbdump.conf.example index e9d56a0..25fd734 100644 --- a/dbdump/dbdump.conf.example +++ b/dbdump/dbdump.conf.example @@ -11,74 +11,92 @@ # Note that the default is the section name, if you do not explicitly set # this in this file. #backend=mysql -# + # The base directory to dump your databases to. # Default is /var/backups/ #datadir=/var/backups/mysql -# + # Use su to drop privileges to the specified user for all SQL related # commands (optional): #su=mysql -# + # Store dumps on a remote location via SSH. REMOTE will be passed to ssh # unchanged (optional): remote = user@backup.example.com -# + +# Store dumps with borgbackup. Specify the common directory, each db will have +# its own repository. +#borg = user@backup.example.com + +# Encrypt borg backups, store the keyfile on the server and encrypt it with the +# password specified in borg-key, if this parameter is given. +#borg-key = add-a-secret-password-here + +# Override the default ssh connect-timeout of 10 seconds: +#ssh-timeout = 10 + +# Space-separated list of any other SSH options to pass to ssh, for example: +#ssh-options = -o Compression=yes -v + # Sign and/or encrypt backups using the specified GPG keys (optional): #sign-key = dbdump@hostname.example.net #recipient = admin@hostname.example.net -# + # You can also use the interpolation feature provided by the # ConfigParser python module. -# + # You can even specify new settings to use in later interpolation: #hostname = example.com -# + # We set the default datadir using dynamic interpolation: #datadir = /var/backups/%(hostname)s/%(backend)s/ -# + #[mysql] # Dump MySQL databases. Because of the settings in the DEFAULT section, # the backend used here is 'mysql' and datadir evaluates to: # /var/backups/example.com/mysql/ # ... and backups will be signed and encrypted (see first lines in the # DEFAULT section). -# + # MYSQL OPTIONS: # # Use a custom defaults-file. This file should be used to specify access # credentials (default given here): #mysql-defaults = ~/.my.cnf -# + # Ignore given tables when dumping. This can be useful if a certain # table does not use InnoDB and the database could not be dumped in a # single transaction otherwise (optional): #mysql-ignore-tables = db_foo.table_bla db_bar.table_hugo -# + #[postgresql] # Dump PostgreSQL databases. -# + # lets override the default datadir, just to be clear: #datadir=/somewhere/else/ -# + # POSTGRESQL OPTIONS: -# + # Additional command-line parameters for psql and pgdump (optional): +#postgresql-connectstring = host=localhost port=5432 #postgresql-psql-opts = --someopt #postgresql-pgdump-opts = --otheropt -# + #[ejabberd] # Dump ejabberd databases. -# + # EJABBERD OPTIONS: # # Dump database from this ejabberd node (optional): #ejabberd-node = ejabberd -# + +# Any other command-line arguments to ejabberd +#ejabberd-options = --no-timeout + # Authenticate with the erlang node. This specifies a normal account on # the jabber server (optional): #ejabberd-auth = user jabber.example.com password -# + # Base directory where the ejabberd database is stored (default given # here): #ejabberd-base-dir = /var/lib/ejabberd diff --git a/dbdump/dbdump.py b/dbdump/dbdump.py index 3e1a695..a787a07 100755 --- a/dbdump/dbdump.py +++ b/dbdump/dbdump.py @@ -1,66 +1,77 @@ #!/usr/bin/env python3 +# +# This program is designed to regulary dump SQL databases into a specified directory for backup +# purposes. Please see the README file for how to use this script and supported features. You might +# also try calling this program with '--help'. +# +# Copyright 2009-2019 Mathias Ertl +# +# This program is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this program. If +# not, see . + +import argparse +import configparser +import os +import sys +import time + +from libdump import ejabberd +from libdump import mysql +from libdump import postgresql + + +def err(msg, *args): + print(msg % args, file=sys.stderr) -""" -This program is designed to regulary dump SQL databases into a -specified directory for backup purposes. Please see the README file -for how to use this script and supported features. You might also -try calling this program with '--help'. - -Copyright 2009-2012 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os, sys, time, argparse, configparser -from libdump import * config_file = ['/etc/dbdump/dbdump.conf', os.path.expanduser('~/.dbdump.conf')] parser = argparse.ArgumentParser(description="Dump databases to a specified directory.") -parser.add_argument('--version', action='version', version="%(prog)s 1.0") -parser.add_argument('-c', '--config', action='append', default=config_file, - help="""Additional config-files to use (default: %(default)s). Can be given multiple times - to name multiple config-files.""") +parser.add_argument('--version', action='version', version="%(prog)s 1.1") +parser.add_argument( + '-c', '--config', action='append', default=config_file, + help="""Additional config-files to use (default: %(default)s). Can be given multiple times to + name multiple config-files.""") parser.add_argument('--verbose', action='store_true', default=False, - help="Print all called commands to stdout.") + help="Print all called commands to stdout.") parser.add_argument('section', action='store', type=str, - help="Section in the config-file to use." ) + help="Section in the config-file to use.") args = parser.parse_args() -if args.section=='DEFAULT': +if args.section == 'DEFAULT': parser.error("--section must not be 'DEFAULT'.") -config = configparser.SafeConfigParser({ +config = configparser.ConfigParser({ 'format': '%%Y-%%m-%%d_%%H:%%M:%%S', 'datadir': '/var/backups/%(backend)s', 'mysql-ignore-tables': '', 'ejabberd-base-dir': '/var/lib/ejabberd', + 'ejabberd-options': '--no-timeout', # https://github.com/processone/ejabberd/issues/866 + 'ssh-timeout': '10', + 'ssh-options': '', }) if not config.read(args.config): parser.error("No config-files could be read.") # check validity of config-file: if args.section not in config: - print("Error: %s: No section found with that name." % args.section, file=sys.stderr) + err("Error: %s: No section found with that name.", args.section) sys.exit(1) if 'datadir' not in config[args.section]: - print("Error: %s: Section does not contain option 'datadir'." % args.section, file=sys.stderr) + err("Error: %s: Section does not contain option 'datadir'.", args.section) sys.exit(1) section = config[args.section] -if 'remote' not in section: +if 'remote' not in section and 'borg' not in section: # Note that if we dump to a remote location, there is no real way to check to check if datadir # exists and is writeable. We have to rely on the competence of the admin in that case. datadir = section['datadir'] @@ -81,8 +92,8 @@ elif section['backend'] == "ejabberd": backend = ejabberd.ejabberd(section, args) else: - print("Error: %s. Unknown backend specified. Only mysql, postgresql and ejabberd are supported." - % section['backend'], file=sys.stderr) + err("Error: %s. Unknown backend specified. Only mysql, postgresql and ejabberd are supported.", + section['backend']) sys.exit(1) databases = backend.get_db_list() diff --git a/dbdump/libdump/__init__.py b/dbdump/libdump/__init__.py index 4704b2f..dcbd346 100644 --- a/dbdump/libdump/__init__.py +++ b/dbdump/libdump/__init__.py @@ -1,20 +1,14 @@ -""" -This file is part of dbdump. - -Copyright 2009-2012 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" +# This file is part of dbdump (https://github.com/mathiasertl/db-backup). +# +# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with dbdump. If not, +# see . __all__ = ['backend', 'mysql', 'postgresql', 'ejabberd'] diff --git a/dbdump/libdump/backend.py b/dbdump/libdump/backend.py index 680aee3..b5a3e4d 100644 --- a/dbdump/libdump/backend.py +++ b/dbdump/libdump/backend.py @@ -1,27 +1,23 @@ -""" -This file is part of dbdump. - -Copyright 2009, 2010 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" +# This file is part of dbdump (https://github.com/mathiasertl/db-backup). +# +# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with dbdump. If not, +# see . import os -from subprocess import Popen, PIPE +import shlex +from subprocess import PIPE +from subprocess import Popen -class backend(object): +class backend: def __init__(self, section, args): self.args = args self.section = section @@ -33,25 +29,35 @@ def __init__(self, section, args): def make_su(self, cmd): if 'su' in self.section: - cmd = ['su', self.section['su'], '-s', + cmd = ['su', '-', self.section['su'], '-s', '/bin/bash', '-c', ' '.join(cmd)] return cmd def get_ssh(self, path, cmds): cmds = [' '.join(cmd) for cmd in cmds] - opts = self.section['remote'].split() prefix = 'umask 077; mkdir -m 0700 -p %s; ' % os.path.dirname(path) - ssh_cmd = prefix + ' | '.join(cmds) + ' > %s.sha1' % path - test = ['ssh'] + opts + [ssh_cmd] - return test + ssh_cmd = prefix + ' | '.join(cmds) + ' > %s.sha256' % path + + ssh = ['ssh'] + timeout = self.section['ssh-timeout'] + if timeout: + ssh += ['-o', 'ConnectTimeout=%s' % timeout, ] + + opts = self.section['ssh-options'] + if opts: + ssh += shlex.split(opts) + + ssh += [self.section['remote'], ssh_cmd] + + return ssh def dump(self, db, timestamp): cmd = self.make_su(self.get_command(db)) if not cmd: return - dirname = os.path.normpath(self.base + '/' + db) - path = os.path.normpath(dirname + '/' + timestamp) + dirname = os.path.abspath(os.path.join(self.base, db)) + path = os.path.join(dirname, '%s.gz' % timestamp) if self.gpg: gpg = ['gpg'] if 'sign_key' in self.section: @@ -60,60 +66,105 @@ def dump(self, db, timestamp): gpg += ['-e', '-r', self.section['recipient']] path += '.gpg' - path += '.gz' - gzip = ['gzip', '-f', '-9', '-', '-'] tee = ['tee', path] - sha1sum = ['sha1sum'] + sha = ['sha256sum'] sed = ['sed', 's/-$/%s/' % os.path.basename(path)] - if 'remote' in self.section: - ssh = self.get_ssh(path, [gzip, tee, sha1sum, sed]) + if 'borg' in self.section: + borg_check = ['borg', 'info'] + borg_init = ['borg', 'init', '--umask', '0077', '--make-parent-dirs'] + borg_create = ['borg', 'create', '--umask', '0077', '--noctime', '--nobirthtime', '--compression', 'zstd', '--files-cache', 'disabled', '--content-from-command', '--'] + borg_repo = self.section['borg'] - cmds = [cmd] - p1 = Popen(cmd, stdout=PIPE) - p = p1 + borg_env = os.environ.copy() + borg_env['BORG_RELOCATED_REPO_ACCESS_IS_OK'] = 'yes' + + if 'borg-key' in self.section: + borg_env['BORG_PASSPHRASE'] = self.section['borg-key'] + borg_init += ['--encryption', 'repokey-blake2'] + else: + borg_env['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = 'yes' + borg_init += ['--encryption', 'none'] + + borg_check += [f"{borg_repo}/{db}"] + borg_init += [f"{borg_repo}/{db}"] + borg_create += [f"{borg_repo}/{db}" + "::" + f"{timestamp}"] + borg_create += cmd + + if self.args.verbose: + str_cmds = [' '.join(c) for c in cmd] + print('# Dump databases:') + print(' | '.join(str_cmds)) + + # check if repo already exists + p_check = Popen(borg_check, env=borg_env, stdout=PIPE) + stdout, stderr = p_check.communicate() + if p_check.returncode != 0: + # repo is missing, create it + p_init = Popen(borg_init, env=borg_env, stdout=PIPE) + stdout, stderr = p_init.communicate() + if p_init.returncode != 0: + raise RuntimeError(f"{borg_init} returned with exit code {p_init.returncode}. (stderr: {stderr})") + + # create backup + p_create = Popen(borg_create, env=borg_env, stdout=PIPE) + stdout, stderr = p_create.communicate() + if self.args.verbose: + print("# borg_create:") + print(' | '.join) + if p_create.returncode != 0: + raise RuntimeError(f"{borg_create} returned with exit code {p_create.returncode}. (stderr: {stderr})") + + elif 'remote' in self.section: + ssh = self.get_ssh(path, [tee, sha, sed]) + + cmds = [cmd, gzip, ] # just for output + p_dump = Popen(cmd, stdout=PIPE) + p_gzip = Popen(gzip, stdin=p_dump.stdout, stdout=PIPE) + ssh_stdin = p_gzip.stdout # what to pipe into SSH if self.gpg: - p = Popen(gpg, stdin=p1.stdout, stdout=PIPE) + p_gpg = Popen(gpg, stdin=p_gzip.stdout, stdout=PIPE) + ssh_stdin = p_gpg.stdout cmds.append(gpg) cmds.append(ssh) if self.args.verbose: - str_cmds = [' '.join(cmd) for cmd in cmds] + str_cmds = [' '.join(c) for c in cmds] print('# Dump databases:') print(' | '.join(str_cmds)) - p2 = Popen(ssh, stdin=p.stdout, stdout=PIPE) - p2.communicate() - if p2.returncode == 255: + p_ssh = Popen(ssh, stdin=ssh_stdin, stdout=PIPE) + p_ssh.communicate() + if p_ssh.returncode == 255: raise RuntimeError("SSH returned with exit code 255.") - elif p2.returncode != 0: - raise RuntimeError("%s returned with exit code %s." - % (ssh, p2.returncode)) + elif p_ssh.returncode != 0: + raise RuntimeError("%s returned with exit code %s." % (ssh, p_ssh.returncode)) else: if not os.path.exists(dirname): os.mkdir(dirname, 0o700) - f = open(path + '.sha1', 'w') - cmds = [cmd] - p1 = Popen(cmd, stdout=PIPE) - p = p1 + f = open(path + '.sha256', 'w') + cmds = [cmd, gzip, ] # just for output + p_dump = Popen(cmd, stdout=PIPE) + p_gzip = Popen(gzip, stdin=p_dump.stdout, stdout=PIPE) + tee_pipe = p_gzip.stdout if self.gpg: - p = Popen(gpg, stdin=p1.stdout, stdout=PIPE) + p_gpg = Popen(gpg, stdin=p_dump.stdout, stdout=PIPE) + tee_pipe = p_gpg.stdout cmds.append(gpg) - p2 = Popen(gzip, stdin=p1.stdout, stdout=PIPE) - p3 = Popen(tee, stdin=p2.stdout, stdout=PIPE) - p4 = Popen(sha1sum, stdin=p3.stdout, stdout=PIPE) - p5 = Popen(sed, stdin=p4.stdout, stdout=f) + p_tee = Popen(tee, stdin=tee_pipe, stdout=PIPE) + p_sha = Popen(sha, stdin=p_tee.stdout, stdout=PIPE) + p_sed = Popen(sed, stdin=p_sha.stdout, stdout=f) - cmds += [p2, p3, p4, p5] + cmds += [tee, sha, sed] if self.args.verbose: - str_cmds = [' '.join(cmd) for cmd in cmds] + str_cmds = [' '.join(c) for c in cmds] print('# Dump databases:') print(' | '.join(str_cmds)) - p5.communicate() + p_sed.communicate() f.close() def prepare(self): diff --git a/dbdump/libdump/ejabberd.py b/dbdump/libdump/ejabberd.py index 777d890..e9b9d60 100644 --- a/dbdump/libdump/ejabberd.py +++ b/dbdump/libdump/ejabberd.py @@ -1,24 +1,18 @@ -""" -This file is part of dbdump. - -Copyright 2009-2012 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - +# This file is part of dbdump (https://github.com/mathiasertl/db-backup). +# +# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with dbdump. If not, +# see . import os +import shlex import subprocess from libdump import backend @@ -30,17 +24,17 @@ def get_db_list(self): def get_command(self, database): path = os.path.normpath(os.path.join( - self.section['ejabberd-base-dir'], '%s.backup' % database)) + self.section['ejabberd-base-dir'], '%s.dump' % database)) return ['cat', path] def prepare_db(self, database): - cmd = ['ejabberdctl'] + cmd = ['ejabberdctl'] + shlex.split(self.section['ejabberd-options']) if 'ejabberd-node' in self.section: cmd += ['--node', self.section['ejabberd-node']] if 'ejabberd-auth' in self.section: cmd += ['--auth', self.section['ejabberd-auth'].split()] - cmd += ['backup', database + '.backup'] + cmd += ['dump', '%s.dump' % database] if self.args.verbose: print('%s # prepare db' % ' '.join(cmd)) p = subprocess.Popen(cmd) @@ -48,7 +42,7 @@ def prepare_db(self, database): def cleanup_db(self, database): path = os.path.normpath(os.path.join( - self.section['ejabberd-base-dir'], '%s.backup' % database)) + self.section['ejabberd-base-dir'], '%s.dump' % database)) if self.args.verbose: print('rm %s # remove local dump' % path) os.remove(path) diff --git a/dbdump/libdump/mysql.py b/dbdump/libdump/mysql.py index 4105845..e3081d2 100644 --- a/dbdump/libdump/mysql.py +++ b/dbdump/libdump/mysql.py @@ -1,27 +1,21 @@ -""" -This file is part of dbdump. - -Copyright 2009-2012 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" +# This file is part of dbdump (https://github.com/mathiasertl/db-backup). +# +# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with dbdump. If not, +# see . import os import stat import sys - -from subprocess import * +from subprocess import PIPE +from subprocess import Popen from libdump import backend @@ -63,7 +57,7 @@ def get_db_list(self): if p_list.returncode != 0: raise Exception("Unable to get list of databases: %s " - % (stderr.decode().strip("\n"))) + % (stderr.decode().strip("\n"))) return [db for db in databases if db not in excluded] @@ -73,7 +67,7 @@ def get_command(self, database): ignored = [t for t in ignored_tables if t.startswith("%s." % database)] # assemble query for used engines in the database - engine_query = "select ENGINE from information_schema.TABLES WHERE TABLE_SCHEMA='%s' AND ENGINE != 'MEMORY'" % database + engine_query = "select ENGINE from information_schema.TABLES WHERE TABLE_SCHEMA='%s' AND ENGINE != 'MEMORY'" % database # NOQA for table in ignored: engine_query += " AND TABLE_NAME != '%s'" % table.split('.')[1] engine_query += ' GROUP BY ENGINE' diff --git a/dbdump/libdump/postgresql.py b/dbdump/libdump/postgresql.py index c2f51a7..2ccdee0 100644 --- a/dbdump/libdump/postgresql.py +++ b/dbdump/libdump/postgresql.py @@ -1,37 +1,36 @@ -""" -This file is part of dbdump. - -Copyright 2009-2012 Mathias Ertl - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -from subprocess import Popen, PIPE +# This file is part of dbdump (https://github.com/mathiasertl/db-backup). +# +# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with dbdump. If not, +# see . + +from subprocess import PIPE +from subprocess import Popen from libdump import backend class postgresql(backend.backend): def get_db_list(self): - cmd = ['psql', '-Aqt', '-c', '"select datname from pg_database"'] + cmd = ['psql', '-Aqt', '-c', 'select datname from pg_database'] if 'postgresql-psql-opts' in self.section: cmd += self.section['postgresql-psql-opts'].split(' ') + if 'postgresql-connectstring' in self.section: + cmd.append(self.section['postgresql-connectstring']) + if 'su' in self.section: + quoted_args = [f"\"{arg}\"" if ' ' in arg else arg for arg in cmd ] cmd = ['su', self.section['su'], '-s', '/bin/bash', '-c', - ' '.join(cmd)] + ' '.join(quoted_args)] p_list = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = p_list.communicate() @@ -41,7 +40,7 @@ def get_db_list(self): p_list.wait() if p_list.returncode != 0: raise Exception("Unable to get list of databases: %s " - % (stderr.decode().strip("\n"))) + % (stderr.decode().strip("\n"))) return databases @@ -49,5 +48,12 @@ def get_command(self, database): cmd = ['pg_dump', '-c'] if 'postgresql-pgdump-opts' in self.section: cmd += self.section['postgresql-pgdump-opts'].split(' ') - cmd.append(database) + + connectstring_options = [] + if 'postgresql-connectstring' in self.section: + connectstring_options = self.section['postgresql-connectstring'].split(' ') + + connectstring_options.append(f"dbname={database}") + cmd.append(' '.join(connectstring_options)) + return cmd diff --git a/dbdump/setup.py b/dbdump/setup.py index f1c29c5..b6a1198 100644 --- a/dbdump/setup.py +++ b/dbdump/setup.py @@ -2,11 +2,12 @@ from distutils.core import setup -setup(name='dbdump', - version='1.0', - description='Database dump utilities', - author='Mathias Ertl', - author_email='mati@er.tl', - url='https://git.fsinf.at/fsinf/db-backup', - packages=['libdump'], - ) +setup( + name='dbdump', + version='1.0', + description='Database dump utilities', + author='Mathias Ertl', + author_email='mati@er.tl', + url='https://github.com/mathiasertl/db-backup.git', + packages=['libdump'], +) diff --git a/debian/changelog b/debian/changelog index fb0185d..2edfeb3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,71 @@ +db-backup (2023-11-08-1) unstable; urgency=low + + [ Mathias Ertl ] + * add --no-timeout option to ejabberdctl + * ignore .tox + * add py38 + * style + * update reqs + + [ David Kaufmann ] + * fix DeprecationWarning for SafeConfigParser in dbclean too + * use connectstring for connecting to databases + * add borg backup support + * update changelog + * update debian/control to debhelper v10 + * add dh-python to control + * update debian-compat to 10 + + -- David Kaufmann Wed, 08 Nov 2023 03:47:28 +0100 + +db-backup (2021-09-09-1) unstable; urgency=medium + + * remove deprecation warning for dbclean too + + -- David Kaufmann Wed, 08 Sep 2021 23:17:36 +0200 + +db-backup (2019-07-20-1) unstable; urgency=low + + [ David Kaufmann ] + * execute commands as database users in a login shell + + [ Mathias Ertl ] + * fix --version parameter + * use tw=99 + * tw=99 + * cosmetics + * add pyenv version + * fix spaces + * add dev reqs + * add tox.ini + * udpate style, unify header + * various style improvs + * switch to sha256 checksums + * minor style + * isort cleanup + + [ somenet ] + * Update dbdump.py + + -- David Kaufmann Sat, 20 Jul 2019 17:32:00 +0200 + +db-backup (2014-12-09-1) unstable; urgency=low + + * Create text-dumps for ejabberd. + * Compress dumps before encrypting them for better compression results. + * Update maintainer address. + * Increase debhelper dependency, update VCS-* fields. + * Update copyright file to up-to-date format. + + -- Mathias Ertl Sun, 07 Dec 2014 14:20:37 +0100 + +db-backup (2013-07-21-1) precise; urgency=low + + * Bump from master-branch (dbdump and dbclean now have their own changelog) + * update cron.d file for dbclean + + -- Mathias Ertl Sun, 21 Jul 2013 21:36:57 +0200 + db-backup (2013-04-27-1) precise; urgency=low * Only add -E when dumping mysql table, and at correct position diff --git a/debian/compat b/debian/compat index 7f8f011..f599e28 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -7 +10 diff --git a/debian/control b/debian/control index 204b187..4e3d9f3 100644 --- a/debian/control +++ b/debian/control @@ -1,11 +1,11 @@ Source: db-backup Section: net Priority: optional -Maintainer: Mathias Ertl +Maintainer: Mathias Ertl Standards-Version: 3.9.2 -Build-Depends: debhelper (>= 7.0.50~), python3-all -Vcs-Browser: http://git.fsinf.at/fsinf/db-backup -Vcs-Git: http://git.fsinf.at/fsinf/db-backup.git +Build-Depends: debhelper (>= 10), dh-python, python3-all, rename +Vcs-Browser: https://github.com/mathiasertl/db-backup +Vcs-Git: https://github.com/mathiasertl/db-backup.git Package: dbdump Architecture: all diff --git a/debian/copyright b/debian/copyright index 8845b03..1044b76 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,7 +1,7 @@ -Format: http://anonscm.debian.org/viewvc/dep/web/deps/dep5.mdwn?revision=173&view=markup -Upstream-Name: fw-rules -Upstream-Contact: Mathias Ertl -Source: https://git.fsinf.at/fsinf/fw-rules +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: db-backup +Upstream-Contact: Mathias Ertl +Source: https://github.com/mathiasertl/db-backup Files: * Copyright: Copyright (c) 2011, Mathias Ertl diff --git a/debian/dbclean.cron.d b/debian/dbclean.cron.d index fb9dd43..07d3c20 100644 --- a/debian/dbclean.cron.d +++ b/debian/dbclean.cron.d @@ -1,4 +1,4 @@ -# Remove old dumps from various hosts. The parameter named with -# --section must be one found in /etc/dbclean/dbclean.conf. +# Remove old dumps from various hosts. The first argument must be a section +# found in /etc/dbclean/dbclean.conf. -#40 * * * * root dbclean --section=example.com +#40 * * * * root dbclean example.com diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..0bf631b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +flake8==3.7.8 +isort==4.3.21 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f5dcc3f --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +# https://devguide.python.org/#status-of-python-branches +envlist = py{35,36,37,38} + +[testenv] +deps = + -rrequirements-dev.txt +commands = + flake8 dbdump dbclean + isort --check-only --diff -rc dbdump dbclean + +[flake8] +max-line-length = 110 +ignore = E265 + +[isort] +force_single_line = true