diff --git a/.gitignore b/.gitignore
index c9b568f..087e8f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
*.pyc
*.swp
+/.tox/
diff --git a/.pep8 b/.pep8
new file mode 100644
index 0000000..c1b094e
--- /dev/null
+++ b/.pep8
@@ -0,0 +1,2 @@
+[pep8]
+max-line-length = 99
diff --git a/.python-version b/.python-version
new file mode 100644
index 0000000..4f2c1d1
--- /dev/null
+++ b/.python-version
@@ -0,0 +1 @@
+3.6.6
diff --git a/dbclean/ChangeLog b/dbclean/ChangeLog
new file mode 100644
index 0000000..25d7878
--- /dev/null
+++ b/dbclean/ChangeLog
@@ -0,0 +1,9 @@
+2016-02-14:
+* Fix --version parameter.
+* Use a text-width of 99 chars.
+
+2013-07-21:
+* Massive pep8 cleanup
+* Update documentation (remove old --section parameter)
+* Add this ChangeLog
+* Add the "last" option, dbclean will always keep the given last backups.
diff --git a/dbclean/README b/dbclean/README
index 443ff84..131645d 100644
--- a/dbclean/README
+++ b/dbclean/README
@@ -8,7 +8,9 @@ are matched to those of dbdump, so they work together.
=== Installation ===
Simply check out the git-repository:
- git clone http://git.git.fsinf.at/fsinf/db-backup.git
+
+ git clone https://github.com/mathiasertl/db-backup.git
+
If you don't want to specify the full path, you can of course copy dbclean.py
somewhere in your path (/usr/local/bin is usually good).
@@ -22,5 +24,5 @@ A sample configuration file is included with the source code, see:
dbclean.conf.example
After that, you can start the script with
- dbclean.py --section=example
+ dbclean.py example
where example is the section in your config-file.
diff --git a/dbclean/dbclean.conf.example b/dbclean/dbclean.conf.example
index ad18a99..2cc52d8 100644
--- a/dbclean/dbclean.conf.example
+++ b/dbclean/dbclean.conf.example
@@ -4,6 +4,8 @@
# sections.
#hourly = 72
#daily = 31
+# Always keep the last three backups
+#last = 3
#
# NOTE: You can also use the interpolation feature provided by the
# ConfigParser python module. The following line is used in the
diff --git a/dbclean/dbclean.py b/dbclean/dbclean.py
index 6985414..1e1c6e7 100755
--- a/dbclean/dbclean.py
+++ b/dbclean/dbclean.py
@@ -1,61 +1,70 @@
#!/usr/bin/env python3
-
-"""
-This program is designed to clean files from a specified directory.
-The program is designed to work together with dbdump.py, so it is
-basically designed to clean out regular database dumps. The files
-are kept at a certain granularity (so daily backups will be kept
-for a month, monthly backups for a year, etc.
-Please see the README file for how to use this script and supported
-features. You might also try calling this program with '--help'.
-
-Copyright 2009 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
-
-import os, time, sys, calendar, configparser, argparse
-
-config_file = ['/etc/dbclean/dbclean.conf', os.path.expanduser('~/.dbclean.conf')]
-
-parser = argparse.ArgumentParser(version="%prog 1.0",
- description = """Cleanup regular database dumps created by dbdump. This script keeps backups
- at given intervals for a given amount of time.""")
-parser.add_argument('-c', '--config', type=str, dest='config', action='append', default=config_file,
- help="""Additional config-files to use (default: %(default)s). Can be given multiple times
- to name multiple config-files.""")
-parser.add_argument('section', action='store', type=str,
- help="Section in the config-file to use." )
+#
+# This program is designed to clean files from a specified directory. The program is designed to
+# work together with dbdump.py, so it is basically designed to clean out regular database dumps.
+# The files are kept at a certain granularity (so daily backups will be kept for a month, monthly
+# backups for a year, etc. Please see the README file for how to use this script and supported
+# features. You might also try calling this program with '--help'.
+#
+# Copyright 2009 - 2019 Mathias Ertl
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the
+# GNU General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import argparse
+import calendar
+import configparser
+import os
+import sys
+import time
+
+
+def err(msg, *args):
+ print(msg % args, file=sys.stderr)
+
+
+config_file = [
+ '/etc/dbclean/dbclean.conf',
+ os.path.expanduser('~/.dbclean.conf')
+]
+
+parser = argparse.ArgumentParser(
+ description="""Cleanup regular database dumps created by dbdump. This script keeps backups at
+given intervals for a given amount of time.""")
+parser.add_argument('--version', action='version', version='%(prog)s 1.1')
+parser.add_argument(
+ '-c', '--config', type=str, dest='config', action='append',
+ default=config_file, help="""Additional config-files to use (default: %(default)s). Can be
+ given multiple times to name multiple config-files.""")
+parser.add_argument('section', action='store', type=str, help="Section in the config-file to use.")
args = parser.parse_args()
-if args.section=='DEFAULT':
+if args.section == 'DEFAULT':
parser.error("--section must not be 'DEFAULT'.")
-config = configparser.SafeConfigParser({
+config = configparser.ConfigParser({
'format': '%%Y-%%m-%%d_%%H:%%M:%%S',
'hourly': '24', 'daily': '31',
- 'monthly': '12', 'yearly': '3'
+ 'monthly': '12', 'yearly': '3',
+ 'last': '3',
})
if not config.read(args.config):
parser.error("No config-files could be read.")
# check validity of config-file:
if args.section not in config:
- print("Error: %s: No section found with that name." % args.section, file=sys.stderr)
+ err("Error: %s: No section found with that name.", args.section)
sys.exit(1)
if 'datadir' not in config[args.section]:
- print("Error: %s: Section does not contain option 'datadir'." % args.section, file=sys.stderr)
+ err("Error: %s: Section does not contain option 'datadir'.", args.section)
sys.exit(1)
# get directory containing backups:
@@ -63,10 +72,10 @@
# check that given directory exists and is a directory:
if not os.path.exists(datadir):
- print("Error: %s: No such directory." % (datadir), file=sys.stderr)
+ err("Error: %s: No such directory.", datadir)
sys.exit(1)
elif not os.path.isdir(datadir):
- print("Error: %s: Not a directory." % (datadir), file=sys.stderr)
+ err("Error: %s: Not a directory.", datadir)
sys.exit(1)
timeformat = config[args.section]['format']
@@ -74,106 +83,98 @@
daily = int(config[args.section]['daily'])
monthly = int(config[args.section]['monthly'])
yearly = int(config[args.section]['yearly'])
+last = int(config[args.section]['last'])
+now = time.time()
+
class backup():
files = []
- def __init__( self, time, base, file ):
+ def __init__(self, time, base, file):
self.time = time
self.base = base
- self.files = [ file ]
+ self.files = [file]
- def add( self, file ):
- self.files.append( file )
+ def add(self, file):
+ self.files.append(file)
- def is_daily( self ):
- if self.time[3] == 0:
+ def is_daily(self):
+ if self.time.tm_hour == 0:
return True
else:
return False
- def is_monthly( self ):
- if self.is_daily() and self.time[2] == 1:
+ def is_monthly(self):
+ if self.is_daily() and self.time.tm_mday == 1:
return True
else:
return False
- def is_yearly( self ):
- if self.is_monthly() and self.time[1] == 1:
+ def is_yearly(self):
+ if self.is_monthly() and self.time.tm_mon == 1:
return True
else:
return False
- def remove( self ):
+ def remove(self):
for file in self.files:
- os.remove( file )
+ os.remove(file)
- def __str__( self ):
- return "%s in %s" %(self.files, self.base)
+ def __str__(self):
+ return "%s in %s" % (self.files, self.base)
-now = time.time()
# loop through each dir in datadir
-for dir in os.listdir( datadir ):
- if dir.startswith( '.' ):
+for dir in os.listdir(datadir):
+ if dir.startswith('.'):
# skip hidden directories
continue
+ if dir == 'lost+found':
+ continue
- fullpath = os.path.normpath( datadir + '/' + dir )
- if not os.path.isdir( fullpath ):
- print( "Warning: %s: Not a directory." % (fullpath) )
+ fullpath = os.path.normpath(datadir + '/' + dir)
+ if not os.path.isdir(fullpath):
+ print("Warning: %s: Not a directory." % fullpath)
continue
- os.chdir( fullpath )
+ os.chdir(fullpath)
backups = {}
- files = os.listdir( '.' )
+ files = os.listdir('.')
files.sort()
for file in files:
- filestamp = file.split( '.' )[0]
+ filestamp = file.split('.')[0]
timestamp = ''
try:
- timestamp = time.strptime( filestamp, timeformat )
+ timestamp = time.strptime(filestamp, timeformat)
except ValueError as e:
- print( '%s: %s' %(file, e) )
+ print('%s: %s' % (file, e))
+ continue
- if timestamp not in list( backups.keys() ):
- backups[timestamp] = backup( timestamp, fullpath, file )
+ if timestamp not in list(backups.keys()):
+ backups[timestamp] = backup(timestamp, fullpath, file)
else:
- backups[timestamp].add( file )
+ backups[timestamp].add(file)
+ backup_items = sorted(backups.items(), key=lambda t: t[0])
+ if last: # NOTE: if last == 0, the slice returns an empty list!
+ backup_items = backup_items[:-last]
- for stamp in list(backups.keys()):
+ for stamp, bck in backup_items:
bck = backups[stamp]
- bck_seconds = calendar.timegm( stamp )
+ bck_seconds = calendar.timegm(stamp)
+
+ if bck_seconds > now - (hourly * 3600):
+ continue
+
+ if bck.is_daily() and bck_seconds > now - (daily * 86400):
+ continue
+
+ if bck.is_monthly() and bck_seconds > now - (monthly * 2678400):
+ continue
- if bck_seconds > now - ( hourly * 3600 ):
-# print ( "%s is hourly and will be kept" % ( time.asctime( stamp ) ) )
+ if bck.is_yearly() and bck_seconds > now - (yearly * 31622400):
continue
-# else:
-# print ("%s is hourly but to old." % ( time.asctime( stamp ) ) )
-
- if bck.is_daily():
- if bck_seconds > now - ( daily * 86400 ):
-# print( "%s is daily and will be kept." % ( time.asctime( stamp ) ) )
- continue
-# else:
-# print ("%s is daily but to old." % ( time.asctime( stamp ) ) )
-
- if bck.is_monthly():
- if bck_seconds > now - ( monthly * 2678400 ):
-# print( "%s is monthly and will be kept." % ( time.asctime( stamp ) ) )
- continue
-# else:
-# print ("%s is monthly but to old." % ( time.asctime( stamp ) ) )
-
- if bck.is_yearly():
- if bck_seconds > now - ( yearly * 31622400 ):
-# print( "%s is yearly and will be kept." % ( time.asctime( stamp ) ) )
- continue
-# else:
-# print ("%s is yearly but to old." % ( time.asctime( stamp ) ) )
-# print( "%s will be removed." % ( time.asctime( stamp ) ) )
bck.remove()
diff --git a/dbdump/ChangeLog b/dbdump/ChangeLog
new file mode 100644
index 0000000..6b06bc6
--- /dev/null
+++ b/dbdump/ChangeLog
@@ -0,0 +1,8 @@
+# 1.2
+* flake8/isort cleanup
+* Add new options: ssh-timeout and ssh-options
+* Switch to generating sha256 checksums
+
+2013-07-21:
+* pep8 cleanup
+* add this ChangeLog
diff --git a/dbdump/README b/dbdump/README
index b1188db..0f0e87a 100644
--- a/dbdump/README
+++ b/dbdump/README
@@ -13,7 +13,7 @@ to get it working.
Simply check out the git-repository:
- git clone https://git.fsinf.at/fsinf/db-backup.git
+ git clone https://github.com/mathiasertl/db-backup.git
If you don't want to specify the full path, you can of course copy dbdump.py
somewhere in your path (/usr/local/bin is usually good). Take care to copy
diff --git a/dbdump/dbdump.conf.example b/dbdump/dbdump.conf.example
index e9d56a0..25fd734 100644
--- a/dbdump/dbdump.conf.example
+++ b/dbdump/dbdump.conf.example
@@ -11,74 +11,92 @@
# Note that the default is the section name, if you do not explicitly set
# this in this file.
#backend=mysql
-#
+
# The base directory to dump your databases to.
# Default is /var/backups/
#datadir=/var/backups/mysql
-#
+
# Use su to drop privileges to the specified user for all SQL related
# commands (optional):
#su=mysql
-#
+
# Store dumps on a remote location via SSH. REMOTE will be passed to ssh
# unchanged (optional):
remote = user@backup.example.com
-#
+
+# Store dumps with borgbackup. Specify the common directory, each db will have
+# its own repository.
+#borg = user@backup.example.com
+
+# Encrypt borg backups, store the keyfile on the server and encrypt it with the
+# password specified in borg-key, if this parameter is given.
+#borg-key = add-a-secret-password-here
+
+# Override the default ssh connect-timeout of 10 seconds:
+#ssh-timeout = 10
+
+# Space-separated list of any other SSH options to pass to ssh, for example:
+#ssh-options = -o Compression=yes -v
+
# Sign and/or encrypt backups using the specified GPG keys (optional):
#sign-key = dbdump@hostname.example.net
#recipient = admin@hostname.example.net
-#
+
# You can also use the interpolation feature provided by the
# ConfigParser python module.
-#
+
# You can even specify new settings to use in later interpolation:
#hostname = example.com
-#
+
# We set the default datadir using dynamic interpolation:
#datadir = /var/backups/%(hostname)s/%(backend)s/
-#
+
#[mysql]
# Dump MySQL databases. Because of the settings in the DEFAULT section,
# the backend used here is 'mysql' and datadir evaluates to:
# /var/backups/example.com/mysql/
# ... and backups will be signed and encrypted (see first lines in the
# DEFAULT section).
-#
+
# MYSQL OPTIONS:
#
# Use a custom defaults-file. This file should be used to specify access
# credentials (default given here):
#mysql-defaults = ~/.my.cnf
-#
+
# Ignore given tables when dumping. This can be useful if a certain
# table does not use InnoDB and the database could not be dumped in a
# single transaction otherwise (optional):
#mysql-ignore-tables = db_foo.table_bla db_bar.table_hugo
-#
+
#[postgresql]
# Dump PostgreSQL databases.
-#
+
# lets override the default datadir, just to be clear:
#datadir=/somewhere/else/
-#
+
# POSTGRESQL OPTIONS:
-#
+
# Additional command-line parameters for psql and pgdump (optional):
+#postgresql-connectstring = host=localhost port=5432
#postgresql-psql-opts = --someopt
#postgresql-pgdump-opts = --otheropt
-#
+
#[ejabberd]
# Dump ejabberd databases.
-#
+
# EJABBERD OPTIONS:
#
# Dump database from this ejabberd node (optional):
#ejabberd-node = ejabberd
-#
+
+# Any other command-line arguments to ejabberd
+#ejabberd-options = --no-timeout
+
# Authenticate with the erlang node. This specifies a normal account on
# the jabber server (optional):
#ejabberd-auth = user jabber.example.com password
-#
+
# Base directory where the ejabberd database is stored (default given
# here):
#ejabberd-base-dir = /var/lib/ejabberd
diff --git a/dbdump/dbdump.py b/dbdump/dbdump.py
index 3e1a695..a787a07 100755
--- a/dbdump/dbdump.py
+++ b/dbdump/dbdump.py
@@ -1,66 +1,77 @@
#!/usr/bin/env python3
+#
+# This program is designed to regulary dump SQL databases into a specified directory for backup
+# purposes. Please see the README file for how to use this script and supported features. You might
+# also try calling this program with '--help'.
+#
+# Copyright 2009-2019 Mathias Ertl
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the
+# GNU General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with this program. If
+# not, see .
+
+import argparse
+import configparser
+import os
+import sys
+import time
+
+from libdump import ejabberd
+from libdump import mysql
+from libdump import postgresql
+
+
+def err(msg, *args):
+ print(msg % args, file=sys.stderr)
-"""
-This program is designed to regulary dump SQL databases into a
-specified directory for backup purposes. Please see the README file
-for how to use this script and supported features. You might also
-try calling this program with '--help'.
-
-Copyright 2009-2012 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
-
-import os, sys, time, argparse, configparser
-from libdump import *
config_file = ['/etc/dbdump/dbdump.conf', os.path.expanduser('~/.dbdump.conf')]
parser = argparse.ArgumentParser(description="Dump databases to a specified directory.")
-parser.add_argument('--version', action='version', version="%(prog)s 1.0")
-parser.add_argument('-c', '--config', action='append', default=config_file,
- help="""Additional config-files to use (default: %(default)s). Can be given multiple times
- to name multiple config-files.""")
+parser.add_argument('--version', action='version', version="%(prog)s 1.1")
+parser.add_argument(
+ '-c', '--config', action='append', default=config_file,
+ help="""Additional config-files to use (default: %(default)s). Can be given multiple times to
+ name multiple config-files.""")
parser.add_argument('--verbose', action='store_true', default=False,
- help="Print all called commands to stdout.")
+ help="Print all called commands to stdout.")
parser.add_argument('section', action='store', type=str,
- help="Section in the config-file to use." )
+ help="Section in the config-file to use.")
args = parser.parse_args()
-if args.section=='DEFAULT':
+if args.section == 'DEFAULT':
parser.error("--section must not be 'DEFAULT'.")
-config = configparser.SafeConfigParser({
+config = configparser.ConfigParser({
'format': '%%Y-%%m-%%d_%%H:%%M:%%S',
'datadir': '/var/backups/%(backend)s',
'mysql-ignore-tables': '',
'ejabberd-base-dir': '/var/lib/ejabberd',
+ 'ejabberd-options': '--no-timeout', # https://github.com/processone/ejabberd/issues/866
+ 'ssh-timeout': '10',
+ 'ssh-options': '',
})
if not config.read(args.config):
parser.error("No config-files could be read.")
# check validity of config-file:
if args.section not in config:
- print("Error: %s: No section found with that name." % args.section, file=sys.stderr)
+ err("Error: %s: No section found with that name.", args.section)
sys.exit(1)
if 'datadir' not in config[args.section]:
- print("Error: %s: Section does not contain option 'datadir'." % args.section, file=sys.stderr)
+ err("Error: %s: Section does not contain option 'datadir'.", args.section)
sys.exit(1)
section = config[args.section]
-if 'remote' not in section:
+if 'remote' not in section and 'borg' not in section:
# Note that if we dump to a remote location, there is no real way to check to check if datadir
# exists and is writeable. We have to rely on the competence of the admin in that case.
datadir = section['datadir']
@@ -81,8 +92,8 @@
elif section['backend'] == "ejabberd":
backend = ejabberd.ejabberd(section, args)
else:
- print("Error: %s. Unknown backend specified. Only mysql, postgresql and ejabberd are supported."
- % section['backend'], file=sys.stderr)
+ err("Error: %s. Unknown backend specified. Only mysql, postgresql and ejabberd are supported.",
+ section['backend'])
sys.exit(1)
databases = backend.get_db_list()
diff --git a/dbdump/libdump/__init__.py b/dbdump/libdump/__init__.py
index 4704b2f..dcbd346 100644
--- a/dbdump/libdump/__init__.py
+++ b/dbdump/libdump/__init__.py
@@ -1,20 +1,14 @@
-"""
-This file is part of dbdump.
-
-Copyright 2009-2012 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
+# This file is part of dbdump (https://github.com/mathiasertl/db-backup).
+#
+# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with dbdump. If not,
+# see .
__all__ = ['backend', 'mysql', 'postgresql', 'ejabberd']
diff --git a/dbdump/libdump/backend.py b/dbdump/libdump/backend.py
index 680aee3..b5a3e4d 100644
--- a/dbdump/libdump/backend.py
+++ b/dbdump/libdump/backend.py
@@ -1,27 +1,23 @@
-"""
-This file is part of dbdump.
-
-Copyright 2009, 2010 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
+# This file is part of dbdump (https://github.com/mathiasertl/db-backup).
+#
+# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with dbdump. If not,
+# see .
import os
-from subprocess import Popen, PIPE
+import shlex
+from subprocess import PIPE
+from subprocess import Popen
-class backend(object):
+class backend:
def __init__(self, section, args):
self.args = args
self.section = section
@@ -33,25 +29,35 @@ def __init__(self, section, args):
def make_su(self, cmd):
if 'su' in self.section:
- cmd = ['su', self.section['su'], '-s',
+ cmd = ['su', '-', self.section['su'], '-s',
'/bin/bash', '-c', ' '.join(cmd)]
return cmd
def get_ssh(self, path, cmds):
cmds = [' '.join(cmd) for cmd in cmds]
- opts = self.section['remote'].split()
prefix = 'umask 077; mkdir -m 0700 -p %s; ' % os.path.dirname(path)
- ssh_cmd = prefix + ' | '.join(cmds) + ' > %s.sha1' % path
- test = ['ssh'] + opts + [ssh_cmd]
- return test
+ ssh_cmd = prefix + ' | '.join(cmds) + ' > %s.sha256' % path
+
+ ssh = ['ssh']
+ timeout = self.section['ssh-timeout']
+ if timeout:
+ ssh += ['-o', 'ConnectTimeout=%s' % timeout, ]
+
+ opts = self.section['ssh-options']
+ if opts:
+ ssh += shlex.split(opts)
+
+ ssh += [self.section['remote'], ssh_cmd]
+
+ return ssh
def dump(self, db, timestamp):
cmd = self.make_su(self.get_command(db))
if not cmd:
return
- dirname = os.path.normpath(self.base + '/' + db)
- path = os.path.normpath(dirname + '/' + timestamp)
+ dirname = os.path.abspath(os.path.join(self.base, db))
+ path = os.path.join(dirname, '%s.gz' % timestamp)
if self.gpg:
gpg = ['gpg']
if 'sign_key' in self.section:
@@ -60,60 +66,105 @@ def dump(self, db, timestamp):
gpg += ['-e', '-r', self.section['recipient']]
path += '.gpg'
- path += '.gz'
-
gzip = ['gzip', '-f', '-9', '-', '-']
tee = ['tee', path]
- sha1sum = ['sha1sum']
+ sha = ['sha256sum']
sed = ['sed', 's/-$/%s/' % os.path.basename(path)]
- if 'remote' in self.section:
- ssh = self.get_ssh(path, [gzip, tee, sha1sum, sed])
+ if 'borg' in self.section:
+ borg_check = ['borg', 'info']
+ borg_init = ['borg', 'init', '--umask', '0077', '--make-parent-dirs']
+ borg_create = ['borg', 'create', '--umask', '0077', '--noctime', '--nobirthtime', '--compression', 'zstd', '--files-cache', 'disabled', '--content-from-command', '--']
+ borg_repo = self.section['borg']
- cmds = [cmd]
- p1 = Popen(cmd, stdout=PIPE)
- p = p1
+ borg_env = os.environ.copy()
+ borg_env['BORG_RELOCATED_REPO_ACCESS_IS_OK'] = 'yes'
+
+ if 'borg-key' in self.section:
+ borg_env['BORG_PASSPHRASE'] = self.section['borg-key']
+ borg_init += ['--encryption', 'repokey-blake2']
+ else:
+ borg_env['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = 'yes'
+ borg_init += ['--encryption', 'none']
+
+ borg_check += [f"{borg_repo}/{db}"]
+ borg_init += [f"{borg_repo}/{db}"]
+ borg_create += [f"{borg_repo}/{db}" + "::" + f"{timestamp}"]
+ borg_create += cmd
+
+ if self.args.verbose:
+ str_cmds = [' '.join(c) for c in cmd]
+ print('# Dump databases:')
+ print(' | '.join(str_cmds))
+
+ # check if repo already exists
+ p_check = Popen(borg_check, env=borg_env, stdout=PIPE)
+ stdout, stderr = p_check.communicate()
+ if p_check.returncode != 0:
+ # repo is missing, create it
+ p_init = Popen(borg_init, env=borg_env, stdout=PIPE)
+ stdout, stderr = p_init.communicate()
+ if p_init.returncode != 0:
+ raise RuntimeError(f"{borg_init} returned with exit code {p_init.returncode}. (stderr: {stderr})")
+
+ # create backup
+ p_create = Popen(borg_create, env=borg_env, stdout=PIPE)
+ stdout, stderr = p_create.communicate()
+ if self.args.verbose:
+ print("# borg_create:")
+ print(' | '.join)
+ if p_create.returncode != 0:
+ raise RuntimeError(f"{borg_create} returned with exit code {p_create.returncode}. (stderr: {stderr})")
+
+ elif 'remote' in self.section:
+ ssh = self.get_ssh(path, [tee, sha, sed])
+
+ cmds = [cmd, gzip, ] # just for output
+ p_dump = Popen(cmd, stdout=PIPE)
+ p_gzip = Popen(gzip, stdin=p_dump.stdout, stdout=PIPE)
+ ssh_stdin = p_gzip.stdout # what to pipe into SSH
if self.gpg:
- p = Popen(gpg, stdin=p1.stdout, stdout=PIPE)
+ p_gpg = Popen(gpg, stdin=p_gzip.stdout, stdout=PIPE)
+ ssh_stdin = p_gpg.stdout
cmds.append(gpg)
cmds.append(ssh)
if self.args.verbose:
- str_cmds = [' '.join(cmd) for cmd in cmds]
+ str_cmds = [' '.join(c) for c in cmds]
print('# Dump databases:')
print(' | '.join(str_cmds))
- p2 = Popen(ssh, stdin=p.stdout, stdout=PIPE)
- p2.communicate()
- if p2.returncode == 255:
+ p_ssh = Popen(ssh, stdin=ssh_stdin, stdout=PIPE)
+ p_ssh.communicate()
+ if p_ssh.returncode == 255:
raise RuntimeError("SSH returned with exit code 255.")
- elif p2.returncode != 0:
- raise RuntimeError("%s returned with exit code %s."
- % (ssh, p2.returncode))
+ elif p_ssh.returncode != 0:
+ raise RuntimeError("%s returned with exit code %s." % (ssh, p_ssh.returncode))
else:
if not os.path.exists(dirname):
os.mkdir(dirname, 0o700)
- f = open(path + '.sha1', 'w')
- cmds = [cmd]
- p1 = Popen(cmd, stdout=PIPE)
- p = p1
+ f = open(path + '.sha256', 'w')
+ cmds = [cmd, gzip, ] # just for output
+ p_dump = Popen(cmd, stdout=PIPE)
+ p_gzip = Popen(gzip, stdin=p_dump.stdout, stdout=PIPE)
+ tee_pipe = p_gzip.stdout
if self.gpg:
- p = Popen(gpg, stdin=p1.stdout, stdout=PIPE)
+ p_gpg = Popen(gpg, stdin=p_dump.stdout, stdout=PIPE)
+ tee_pipe = p_gpg.stdout
cmds.append(gpg)
- p2 = Popen(gzip, stdin=p1.stdout, stdout=PIPE)
- p3 = Popen(tee, stdin=p2.stdout, stdout=PIPE)
- p4 = Popen(sha1sum, stdin=p3.stdout, stdout=PIPE)
- p5 = Popen(sed, stdin=p4.stdout, stdout=f)
+ p_tee = Popen(tee, stdin=tee_pipe, stdout=PIPE)
+ p_sha = Popen(sha, stdin=p_tee.stdout, stdout=PIPE)
+ p_sed = Popen(sed, stdin=p_sha.stdout, stdout=f)
- cmds += [p2, p3, p4, p5]
+ cmds += [tee, sha, sed]
if self.args.verbose:
- str_cmds = [' '.join(cmd) for cmd in cmds]
+ str_cmds = [' '.join(c) for c in cmds]
print('# Dump databases:')
print(' | '.join(str_cmds))
- p5.communicate()
+ p_sed.communicate()
f.close()
def prepare(self):
diff --git a/dbdump/libdump/ejabberd.py b/dbdump/libdump/ejabberd.py
index 777d890..e9b9d60 100644
--- a/dbdump/libdump/ejabberd.py
+++ b/dbdump/libdump/ejabberd.py
@@ -1,24 +1,18 @@
-"""
-This file is part of dbdump.
-
-Copyright 2009-2012 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
-
+# This file is part of dbdump (https://github.com/mathiasertl/db-backup).
+#
+# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with dbdump. If not,
+# see .
import os
+import shlex
import subprocess
from libdump import backend
@@ -30,17 +24,17 @@ def get_db_list(self):
def get_command(self, database):
path = os.path.normpath(os.path.join(
- self.section['ejabberd-base-dir'], '%s.backup' % database))
+ self.section['ejabberd-base-dir'], '%s.dump' % database))
return ['cat', path]
def prepare_db(self, database):
- cmd = ['ejabberdctl']
+ cmd = ['ejabberdctl'] + shlex.split(self.section['ejabberd-options'])
if 'ejabberd-node' in self.section:
cmd += ['--node', self.section['ejabberd-node']]
if 'ejabberd-auth' in self.section:
cmd += ['--auth', self.section['ejabberd-auth'].split()]
- cmd += ['backup', database + '.backup']
+ cmd += ['dump', '%s.dump' % database]
if self.args.verbose:
print('%s # prepare db' % ' '.join(cmd))
p = subprocess.Popen(cmd)
@@ -48,7 +42,7 @@ def prepare_db(self, database):
def cleanup_db(self, database):
path = os.path.normpath(os.path.join(
- self.section['ejabberd-base-dir'], '%s.backup' % database))
+ self.section['ejabberd-base-dir'], '%s.dump' % database))
if self.args.verbose:
print('rm %s # remove local dump' % path)
os.remove(path)
diff --git a/dbdump/libdump/mysql.py b/dbdump/libdump/mysql.py
index 4105845..e3081d2 100644
--- a/dbdump/libdump/mysql.py
+++ b/dbdump/libdump/mysql.py
@@ -1,27 +1,21 @@
-"""
-This file is part of dbdump.
-
-Copyright 2009-2012 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
+# This file is part of dbdump (https://github.com/mathiasertl/db-backup).
+#
+# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with dbdump. If not,
+# see .
import os
import stat
import sys
-
-from subprocess import *
+from subprocess import PIPE
+from subprocess import Popen
from libdump import backend
@@ -63,7 +57,7 @@ def get_db_list(self):
if p_list.returncode != 0:
raise Exception("Unable to get list of databases: %s "
- % (stderr.decode().strip("\n")))
+ % (stderr.decode().strip("\n")))
return [db for db in databases if db not in excluded]
@@ -73,7 +67,7 @@ def get_command(self, database):
ignored = [t for t in ignored_tables if t.startswith("%s." % database)]
# assemble query for used engines in the database
- engine_query = "select ENGINE from information_schema.TABLES WHERE TABLE_SCHEMA='%s' AND ENGINE != 'MEMORY'" % database
+ engine_query = "select ENGINE from information_schema.TABLES WHERE TABLE_SCHEMA='%s' AND ENGINE != 'MEMORY'" % database # NOQA
for table in ignored:
engine_query += " AND TABLE_NAME != '%s'" % table.split('.')[1]
engine_query += ' GROUP BY ENGINE'
diff --git a/dbdump/libdump/postgresql.py b/dbdump/libdump/postgresql.py
index c2f51a7..2ccdee0 100644
--- a/dbdump/libdump/postgresql.py
+++ b/dbdump/libdump/postgresql.py
@@ -1,37 +1,36 @@
-"""
-This file is part of dbdump.
-
-Copyright 2009-2012 Mathias Ertl
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
-
-from subprocess import Popen, PIPE
+# This file is part of dbdump (https://github.com/mathiasertl/db-backup).
+#
+# dbdump is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# dbdump is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with dbdump. If not,
+# see .
+
+from subprocess import PIPE
+from subprocess import Popen
from libdump import backend
class postgresql(backend.backend):
def get_db_list(self):
- cmd = ['psql', '-Aqt', '-c', '"select datname from pg_database"']
+ cmd = ['psql', '-Aqt', '-c', 'select datname from pg_database']
if 'postgresql-psql-opts' in self.section:
cmd += self.section['postgresql-psql-opts'].split(' ')
+ if 'postgresql-connectstring' in self.section:
+ cmd.append(self.section['postgresql-connectstring'])
+
if 'su' in self.section:
+ quoted_args = [f"\"{arg}\"" if ' ' in arg else arg for arg in cmd ]
cmd = ['su', self.section['su'], '-s', '/bin/bash', '-c',
- ' '.join(cmd)]
+ ' '.join(quoted_args)]
p_list = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p_list.communicate()
@@ -41,7 +40,7 @@ def get_db_list(self):
p_list.wait()
if p_list.returncode != 0:
raise Exception("Unable to get list of databases: %s "
- % (stderr.decode().strip("\n")))
+ % (stderr.decode().strip("\n")))
return databases
@@ -49,5 +48,12 @@ def get_command(self, database):
cmd = ['pg_dump', '-c']
if 'postgresql-pgdump-opts' in self.section:
cmd += self.section['postgresql-pgdump-opts'].split(' ')
- cmd.append(database)
+
+ connectstring_options = []
+ if 'postgresql-connectstring' in self.section:
+ connectstring_options = self.section['postgresql-connectstring'].split(' ')
+
+ connectstring_options.append(f"dbname={database}")
+ cmd.append(' '.join(connectstring_options))
+
return cmd
diff --git a/dbdump/setup.py b/dbdump/setup.py
index f1c29c5..b6a1198 100644
--- a/dbdump/setup.py
+++ b/dbdump/setup.py
@@ -2,11 +2,12 @@
from distutils.core import setup
-setup(name='dbdump',
- version='1.0',
- description='Database dump utilities',
- author='Mathias Ertl',
- author_email='mati@er.tl',
- url='https://git.fsinf.at/fsinf/db-backup',
- packages=['libdump'],
- )
+setup(
+ name='dbdump',
+ version='1.0',
+ description='Database dump utilities',
+ author='Mathias Ertl',
+ author_email='mati@er.tl',
+ url='https://github.com/mathiasertl/db-backup.git',
+ packages=['libdump'],
+)
diff --git a/debian/changelog b/debian/changelog
index fb0185d..2edfeb3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,71 @@
+db-backup (2023-11-08-1) unstable; urgency=low
+
+ [ Mathias Ertl ]
+ * add --no-timeout option to ejabberdctl
+ * ignore .tox
+ * add py38
+ * style
+ * update reqs
+
+ [ David Kaufmann ]
+ * fix DeprecationWarning for SafeConfigParser in dbclean too
+ * use connectstring for connecting to databases
+ * add borg backup support
+ * update changelog
+ * update debian/control to debhelper v10
+ * add dh-python to control
+ * update debian-compat to 10
+
+ -- David Kaufmann Wed, 08 Nov 2023 03:47:28 +0100
+
+db-backup (2021-09-09-1) unstable; urgency=medium
+
+ * remove deprecation warning for dbclean too
+
+ -- David Kaufmann Wed, 08 Sep 2021 23:17:36 +0200
+
+db-backup (2019-07-20-1) unstable; urgency=low
+
+ [ David Kaufmann ]
+ * execute commands as database users in a login shell
+
+ [ Mathias Ertl ]
+ * fix --version parameter
+ * use tw=99
+ * tw=99
+ * cosmetics
+ * add pyenv version
+ * fix spaces
+ * add dev reqs
+ * add tox.ini
+ * udpate style, unify header
+ * various style improvs
+ * switch to sha256 checksums
+ * minor style
+ * isort cleanup
+
+ [ somenet ]
+ * Update dbdump.py
+
+ -- David Kaufmann Sat, 20 Jul 2019 17:32:00 +0200
+
+db-backup (2014-12-09-1) unstable; urgency=low
+
+ * Create text-dumps for ejabberd.
+ * Compress dumps before encrypting them for better compression results.
+ * Update maintainer address.
+ * Increase debhelper dependency, update VCS-* fields.
+ * Update copyright file to up-to-date format.
+
+ -- Mathias Ertl Sun, 07 Dec 2014 14:20:37 +0100
+
+db-backup (2013-07-21-1) precise; urgency=low
+
+ * Bump from master-branch (dbdump and dbclean now have their own changelog)
+ * update cron.d file for dbclean
+
+ -- Mathias Ertl Sun, 21 Jul 2013 21:36:57 +0200
+
db-backup (2013-04-27-1) precise; urgency=low
* Only add -E when dumping mysql table, and at correct position
diff --git a/debian/compat b/debian/compat
index 7f8f011..f599e28 100644
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-7
+10
diff --git a/debian/control b/debian/control
index 204b187..4e3d9f3 100644
--- a/debian/control
+++ b/debian/control
@@ -1,11 +1,11 @@
Source: db-backup
Section: net
Priority: optional
-Maintainer: Mathias Ertl
+Maintainer: Mathias Ertl
Standards-Version: 3.9.2
-Build-Depends: debhelper (>= 7.0.50~), python3-all
-Vcs-Browser: http://git.fsinf.at/fsinf/db-backup
-Vcs-Git: http://git.fsinf.at/fsinf/db-backup.git
+Build-Depends: debhelper (>= 10), dh-python, python3-all, rename
+Vcs-Browser: https://github.com/mathiasertl/db-backup
+Vcs-Git: https://github.com/mathiasertl/db-backup.git
Package: dbdump
Architecture: all
diff --git a/debian/copyright b/debian/copyright
index 8845b03..1044b76 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -1,7 +1,7 @@
-Format: http://anonscm.debian.org/viewvc/dep/web/deps/dep5.mdwn?revision=173&view=markup
-Upstream-Name: fw-rules
-Upstream-Contact: Mathias Ertl
-Source: https://git.fsinf.at/fsinf/fw-rules
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: db-backup
+Upstream-Contact: Mathias Ertl
+Source: https://github.com/mathiasertl/db-backup
Files: *
Copyright: Copyright (c) 2011, Mathias Ertl
diff --git a/debian/dbclean.cron.d b/debian/dbclean.cron.d
index fb9dd43..07d3c20 100644
--- a/debian/dbclean.cron.d
+++ b/debian/dbclean.cron.d
@@ -1,4 +1,4 @@
-# Remove old dumps from various hosts. The parameter named with
-# --section must be one found in /etc/dbclean/dbclean.conf.
+# Remove old dumps from various hosts. The first argument must be a section
+# found in /etc/dbclean/dbclean.conf.
-#40 * * * * root dbclean --section=example.com
+#40 * * * * root dbclean example.com
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..0bf631b
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,2 @@
+flake8==3.7.8
+isort==4.3.21
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..f5dcc3f
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,17 @@
+[tox]
+# https://devguide.python.org/#status-of-python-branches
+envlist = py{35,36,37,38}
+
+[testenv]
+deps =
+ -rrequirements-dev.txt
+commands =
+ flake8 dbdump dbclean
+ isort --check-only --diff -rc dbdump dbclean
+
+[flake8]
+max-line-length = 110
+ignore = E265
+
+[isort]
+force_single_line = true