Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Apr 29, 2022
1 parent cab3cbb commit d2e9683
Show file tree
Hide file tree
Showing 38 changed files with 1,531 additions and 1,136 deletions.
2 changes: 1 addition & 1 deletion dbbackup/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
import django

if django.VERSION < (3, 2):
default_app_config = 'dbbackup.apps.DbbackupConfig'
default_app_config = "dbbackup.apps.DbbackupConfig"
9 changes: 5 additions & 4 deletions dbbackup/apps.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ class DbbackupConfig(AppConfig):
"""
Config for DBBackup application.
"""
name = 'dbbackup'
label = 'dbbackup'
verbose_name = gettext_lazy('Backup and restore')
default_auto_field = 'django.db.models.AutoField'

name = "dbbackup"
label = "dbbackup"
verbose_name = gettext_lazy("Backup and restore")
default_auto_field = "django.db.models.AutoField"

def ready(self):
log.load()
64 changes: 41 additions & 23 deletions dbbackup/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,37 @@

from dbbackup import settings

W001 = Warning('Invalid HOSTNAME parameter',
hint='Set a non empty string to this settings.DBBACKUP_HOSTNAME',
id='dbbackup.W001')
W002 = Warning('Invalid STORAGE parameter',
hint='Set a valid path to a storage in settings.DBBACKUP_STORAGE',
id='dbbackup.W002')
W003 = Warning('Invalid FILENAME_TEMPLATE parameter',
hint='Include {datetime} to settings.DBBACKUP_FILENAME_TEMPLATE',
id='dbbackup.W003')
W004 = Warning('Invalid MEDIA_FILENAME_TEMPLATE parameter',
hint='Include {datetime} to settings.DBBACKUP_MEDIA_FILENAME_TEMPLATE',
id='dbbackup.W004')
W005 = Warning('Invalid DATE_FORMAT parameter',
hint='settings.DBBACKUP_DATE_FORMAT can contain only [A-Za-z0-9%_-]',
id='dbbackup.W005')
W006 = Warning('FAILURE_RECIPIENTS has been deprecated',
hint='settings.DBBACKUP_FAILURE_RECIPIENTS is replaced by '
'settings.DBBACKUP_ADMINS',
id='dbbackup.W006')
W001 = Warning(
"Invalid HOSTNAME parameter",
hint="Set a non empty string to this settings.DBBACKUP_HOSTNAME",
id="dbbackup.W001",
)
W002 = Warning(
"Invalid STORAGE parameter",
hint="Set a valid path to a storage in settings.DBBACKUP_STORAGE",
id="dbbackup.W002",
)
W003 = Warning(
"Invalid FILENAME_TEMPLATE parameter",
hint="Include {datetime} to settings.DBBACKUP_FILENAME_TEMPLATE",
id="dbbackup.W003",
)
W004 = Warning(
"Invalid MEDIA_FILENAME_TEMPLATE parameter",
hint="Include {datetime} to settings.DBBACKUP_MEDIA_FILENAME_TEMPLATE",
id="dbbackup.W004",
)
W005 = Warning(
"Invalid DATE_FORMAT parameter",
hint="settings.DBBACKUP_DATE_FORMAT can contain only [A-Za-z0-9%_-]",
id="dbbackup.W005",
)
W006 = Warning(
"FAILURE_RECIPIENTS has been deprecated",
hint="settings.DBBACKUP_FAILURE_RECIPIENTS is replaced by "
"settings.DBBACKUP_ADMINS",
id="dbbackup.W006",
)


@register(Tags.compatibility)
Expand All @@ -34,16 +46,22 @@ def check_settings(app_configs, **kwargs):
if not settings.STORAGE or not isinstance(settings.STORAGE, str):
errors.append(W002)

if not callable(settings.FILENAME_TEMPLATE) and '{datetime}' not in settings.FILENAME_TEMPLATE:
if (
not callable(settings.FILENAME_TEMPLATE)
and "{datetime}" not in settings.FILENAME_TEMPLATE
):
errors.append(W003)

if not callable(settings.MEDIA_FILENAME_TEMPLATE) and '{datetime}' not in settings.MEDIA_FILENAME_TEMPLATE:
if (
not callable(settings.MEDIA_FILENAME_TEMPLATE)
and "{datetime}" not in settings.MEDIA_FILENAME_TEMPLATE
):
errors.append(W004)

if re.search(r'[^A-Za-z0-9%_-]', settings.DATE_FORMAT):
if re.search(r"[^A-Za-z0-9%_-]", settings.DATE_FORMAT):
errors.append(W005)

if getattr(settings, 'FAILURE_RECIPIENTS', None) is not None:
if getattr(settings, "FAILURE_RECIPIENTS", None) is not None:
errors.append(W006)

return errors
79 changes: 46 additions & 33 deletions dbbackup/db/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,22 @@

from . import exceptions

logger = logging.getLogger('dbbackup.command')
logger = logging.getLogger("dbbackup.command")
logger.setLevel(logging.DEBUG)


CONNECTOR_MAPPING = {
'django.db.backends.sqlite3': 'dbbackup.db.sqlite.SqliteConnector',
'django.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector',
'django.db.backends.postgresql': 'dbbackup.db.postgresql.PgDumpBinaryConnector',
'django.db.backends.postgresql_psycopg2': 'dbbackup.db.postgresql.PgDumpBinaryConnector',
'django.db.backends.oracle': None,
'django_mongodb_engine': 'dbbackup.db.mongodb.MongoDumpConnector',
'djongo': 'dbbackup.db.mongodb.MongoDumpConnector',
'django.contrib.gis.db.backends.postgis': 'dbbackup.db.postgresql.PgDumpGisConnector',
'django.contrib.gis.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector',
'django.contrib.gis.db.backends.oracle': None,
'django.contrib.gis.db.backends.spatialite': 'dbbackup.db.sqlite.SqliteConnector',
"django.db.backends.sqlite3": "dbbackup.db.sqlite.SqliteConnector",
"django.db.backends.mysql": "dbbackup.db.mysql.MysqlDumpConnector",
"django.db.backends.postgresql": "dbbackup.db.postgresql.PgDumpBinaryConnector",
"django.db.backends.postgresql_psycopg2": "dbbackup.db.postgresql.PgDumpBinaryConnector",
"django.db.backends.oracle": None,
"django_mongodb_engine": "dbbackup.db.mongodb.MongoDumpConnector",
"djongo": "dbbackup.db.mongodb.MongoDumpConnector",
"django.contrib.gis.db.backends.postgis": "dbbackup.db.postgresql.PgDumpGisConnector",
"django.contrib.gis.db.backends.mysql": "dbbackup.db.mysql.MysqlDumpConnector",
"django.contrib.gis.db.backends.oracle": None,
"django.contrib.gis.db.backends.spatialite": "dbbackup.db.sqlite.SqliteConnector",
}

if settings.CUSTOM_CONNECTOR_MAPPING:
Expand All @@ -45,12 +45,12 @@ def get_connector(database_name=None):
# Get DB
database_name = database_name or DEFAULT_DB_ALIAS
connection = connections[database_name]
engine = connection.settings_dict['ENGINE']
engine = connection.settings_dict["ENGINE"]
connector_settings = settings.CONNECTORS.get(database_name, {})
connector_path = connector_settings.get('CONNECTOR', CONNECTOR_MAPPING[engine])
connector_module_path = '.'.join(connector_path.split('.')[:-1])
connector_path = connector_settings.get("CONNECTOR", CONNECTOR_MAPPING[engine])
connector_module_path = ".".join(connector_path.split(".")[:-1])
module = import_module(connector_module_path)
connector_name = connector_path.split('.')[-1]
connector_name = connector_path.split(".")[-1]
connector = getattr(module, connector_name)
return connector(database_name, **connector_settings)

Expand All @@ -60,11 +60,13 @@ class BaseDBConnector:
Base class for create database connector. This kind of object creates
interaction with database and allow backup and restore operations.
"""
extension = 'dump'

extension = "dump"
exclude = []

def __init__(self, database_name=None, **kwargs):
from django.db import DEFAULT_DB_ALIAS, connections

self.database_name = database_name or DEFAULT_DB_ALIAS
self.connection = connections[self.database_name]
for attr, value in kwargs.items():
Expand All @@ -73,15 +75,14 @@ def __init__(self, database_name=None, **kwargs):
@property
def settings(self):
"""Mix of database and connector settings."""
if not hasattr(self, '_settings'):
if not hasattr(self, "_settings"):
sett = self.connection.settings_dict.copy()
sett.update(settings.CONNECTORS.get(self.database_name, {}))
self._settings = sett
return self._settings

def generate_filename(self, server_name=None):
return utils.filename_generate(self.extension, self.database_name,
server_name)
return utils.filename_generate(self.extension, self.database_name, server_name)

def create_dump(self):
return self._create_dump()
Expand Down Expand Up @@ -112,10 +113,11 @@ class BaseCommandDBConnector(BaseDBConnector):
"""
Base class for create database connector based on command line tools.
"""
dump_prefix = ''
dump_suffix = ''
restore_prefix = ''
restore_suffix = ''

dump_prefix = ""
dump_suffix = ""
restore_prefix = ""
restore_suffix = ""

use_parent_env = True
env = {}
Expand All @@ -137,29 +139,40 @@ def run_command(self, command, stdin=None, env=None):
"""
logger.debug(command)
cmd = shlex.split(command)
stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
stdout = SpooledTemporaryFile(
max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR
)
stderr = SpooledTemporaryFile(
max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR
)
full_env = os.environ.copy() if self.use_parent_env else {}
full_env.update(self.env)
full_env.update(env or {})
try:
if isinstance(stdin, File):
process = Popen(
cmd, stdin=stdin.open("rb"), stdout=stdout, stderr=stderr,
env=full_env
cmd,
stdin=stdin.open("rb"),
stdout=stdout,
stderr=stderr,
env=full_env,
)
else:
process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env)
process = Popen(
cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env
)
process.wait()
if process.poll():
stderr.seek(0)
raise exceptions.CommandConnectorError(
"Error running: {}\n{}".format(command, stderr.read().decode('utf-8')))
"Error running: {}\n{}".format(
command, stderr.read().decode("utf-8")
)
)
stdout.seek(0)
stderr.seek(0)
return stdout, stderr
except OSError as err:
raise exceptions.CommandConnectorError(
f"Error running: {command}\n{str(err)}")
f"Error running: {command}\n{str(err)}"
)
43 changes: 22 additions & 21 deletions dbbackup/db/mongodb.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,46 +8,47 @@ class MongoDumpConnector(BaseCommandDBConnector):
MongoDB connector, creates dump with ``mongodump`` and restore with
``mongorestore``.
"""
dump_cmd = 'mongodump'
restore_cmd = 'mongorestore'

dump_cmd = "mongodump"
restore_cmd = "mongorestore"
object_check = True
drop = True

def _create_dump(self):
cmd = f"{self.dump_cmd} --db {self.settings['NAME']}"
host = self.settings.get('HOST') or 'localhost'
port = self.settings.get('PORT') or 27017
cmd += f' --host {host}:{port}'
if self.settings.get('USER'):
host = self.settings.get("HOST") or "localhost"
port = self.settings.get("PORT") or 27017
cmd += f" --host {host}:{port}"
if self.settings.get("USER"):
cmd += f" --username {self.settings['USER']}"
if self.settings.get('PASSWORD'):
if self.settings.get("PASSWORD"):
cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}"

if self.settings.get('AUTH_SOURCE'):
if self.settings.get("AUTH_SOURCE"):
cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}"
for collection in self.exclude:
cmd += f' --excludeCollection {collection}'
cmd += ' --archive'
cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}'
cmd += f" --excludeCollection {collection}"
cmd += " --archive"
cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
stdout, stderr = self.run_command(cmd, env=self.dump_env)
return stdout

def _restore_dump(self, dump):
cmd = self.restore_cmd
host = self.settings.get('HOST') or 'localhost'
port = self.settings.get('PORT') or 27017
cmd += f' --host {host}:{port}'
if self.settings.get('USER'):
host = self.settings.get("HOST") or "localhost"
port = self.settings.get("PORT") or 27017
cmd += f" --host {host}:{port}"
if self.settings.get("USER"):
cmd += f" --username {self.settings['USER']}"
if self.settings.get('PASSWORD'):
if self.settings.get("PASSWORD"):
cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}"

if self.settings.get('AUTH_SOURCE'):
if self.settings.get("AUTH_SOURCE"):
cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}"
if self.object_check:
cmd += ' --objcheck'
cmd += " --objcheck"
if self.drop:
cmd += ' --drop'
cmd += ' --archive'
cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}'
cmd += " --drop"
cmd += " --archive"
cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
return self.run_command(cmd, stdin=dump, env=self.restore_env)
25 changes: 13 additions & 12 deletions dbbackup/db/mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,38 @@ class MysqlDumpConnector(BaseCommandDBConnector):
MySQL connector, creates dump with ``mysqldump`` and restore with
``mysql``.
"""
dump_cmd = 'mysqldump'
restore_cmd = 'mysql'

dump_cmd = "mysqldump"
restore_cmd = "mysql"

def _create_dump(self):
cmd = f"{self.dump_cmd} {self.settings['NAME']} --quick"
if self.settings.get('HOST'):
if self.settings.get("HOST"):
cmd += f" --host={self.settings['HOST']}"
if self.settings.get('PORT'):
if self.settings.get("PORT"):
cmd += f" --port={self.settings['PORT']}"
if self.settings.get('USER'):
if self.settings.get("USER"):
cmd += f" --user={self.settings['USER']}"
if self.settings.get('PASSWORD'):
if self.settings.get("PASSWORD"):
cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}"

for table in self.exclude:
cmd += f" --ignore-table={self.settings['NAME']}.{table}"
cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}'
cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
stdout, stderr = self.run_command(cmd, env=self.dump_env)
return stdout

def _restore_dump(self, dump):
cmd = f"{self.restore_cmd} {self.settings['NAME']}"
if self.settings.get('HOST'):
if self.settings.get("HOST"):
cmd += f" --host={self.settings['HOST']}"
if self.settings.get('PORT'):
if self.settings.get("PORT"):
cmd += f" --port={self.settings['PORT']}"
if self.settings.get('USER'):
if self.settings.get("USER"):
cmd += f" --user={self.settings['USER']}"
if self.settings.get('PASSWORD'):
if self.settings.get("PASSWORD"):
cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}"

cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}'
cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
return stdout, stderr
Loading

0 comments on commit d2e9683

Please sign in to comment.