diff --git a/.editorconfig b/.editorconfig index 6a1ab45..4c663f3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,4 +6,7 @@ indent_size = 4 end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true -max_line_length = 79 +max_line_length = 79 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/ctrl_z/__init__.py b/ctrl_z/__init__.py index 75eb1af..6ec5b77 100644 --- a/ctrl_z/__init__.py +++ b/ctrl_z/__init__.py @@ -1,7 +1,7 @@ from pkg_resources import get_distribution +from ._cli import cli # noqa from .backup import Backup, configure_logging # noqa -from .cli import cli # noqa __version__ = get_distribution('CTRL-Z').version diff --git a/ctrl_z/cli.py b/ctrl_z/_cli.py similarity index 79% rename from ctrl_z/cli.py rename to ctrl_z/_cli.py index e507e68..4b2598f 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/_cli.py @@ -7,13 +7,18 @@ import sys import django +from django.utils.module_loading import import_string from .backup import Backup, configure_logging -from .config import DEFAULT_CONFIG_FILE +from .config import DEFAULT_CONFIG_FILE, Config +from .transfer import BackupTransfer logger = logging.getLogger(__name__) +HEADER = "CTRL-Z {version} - Backup and recovery tool\n" + + def noop(*args, **kwargs): pass @@ -61,8 +66,6 @@ class CLI: def __init__(self): parser = argparse.ArgumentParser(description="CTRL-Z CLI") - parser.add_argument('--config-file', help="Config file to use") - parser.add_argument('--base-dir', help="Base directory override") subparsers = parser.add_subparsers( help="Sub commands", dest='subcommand' @@ -117,6 +120,11 @@ def __init__(self): default=True, help="Do not restore files" ) + # backup transfer + self.parser_transfer = subparsers.add_parser( + 'transfer', help='Transfer the backups to an off-site location' + ) + self.parser = parser def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, stderr=None): @@ -128,10 +136,14 @@ def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, if stderr: self.stderr = stderr - self.stderr.write(f"CTRL-Z {__version__} - Backup and recovery tool\n") + self.stderr.write(HEADER.format(version=__version__)) - args = self.parser.parse_args(args or sys.argv[1:]) - config_file = args.config_file or config_file + # load the command line args for transfers + config = Config.from_file(config_file) + TransferBackend = import_string(config.transfer_backend) + TransferBackend.add_arguments(self.parser_transfer) + + args = self.parser.parse_args(args if args is not None else sys.argv[1:]) self._setup() @@ -145,8 +157,6 @@ def _setup(self): def run(self, options, config_file: str): subcommand = options.subcommand conf_overrides = {} - if options.base_dir: - conf_overrides['base_dir'] = options.base_dir if subcommand == 'restore': self._backup = Backup.prepare_restore( @@ -166,8 +176,10 @@ def run(self, options, config_file: str): self.backup(options) elif subcommand == 'restore': self.restore(options) + elif subcommand == 'transfer': + self.transfer(options, config_file, conf_overrides) else: - self.parser.print_help() + self.parser.print_help(file=self.stdout) def generate_config(self, options): """ @@ -219,5 +231,26 @@ def restore(self, options): finally: backup.report(has_errors) + def transfer(self, options, config_file, conf_overrides): + """ + Relay the command to the transfer backend or initiate the actual transfer. + """ + transfer = BackupTransfer.from_config(config_file) + + # handle potential backend specific subcommands + handled = transfer.backend.handle_command(self, transfer, options) + if handled: + return + + has_errors = False + try: + transfer.show_info() + transfer.sync_to_remote() + except Exception: + has_errors = True + raise + finally: + self._backup.report(has_errors) + cli = CLI() diff --git a/ctrl_z/backup.py b/ctrl_z/backup.py index b6e7afc..0d8a31e 100644 --- a/ctrl_z/backup.py +++ b/ctrl_z/backup.py @@ -33,7 +33,7 @@ def from_config(cls, config_file): @classmethod def prepare_restore(cls, config_file, base_dir: str): - config = Config.from_file(config_file, base_dir=base_dir, restore=True) + config = Config.from_file(config_file, base_dir=base_dir, use_parent_dir=True) return cls(config=config) def restore(self, db=True, skip_db=None, files=True): diff --git a/ctrl_z/config.default.yml b/ctrl_z/config.default.yml index 9ab60b9..03cd485 100644 --- a/ctrl_z/config.default.yml +++ b/ctrl_z/config.default.yml @@ -33,3 +33,8 @@ files: # Which binaries to use for backup creation/restore pg_dump_binary: /usr/bin/pg_dump pg_restore_binary: /usr/bin/pg_restore + +# where are backups transferred (off-site) +transfer_backend: ctrl_z.transfer.backends.base.Base +transfer_backend_init_kwargs: {} +transfer_path: / diff --git a/ctrl_z/config.py b/ctrl_z/config.py index 76748c9..e2cb198 100644 --- a/ctrl_z/config.py +++ b/ctrl_z/config.py @@ -16,7 +16,7 @@ class Config: __slots__ = [ - 'restore', + 'use_parent_dir', 'base_dir', 'logging', 'database', @@ -25,10 +25,13 @@ class Config: 'files', 'pg_dump_binary', 'pg_restore_binary', + 'transfer_backend', + 'transfer_backend_init_kwargs', + 'transfer_path', ] def __init__(self, **kwargs): - self.restore = kwargs.pop('restore', False) + self.use_parent_dir = kwargs.pop('use_parent_dir', False) for key, value in kwargs.items(): setattr(self, key, value) @@ -54,14 +57,14 @@ def from_file(cls, config_file: str, **overrides): def write_to(self, path: str): as_dict = {key: getattr(self, key) for key in self.__slots__} - if not self.restore: + if not self.use_parent_dir: as_dict['base_dir'] = os.path.dirname(as_dict['base_dir']) as_dict['retention_policy'] = as_dict['retention_policy'].serialize() with open(path, 'w') as stream: yaml.dump(as_dict, stream=stream) def set_base_dir(self): - if self.restore: # should be set via overrides + if self.use_parent_dir: # should be set via overrides return self.base_dir = self.retention_policy.get_base_dir(self.base_dir) diff --git a/ctrl_z/constants.py b/ctrl_z/constants.py new file mode 100644 index 0000000..1becf82 --- /dev/null +++ b/ctrl_z/constants.py @@ -0,0 +1,5 @@ +import re + +DATE_FORMAT = "%Y-%m-%d" + +BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)') diff --git a/ctrl_z/retention.py b/ctrl_z/retention.py index b51a555..78b3f1d 100644 --- a/ctrl_z/retention.py +++ b/ctrl_z/retention.py @@ -1,6 +1,5 @@ import logging import os -import re import shutil from datetime import date, datetime from itertools import chain @@ -9,16 +8,15 @@ from dateutil.relativedelta import relativedelta from dateutil.rrule import DAILY, WEEKLY, rrule +from .constants import DATE_FORMAT +from .utils import is_backup_dir + logger = logging.getLogger(__name__) class RetentionPolicy: __slots__ = ['day_of_week', 'days_to_keep', 'weeks_to_keep'] - DATE_FORMAT = "%Y-%m-%d" - - BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)') - def __init__(self, **config): for key, value in config.items(): setattr(self, key, value) @@ -26,16 +24,6 @@ def __init__(self, **config): def serialize(self): return {key: getattr(self, key) for key in self.__slots__} - def is_backup_dir(self, dir_name: str) -> bool: - """ - Test if a directory name fits the pattern of backup folder names. - - The pattern is YYYY-MM-DD-suffix, where suffix is either 'daily' or 'weekly'. - """ - if self.BACKUP_DIR_PATTERN.match(dir_name): - return True - return False - def get_suffix(self, dt: Union[date, datetime]) -> str: return 'weekly' if dt.weekday() == self.day_of_week else 'daily' @@ -44,7 +32,7 @@ def get_base_dir(self, base: str) -> str: Figure out the folder name for the current backup. """ now = datetime.utcnow() - datestamp = now.strftime(self.DATE_FORMAT) + datestamp = now.strftime(DATE_FORMAT) suffix = self.get_suffix(now) return os.path.join(base, f"{datestamp}-{suffix}") @@ -67,14 +55,14 @@ def rotate(self, base: str): weeklies = rrule(WEEKLY, dtstart=weekly_start, count=self.weeks_to_keep) to_keep = sorted({ - "{}-{}".format(dt.strftime(self.DATE_FORMAT), self.get_suffix(dt)) + "{}-{}".format(dt.strftime(DATE_FORMAT), self.get_suffix(dt)) for dt in chain(dailies, weeklies) }) logger.debug("Keeping backups from: %r", to_keep) to_delete = [] for dir_name in os.listdir(base): - if not self.is_backup_dir(dir_name): + if not is_backup_dir(dir_name): logger.debug("%s doesn't look like a backup directory, keeping it.", dir_name) continue diff --git a/ctrl_z/transfer/__init__.py b/ctrl_z/transfer/__init__.py new file mode 100644 index 0000000..d001933 --- /dev/null +++ b/ctrl_z/transfer/__init__.py @@ -0,0 +1,2 @@ +from .archive import BackupArchive # noqa +from .transfer import BackupTransfer, UploadError # noqa diff --git a/ctrl_z/transfer/archive.py b/ctrl_z/transfer/archive.py new file mode 100644 index 0000000..81d7173 --- /dev/null +++ b/ctrl_z/transfer/archive.py @@ -0,0 +1,26 @@ +import hashlib +import os + + +class BackupArchive: + def __init__(self, archive: str): + """ + A single backup as an archive. + + The archive format depends on the backend, it could be tar.gz, zip... + + :param archive: full path to the archive file + """ + self.archive = archive + + @property + def md5_checksum(self): + """ + Calculate the md5 checksum of the archive + """ + assert os.path.isfile(self.archive), "%s is not a file" + hash_md5 = hashlib.md5() + with open(self.archive, "rb") as archive: + for chunk in iter(lambda: archive.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() diff --git a/ctrl_z/transfer/backends/__init__.py b/ctrl_z/transfer/backends/__init__.py new file mode 100644 index 0000000..e461018 --- /dev/null +++ b/ctrl_z/transfer/backends/__init__.py @@ -0,0 +1 @@ +from .base import Base # noqa diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py new file mode 100644 index 0000000..f62a6d6 --- /dev/null +++ b/ctrl_z/transfer/backends/base.py @@ -0,0 +1,74 @@ +import argparse + +from .. import BackupArchive, BackupTransfer +from ..._cli import CLI + + +class Base: + """ + Define the public API that a backend must implement. + """ + + def show_quota(self): + """ + Show information about remote storage quota. + """ + raise NotImplementedError("Transfer backends must implement a show_quota() method") + + def ensure_dirs(self, path: str): + """ + Ensure the folders in path exist on the remote. + + :param path: the path to the base directory on the remote. Folder names + are separated by forward slashes, e.g. /var/backups/staging + """ + pass + + def prepare(self, full_path: str) -> BackupArchive: + """ + Prepare the archive on the local drive for upload/check. + + :param full_path: full path of the directory to archive + :return: BackupArchive instance + """ + raise NotImplementedError("Transfer backends must implement a prepare() method") + + def exists(self, backup_archive: BackupArchive) -> bool: + """ + Check if the file exists on the remote + """ + raise NotImplementedError("Transfer backends must implement a exists() method") + + def upload(self, backup_archive): + """ + Upload the prepared archive to the remote + + :raises UploadError: if the integrity check failed, a + :class:`ctrl_z.transfer.UploadError` should be raised. + """ + raise NotImplementedError("Transfer backends must implement a upload() method") + + @classmethod + def add_arguments(cls, parser): + """ + Add optional command line arguments to extend the CLI. + + You may want to facilitate interacting with the remote through the CLI. + + :param parser: the ``transfer`` subparser of the main CLI, this is an + argparse parser. + """ + pass + + def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.Namespace) -> bool: + """ + Handle backend specific commands + + :param cli: the CLI instance asking to handle the command + :param transfer: the :class:`ctrl_z.transfer.BackupTransfer` instance + holding the config. + :param options: the CLI arguments parsed into argparse.Namespace + :return: boolean indicating if a command was handled or not, if not, + the default behaviour is to transfer the backup. + """ + return False diff --git a/ctrl_z/transfer/backends/google_drive.py b/ctrl_z/transfer/backends/google_drive.py new file mode 100644 index 0000000..e65bba0 --- /dev/null +++ b/ctrl_z/transfer/backends/google_drive.py @@ -0,0 +1,260 @@ +""" +Remote/offsite backup storage backend using Google Drive. + +Auth is done through service accounts, see +https://developers.google.com/identity/protocols/OAuth2ServiceAccount +""" +import argparse +import os +import sys +import tarfile + +import googleapiclient.discovery +from google.oauth2 import service_account +from googleapiclient.http import MediaFileUpload + +from .. import BackupArchive, BackupTransfer, UploadError +from ..._cli import CLI +from .base import Base + + +def sizeof_fmt(num, suffix='B'): + # see https://stackoverflow.com/a/1094933/973537 + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + + +def make_tarfile(output_filename: str, source_dir: str) -> None: + with tarfile.open(output_filename, "w:gz") as tar: + for name in os.listdir(source_dir): + full_path = os.path.join(source_dir, name) + tar.add(full_path, arcname=os.path.basename(full_path)) + assert os.path.exists(output_filename), "Archive creation (%s) failed" % output_filename + + +def pprint_key_value(mapping, formatter=None, stdout=None): + for key, value in mapping.items(): + value = formatter(value) if formatter else value + print(f" {key}: {value}", file=stdout) + + +class Backend(Base): + + SCOPES = [ + 'https://www.googleapis.com/auth/drive.file' + ] + + _folder_id = None + _http = None + + stdout = sys.stdout + + def __init__(self, client_secrets: str): + """ + A remote backup transfer backend using Google Drive. + + :param client_secrets: (absolute) path to the client_secrets.json file + containing the credentials to connect to the API. Note that a service + account is required for non-interactive behaviour (and the only way + implemented currently). + """ + self.client_secrets = client_secrets + + # Backend + @property + def service(self): + if not hasattr(self, '_service'): + credentials = service_account.Credentials.from_service_account_file( + self.client_secrets, + scopes=self.SCOPES + ) + + self._service = googleapiclient.discovery.build('drive', 'v3', credentials=credentials) + return self._service + + @property + def http(self): + return self._http + + def show_quota(self): + """ + Print the storage quota. + """ + quota = self.service.about().get(fields='storageQuota').execute(http=self.http)['storageQuota'] + print("Quota:", file=self.stdout) + pprint_key_value(quota, lambda x: sizeof_fmt(int(x)), stdout=self.stdout) + + def _get_or_create_dir(self, name, parent=None) -> str: + # search if the folder exists + search_q = "name='{}' and mimeType='application/vnd.google-apps.folder'".format(name) + if parent: + search_q += "and '{}' in parents".format(parent) + else: + search_q += "and 'root' in parents" + + folders = ( + self.service.files() + .list(q=search_q, fields='files(id, parents)') + .execute(http=self.http) + .get('files', []) + ) + if folders: + return folders[0]['id'] + + _folder = ( + self.service + .files() + .create(body={ + 'name': name, + 'mimeType': 'application/vnd.google-apps.folder', + 'parents': [parent] if parent else [], + }, fields='id') + .execute(http=self.http) + ) + return _folder['id'] + + def ensure_dirs(self, path: str): + """ + Ensure the folders in path exist on Drive. + """ + if path.startswith('/'): + path = path[1:] + if path.endswith('/'): + path = path[:-1] + + folders = path.split('/') + + _folder_id = None + for folder in folders: + _folder_id = self._get_or_create_dir(folder, _folder_id) + + self._folder_id = _folder_id + + def prepare(self, full_path: str) -> BackupArchive: + """ + Prepare the archive on the local drive. + + :param full_path: full path to the directory to archive + :return: BackupArchive instance + """ + output_filename = "{}.tar.gz".format(full_path) + + if not os.path.exists(output_filename): + make_tarfile(output_filename, full_path) + + return BackupArchive(output_filename) + + def exists(self, backup_archive: BackupArchive) -> bool: + """ + Check if the file exists on the remote + """ + assert self._folder_id, "ensure_dirs() must be called before uploading files" + + name = os.path.basename(backup_archive.archive) + search_q = ( + "name='{}' " + "and mimeType='application/gzip' " + "and '{}' in parents" + ).format(name, self._folder_id) + + files = ( + self.service + .files() + .list(q=search_q, fields='files(md5Checksum)') + .execute(http=self.http) + .get('files', []) + ) + if files: + checksum = backup_archive.md5_checksum + return any([ + file_['md5Checksum'] == checksum for file_ in files + ]) + + return False + + def upload(self, backup_archive): + assert self._folder_id, "ensure_dirs() must be called before uploading files" + file_name = os.path.basename(backup_archive.archive) + metadata = { + 'name': file_name, + 'originalFilename': backup_archive.archive, + 'parents': [self._folder_id] + } + media = MediaFileUpload(backup_archive.archive, mimetype='application/gzip') + + response = ( + self.service + .files() + .create( + body=metadata, + media_body=media, + fields='md5Checksum' + ) + .execute(http=self.http) + ) + + if response['md5Checksum'] != backup_archive.md5_checksum: + raise UploadError("Uploaded archive checksum does not match") + + # CLI extension + @classmethod + def add_arguments(cls, parser): + BackendCli.add_arguments(parser) + + def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.Namespace) -> bool: + self.stdout = cli.stdout + return BackendCli.handle_command(transfer, options) + + def test_connection(self): + user = self.service.about().get(fields='user').execute(http=self.http)['user'] + print("User:", file=self.stdout) + pprint_key_value(user, stdout=self.stdout) + + def give_permissions(self, transfer: BackupTransfer, email: str): + """ + Assign read permissions to the e-mail address. + """ + bits = [bit for bit in transfer.config.transfer_path.split('/') if bit] + folder_id = self._get_or_create_dir(bits[0], parent=None) + + self.service.permissions().create( + fileId=folder_id, + body={ + 'type': 'user', + 'role': 'reader', + 'emailAddress': email, + } + ).execute(http=self.http) + print("Folder {} can now be read".format(bits[0]), file=self.stdout) + + +class BackendCli: + + @staticmethod + def add_arguments(parser): + drive_parser = parser.add_subparsers(help='Google Drive commands', dest='drive_command') + + drive_parser.add_parser('test_connection', help="Test Google Drive connection/credentials") + + drive_parser.add_parser('show_quota', help="Show the Drive quota") + + give_permissions = drive_parser.add_parser('give_permissions', help="Give read permissions to a user") + give_permissions.add_argument('email', help="E-mail address of user to get read permission") + + @staticmethod + def handle_command(transfer: BackupTransfer, options: argparse.Namespace) -> bool: + subcommand = options.drive_command + + if not subcommand: + return False + + if subcommand == 'give_permissions': + transfer.backend.give_permissions(transfer, options.email) + elif subcommand == 'test_connection': + transfer.backend.test_connection() + elif subcommand == 'show_quota': + transfer.backend.show_quota() + return True diff --git a/ctrl_z/transfer/transfer.py b/ctrl_z/transfer/transfer.py new file mode 100644 index 0000000..f0242e5 --- /dev/null +++ b/ctrl_z/transfer/transfer.py @@ -0,0 +1,79 @@ +import logging +import os + +from django.utils.module_loading import import_string + +from ..config import Config +from ..utils import is_backup_dir + +logger = logging.getLogger(__name__) + + +class UploadError(Exception): + pass + + +class BackupTransfer: + + def __init__(self, config: Config): + self.config = config + + cls = import_string(config.transfer_backend) + self.backend = cls(**config.transfer_backend_init_kwargs) + + @classmethod + def from_config(cls, config_file, **overrides): + overrides['use_parent_dir'] = True + config = Config.from_file(config_file, **overrides) + return cls(config=config) + + def show_info(self): + """ + Inventarize the operations to be done. + """ + self.backend.show_quota() + + def sync_to_remote(self): + """ + Transfer the backups to the remote. + """ + logger.info("Scanning %s for backups to transfer...", self.config.base_dir) + + backup_dirs = [] + + for dirname in sorted(os.listdir(self.config.base_dir)): + full_path = os.path.join(self.config.base_dir, dirname) + if not os.path.isdir(full_path): + continue + if not is_backup_dir(dirname): + continue + backup_dirs.append(dirname) + + # make sure the (relative) directory structure exists on the remote + # this depends on the backend - for rsync for example this may not be + # needed as it creates them on the fly + self.backend.ensure_dirs(self.config.transfer_path) + + logger.info("Syncing backups %s to remote...", backup_dirs) + + has_failures = False + for dirname in backup_dirs: + full_path = os.path.join(self.config.base_dir, dirname) + backup_archive = self.backend.prepare(full_path) + if self.backend.exists(backup_archive): + logger.info("Backup %s exists on remote, skipping", dirname) + continue + + logger.info("Uploading %s to remote...", dirname) + try: + self.backend.upload(backup_archive) + except UploadError: + logger.exception("%s upload failed", dirname) + has_failures = True + continue + logger.info("Uploaded %s to remote", dirname) + + if not has_failures: + logger.info("Done syncing backups to remote") + else: + raise UploadError("Some backups failed to upload, check the error log") diff --git a/ctrl_z/utils.py b/ctrl_z/utils.py new file mode 100644 index 0000000..bda20b6 --- /dev/null +++ b/ctrl_z/utils.py @@ -0,0 +1,12 @@ +from .constants import BACKUP_DIR_PATTERN + + +def is_backup_dir(dir_name: str) -> bool: + """ + Test if a directory name fits the pattern of backup folder names. + + The pattern is YYYY-MM-DD-suffix, where suffix is either 'daily' or 'weekly'. + """ + if BACKUP_DIR_PATTERN.match(dir_name): + return True + return False diff --git a/doc/conf.py b/doc/conf.py index ac3316f..dac2500 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,9 +12,10 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- @@ -40,6 +41,7 @@ # ones. extensions = [ 'sphinx.ext.todo', + 'sphinx.ext.autodoc', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/configuration.rst b/doc/configuration.rst index 89b5c67..d72db94 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -5,8 +5,7 @@ Configuration ============= Configuration is done in YAML format. You can pass the config file to use -to the ``cli`` object (see :ref:`usage`), or pass it as a global CLI option -``--config-file``. +to the ``cli`` object (see :ref:`usage`). Global config options ===================== @@ -130,3 +129,26 @@ Which binary to use to dump the database. Defaults to ``/usr/bin/pg_dump``. --------------------- Which binary to use to dump the database. Defaults to ``/usr/bin/pg_restore``. + +``transfer_backend`` +-------------------- + +The backend class to use to transfer backups from the local file system to a +remote/off-site location. Specify a dotted Python path to import. + +Available options: + +* ``ctrl_z.transfer.backends.google_drive.Backend`` - see :ref:`transfer-backend-drive` + +``transfer_backend_init_kwargs`` +-------------------------------- + +A mapping, specifying any keyword arguments needed to initialize the backend. +See :ref:`transfer-backends` for detailed information. + +``transfer_path`` +----------------- + +A string representing the base path to transfer backends to on the remote +system. Always use forward slashes. Defaults to ``/``. The backend is +responsible for creating the directory tree if it doesn't exist yet. diff --git a/doc/index.rst b/doc/index.rst index 5f8a3a8..bf05403 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -22,6 +22,7 @@ while being flexible through a yaml configuration file. quickstart configuration + transfer_backends Indices and tables diff --git a/doc/quickstart.rst b/doc/quickstart.rst index b6f3c71..511bc10 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -19,6 +19,13 @@ Install pip install ctrl-z +If you're using a built in backend to transfer the backups to a remote/offsite +location (currently only Google Drive is supported), you can install the +extra dependencies as well: + +.. code-block:: bash + + pip install ctrl-z[transfer_backend.drive] .. _usage: @@ -55,6 +62,8 @@ CTRL-Z exposes a CLI object to hook into your project, for example: Once the setup around the CLI is done, you can use it. +.. _cli-help: + CLI help -------- @@ -136,3 +145,16 @@ Restore the backup at the specified path. for. Useful if you have a multi-db setup and only the ``default`` is important, for example. Use multiple times for each alias to skip. * ``--no-files``: do not restore the (uploaded) files (e.g. ``settings.MEDIA_ROOT``) + + +Transfer a backup +----------------- + +.. code-block:: bash + + python backup/cli.py backup transfer + +Depending on the backend used, more CLI options may be available. These show +up via the built-in :ref:`cli-help`. + +The transfer is done according to the configured backend. diff --git a/doc/transfer_backends.rst b/doc/transfer_backends.rst new file mode 100644 index 0000000..537d136 --- /dev/null +++ b/doc/transfer_backends.rst @@ -0,0 +1,153 @@ +.. _transfer-backends: + +================= +Transfer backends +================= + +Just creating backups and leaving them on the local file system is not enough. +To deal with hardware failure, you should transfer your backups to *off-site* +systems. In CTRL-Z, you can achieve this through transfer backends. + +A transfer backend must: + +* prepare a local backup directory, for example creating a single-file archive + for ease of transfer. +* create the target directory hierarchy on the remote system, if needed +* be able to say if a local backup already exists on the remote or not +* be able to upload the prepared backup to the remote (and verify it's integrity) + +A transfer backend may optionally define subcommands for the main command line +interface. + +Available backends: + +* :ref:`transfer-backend-drive` + +Third party backends can be developed if needed, and installed as a separate +package. The YAML configuration is used to specify which backend to use and +which ``__init__`` parameters are required for the backend to configure +runtime options. + +Base backend +============ + +Custom backends should inherit from the base backend to guarantee API +compatibility. + +This backend is not functional by itself - it's an abstract base class. + +.. autoclass:: ctrl_z.transfer.backends.base.Base + :members: + + +.. _transfer-backend-drive: + +Google Drive +============ + +The Google Drive backend uploads the documents to Google Drive through the +Drive API. You need to create a `service account `_ +(see :ref:`drive-getting-started`) to be able to run the transfer without +interaction. + +.. todo:: See how to integrate with `team drives`_. + +The Drive backend: + +* creates the subfolders in Drive to store the backups if needed +* creates a gzipped tar-archive of the backup on the local file system +* checks if the backup exists by matching the name of the archive and the md5 + checksum against the (potentially) existing file on Drive +* uploads the archive to Drive and verifies integrity by comparing the + reported checksum with the local checksum + +Additionally, this backend provides some :ref:`drive-cli-extensions`. + +.. _drive-getting-started: + +Getting started +--------------- + +`google service accounts`_ are Google's way to interact with APIs on behalf +of a single user/entity. This is in contrast with OAUTH2 based flows where the +application using the Drive API typically interacts with the *account of the +end user*. For transfer backends, there are no end-users. + +One drawback is that you cannot have the Google Drive interface as the service +account, there is only API access. You can work around this by sharing the +folder holding the backups in Drive with yourself, see the +:ref:`drive-cli-extensions`. + +#. In the `google cloud console`_, create or select the project to use for the + backup transfer. +#. Next, click **+ Create service account**, and fill out the fields, e.g.: + + * Service account name: isp-backups + * Project role: you don't need to specify any role + * Check 'Furnish a private key' and check the default 'JSON' + +#. Backup the key file download to your browser - there's no way to obtain this + again. You can generate new keys of course. + +#. Configure CTRL-Z - in your ``config.yml``, ensure you have: + + .. code-block:: yaml + + transfer_backend: ctrl_z.transfer.backends.google_drive.Backend + transfer_backend_init_kwargs: + client_secrets: /path/to/the/downloaded/key/file.json + transfer_path: /some/root/dir + + + It's recommended to create at least one root directory to facilitate + sharing the folder with other people, since it's not possible to share the + root folder of Drive. + +#. Test the connection: + + .. code-block:: bash + + $ python cli.py transfer test_connection + + +.. _drive-cli-extensions: + +CLI extensions +-------------- + +The Drive backend extends the CLI with some subcommands. + +Connection test ++++++++++++++++ + +To verify that your credentials are correct, you can do a connection test: + +.. code-block:: bash + + $ python cli.py transfer test_connection + +Show quota +++++++++++ + +You can display the quota usage without needing to initiate a transfer: + +.. code-block:: bash + + $ python cli.py transfer show_quota + +Give read access +++++++++++++++++ + +You can share the root foldder with: + +.. code-block:: bash + + $ python cli.py transfer give_permissions hello@example.com + +The owner of the e-mail address then has read access to the folder in their +Drive via the 'Shared with me' tab. + + +.. _google service accounts: https://cloud.google.com/iam/docs/service-accounts +.. _google cloud console: https://console.cloud.google.com +.. _team drives: https://stackoverflow.com/questions/43243865/how-to-access-team-drive-using-service-account-with-google-drive-net-api-v3 diff --git a/setup.cfg b/setup.cfg index 33bd324..b9651b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,15 +34,24 @@ tests_require = pytest pytest-django pytest-freezegun + pytest-mock tox isort +[options.packages.find] +exclude = + tests + [options.extras_require] +transfer_backend.drive = + google-api-python-client + google-auth tests = psycopg2-binary pytest pytest-django pytest-freezegun + pytest-mock tox isort pep8 = flake8 diff --git a/tests/backends/test_google_drive.py b/tests/backends/test_google_drive.py new file mode 100644 index 0000000..feb0aa8 --- /dev/null +++ b/tests/backends/test_google_drive.py @@ -0,0 +1,265 @@ +import json +import logging +import tarfile +from io import StringIO + +import googleapiclient.discovery +import pytest +from apiclient.http import HttpMock, HttpMockSequence + +from ctrl_z._cli import CLI +from ctrl_z.transfer import UploadError +from ctrl_z.transfer.backends.google_drive import Backend + + +class Service: + _service = None + + @classmethod + def get(cls): + if not cls._service: + # use the real discovery file - makes an actual HTTP request + # this is a conscious decision - we don't want to be testing + # against a no-longer functional discovery file + cls._service = googleapiclient.discovery.build('drive', 'v3', developerKey='dummy') + return cls._service + + +def _get_backend(response: dict=None, responses=None) -> Backend: + logging.disable(logging.CRITICAL) + backend = Backend('dummy') + backend.stdout = StringIO() + backend._service = Service.get() + logging.disable(logging.NOTSET) + + # set the response data + backend._http = HttpMock() + if response: + backend._http.data = json.dumps(response) + + elif responses: + iterable = [ + ({'status': '200'}, json.dumps(response)) + for response in responses + ] + backend._http = HttpMockSequence(iterable) + + return backend + + +class CLIBackend(Backend): + + def __init__(self, response): + logging.disable(logging.CRITICAL) + self._service = Service.get() + self._http = HttpMock() + self._http.data = json.dumps(response) + logging.disable(logging.NOTSET) + + def _get_or_create_dir(self, *args, **kwargs) -> str: + return 'dummy' + + +def test_show_quota(capsys): + backend = _get_backend(response={ + 'storageQuota': { + 'limit': '16106127360', + 'usage': '13344214', + 'usageInDrive': '13344214', + 'usageInDriveTrash': '0' + } + }) + + backend.show_quota() + + out = backend.stdout + out.seek(0) + assert out.read() == """Quota: + limit: 15.0GiB + usage: 12.7MiB + usageInDrive: 12.7MiB + usageInDriveTrash: 0.0B +""" + + +def test_prepare(tmpdir): + """ + Test that the prepare method creates a tar.gz archive + """ + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + + archive = backend.prepare(str(tmpdir)) + + with tarfile.open(archive.archive, 'r:gz') as arc: + members = arc.getmembers() + + assert len(members) == 1 + assert members[0].name == 'some_file.sql' + + +@pytest.mark.parametrize("files,exists", [ + ([], False), + ([{'md5Checksum': '5bd245d3a0f0183dcdf4ae4893bf1312'}, + {'md5Checksum': '865689b06759eb81cda2e0bcdf742d47'}], False), +]) +def test_archive_doesnt_exists(tmpdir, files, exists): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend(response={'files': files}) + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + assert backend.exists(archive) == exists + + +def test_archive_exists(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + backend._http.data = json.dumps({ + # need to use the actual checksum here, since gzip includes the timestamp + # in the header + 'files': [{'md5Checksum': archive.md5_checksum}] + }) + + assert backend.exists(archive) + + +def test_upload_success(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + backend._http.data = json.dumps( + # need to use the actual checksum here, since gzip includes the + # timestamp in the header + {'md5Checksum': archive.md5_checksum} + ) + + try: + backend.upload(archive) + except UploadError: + pytest.fail("Upload should succeed when md5_checksums match") + + +def test_upload_fails(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend({'md5Checksum': 'incorrect'}) + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + with pytest.raises(UploadError): + backend.upload(archive) + + +def test_ensure_dirs_create(): + """ + Check that the directory structure is created in Drive. + """ + backend = _get_backend(responses=[ + { + 'files': [], + }, + { + 'id': 'dummy-123' + } + ]) + + backend.ensure_dirs('/sub') + + assert backend._folder_id == 'dummy-123' + + +def test_ensure_dirs_exists(): + """ + Check that the directory structure is created in Drive. + """ + backend = _get_backend(responses=[ + { + 'files': [{'id': 'dummy-123'}], + }, + ]) + + backend.ensure_dirs('/sub') + + assert backend._folder_id == 'dummy-123' + + +def test_cli_extension_test_connection(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': { + 'user': { + 'foo': 'bar' + } + }} + ) + stdout = StringIO() + cli = CLI() + + cli(['transfer', 'test_connection'], config_file=config_path, stdout=stdout) + + stdout.seek(0) + assert stdout.read() == "User:\n foo: bar\n" + + +def test_cli_extension_show_quota(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': { + 'storageQuota': { + 'limit': '16106127360', + 'usage': '13344214', + 'usageInDrive': '13344214', + 'usageInDriveTrash': '0' + } + }} + ) + stdout = StringIO() + cli = CLI() + + cli(['transfer', 'show_quota'], config_file=config_path, stdout=stdout) + + stdout.seek(0) + assert stdout.read() == """Quota: + limit: 15.0GiB + usage: 12.7MiB + usageInDrive: 12.7MiB + usageInDriveTrash: 0.0B +""" + + +def test_cli_extension_give_permissions(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': {}}, + transfer_path='/root/sub/dir' + ) + stdout = StringIO() + cli = CLI() + + cli( + ['transfer', 'give_permissions', 'hello@example.com'], + config_file=config_path, stdout=stdout + ) + + stdout.seek(0) + assert stdout.read() == 'Folder root can now be read\n' diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fb1b20..ab07096 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,100 +10,154 @@ import pytest -from ctrl_z import cli +from ctrl_z import __version__, cli +from ctrl_z._cli import HEADER from ctrl_z.config import DEFAULT_CONFIG_FILE -def test_config_generation(config_path): - cli(['generate_config'], stdout=StringIO(), config_file=config_path) +class TestConfigGeneration: - cli.stdout.seek(0) - result = cli.stdout.read() - with open(DEFAULT_CONFIG_FILE, 'r') as default_config: - assert result == default_config.read() + def test_config_generation(self, config_path): + cli(['generate_config'], stdout=StringIO(), stderr=StringIO(), config_file=config_path) + cli.stdout.seek(0) + result = cli.stdout.read() + with open(DEFAULT_CONFIG_FILE, 'r') as default_config: + assert result == default_config.read() -def test_config_generation_external_file(tmpdir, config_path): - tempfile = str(tmpdir.join("some_config.yml")) + # when piping stdout to a file, we don't want the header to end there + # as well, so it should be in stderr + cli.stderr.seek(0) + assert HEADER.format(version=__version__) in cli.stderr.read() - cli( - ['generate_config', '-o', tempfile], - stdout=StringIO(), config_file=config_path - ) + def test_config_generation_external_file(self, tmpdir, config_path): + tempfile = str(tmpdir.join("some_config.yml")) - cli.stdout.seek(0) - assert cli.stdout.read() == '' - with open(DEFAULT_CONFIG_FILE, 'r') as default_config: - with open(tempfile, 'r') as config: - assert config.read() == default_config.read() + cli( + ['generate_config', '-o', tempfile], + stdout=StringIO(), config_file=config_path + ) + cli.stdout.seek(0) + assert cli.stdout.read() == '' + with open(DEFAULT_CONFIG_FILE, 'r') as default_config: + with open(tempfile, 'r') as config: + assert config.read() == default_config.read() -def test_full_backup(tmpdir, settings, config_writer): - config_path = str(tmpdir.join('config.yml')) - backups_base = tmpdir.join('backups') - config_writer(config_path, base_dir=str(backups_base)) +class TestBackup: - # prevent actual db access - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning) - settings.DATABASES = {} - settings.MEDIA_ROOT = str(tmpdir.join('media')) + def test_full_backup(self, tmpdir, settings, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') - cli(args=['backup'], config_file=config_path, stdout=StringIO()) + config_writer(config_path, base_dir=str(backups_base)) - expected_date = datetime.utcnow().strftime("%Y-%m-%d") + # prevent actual db access + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning) + settings.DATABASES = {} + settings.MEDIA_ROOT = str(tmpdir.join('media')) - # assert that the backup directory was created - children = os.listdir(backups_base) - assert len(children) == 1 - backup_dir = children[0] - assert backup_dir.startswith(expected_date) + cli(args=['backup'], config_file=config_path, stdout=StringIO()) - full_path = backups_base.join(backup_dir) - subdirs = os.listdir(full_path) - assert sorted(subdirs) == ['backup.log', 'db', 'files'] + expected_date = datetime.utcnow().strftime("%Y-%m-%d") + # assert that the backup directory was created + children = os.listdir(backups_base) + assert len(children) == 1 + backup_dir = children[0] + assert backup_dir.startswith(expected_date) -def test_full_restore(tmpdir, settings, config_writer): - config_path = str(tmpdir.join('config.yml')) - backups_base = tmpdir.mkdir('backups') - backup_dir = backups_base.mkdir('2018-05-29-daily') - backup_dir.mkdir('db') - backup_dir.mkdir('files') + full_path = backups_base.join(backup_dir) + subdirs = os.listdir(full_path) + assert sorted(subdirs) == ['backup.log', 'db', 'files'] - config_writer(config_path, base_dir=str(backups_base)) - # prevent actual db access - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning) - settings.DATABASES = {} - settings.MEDIA_ROOT = str(tmpdir.join('media')) +class TestRestore: + + def test_full_restore(self, tmpdir, settings, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.mkdir('backups') + backup_dir = backups_base.mkdir('2018-05-29-daily') + backup_dir.mkdir('db') + backup_dir.mkdir('files') + + config_writer(config_path, base_dir=str(backups_base)) + + # prevent actual db access + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning) + settings.DATABASES = {} + settings.MEDIA_ROOT = str(tmpdir.join('media')) + + cli(args=['restore', str(backup_dir)], config_file=config_path, stdout=StringIO()) + + # verify that the log file was created + assert 'backup.log' in os.listdir(backup_dir) - cli(args=['restore', str(backup_dir)], config_file=config_path, stdout=StringIO()) + def test_full_restore_bad_directory(self): + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', '/i/dont/exist/'], stdout=StringIO()) - # verify that the log file was created - assert 'backup.log' in os.listdir(backup_dir) + @pytest.mark.skipif(sys.platform == 'win32', + reason="does not run on windows") + def test_full_restore_bad_directory2(self, tmpdir): + bad_permissions_dir = str(tmpdir.mkdir('nope')) + os.chmod(bad_permissions_dir, 0o000) + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', bad_permissions_dir], stdout=StringIO()) -def test_full_restore_bad_directory(): - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', '/i/dont/exist/'], stdout=StringIO()) + def test_full_restore_not_directory(self, tmpdir): + some_file = tmpdir.join('not_a_dir.txt') + some_file.write("not a dir!\n") + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', str(some_file)], stdout=StringIO()) -@pytest.mark.skipif(sys.platform == 'win32', - reason="does not run on windows") -def test_full_restore_bad_directory2(tmpdir): - bad_permissions_dir = str(tmpdir.mkdir('nope')) - os.chmod(bad_permissions_dir, 0o000) - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', bad_permissions_dir], stdout=StringIO()) +class TestTransfer: + def test_transfer_main_command(self, tmpdir, config_writer, mocker): + """ + Assert that the transfer is initiated if no backend-specific subcommand was handled. + """ + mocked_BackupTransfer = mocker.patch('ctrl_z._cli.BackupTransfer') + transfer = mocked_BackupTransfer.from_config.return_value + transfer.backend.handle_command.return_value = False -def test_full_restore_not_directory(tmpdir): - some_file = tmpdir.join('not_a_dir.txt') - some_file.write("not a dir!\n") + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli(['transfer'], config_file=config_path) + transfer.sync_to_remote.assert_called_once() + + def test_transfer_subcommand_handled(self, tmpdir, config_writer, mocker): + """ + Assert that the transfer is not initiated if a backend-specific subcommand was handled. + """ + mocked_BackupTransfer = mocker.patch('ctrl_z._cli.BackupTransfer') + transfer = mocked_BackupTransfer.from_config.return_value + transfer.backend.handle_command.return_value = True + + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli(['transfer'], config_file=config_path) + transfer.sync_to_remote.assert_not_called() + + +def test_no_subcommand(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli([], stdout=StringIO(), config_file=config_path) + + cli.stdout.seek(0) + output = cli.stdout.read() - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', str(some_file)], stdout=StringIO()) + assert 'usage: ' in output diff --git a/tests/test_transfer.py b/tests/test_transfer.py new file mode 100644 index 0000000..6345c53 --- /dev/null +++ b/tests/test_transfer.py @@ -0,0 +1,110 @@ +import pytest + +from ctrl_z.config import DEFAULT_CONFIG_FILE +from ctrl_z.transfer import BackupArchive, BackupTransfer, UploadError +from ctrl_z.transfer.backends import Base + + +def test_public_api(tmpdir): + """ + Test that the base backend performs the appropriate checks for subclasses. + """ + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + + transfer = BackupTransfer.from_config(DEFAULT_CONFIG_FILE, base_dir=str(base_dir)) + + with pytest.raises(NotImplementedError): + transfer.show_info() + + with pytest.raises(NotImplementedError): + transfer.sync_to_remote() + + +class DummyBackend(Base): + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def exists(self, *args, **kwargs): + return True + + def prepare(self, full_path: str) -> BackupArchive: + return BackupArchive(full_path) + + def upload(self, archive): + raise AssertionError("No uploads should happen if there are no backups") + + +def test_load_backend_from_settings(): + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + transfer_backend='tests.test_transfer.DummyBackend', + transfer_backend_init_kwargs={'foo': 'bar'} + ) + + assert isinstance(transfer.backend, DummyBackend) + assert transfer.backend.kwargs == {'foo': 'bar'} + + +def test_sync_nothing_to_sync(tmpdir): + """ + Test that the backend is not called if there's nothing to sync. + """ + base_dir = tmpdir.mkdir('backups') + # create some files that should be ignored + base_dir.join('2018-07-17-daily').write('not a directory') + base_dir.mkdir('ignore-me') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + try: + transfer.sync_to_remote() + except AssertionError: + pytest.fail("Upload should NOT have been called") + + +def test_sync_skip_existing(tmpdir): + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + try: + transfer.sync_to_remote() + except AssertionError: + pytest.fail("Upload should NOT have been called") + + +def test_upload_fails_continue_others(tmpdir, mocker): + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + base_dir.mkdir('2018-07-18-daily') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + def side_effect(archive): + if archive.archive.endswith('2018-07-17-daily'): + raise UploadError("'Random' failure") + else: + pass + + mocker.patch.object(transfer.backend, 'exists', return_value=False) + mock_upload = mocker.patch.object(transfer.backend, 'upload', side_effect=side_effect) + + with pytest.raises(UploadError): + transfer.sync_to_remote() + + assert mock_upload.call_count == 2 diff --git a/tox.ini b/tox.ini index 69df151..54a4110 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ DJANGO = extras = tests coverage + transfer_backend.drive deps = django20: Django>=2.0,<2.1 passenv =