From 1a14e2635c7ad65373f083f9fd9b0c0a3ab27085 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 10 Jul 2018 18:07:16 +0200 Subject: [PATCH 01/10] Google Drive backend for remote storage --- ctrl_z/backup.py | 2 +- ctrl_z/cli.py | 15 ++ ctrl_z/config.default.yml | 4 + ctrl_z/config.py | 10 +- ctrl_z/constants.py | 5 + ctrl_z/retention.py | 24 +--- ctrl_z/transfer/__init__.py | 98 +++++++++++++ ctrl_z/transfer/backends/__init__.py | 0 ctrl_z/transfer/backends/base.py | 0 ctrl_z/transfer/backends/google_drive.py | 171 +++++++++++++++++++++++ ctrl_z/utils.py | 12 ++ setup.cfg | 4 + 12 files changed, 322 insertions(+), 23 deletions(-) create mode 100644 ctrl_z/constants.py create mode 100644 ctrl_z/transfer/__init__.py create mode 100644 ctrl_z/transfer/backends/__init__.py create mode 100644 ctrl_z/transfer/backends/base.py create mode 100644 ctrl_z/transfer/backends/google_drive.py create mode 100644 ctrl_z/utils.py diff --git a/ctrl_z/backup.py b/ctrl_z/backup.py index b6e7afc..0d8a31e 100644 --- a/ctrl_z/backup.py +++ b/ctrl_z/backup.py @@ -33,7 +33,7 @@ def from_config(cls, config_file): @classmethod def prepare_restore(cls, config_file, base_dir: str): - config = Config.from_file(config_file, base_dir=base_dir, restore=True) + config = Config.from_file(config_file, base_dir=base_dir, use_parent_dir=True) return cls(config=config) def restore(self, db=True, skip_db=None, files=True): diff --git a/ctrl_z/cli.py b/ctrl_z/cli.py index e507e68..8dcf313 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/cli.py @@ -10,6 +10,7 @@ from .backup import Backup, configure_logging from .config import DEFAULT_CONFIG_FILE +from .transfer import BackupTransfer logger = logging.getLogger(__name__) @@ -117,6 +118,11 @@ def __init__(self): default=True, help="Do not restore files" ) + # backup transfer + parser_transfer = subparsers.add_parser( + 'transfer', help='Transfer the backups to an off-site location' + ) + self.parser = parser def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, stderr=None): @@ -166,6 +172,8 @@ def run(self, options, config_file: str): self.backup(options) elif subcommand == 'restore': self.restore(options) + elif subcommand == 'transfer': + self.transfer(options, config_file, conf_overrides) else: self.parser.print_help() @@ -219,5 +227,12 @@ def restore(self, options): finally: backup.report(has_errors) + def transfer(self, options, config_file, conf_overrides): + conf_overrides['use_parent_dir'] = True + + transfer = BackupTransfer.from_config(config_file, **conf_overrides) + transfer.show_info() + transfer.sync_to_remote() + cli = CLI() diff --git a/ctrl_z/config.default.yml b/ctrl_z/config.default.yml index 9ab60b9..112e436 100644 --- a/ctrl_z/config.default.yml +++ b/ctrl_z/config.default.yml @@ -33,3 +33,7 @@ files: # Which binaries to use for backup creation/restore pg_dump_binary: /usr/bin/pg_dump pg_restore_binary: /usr/bin/pg_restore + +# where are backups transferred (off-site) +transfer_backend: ctrl_z.transfer.backends.google_drive.Backend +transfer_path: / diff --git a/ctrl_z/config.py b/ctrl_z/config.py index 76748c9..3c5f4bb 100644 --- a/ctrl_z/config.py +++ b/ctrl_z/config.py @@ -16,7 +16,7 @@ class Config: __slots__ = [ - 'restore', + 'use_parent_dir', 'base_dir', 'logging', 'database', @@ -25,10 +25,12 @@ class Config: 'files', 'pg_dump_binary', 'pg_restore_binary', + 'transfer_backend', + 'transfer_path', ] def __init__(self, **kwargs): - self.restore = kwargs.pop('restore', False) + self.use_parent_dir = kwargs.pop('use_parent_dir', False) for key, value in kwargs.items(): setattr(self, key, value) @@ -54,14 +56,14 @@ def from_file(cls, config_file: str, **overrides): def write_to(self, path: str): as_dict = {key: getattr(self, key) for key in self.__slots__} - if not self.restore: + if not self.use_parent_dir: as_dict['base_dir'] = os.path.dirname(as_dict['base_dir']) as_dict['retention_policy'] = as_dict['retention_policy'].serialize() with open(path, 'w') as stream: yaml.dump(as_dict, stream=stream) def set_base_dir(self): - if self.restore: # should be set via overrides + if self.use_parent_dir: # should be set via overrides return self.base_dir = self.retention_policy.get_base_dir(self.base_dir) diff --git a/ctrl_z/constants.py b/ctrl_z/constants.py new file mode 100644 index 0000000..1becf82 --- /dev/null +++ b/ctrl_z/constants.py @@ -0,0 +1,5 @@ +import re + +DATE_FORMAT = "%Y-%m-%d" + +BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)') diff --git a/ctrl_z/retention.py b/ctrl_z/retention.py index b51a555..78b3f1d 100644 --- a/ctrl_z/retention.py +++ b/ctrl_z/retention.py @@ -1,6 +1,5 @@ import logging import os -import re import shutil from datetime import date, datetime from itertools import chain @@ -9,16 +8,15 @@ from dateutil.relativedelta import relativedelta from dateutil.rrule import DAILY, WEEKLY, rrule +from .constants import DATE_FORMAT +from .utils import is_backup_dir + logger = logging.getLogger(__name__) class RetentionPolicy: __slots__ = ['day_of_week', 'days_to_keep', 'weeks_to_keep'] - DATE_FORMAT = "%Y-%m-%d" - - BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)') - def __init__(self, **config): for key, value in config.items(): setattr(self, key, value) @@ -26,16 +24,6 @@ def __init__(self, **config): def serialize(self): return {key: getattr(self, key) for key in self.__slots__} - def is_backup_dir(self, dir_name: str) -> bool: - """ - Test if a directory name fits the pattern of backup folder names. - - The pattern is YYYY-MM-DD-suffix, where suffix is either 'daily' or 'weekly'. - """ - if self.BACKUP_DIR_PATTERN.match(dir_name): - return True - return False - def get_suffix(self, dt: Union[date, datetime]) -> str: return 'weekly' if dt.weekday() == self.day_of_week else 'daily' @@ -44,7 +32,7 @@ def get_base_dir(self, base: str) -> str: Figure out the folder name for the current backup. """ now = datetime.utcnow() - datestamp = now.strftime(self.DATE_FORMAT) + datestamp = now.strftime(DATE_FORMAT) suffix = self.get_suffix(now) return os.path.join(base, f"{datestamp}-{suffix}") @@ -67,14 +55,14 @@ def rotate(self, base: str): weeklies = rrule(WEEKLY, dtstart=weekly_start, count=self.weeks_to_keep) to_keep = sorted({ - "{}-{}".format(dt.strftime(self.DATE_FORMAT), self.get_suffix(dt)) + "{}-{}".format(dt.strftime(DATE_FORMAT), self.get_suffix(dt)) for dt in chain(dailies, weeklies) }) logger.debug("Keeping backups from: %r", to_keep) to_delete = [] for dir_name in os.listdir(base): - if not self.is_backup_dir(dir_name): + if not is_backup_dir(dir_name): logger.debug("%s doesn't look like a backup directory, keeping it.", dir_name) continue diff --git a/ctrl_z/transfer/__init__.py b/ctrl_z/transfer/__init__.py new file mode 100644 index 0000000..eb8165f --- /dev/null +++ b/ctrl_z/transfer/__init__.py @@ -0,0 +1,98 @@ +import hashlib +import logging +import os + +from django.utils.module_loading import import_string + +from ..config import Config +from ..utils import is_backup_dir + +logger = logging.getLogger(__name__) + + +class BackupArchive: + def __init__(self, archive: str): + """ + A single backup as an archive. + + The archive format depends on the backend, it could be tar.gz, zip... + + :param archive: full path to the archive file + """ + assert os.path.isfile(archive), "%s is not a file" + self.archive = archive + + @property + def md5_checksum(self): + """ + Calculate the md5 checksum of the archive + """ + hash_md5 = hashlib.md5() + with open(self.archive, "rb") as archive: + for chunk in iter(lambda: archive.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +class UploadError(Exception): + pass + + +class BackupTransfer: + + def __init__(self, config: Config): + self.config = config + self.backend = import_string(config.transfer_backend)() + + @classmethod + def from_config(cls, config_file, **overrides): + config = Config.from_file(config_file, **overrides) + return cls(config=config) + + def show_info(self): + """ + Inventarize the operations to be done. + """ + self.backend.show_quota() + + def sync_to_remote(self): + """ + Transfer the backups to the remote. + """ + logger.info("Scanning %s for backups to transfer...", self.config.base_dir) + + backup_dirs = [] + + for dirname in os.listdir(self.config.base_dir): + full_path = os.path.join(self.config.base_dir, dirname) + if not os.path.isdir(full_path): + continue + if not is_backup_dir(dirname): + continue + backup_dirs.append(dirname) + + self.backend.ensure_dirs(self.config.transfer_path) + + logger.info("Syncing backups %s to remote...", backup_dirs) + + has_failures = False + for dirname in backup_dirs: + full_path = os.path.join(self.config.base_dir, dirname) + backup_archive = self.backend.prepare(full_path) + if self.backend.exists(backup_archive): + logger.info("Backup %s exists on remote, skipping", dirname) + continue + + logger.info("Uploading %s to remote...", dirname) + try: + self.backend.upload(backup_archive) + except UploadError: + logger.exception("%s upload failed", dirname) + has_failures = True + continue + logger.info("Uploaded %s to remote", dirname) + + if not has_failures: + logger.info("Done syncing backups to remote") + else: + raise UploadError("Some backups failed to upload, check the error log") diff --git a/ctrl_z/transfer/backends/__init__.py b/ctrl_z/transfer/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py new file mode 100644 index 0000000..e69de29 diff --git a/ctrl_z/transfer/backends/google_drive.py b/ctrl_z/transfer/backends/google_drive.py new file mode 100644 index 0000000..a222e9c --- /dev/null +++ b/ctrl_z/transfer/backends/google_drive.py @@ -0,0 +1,171 @@ +""" +Remote/offsite backup storage backend using Google Drive. + +Auth is done through service accounts, see +https://developers.google.com/identity/protocols/OAuth2ServiceAccount +""" +import os +import tarfile + +import googleapiclient.discovery +from google.oauth2 import service_account +from googleapiclient.http import MediaFileUpload + +from .. import BackupArchive, UploadError + + +def sizeof_fmt(num, suffix='B'): + # see https://stackoverflow.com/a/1094933/973537 + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + + +def make_tarfile(output_filename: str, source_dir: str) -> None: + with tarfile.open(output_filename, "w:gz") as tar: + tar.add(source_dir, arcname=os.path.basename(source_dir)) + assert os.path.exists(output_filename), "Archive creation (%s) failed" % output_filename + + +class Backend: + + SERVICE_ACCOUNT_FILE = '/home/bbt/code/isprojects/ispnext/backend/backup/client_secrets.json' + + SCOPES = [ + 'https://www.googleapis.com/auth/drive.file' + ] + + _folder_id = None + + @property + def service(self): + if not hasattr(self, '_service'): + credentials = service_account.Credentials.from_service_account_file( + self.SERVICE_ACCOUNT_FILE, + scopes=self.SCOPES + ) + + self._service = googleapiclient.discovery.build('drive', 'v3', credentials=credentials) + return self._service + + def show_quota(self): + quota = self.service.about().get(fields='storageQuota').execute()['storageQuota'] + print("\nQuota:") + for key, value in quota.items(): + size = sizeof_fmt(int(value)) + print(f" {key}: {size}") + print("\n") + + def _get_or_create_dir(self, name, parent=None) -> str: + # search if the folder exists + search_q = "name='{}' and mimeType='application/vnd.google-apps.folder'".format(name) + if parent: + search_q += "and '{}' in parents".format(parent) + else: + search_q += "and 'root' in parents" + + folders = ( + self.service.files() + .list(q=search_q, fields='files(id, parents)') + .execute() + .get('files', []) + ) + if folders: + return folders[0]['id'] + + _folder = ( + self.service + .files() + .create(body={ + 'name': name, + 'mimeType': 'application/vnd.google-apps.folder', + 'parents': [parent] if parent else [], + }, fields='id') + .execute() + ) + return _folder['id'] + + def ensure_dirs(self, path: str): + """ + Ensure the folders in path exist on Drive. + """ + if path.startswith('/'): + path = path[1:] + if path.endswith('/'): + path = path[:-1] + + folders = path.split('/') + + _folder_id = None + for folder in folders: + _folder_id = self._get_or_create_dir(folder, _folder_id) + + self._folder_id = _folder_id + + def prepare(self, full_path: str) -> BackupArchive: + """ + Prepare the archive on the local drive. + + :param full_path: full path to the directory to archive + :return: BackupArchive instance + """ + output_filename = "{}.tar.gz".format(full_path) + + if not os.path.exists(output_filename): + make_tarfile(output_filename, full_path) + + return BackupArchive(output_filename) + + def exists(self, backup_archive: BackupArchive) -> bool: + """ + Check if the file exists on the remote + """ + assert self._folder_id, "ensure_dirs() must be called before uploading files" + + name = os.path.basename(backup_archive.archive) + search_q = ( + "name='{}' " + "and mimeType='application/gzip' " + "and '{}' in parents" + ).format(name, self._folder_id) + + files = ( + self.service + .files() + .list(q=search_q, fields='files(md5Checksum)') + .execute() + .get('files', []) + ) + if files: + checksum = backup_archive.md5_checksum + return any([ + file_['md5Checksum'] == checksum for file_ in files + ]) + + return False + + def upload(self, backup_archive): + assert self._folder_id, "ensure_dirs() must be called before uploading files" + file_name = os.path.basename(backup_archive.archive) + metadata = { + 'name': file_name, + 'originalFilename': backup_archive.archive, + 'parents': [self._folder_id] + } + media = MediaFileUpload(backup_archive.archive, mimetype='application/gzip') + + response = ( + self.service + .files() + .create( + body=metadata, + media_body=media, + fields='md5Checksum' + ) + .execute() + ) + + if response['md5Checksum'] != backup_archive.md5_checksum: + raise UploadError("Uploaded archive checksum does not match") diff --git a/ctrl_z/utils.py b/ctrl_z/utils.py new file mode 100644 index 0000000..bda20b6 --- /dev/null +++ b/ctrl_z/utils.py @@ -0,0 +1,12 @@ +from .constants import BACKUP_DIR_PATTERN + + +def is_backup_dir(dir_name: str) -> bool: + """ + Test if a directory name fits the pattern of backup folder names. + + The pattern is YYYY-MM-DD-suffix, where suffix is either 'daily' or 'weekly'. + """ + if BACKUP_DIR_PATTERN.match(dir_name): + return True + return False diff --git a/setup.cfg b/setup.cfg index 33bd324..666ca65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,10 @@ tests_require = tox isort +[options.packages.find] +exclude = + tests + [options.extras_require] tests = psycopg2-binary From c544d45017750631ffa0de39c447d749482f4761 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Fri, 13 Jul 2018 17:28:47 +0200 Subject: [PATCH 02/10] Report after backup transfer --- ctrl_z/cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ctrl_z/cli.py b/ctrl_z/cli.py index 8dcf313..67d4ef7 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/cli.py @@ -231,8 +231,15 @@ def transfer(self, options, config_file, conf_overrides): conf_overrides['use_parent_dir'] = True transfer = BackupTransfer.from_config(config_file, **conf_overrides) - transfer.show_info() - transfer.sync_to_remote() + has_errors = False + try: + transfer.show_info() + transfer.sync_to_remote() + except Exception: + has_errors = True + raise + finally: + self._backup.report(has_errors) cli = CLI() From 314e374ed2f7b7b8e3bd53577d23e2da358ad9c2 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 17 Jul 2018 16:09:06 +0200 Subject: [PATCH 03/10] Provide CLI to share folder for easier debugging/inspection --- ctrl_z/cli.py | 20 ++++++++- ctrl_z/transfer/__init__.py | 6 +++ ctrl_z/transfer/backends/base.py | 47 ++++++++++++++++++++ ctrl_z/transfer/backends/google_drive.py | 55 +++++++++++++++++++++++- 4 files changed, 124 insertions(+), 4 deletions(-) diff --git a/ctrl_z/cli.py b/ctrl_z/cli.py index 67d4ef7..98a4c72 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/cli.py @@ -7,9 +7,10 @@ import sys import django +from django.utils.module_loading import import_string from .backup import Backup, configure_logging -from .config import DEFAULT_CONFIG_FILE +from .config import DEFAULT_CONFIG_FILE, Config from .transfer import BackupTransfer logger = logging.getLogger(__name__) @@ -119,7 +120,7 @@ def __init__(self): ) # backup transfer - parser_transfer = subparsers.add_parser( + self.parser_transfer = subparsers.add_parser( 'transfer', help='Transfer the backups to an off-site location' ) @@ -136,6 +137,12 @@ def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, self.stderr.write(f"CTRL-Z {__version__} - Backup and recovery tool\n") + # load the command line args for transfers + # FIXME: handle the global --config-file option? + config = Config.from_file(config_file) + transfer_backend = import_string(config.transfer_backend)() + transfer_backend.add_arguments(self.parser_transfer) + args = self.parser.parse_args(args or sys.argv[1:]) config_file = args.config_file or config_file @@ -228,9 +235,18 @@ def restore(self, options): backup.report(has_errors) def transfer(self, options, config_file, conf_overrides): + """ + Relay the command to the transfer backend or initiate the actual transfer. + """ conf_overrides['use_parent_dir'] = True transfer = BackupTransfer.from_config(config_file, **conf_overrides) + + # handle potential backend specific subcommands + handled = transfer.handle_command(options) + if handled: + return + has_errors = False try: transfer.show_info() diff --git a/ctrl_z/transfer/__init__.py b/ctrl_z/transfer/__init__.py index eb8165f..8e9ad6a 100644 --- a/ctrl_z/transfer/__init__.py +++ b/ctrl_z/transfer/__init__.py @@ -49,6 +49,9 @@ def from_config(cls, config_file, **overrides): config = Config.from_file(config_file, **overrides) return cls(config=config) + def handle_command(self, options): + return self.backend.handle_command(self, options) + def show_info(self): """ Inventarize the operations to be done. @@ -71,6 +74,9 @@ def sync_to_remote(self): continue backup_dirs.append(dirname) + # make sure the (relative) directory structure exists on the remote + # this depends on the backend - for rsync for example this may not be + # needed as it creates them on the fly self.backend.ensure_dirs(self.config.transfer_path) logger.info("Syncing backups %s to remote...", backup_dirs) diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index e69de29..1266c06 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -0,0 +1,47 @@ +from .. import BackupArchive + + +class Base: + + def show_quota(self): + raise NotImplementedError # noqa + + def ensure_dirs(self): + """ + Ensure the folders in path exist on the remote. + """ + pass + + def prepare(self, full_path: str) -> BackupArchive: + """ + Prepare the archive on the local drive. + + :param full_path: full path to the directory to archive + :return: BackupArchive instance + """ + raise NotImplementedError # noqa + + def exists(self, backup_archive: BackupArchive) -> bool: + """ + Check if the file exists on the remote + """ + raise NotImplementedError # noqa + + def upload(self, backup_archive): + raise NotImplementedError # noqa + + @staticmethod + def add_arguments(parser): + """ + Add optional command line arguments. + """ + pass + + def handle_command(self, options) -> bool: + """ + Handle backend specific commands + + :return: boolean indicating if a command was handled or not, if not, + the default behaviour is to transfer the backup. + """ + return False diff --git a/ctrl_z/transfer/backends/google_drive.py b/ctrl_z/transfer/backends/google_drive.py index a222e9c..2f229ea 100644 --- a/ctrl_z/transfer/backends/google_drive.py +++ b/ctrl_z/transfer/backends/google_drive.py @@ -11,7 +11,8 @@ from google.oauth2 import service_account from googleapiclient.http import MediaFileUpload -from .. import BackupArchive, UploadError +from .. import BackupArchive, BackupTransfer, UploadError +from .base import Base def sizeof_fmt(num, suffix='B'): @@ -29,7 +30,7 @@ def make_tarfile(output_filename: str, source_dir: str) -> None: assert os.path.exists(output_filename), "Archive creation (%s) failed" % output_filename -class Backend: +class Backend(Base): SERVICE_ACCOUNT_FILE = '/home/bbt/code/isprojects/ispnext/backend/backup/client_secrets.json' @@ -39,6 +40,14 @@ class Backend: _folder_id = None + @classmethod + def add_arguments(cls, parser): + BackendCli.add_arguments(parser) + cls.parser = parser + + def handle_command(self, transfer, options) -> bool: + return BackendCli.handle_command(transfer, options) + @property def service(self): if not hasattr(self, '_service'): @@ -169,3 +178,45 @@ def upload(self, backup_archive): if response['md5Checksum'] != backup_archive.md5_checksum: raise UploadError("Uploaded archive checksum does not match") + + # custom CLI action implementations + + def give_permissions(self, transfer: BackupTransfer, email: str): + """ + Assign read permissions to the e-mail address. + """ + bits = [bit for bit in transfer.config.transfer_path.split('/') if bit] + folder_id = self._get_or_create_dir(bits[0], parent=None) + + self.service.permissions().create( + fileId=folder_id, + body={ + 'type': 'user', + 'role': 'reader', + 'emailAddress': email, + } + ).execute() + print("Folder {} can now be read".format(bits[0])) + + +class BackendCli: + + @staticmethod + def add_arguments(parser): + drive_parser = parser.add_subparsers(help='Google Drive commands', dest='drive_command') + + give_permissions = drive_parser.add_parser('give_permissions', help="Give read permissions to a user") + give_permissions.add_argument('email', help="E-mail address of user to get read permission") + + @staticmethod + def handle_command(transfer, options) -> bool: + subcommand = options.drive_command + + if not subcommand: + return False + + if subcommand == 'give_permissions': + transfer.backend.give_permissions(transfer, options.email) + else: + transfer.backend.parser.print_help() + return True From 4d472d9312c343453fa3df88960c23a7e7947324 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 17 Jul 2018 16:45:55 +0200 Subject: [PATCH 04/10] Cleaning up... --- .editorconfig | 5 +- ctrl_z/cli.py | 13 ++---- ctrl_z/config.default.yml | 3 +- ctrl_z/config.py | 1 + ctrl_z/transfer/__init__.py | 7 ++- ctrl_z/transfer/backends/base.py | 4 +- ctrl_z/transfer/backends/google_drive.py | 59 +++++++++++++++++------- doc/configuration.rst | 3 +- 8 files changed, 59 insertions(+), 36 deletions(-) diff --git a/.editorconfig b/.editorconfig index 6a1ab45..4c663f3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,4 +6,7 @@ indent_size = 4 end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true -max_line_length = 79 +max_line_length = 79 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/ctrl_z/cli.py b/ctrl_z/cli.py index 98a4c72..9041f3c 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/cli.py @@ -63,7 +63,6 @@ class CLI: def __init__(self): parser = argparse.ArgumentParser(description="CTRL-Z CLI") - parser.add_argument('--config-file', help="Config file to use") parser.add_argument('--base-dir', help="Base directory override") subparsers = parser.add_subparsers( @@ -138,13 +137,11 @@ def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, self.stderr.write(f"CTRL-Z {__version__} - Backup and recovery tool\n") # load the command line args for transfers - # FIXME: handle the global --config-file option? config = Config.from_file(config_file) - transfer_backend = import_string(config.transfer_backend)() - transfer_backend.add_arguments(self.parser_transfer) + TransferBackend = import_string(config.transfer_backend) + TransferBackend.add_arguments(self.parser_transfer) args = self.parser.parse_args(args or sys.argv[1:]) - config_file = args.config_file or config_file self._setup() @@ -238,12 +235,10 @@ def transfer(self, options, config_file, conf_overrides): """ Relay the command to the transfer backend or initiate the actual transfer. """ - conf_overrides['use_parent_dir'] = True - - transfer = BackupTransfer.from_config(config_file, **conf_overrides) + transfer = BackupTransfer.from_config(config_file, use_parent_dir=True) # handle potential backend specific subcommands - handled = transfer.handle_command(options) + handled = transfer.backend.handle_command(transfer, options) if handled: return diff --git a/ctrl_z/config.default.yml b/ctrl_z/config.default.yml index 112e436..03cd485 100644 --- a/ctrl_z/config.default.yml +++ b/ctrl_z/config.default.yml @@ -35,5 +35,6 @@ pg_dump_binary: /usr/bin/pg_dump pg_restore_binary: /usr/bin/pg_restore # where are backups transferred (off-site) -transfer_backend: ctrl_z.transfer.backends.google_drive.Backend +transfer_backend: ctrl_z.transfer.backends.base.Base +transfer_backend_init_kwargs: {} transfer_path: / diff --git a/ctrl_z/config.py b/ctrl_z/config.py index 3c5f4bb..e2cb198 100644 --- a/ctrl_z/config.py +++ b/ctrl_z/config.py @@ -26,6 +26,7 @@ class Config: 'pg_dump_binary', 'pg_restore_binary', 'transfer_backend', + 'transfer_backend_init_kwargs', 'transfer_path', ] diff --git a/ctrl_z/transfer/__init__.py b/ctrl_z/transfer/__init__.py index 8e9ad6a..be6ee4d 100644 --- a/ctrl_z/transfer/__init__.py +++ b/ctrl_z/transfer/__init__.py @@ -42,16 +42,15 @@ class BackupTransfer: def __init__(self, config: Config): self.config = config - self.backend = import_string(config.transfer_backend)() + + cls = import_string(config.transfer_backend) + self.backend = cls(**config.transfer_backend_init_kwargs) @classmethod def from_config(cls, config_file, **overrides): config = Config.from_file(config_file, **overrides) return cls(config=config) - def handle_command(self, options): - return self.backend.handle_command(self, options) - def show_info(self): """ Inventarize the operations to be done. diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index 1266c06..aa4fd1b 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -30,8 +30,8 @@ def exists(self, backup_archive: BackupArchive) -> bool: def upload(self, backup_archive): raise NotImplementedError # noqa - @staticmethod - def add_arguments(parser): + @classmethod + def add_arguments(cls, parser): """ Add optional command line arguments. """ diff --git a/ctrl_z/transfer/backends/google_drive.py b/ctrl_z/transfer/backends/google_drive.py index 2f229ea..e15e2b5 100644 --- a/ctrl_z/transfer/backends/google_drive.py +++ b/ctrl_z/transfer/backends/google_drive.py @@ -30,9 +30,13 @@ def make_tarfile(output_filename: str, source_dir: str) -> None: assert os.path.exists(output_filename), "Archive creation (%s) failed" % output_filename -class Backend(Base): +def pprint_key_value(mapping, formatter=None): + for key, value in mapping.items(): + value = formatter(value) if formatter else value + print(f" {key}: {value}") + - SERVICE_ACCOUNT_FILE = '/home/bbt/code/isprojects/ispnext/backend/backup/client_secrets.json' +class Backend(Base): SCOPES = [ 'https://www.googleapis.com/auth/drive.file' @@ -40,19 +44,23 @@ class Backend(Base): _folder_id = None - @classmethod - def add_arguments(cls, parser): - BackendCli.add_arguments(parser) - cls.parser = parser + def __init__(self, client_secrets: str): + """ + A remote backup transfer backend using Google Drive. - def handle_command(self, transfer, options) -> bool: - return BackendCli.handle_command(transfer, options) + :param client_secrets: (absolute) path to the client_secrets.json file + containing the credentials to connect to the API. Note that a service + account is required for non-interactive behaviour (and the only way + implemented currently). + """ + self.client_secrets = client_secrets + # Backend @property def service(self): if not hasattr(self, '_service'): credentials = service_account.Credentials.from_service_account_file( - self.SERVICE_ACCOUNT_FILE, + self.client_secrets, scopes=self.SCOPES ) @@ -60,12 +68,12 @@ def service(self): return self._service def show_quota(self): + """ + Print the storage quota. + """ quota = self.service.about().get(fields='storageQuota').execute()['storageQuota'] - print("\nQuota:") - for key, value in quota.items(): - size = sizeof_fmt(int(value)) - print(f" {key}: {size}") - print("\n") + print("Quota:") + pprint_key_value(quota, lambda x: sizeof_fmt(int(x))) def _get_or_create_dir(self, name, parent=None) -> str: # search if the folder exists @@ -179,7 +187,18 @@ def upload(self, backup_archive): if response['md5Checksum'] != backup_archive.md5_checksum: raise UploadError("Uploaded archive checksum does not match") - # custom CLI action implementations + # CLI extension + @classmethod + def add_arguments(cls, parser): + BackendCli.add_arguments(parser) + + def handle_command(self, transfer, options) -> bool: + return BackendCli.handle_command(transfer, options) + + def test_connection(self): + user = self.service.about().get(fields='user').execute()['user'] + print("User:") + pprint_key_value(user) def give_permissions(self, transfer: BackupTransfer, email: str): """ @@ -205,6 +224,10 @@ class BackendCli: def add_arguments(parser): drive_parser = parser.add_subparsers(help='Google Drive commands', dest='drive_command') + drive_parser.add_parser('test_connection', help="Test Google Drive connection/credentials") + + drive_parser.add_parser('show_quota', help="Show the Drive quota") + give_permissions = drive_parser.add_parser('give_permissions', help="Give read permissions to a user") give_permissions.add_argument('email', help="E-mail address of user to get read permission") @@ -217,6 +240,8 @@ def handle_command(transfer, options) -> bool: if subcommand == 'give_permissions': transfer.backend.give_permissions(transfer, options.email) - else: - transfer.backend.parser.print_help() + elif subcommand == 'test_connection': + transfer.backend.test_connection() + elif subcommand == 'show_quota': + transfer.backend.show_quota() return True diff --git a/doc/configuration.rst b/doc/configuration.rst index 89b5c67..df7fcea 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -5,8 +5,7 @@ Configuration ============= Configuration is done in YAML format. You can pass the config file to use -to the ``cli`` object (see :ref:`usage`), or pass it as a global CLI option -``--config-file``. +to the ``cli`` object (see :ref:`usage`). Global config options ===================== From c747d9ea894405014caac0556033a2999e2d47b9 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 17 Jul 2018 16:51:19 +0200 Subject: [PATCH 05/10] Add optional deps for Drive backend --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 666ca65..0b30fc3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,9 @@ exclude = tests [options.extras_require] +transfer_backend.drive = + google-api-python-client + google-auth tests = psycopg2-binary pytest From 4ad0e9f40ae6e2cd258f14972f43dec6b65badfd Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 17 Jul 2018 17:52:37 +0200 Subject: [PATCH 06/10] Add CLI tests for backup transfer --- ctrl_z/__init__.py | 2 +- ctrl_z/{cli.py => _cli.py} | 12 +-- ctrl_z/transfer/backends/base.py | 4 +- setup.cfg | 2 + tests/test_cli.py | 176 +++++++++++++++++++------------ 5 files changed, 120 insertions(+), 76 deletions(-) rename ctrl_z/{cli.py => _cli.py} (95%) diff --git a/ctrl_z/__init__.py b/ctrl_z/__init__.py index 75eb1af..6ec5b77 100644 --- a/ctrl_z/__init__.py +++ b/ctrl_z/__init__.py @@ -1,7 +1,7 @@ from pkg_resources import get_distribution +from ._cli import cli # noqa from .backup import Backup, configure_logging # noqa -from .cli import cli # noqa __version__ = get_distribution('CTRL-Z').version diff --git a/ctrl_z/cli.py b/ctrl_z/_cli.py similarity index 95% rename from ctrl_z/cli.py rename to ctrl_z/_cli.py index 9041f3c..2f5cdbb 100644 --- a/ctrl_z/cli.py +++ b/ctrl_z/_cli.py @@ -16,6 +16,9 @@ logger = logging.getLogger(__name__) +HEADER = "CTRL-Z {version} - Backup and recovery tool\n" + + def noop(*args, **kwargs): pass @@ -63,7 +66,6 @@ class CLI: def __init__(self): parser = argparse.ArgumentParser(description="CTRL-Z CLI") - parser.add_argument('--base-dir', help="Base directory override") subparsers = parser.add_subparsers( help="Sub commands", dest='subcommand' @@ -134,14 +136,14 @@ def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, if stderr: self.stderr = stderr - self.stderr.write(f"CTRL-Z {__version__} - Backup and recovery tool\n") + self.stderr.write(HEADER.format(version=__version__)) # load the command line args for transfers config = Config.from_file(config_file) TransferBackend = import_string(config.transfer_backend) TransferBackend.add_arguments(self.parser_transfer) - args = self.parser.parse_args(args or sys.argv[1:]) + args = self.parser.parse_args(args if args is not None else sys.argv[1:]) self._setup() @@ -155,8 +157,6 @@ def _setup(self): def run(self, options, config_file: str): subcommand = options.subcommand conf_overrides = {} - if options.base_dir: - conf_overrides['base_dir'] = options.base_dir if subcommand == 'restore': self._backup = Backup.prepare_restore( @@ -179,7 +179,7 @@ def run(self, options, config_file: str): elif subcommand == 'transfer': self.transfer(options, config_file, conf_overrides) else: - self.parser.print_help() + self.parser.print_help(file=self.stdout) def generate_config(self, options): """ diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index aa4fd1b..3c3f48a 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -1,4 +1,4 @@ -from .. import BackupArchive +from .. import BackupArchive, BackupTransfer class Base: @@ -37,7 +37,7 @@ def add_arguments(cls, parser): """ pass - def handle_command(self, options) -> bool: + def handle_command(self, transfer: BackupTransfer, options) -> bool: """ Handle backend specific commands diff --git a/setup.cfg b/setup.cfg index 0b30fc3..b9651b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ tests_require = pytest pytest-django pytest-freezegun + pytest-mock tox isort @@ -50,6 +51,7 @@ tests = pytest pytest-django pytest-freezegun + pytest-mock tox isort pep8 = flake8 diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fb1b20..55c51f0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,100 +10,142 @@ import pytest -from ctrl_z import cli +from ctrl_z import __version__, cli +from ctrl_z._cli import HEADER from ctrl_z.config import DEFAULT_CONFIG_FILE -def test_config_generation(config_path): - cli(['generate_config'], stdout=StringIO(), config_file=config_path) +class TestConfigGeneration: - cli.stdout.seek(0) - result = cli.stdout.read() - with open(DEFAULT_CONFIG_FILE, 'r') as default_config: - assert result == default_config.read() + def test_config_generation(self, config_path): + cli(['generate_config'], stdout=StringIO(), stderr=StringIO(), config_file=config_path) + cli.stdout.seek(0) + result = cli.stdout.read() + with open(DEFAULT_CONFIG_FILE, 'r') as default_config: + assert result == default_config.read() -def test_config_generation_external_file(tmpdir, config_path): - tempfile = str(tmpdir.join("some_config.yml")) + # when piping stdout to a file, we don't want the header to end there + # as well, so it should be in stderr + cli.stderr.seek(0) + assert HEADER.format(version=__version__) in cli.stderr.read() - cli( - ['generate_config', '-o', tempfile], - stdout=StringIO(), config_file=config_path - ) + def test_config_generation_external_file(self, tmpdir, config_path): + tempfile = str(tmpdir.join("some_config.yml")) - cli.stdout.seek(0) - assert cli.stdout.read() == '' - with open(DEFAULT_CONFIG_FILE, 'r') as default_config: - with open(tempfile, 'r') as config: - assert config.read() == default_config.read() + cli( + ['generate_config', '-o', tempfile], + stdout=StringIO(), config_file=config_path + ) + + cli.stdout.seek(0) + assert cli.stdout.read() == '' + with open(DEFAULT_CONFIG_FILE, 'r') as default_config: + with open(tempfile, 'r') as config: + assert config.read() == default_config.read() + + +class TestBackup: + + def test_full_backup(self, tmpdir, settings, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer(config_path, base_dir=str(backups_base)) + # prevent actual db access + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning) + settings.DATABASES = {} + settings.MEDIA_ROOT = str(tmpdir.join('media')) -def test_full_backup(tmpdir, settings, config_writer): - config_path = str(tmpdir.join('config.yml')) - backups_base = tmpdir.join('backups') + cli(args=['backup'], config_file=config_path, stdout=StringIO()) - config_writer(config_path, base_dir=str(backups_base)) + expected_date = datetime.utcnow().strftime("%Y-%m-%d") - # prevent actual db access - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning) - settings.DATABASES = {} - settings.MEDIA_ROOT = str(tmpdir.join('media')) + # assert that the backup directory was created + children = os.listdir(backups_base) + assert len(children) == 1 + backup_dir = children[0] + assert backup_dir.startswith(expected_date) - cli(args=['backup'], config_file=config_path, stdout=StringIO()) + full_path = backups_base.join(backup_dir) + subdirs = os.listdir(full_path) + assert sorted(subdirs) == ['backup.log', 'db', 'files'] - expected_date = datetime.utcnow().strftime("%Y-%m-%d") - # assert that the backup directory was created - children = os.listdir(backups_base) - assert len(children) == 1 - backup_dir = children[0] - assert backup_dir.startswith(expected_date) +class TestRestore: - full_path = backups_base.join(backup_dir) - subdirs = os.listdir(full_path) - assert sorted(subdirs) == ['backup.log', 'db', 'files'] + def test_full_restore(self, tmpdir, settings, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.mkdir('backups') + backup_dir = backups_base.mkdir('2018-05-29-daily') + backup_dir.mkdir('db') + backup_dir.mkdir('files') + config_writer(config_path, base_dir=str(backups_base)) -def test_full_restore(tmpdir, settings, config_writer): - config_path = str(tmpdir.join('config.yml')) - backups_base = tmpdir.mkdir('backups') - backup_dir = backups_base.mkdir('2018-05-29-daily') - backup_dir.mkdir('db') - backup_dir.mkdir('files') + # prevent actual db access + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=UserWarning) + settings.DATABASES = {} + settings.MEDIA_ROOT = str(tmpdir.join('media')) - config_writer(config_path, base_dir=str(backups_base)) + cli(args=['restore', str(backup_dir)], config_file=config_path, stdout=StringIO()) - # prevent actual db access - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=UserWarning) - settings.DATABASES = {} - settings.MEDIA_ROOT = str(tmpdir.join('media')) + # verify that the log file was created + assert 'backup.log' in os.listdir(backup_dir) - cli(args=['restore', str(backup_dir)], config_file=config_path, stdout=StringIO()) + def test_full_restore_bad_directory(self): + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', '/i/dont/exist/'], stdout=StringIO()) - # verify that the log file was created - assert 'backup.log' in os.listdir(backup_dir) + @pytest.mark.skipif(sys.platform == 'win32', + reason="does not run on windows") + def test_full_restore_bad_directory2(self, tmpdir): + bad_permissions_dir = str(tmpdir.mkdir('nope')) + os.chmod(bad_permissions_dir, 0o000) + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', bad_permissions_dir], stdout=StringIO()) -def test_full_restore_bad_directory(): - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', '/i/dont/exist/'], stdout=StringIO()) + def test_full_restore_not_directory(self, tmpdir): + some_file = tmpdir.join('not_a_dir.txt') + some_file.write("not a dir!\n") + with pytest.raises(argparse.ArgumentTypeError): + cli(args=['restore', str(some_file)], stdout=StringIO()) -@pytest.mark.skipif(sys.platform == 'win32', - reason="does not run on windows") -def test_full_restore_bad_directory2(tmpdir): - bad_permissions_dir = str(tmpdir.mkdir('nope')) - os.chmod(bad_permissions_dir, 0o000) - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', bad_permissions_dir], stdout=StringIO()) +class TestTransfer: + def test_transfer_main_command(self, mocker): + """ + Assert that the transfer is initiated if no backend-specific subcommand was handled. + """ + mocked_BackupTransfer = mocker.patch('ctrl_z._cli.BackupTransfer') + transfer = mocked_BackupTransfer.from_config.return_value + transfer.backend.handle_command.return_value = False -def test_full_restore_not_directory(tmpdir): - some_file = tmpdir.join('not_a_dir.txt') - some_file.write("not a dir!\n") + cli(['transfer']) + transfer.sync_to_remote.assert_called_once() + + def test_transfer_subcommand_handled(self, mocker): + """ + Assert that the transfer is not initiated if a backend-specific subcommand was handled. + """ + mocked_BackupTransfer = mocker.patch('ctrl_z._cli.BackupTransfer') + transfer = mocked_BackupTransfer.from_config.return_value + transfer.backend.handle_command.return_value = True + + cli(['transfer']) + transfer.sync_to_remote.assert_not_called() + + +def test_no_subcommand(): + cli([], stdout=StringIO()) + + cli.stdout.seek(0) + output = cli.stdout.read() - with pytest.raises(argparse.ArgumentTypeError): - cli(args=['restore', str(some_file)], stdout=StringIO()) + assert 'usage: ' in output From 74666b5dc9e274ddc09c05cfd077ff0e494edbe5 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 17 Jul 2018 18:23:29 +0200 Subject: [PATCH 07/10] Test the BackupTransfer API --- ctrl_z/_cli.py | 2 +- ctrl_z/transfer/__init__.py | 105 +------------------------ ctrl_z/transfer/archive.py | 26 +++++++ ctrl_z/transfer/backends/__init__.py | 1 + ctrl_z/transfer/backends/base.py | 13 ++-- ctrl_z/transfer/transfer.py | 79 +++++++++++++++++++ tests/test_transfer.py | 110 +++++++++++++++++++++++++++ 7 files changed, 227 insertions(+), 109 deletions(-) create mode 100644 ctrl_z/transfer/archive.py create mode 100644 ctrl_z/transfer/transfer.py create mode 100644 tests/test_transfer.py diff --git a/ctrl_z/_cli.py b/ctrl_z/_cli.py index 2f5cdbb..d1b6171 100644 --- a/ctrl_z/_cli.py +++ b/ctrl_z/_cli.py @@ -235,7 +235,7 @@ def transfer(self, options, config_file, conf_overrides): """ Relay the command to the transfer backend or initiate the actual transfer. """ - transfer = BackupTransfer.from_config(config_file, use_parent_dir=True) + transfer = BackupTransfer.from_config(config_file) # handle potential backend specific subcommands handled = transfer.backend.handle_command(transfer, options) diff --git a/ctrl_z/transfer/__init__.py b/ctrl_z/transfer/__init__.py index be6ee4d..d001933 100644 --- a/ctrl_z/transfer/__init__.py +++ b/ctrl_z/transfer/__init__.py @@ -1,103 +1,2 @@ -import hashlib -import logging -import os - -from django.utils.module_loading import import_string - -from ..config import Config -from ..utils import is_backup_dir - -logger = logging.getLogger(__name__) - - -class BackupArchive: - def __init__(self, archive: str): - """ - A single backup as an archive. - - The archive format depends on the backend, it could be tar.gz, zip... - - :param archive: full path to the archive file - """ - assert os.path.isfile(archive), "%s is not a file" - self.archive = archive - - @property - def md5_checksum(self): - """ - Calculate the md5 checksum of the archive - """ - hash_md5 = hashlib.md5() - with open(self.archive, "rb") as archive: - for chunk in iter(lambda: archive.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - - -class UploadError(Exception): - pass - - -class BackupTransfer: - - def __init__(self, config: Config): - self.config = config - - cls = import_string(config.transfer_backend) - self.backend = cls(**config.transfer_backend_init_kwargs) - - @classmethod - def from_config(cls, config_file, **overrides): - config = Config.from_file(config_file, **overrides) - return cls(config=config) - - def show_info(self): - """ - Inventarize the operations to be done. - """ - self.backend.show_quota() - - def sync_to_remote(self): - """ - Transfer the backups to the remote. - """ - logger.info("Scanning %s for backups to transfer...", self.config.base_dir) - - backup_dirs = [] - - for dirname in os.listdir(self.config.base_dir): - full_path = os.path.join(self.config.base_dir, dirname) - if not os.path.isdir(full_path): - continue - if not is_backup_dir(dirname): - continue - backup_dirs.append(dirname) - - # make sure the (relative) directory structure exists on the remote - # this depends on the backend - for rsync for example this may not be - # needed as it creates them on the fly - self.backend.ensure_dirs(self.config.transfer_path) - - logger.info("Syncing backups %s to remote...", backup_dirs) - - has_failures = False - for dirname in backup_dirs: - full_path = os.path.join(self.config.base_dir, dirname) - backup_archive = self.backend.prepare(full_path) - if self.backend.exists(backup_archive): - logger.info("Backup %s exists on remote, skipping", dirname) - continue - - logger.info("Uploading %s to remote...", dirname) - try: - self.backend.upload(backup_archive) - except UploadError: - logger.exception("%s upload failed", dirname) - has_failures = True - continue - logger.info("Uploaded %s to remote", dirname) - - if not has_failures: - logger.info("Done syncing backups to remote") - else: - raise UploadError("Some backups failed to upload, check the error log") +from .archive import BackupArchive # noqa +from .transfer import BackupTransfer, UploadError # noqa diff --git a/ctrl_z/transfer/archive.py b/ctrl_z/transfer/archive.py new file mode 100644 index 0000000..81d7173 --- /dev/null +++ b/ctrl_z/transfer/archive.py @@ -0,0 +1,26 @@ +import hashlib +import os + + +class BackupArchive: + def __init__(self, archive: str): + """ + A single backup as an archive. + + The archive format depends on the backend, it could be tar.gz, zip... + + :param archive: full path to the archive file + """ + self.archive = archive + + @property + def md5_checksum(self): + """ + Calculate the md5 checksum of the archive + """ + assert os.path.isfile(self.archive), "%s is not a file" + hash_md5 = hashlib.md5() + with open(self.archive, "rb") as archive: + for chunk in iter(lambda: archive.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() diff --git a/ctrl_z/transfer/backends/__init__.py b/ctrl_z/transfer/backends/__init__.py index e69de29..e461018 100644 --- a/ctrl_z/transfer/backends/__init__.py +++ b/ctrl_z/transfer/backends/__init__.py @@ -0,0 +1 @@ +from .base import Base # noqa diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index 3c3f48a..d36ba8c 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -2,11 +2,14 @@ class Base: + """ + Define the public API that a backend must implement. + """ def show_quota(self): - raise NotImplementedError # noqa + raise NotImplementedError("Transfer backends must implement a show_quota() method") - def ensure_dirs(self): + def ensure_dirs(self, path: str): """ Ensure the folders in path exist on the remote. """ @@ -19,16 +22,16 @@ def prepare(self, full_path: str) -> BackupArchive: :param full_path: full path to the directory to archive :return: BackupArchive instance """ - raise NotImplementedError # noqa + raise NotImplementedError("Transfer backends must implement a prepare() method") def exists(self, backup_archive: BackupArchive) -> bool: """ Check if the file exists on the remote """ - raise NotImplementedError # noqa + raise NotImplementedError("Transfer backends must implement a exists() method") def upload(self, backup_archive): - raise NotImplementedError # noqa + raise NotImplementedError("Transfer backends must implement a upload() method") @classmethod def add_arguments(cls, parser): diff --git a/ctrl_z/transfer/transfer.py b/ctrl_z/transfer/transfer.py new file mode 100644 index 0000000..f0242e5 --- /dev/null +++ b/ctrl_z/transfer/transfer.py @@ -0,0 +1,79 @@ +import logging +import os + +from django.utils.module_loading import import_string + +from ..config import Config +from ..utils import is_backup_dir + +logger = logging.getLogger(__name__) + + +class UploadError(Exception): + pass + + +class BackupTransfer: + + def __init__(self, config: Config): + self.config = config + + cls = import_string(config.transfer_backend) + self.backend = cls(**config.transfer_backend_init_kwargs) + + @classmethod + def from_config(cls, config_file, **overrides): + overrides['use_parent_dir'] = True + config = Config.from_file(config_file, **overrides) + return cls(config=config) + + def show_info(self): + """ + Inventarize the operations to be done. + """ + self.backend.show_quota() + + def sync_to_remote(self): + """ + Transfer the backups to the remote. + """ + logger.info("Scanning %s for backups to transfer...", self.config.base_dir) + + backup_dirs = [] + + for dirname in sorted(os.listdir(self.config.base_dir)): + full_path = os.path.join(self.config.base_dir, dirname) + if not os.path.isdir(full_path): + continue + if not is_backup_dir(dirname): + continue + backup_dirs.append(dirname) + + # make sure the (relative) directory structure exists on the remote + # this depends on the backend - for rsync for example this may not be + # needed as it creates them on the fly + self.backend.ensure_dirs(self.config.transfer_path) + + logger.info("Syncing backups %s to remote...", backup_dirs) + + has_failures = False + for dirname in backup_dirs: + full_path = os.path.join(self.config.base_dir, dirname) + backup_archive = self.backend.prepare(full_path) + if self.backend.exists(backup_archive): + logger.info("Backup %s exists on remote, skipping", dirname) + continue + + logger.info("Uploading %s to remote...", dirname) + try: + self.backend.upload(backup_archive) + except UploadError: + logger.exception("%s upload failed", dirname) + has_failures = True + continue + logger.info("Uploaded %s to remote", dirname) + + if not has_failures: + logger.info("Done syncing backups to remote") + else: + raise UploadError("Some backups failed to upload, check the error log") diff --git a/tests/test_transfer.py b/tests/test_transfer.py new file mode 100644 index 0000000..6345c53 --- /dev/null +++ b/tests/test_transfer.py @@ -0,0 +1,110 @@ +import pytest + +from ctrl_z.config import DEFAULT_CONFIG_FILE +from ctrl_z.transfer import BackupArchive, BackupTransfer, UploadError +from ctrl_z.transfer.backends import Base + + +def test_public_api(tmpdir): + """ + Test that the base backend performs the appropriate checks for subclasses. + """ + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + + transfer = BackupTransfer.from_config(DEFAULT_CONFIG_FILE, base_dir=str(base_dir)) + + with pytest.raises(NotImplementedError): + transfer.show_info() + + with pytest.raises(NotImplementedError): + transfer.sync_to_remote() + + +class DummyBackend(Base): + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def exists(self, *args, **kwargs): + return True + + def prepare(self, full_path: str) -> BackupArchive: + return BackupArchive(full_path) + + def upload(self, archive): + raise AssertionError("No uploads should happen if there are no backups") + + +def test_load_backend_from_settings(): + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + transfer_backend='tests.test_transfer.DummyBackend', + transfer_backend_init_kwargs={'foo': 'bar'} + ) + + assert isinstance(transfer.backend, DummyBackend) + assert transfer.backend.kwargs == {'foo': 'bar'} + + +def test_sync_nothing_to_sync(tmpdir): + """ + Test that the backend is not called if there's nothing to sync. + """ + base_dir = tmpdir.mkdir('backups') + # create some files that should be ignored + base_dir.join('2018-07-17-daily').write('not a directory') + base_dir.mkdir('ignore-me') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + try: + transfer.sync_to_remote() + except AssertionError: + pytest.fail("Upload should NOT have been called") + + +def test_sync_skip_existing(tmpdir): + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + try: + transfer.sync_to_remote() + except AssertionError: + pytest.fail("Upload should NOT have been called") + + +def test_upload_fails_continue_others(tmpdir, mocker): + base_dir = tmpdir.mkdir('backups') + base_dir.mkdir('2018-07-17-daily') + base_dir.mkdir('2018-07-18-daily') + + transfer = BackupTransfer.from_config( + DEFAULT_CONFIG_FILE, + base_dir=str(base_dir), + transfer_backend='tests.test_transfer.DummyBackend', + ) + + def side_effect(archive): + if archive.archive.endswith('2018-07-17-daily'): + raise UploadError("'Random' failure") + else: + pass + + mocker.patch.object(transfer.backend, 'exists', return_value=False) + mock_upload = mocker.patch.object(transfer.backend, 'upload', side_effect=side_effect) + + with pytest.raises(UploadError): + transfer.sync_to_remote() + + assert mock_upload.call_count == 2 From 54a3203ac661236f4cb6cb443dac240cc8011909 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Fri, 20 Jul 2018 12:11:14 +0200 Subject: [PATCH 08/10] Added Drive backend tests --- ctrl_z/_cli.py | 2 +- ctrl_z/transfer/backends/base.py | 9 +- ctrl_z/transfer/backends/google_drive.py | 47 ++-- tests/backends/test_google_drive.py | 265 +++++++++++++++++++++++ tox.ini | 1 + 5 files changed, 305 insertions(+), 19 deletions(-) create mode 100644 tests/backends/test_google_drive.py diff --git a/ctrl_z/_cli.py b/ctrl_z/_cli.py index d1b6171..4b2598f 100644 --- a/ctrl_z/_cli.py +++ b/ctrl_z/_cli.py @@ -238,7 +238,7 @@ def transfer(self, options, config_file, conf_overrides): transfer = BackupTransfer.from_config(config_file) # handle potential backend specific subcommands - handled = transfer.backend.handle_command(transfer, options) + handled = transfer.backend.handle_command(self, transfer, options) if handled: return diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index d36ba8c..80c9a95 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -1,4 +1,7 @@ +import argparse + from .. import BackupArchive, BackupTransfer +from ..._cli import CLI class Base: @@ -40,10 +43,14 @@ def add_arguments(cls, parser): """ pass - def handle_command(self, transfer: BackupTransfer, options) -> bool: + def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.Namespace) -> bool: """ Handle backend specific commands + :param cli: the CLI instance asking to handle the command + :param transfer: the :class:`ctrl_z.transfer.BackupTransfer` instance + holding the config. + :param options: the CLI arguments parsed into argparse.Namespace :return: boolean indicating if a command was handled or not, if not, the default behaviour is to transfer the backup. """ diff --git a/ctrl_z/transfer/backends/google_drive.py b/ctrl_z/transfer/backends/google_drive.py index e15e2b5..e65bba0 100644 --- a/ctrl_z/transfer/backends/google_drive.py +++ b/ctrl_z/transfer/backends/google_drive.py @@ -4,7 +4,9 @@ Auth is done through service accounts, see https://developers.google.com/identity/protocols/OAuth2ServiceAccount """ +import argparse import os +import sys import tarfile import googleapiclient.discovery @@ -12,6 +14,7 @@ from googleapiclient.http import MediaFileUpload from .. import BackupArchive, BackupTransfer, UploadError +from ..._cli import CLI from .base import Base @@ -26,14 +29,16 @@ def sizeof_fmt(num, suffix='B'): def make_tarfile(output_filename: str, source_dir: str) -> None: with tarfile.open(output_filename, "w:gz") as tar: - tar.add(source_dir, arcname=os.path.basename(source_dir)) + for name in os.listdir(source_dir): + full_path = os.path.join(source_dir, name) + tar.add(full_path, arcname=os.path.basename(full_path)) assert os.path.exists(output_filename), "Archive creation (%s) failed" % output_filename -def pprint_key_value(mapping, formatter=None): +def pprint_key_value(mapping, formatter=None, stdout=None): for key, value in mapping.items(): value = formatter(value) if formatter else value - print(f" {key}: {value}") + print(f" {key}: {value}", file=stdout) class Backend(Base): @@ -43,6 +48,9 @@ class Backend(Base): ] _folder_id = None + _http = None + + stdout = sys.stdout def __init__(self, client_secrets: str): """ @@ -67,13 +75,17 @@ def service(self): self._service = googleapiclient.discovery.build('drive', 'v3', credentials=credentials) return self._service + @property + def http(self): + return self._http + def show_quota(self): """ Print the storage quota. """ - quota = self.service.about().get(fields='storageQuota').execute()['storageQuota'] - print("Quota:") - pprint_key_value(quota, lambda x: sizeof_fmt(int(x))) + quota = self.service.about().get(fields='storageQuota').execute(http=self.http)['storageQuota'] + print("Quota:", file=self.stdout) + pprint_key_value(quota, lambda x: sizeof_fmt(int(x)), stdout=self.stdout) def _get_or_create_dir(self, name, parent=None) -> str: # search if the folder exists @@ -86,7 +98,7 @@ def _get_or_create_dir(self, name, parent=None) -> str: folders = ( self.service.files() .list(q=search_q, fields='files(id, parents)') - .execute() + .execute(http=self.http) .get('files', []) ) if folders: @@ -100,7 +112,7 @@ def _get_or_create_dir(self, name, parent=None) -> str: 'mimeType': 'application/vnd.google-apps.folder', 'parents': [parent] if parent else [], }, fields='id') - .execute() + .execute(http=self.http) ) return _folder['id'] @@ -152,7 +164,7 @@ def exists(self, backup_archive: BackupArchive) -> bool: self.service .files() .list(q=search_q, fields='files(md5Checksum)') - .execute() + .execute(http=self.http) .get('files', []) ) if files: @@ -181,7 +193,7 @@ def upload(self, backup_archive): media_body=media, fields='md5Checksum' ) - .execute() + .execute(http=self.http) ) if response['md5Checksum'] != backup_archive.md5_checksum: @@ -192,13 +204,14 @@ def upload(self, backup_archive): def add_arguments(cls, parser): BackendCli.add_arguments(parser) - def handle_command(self, transfer, options) -> bool: + def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.Namespace) -> bool: + self.stdout = cli.stdout return BackendCli.handle_command(transfer, options) def test_connection(self): - user = self.service.about().get(fields='user').execute()['user'] - print("User:") - pprint_key_value(user) + user = self.service.about().get(fields='user').execute(http=self.http)['user'] + print("User:", file=self.stdout) + pprint_key_value(user, stdout=self.stdout) def give_permissions(self, transfer: BackupTransfer, email: str): """ @@ -214,8 +227,8 @@ def give_permissions(self, transfer: BackupTransfer, email: str): 'role': 'reader', 'emailAddress': email, } - ).execute() - print("Folder {} can now be read".format(bits[0])) + ).execute(http=self.http) + print("Folder {} can now be read".format(bits[0]), file=self.stdout) class BackendCli: @@ -232,7 +245,7 @@ def add_arguments(parser): give_permissions.add_argument('email', help="E-mail address of user to get read permission") @staticmethod - def handle_command(transfer, options) -> bool: + def handle_command(transfer: BackupTransfer, options: argparse.Namespace) -> bool: subcommand = options.drive_command if not subcommand: diff --git a/tests/backends/test_google_drive.py b/tests/backends/test_google_drive.py new file mode 100644 index 0000000..feb0aa8 --- /dev/null +++ b/tests/backends/test_google_drive.py @@ -0,0 +1,265 @@ +import json +import logging +import tarfile +from io import StringIO + +import googleapiclient.discovery +import pytest +from apiclient.http import HttpMock, HttpMockSequence + +from ctrl_z._cli import CLI +from ctrl_z.transfer import UploadError +from ctrl_z.transfer.backends.google_drive import Backend + + +class Service: + _service = None + + @classmethod + def get(cls): + if not cls._service: + # use the real discovery file - makes an actual HTTP request + # this is a conscious decision - we don't want to be testing + # against a no-longer functional discovery file + cls._service = googleapiclient.discovery.build('drive', 'v3', developerKey='dummy') + return cls._service + + +def _get_backend(response: dict=None, responses=None) -> Backend: + logging.disable(logging.CRITICAL) + backend = Backend('dummy') + backend.stdout = StringIO() + backend._service = Service.get() + logging.disable(logging.NOTSET) + + # set the response data + backend._http = HttpMock() + if response: + backend._http.data = json.dumps(response) + + elif responses: + iterable = [ + ({'status': '200'}, json.dumps(response)) + for response in responses + ] + backend._http = HttpMockSequence(iterable) + + return backend + + +class CLIBackend(Backend): + + def __init__(self, response): + logging.disable(logging.CRITICAL) + self._service = Service.get() + self._http = HttpMock() + self._http.data = json.dumps(response) + logging.disable(logging.NOTSET) + + def _get_or_create_dir(self, *args, **kwargs) -> str: + return 'dummy' + + +def test_show_quota(capsys): + backend = _get_backend(response={ + 'storageQuota': { + 'limit': '16106127360', + 'usage': '13344214', + 'usageInDrive': '13344214', + 'usageInDriveTrash': '0' + } + }) + + backend.show_quota() + + out = backend.stdout + out.seek(0) + assert out.read() == """Quota: + limit: 15.0GiB + usage: 12.7MiB + usageInDrive: 12.7MiB + usageInDriveTrash: 0.0B +""" + + +def test_prepare(tmpdir): + """ + Test that the prepare method creates a tar.gz archive + """ + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + + archive = backend.prepare(str(tmpdir)) + + with tarfile.open(archive.archive, 'r:gz') as arc: + members = arc.getmembers() + + assert len(members) == 1 + assert members[0].name == 'some_file.sql' + + +@pytest.mark.parametrize("files,exists", [ + ([], False), + ([{'md5Checksum': '5bd245d3a0f0183dcdf4ae4893bf1312'}, + {'md5Checksum': '865689b06759eb81cda2e0bcdf742d47'}], False), +]) +def test_archive_doesnt_exists(tmpdir, files, exists): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend(response={'files': files}) + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + assert backend.exists(archive) == exists + + +def test_archive_exists(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + backend._http.data = json.dumps({ + # need to use the actual checksum here, since gzip includes the timestamp + # in the header + 'files': [{'md5Checksum': archive.md5_checksum}] + }) + + assert backend.exists(archive) + + +def test_upload_success(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend() + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + backend._http.data = json.dumps( + # need to use the actual checksum here, since gzip includes the + # timestamp in the header + {'md5Checksum': archive.md5_checksum} + ) + + try: + backend.upload(archive) + except UploadError: + pytest.fail("Upload should succeed when md5_checksums match") + + +def test_upload_fails(tmpdir): + tmpdir.join('some_file.sql').write("SELECT 1;") + backend = _get_backend({'md5Checksum': 'incorrect'}) + backend._folder_id = 'dummy' + + archive = backend.prepare(str(tmpdir)) + with pytest.raises(UploadError): + backend.upload(archive) + + +def test_ensure_dirs_create(): + """ + Check that the directory structure is created in Drive. + """ + backend = _get_backend(responses=[ + { + 'files': [], + }, + { + 'id': 'dummy-123' + } + ]) + + backend.ensure_dirs('/sub') + + assert backend._folder_id == 'dummy-123' + + +def test_ensure_dirs_exists(): + """ + Check that the directory structure is created in Drive. + """ + backend = _get_backend(responses=[ + { + 'files': [{'id': 'dummy-123'}], + }, + ]) + + backend.ensure_dirs('/sub') + + assert backend._folder_id == 'dummy-123' + + +def test_cli_extension_test_connection(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': { + 'user': { + 'foo': 'bar' + } + }} + ) + stdout = StringIO() + cli = CLI() + + cli(['transfer', 'test_connection'], config_file=config_path, stdout=stdout) + + stdout.seek(0) + assert stdout.read() == "User:\n foo: bar\n" + + +def test_cli_extension_show_quota(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': { + 'storageQuota': { + 'limit': '16106127360', + 'usage': '13344214', + 'usageInDrive': '13344214', + 'usageInDriveTrash': '0' + } + }} + ) + stdout = StringIO() + cli = CLI() + + cli(['transfer', 'show_quota'], config_file=config_path, stdout=stdout) + + stdout.seek(0) + assert stdout.read() == """Quota: + limit: 15.0GiB + usage: 12.7MiB + usageInDrive: 12.7MiB + usageInDriveTrash: 0.0B +""" + + +def test_cli_extension_give_permissions(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + + config_writer( + config_path, + base_dir=str(backups_base), + transfer_backend='tests.backends.test_google_drive.CLIBackend', + transfer_backend_init_kwargs={'response': {}}, + transfer_path='/root/sub/dir' + ) + stdout = StringIO() + cli = CLI() + + cli( + ['transfer', 'give_permissions', 'hello@example.com'], + config_file=config_path, stdout=stdout + ) + + stdout.seek(0) + assert stdout.read() == 'Folder root can now be read\n' diff --git a/tox.ini b/tox.ini index 69df151..54a4110 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ DJANGO = extras = tests coverage + transfer_backend.drive deps = django20: Django>=2.0,<2.1 passenv = From bb8ce0c63af8bf3d11c5f28a3a525fde415e5027 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Fri, 20 Jul 2018 16:27:34 +0200 Subject: [PATCH 09/10] Update documentation --- ctrl_z/transfer/backends/base.py | 25 ++++- doc/conf.py | 8 +- doc/configuration.rst | 23 +++++ doc/index.rst | 1 + doc/quickstart.rst | 22 +++++ doc/transfer_backends.rst | 153 +++++++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 doc/transfer_backends.rst diff --git a/ctrl_z/transfer/backends/base.py b/ctrl_z/transfer/backends/base.py index 80c9a95..f62a6d6 100644 --- a/ctrl_z/transfer/backends/base.py +++ b/ctrl_z/transfer/backends/base.py @@ -10,19 +10,25 @@ class Base: """ def show_quota(self): + """ + Show information about remote storage quota. + """ raise NotImplementedError("Transfer backends must implement a show_quota() method") def ensure_dirs(self, path: str): """ Ensure the folders in path exist on the remote. + + :param path: the path to the base directory on the remote. Folder names + are separated by forward slashes, e.g. /var/backups/staging """ pass def prepare(self, full_path: str) -> BackupArchive: """ - Prepare the archive on the local drive. + Prepare the archive on the local drive for upload/check. - :param full_path: full path to the directory to archive + :param full_path: full path of the directory to archive :return: BackupArchive instance """ raise NotImplementedError("Transfer backends must implement a prepare() method") @@ -34,12 +40,23 @@ def exists(self, backup_archive: BackupArchive) -> bool: raise NotImplementedError("Transfer backends must implement a exists() method") def upload(self, backup_archive): + """ + Upload the prepared archive to the remote + + :raises UploadError: if the integrity check failed, a + :class:`ctrl_z.transfer.UploadError` should be raised. + """ raise NotImplementedError("Transfer backends must implement a upload() method") @classmethod def add_arguments(cls, parser): """ - Add optional command line arguments. + Add optional command line arguments to extend the CLI. + + You may want to facilitate interacting with the remote through the CLI. + + :param parser: the ``transfer`` subparser of the main CLI, this is an + argparse parser. """ pass @@ -52,6 +69,6 @@ def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.N holding the config. :param options: the CLI arguments parsed into argparse.Namespace :return: boolean indicating if a command was handled or not, if not, - the default behaviour is to transfer the backup. + the default behaviour is to transfer the backup. """ return False diff --git a/doc/conf.py b/doc/conf.py index ac3316f..dac2500 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,9 +12,10 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- @@ -40,6 +41,7 @@ # ones. extensions = [ 'sphinx.ext.todo', + 'sphinx.ext.autodoc', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/configuration.rst b/doc/configuration.rst index df7fcea..d72db94 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -129,3 +129,26 @@ Which binary to use to dump the database. Defaults to ``/usr/bin/pg_dump``. --------------------- Which binary to use to dump the database. Defaults to ``/usr/bin/pg_restore``. + +``transfer_backend`` +-------------------- + +The backend class to use to transfer backups from the local file system to a +remote/off-site location. Specify a dotted Python path to import. + +Available options: + +* ``ctrl_z.transfer.backends.google_drive.Backend`` - see :ref:`transfer-backend-drive` + +``transfer_backend_init_kwargs`` +-------------------------------- + +A mapping, specifying any keyword arguments needed to initialize the backend. +See :ref:`transfer-backends` for detailed information. + +``transfer_path`` +----------------- + +A string representing the base path to transfer backends to on the remote +system. Always use forward slashes. Defaults to ``/``. The backend is +responsible for creating the directory tree if it doesn't exist yet. diff --git a/doc/index.rst b/doc/index.rst index 5f8a3a8..bf05403 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -22,6 +22,7 @@ while being flexible through a yaml configuration file. quickstart configuration + transfer_backends Indices and tables diff --git a/doc/quickstart.rst b/doc/quickstart.rst index b6f3c71..511bc10 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -19,6 +19,13 @@ Install pip install ctrl-z +If you're using a built in backend to transfer the backups to a remote/offsite +location (currently only Google Drive is supported), you can install the +extra dependencies as well: + +.. code-block:: bash + + pip install ctrl-z[transfer_backend.drive] .. _usage: @@ -55,6 +62,8 @@ CTRL-Z exposes a CLI object to hook into your project, for example: Once the setup around the CLI is done, you can use it. +.. _cli-help: + CLI help -------- @@ -136,3 +145,16 @@ Restore the backup at the specified path. for. Useful if you have a multi-db setup and only the ``default`` is important, for example. Use multiple times for each alias to skip. * ``--no-files``: do not restore the (uploaded) files (e.g. ``settings.MEDIA_ROOT``) + + +Transfer a backup +----------------- + +.. code-block:: bash + + python backup/cli.py backup transfer + +Depending on the backend used, more CLI options may be available. These show +up via the built-in :ref:`cli-help`. + +The transfer is done according to the configured backend. diff --git a/doc/transfer_backends.rst b/doc/transfer_backends.rst new file mode 100644 index 0000000..537d136 --- /dev/null +++ b/doc/transfer_backends.rst @@ -0,0 +1,153 @@ +.. _transfer-backends: + +================= +Transfer backends +================= + +Just creating backups and leaving them on the local file system is not enough. +To deal with hardware failure, you should transfer your backups to *off-site* +systems. In CTRL-Z, you can achieve this through transfer backends. + +A transfer backend must: + +* prepare a local backup directory, for example creating a single-file archive + for ease of transfer. +* create the target directory hierarchy on the remote system, if needed +* be able to say if a local backup already exists on the remote or not +* be able to upload the prepared backup to the remote (and verify it's integrity) + +A transfer backend may optionally define subcommands for the main command line +interface. + +Available backends: + +* :ref:`transfer-backend-drive` + +Third party backends can be developed if needed, and installed as a separate +package. The YAML configuration is used to specify which backend to use and +which ``__init__`` parameters are required for the backend to configure +runtime options. + +Base backend +============ + +Custom backends should inherit from the base backend to guarantee API +compatibility. + +This backend is not functional by itself - it's an abstract base class. + +.. autoclass:: ctrl_z.transfer.backends.base.Base + :members: + + +.. _transfer-backend-drive: + +Google Drive +============ + +The Google Drive backend uploads the documents to Google Drive through the +Drive API. You need to create a `service account `_ +(see :ref:`drive-getting-started`) to be able to run the transfer without +interaction. + +.. todo:: See how to integrate with `team drives`_. + +The Drive backend: + +* creates the subfolders in Drive to store the backups if needed +* creates a gzipped tar-archive of the backup on the local file system +* checks if the backup exists by matching the name of the archive and the md5 + checksum against the (potentially) existing file on Drive +* uploads the archive to Drive and verifies integrity by comparing the + reported checksum with the local checksum + +Additionally, this backend provides some :ref:`drive-cli-extensions`. + +.. _drive-getting-started: + +Getting started +--------------- + +`google service accounts`_ are Google's way to interact with APIs on behalf +of a single user/entity. This is in contrast with OAUTH2 based flows where the +application using the Drive API typically interacts with the *account of the +end user*. For transfer backends, there are no end-users. + +One drawback is that you cannot have the Google Drive interface as the service +account, there is only API access. You can work around this by sharing the +folder holding the backups in Drive with yourself, see the +:ref:`drive-cli-extensions`. + +#. In the `google cloud console`_, create or select the project to use for the + backup transfer. +#. Next, click **+ Create service account**, and fill out the fields, e.g.: + + * Service account name: isp-backups + * Project role: you don't need to specify any role + * Check 'Furnish a private key' and check the default 'JSON' + +#. Backup the key file download to your browser - there's no way to obtain this + again. You can generate new keys of course. + +#. Configure CTRL-Z - in your ``config.yml``, ensure you have: + + .. code-block:: yaml + + transfer_backend: ctrl_z.transfer.backends.google_drive.Backend + transfer_backend_init_kwargs: + client_secrets: /path/to/the/downloaded/key/file.json + transfer_path: /some/root/dir + + + It's recommended to create at least one root directory to facilitate + sharing the folder with other people, since it's not possible to share the + root folder of Drive. + +#. Test the connection: + + .. code-block:: bash + + $ python cli.py transfer test_connection + + +.. _drive-cli-extensions: + +CLI extensions +-------------- + +The Drive backend extends the CLI with some subcommands. + +Connection test ++++++++++++++++ + +To verify that your credentials are correct, you can do a connection test: + +.. code-block:: bash + + $ python cli.py transfer test_connection + +Show quota +++++++++++ + +You can display the quota usage without needing to initiate a transfer: + +.. code-block:: bash + + $ python cli.py transfer show_quota + +Give read access +++++++++++++++++ + +You can share the root foldder with: + +.. code-block:: bash + + $ python cli.py transfer give_permissions hello@example.com + +The owner of the e-mail address then has read access to the folder in their +Drive via the 'Shared with me' tab. + + +.. _google service accounts: https://cloud.google.com/iam/docs/service-accounts +.. _google cloud console: https://console.cloud.google.com +.. _team drives: https://stackoverflow.com/questions/43243865/how-to-access-team-drive-using-service-account-with-google-drive-net-api-v3 From ef95c481ed884a7b5ec3816aec410ceff96bce5c Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Fri, 20 Jul 2018 16:42:41 +0200 Subject: [PATCH 10/10] Travis permissions... --- tests/test_cli.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 55c51f0..ab07096 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -119,7 +119,7 @@ def test_full_restore_not_directory(self, tmpdir): class TestTransfer: - def test_transfer_main_command(self, mocker): + def test_transfer_main_command(self, tmpdir, config_writer, mocker): """ Assert that the transfer is initiated if no backend-specific subcommand was handled. """ @@ -127,10 +127,14 @@ def test_transfer_main_command(self, mocker): transfer = mocked_BackupTransfer.from_config.return_value transfer.backend.handle_command.return_value = False - cli(['transfer']) + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli(['transfer'], config_file=config_path) transfer.sync_to_remote.assert_called_once() - def test_transfer_subcommand_handled(self, mocker): + def test_transfer_subcommand_handled(self, tmpdir, config_writer, mocker): """ Assert that the transfer is not initiated if a backend-specific subcommand was handled. """ @@ -138,12 +142,20 @@ def test_transfer_subcommand_handled(self, mocker): transfer = mocked_BackupTransfer.from_config.return_value transfer.backend.handle_command.return_value = True - cli(['transfer']) + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli(['transfer'], config_file=config_path) transfer.sync_to_remote.assert_not_called() -def test_no_subcommand(): - cli([], stdout=StringIO()) +def test_no_subcommand(tmpdir, config_writer): + config_path = str(tmpdir.join('config.yml')) + backups_base = tmpdir.join('backups') + config_writer(config_path, base_dir=str(backups_base)) + + cli([], stdout=StringIO(), config_file=config_path) cli.stdout.seek(0) output = cli.stdout.read()