Google Drive backend for remote storage

.editorconfig

@@ -6,4 +6,7 @@ indent_size = 4
 end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
-max_line_length = 79 
+max_line_length = 79
+
+[*.{yml,yaml}]
+indent_size = 2

ctrl_z/__init__.py

@@ -1,7 +1,7 @@
 from pkg_resources import get_distribution
 
+from ._cli import cli  # noqa
 from .backup import Backup, configure_logging  # noqa
-from .cli import cli  # noqa
 
 __version__ = get_distribution('CTRL-Z').version
 

ctrl_z/cli.py → ctrl_z/_cli.py

@@ -7,13 +7,18 @@
 import sys
 
 import django
+from django.utils.module_loading import import_string
 
 from .backup import Backup, configure_logging
-from .config import DEFAULT_CONFIG_FILE
+from .config import DEFAULT_CONFIG_FILE, Config
+from .transfer import BackupTransfer
 
 logger = logging.getLogger(__name__)
 
 
+HEADER = "CTRL-Z {version} - Backup and recovery tool\n"
+
+
 def noop(*args, **kwargs):
     pass
 
@@ -61,8 +66,6 @@ class CLI:
 
     def __init__(self):
         parser = argparse.ArgumentParser(description="CTRL-Z CLI")
-        parser.add_argument('--config-file', help="Config file to use")
-        parser.add_argument('--base-dir', help="Base directory override")
 
         subparsers = parser.add_subparsers(
             help="Sub commands", dest='subcommand'
@@ -117,6 +120,11 @@ def __init__(self):
             default=True, help="Do not restore files"
         )
 
+        # backup transfer
+        self.parser_transfer = subparsers.add_parser(
+            'transfer', help='Transfer the backups to an off-site location'
+        )
+
         self.parser = parser
 
     def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None, stderr=None):
@@ -128,10 +136,14 @@ def __call__(self, args=None, config_file: str=DEFAULT_CONFIG_FILE, stdout=None,
         if stderr:
             self.stderr = stderr
 
-        self.stderr.write(f"CTRL-Z {__version__} - Backup and recovery tool\n")
+        self.stderr.write(HEADER.format(version=__version__))
 
-        args = self.parser.parse_args(args or sys.argv[1:])
-        config_file = args.config_file or config_file
+        # load the command line args for transfers
+        config = Config.from_file(config_file)
+        TransferBackend = import_string(config.transfer_backend)
+        TransferBackend.add_arguments(self.parser_transfer)
+
+        args = self.parser.parse_args(args if args is not None else sys.argv[1:])
 
         self._setup()
 
@@ -145,8 +157,6 @@ def _setup(self):
     def run(self, options, config_file: str):
         subcommand = options.subcommand
         conf_overrides = {}
-        if options.base_dir:
-            conf_overrides['base_dir'] = options.base_dir
 
         if subcommand == 'restore':
             self._backup = Backup.prepare_restore(
@@ -166,8 +176,10 @@ def run(self, options, config_file: str):
             self.backup(options)
         elif subcommand == 'restore':
             self.restore(options)
+        elif subcommand == 'transfer':
+            self.transfer(options, config_file, conf_overrides)
         else:
-            self.parser.print_help()
+            self.parser.print_help(file=self.stdout)
 
     def generate_config(self, options):
         """
@@ -219,5 +231,26 @@ def restore(self, options):
         finally:
             backup.report(has_errors)
 
+    def transfer(self, options, config_file, conf_overrides):
+        """
+        Relay the command to the transfer backend or initiate the actual transfer.
+        """
+        transfer = BackupTransfer.from_config(config_file)
+
+        # handle potential backend specific subcommands
+        handled = transfer.backend.handle_command(self, transfer, options)
+        if handled:
+            return
+
+        has_errors = False
+        try:
+            transfer.show_info()
+            transfer.sync_to_remote()
+        except Exception:
+            has_errors = True
+            raise
+        finally:
+            self._backup.report(has_errors)
+
 
 cli = CLI()

ctrl_z/backup.py

@@ -33,7 +33,7 @@ def from_config(cls, config_file):
 
     @classmethod
     def prepare_restore(cls, config_file, base_dir: str):
-        config = Config.from_file(config_file, base_dir=base_dir, restore=True)
+        config = Config.from_file(config_file, base_dir=base_dir, use_parent_dir=True)
         return cls(config=config)
 
     def restore(self, db=True, skip_db=None, files=True):

ctrl_z/config.default.yml

@@ -33,3 +33,8 @@ files:
 # Which binaries to use for backup creation/restore
 pg_dump_binary: /usr/bin/pg_dump
 pg_restore_binary: /usr/bin/pg_restore
+
+# where are backups transferred (off-site)
+transfer_backend: ctrl_z.transfer.backends.base.Base
+transfer_backend_init_kwargs: {}
+transfer_path: /

ctrl_z/config.py

@@ -16,7 +16,7 @@
 
 class Config:
     __slots__ = [
-        'restore',
+        'use_parent_dir',
         'base_dir',
         'logging',
         'database',
@@ -25,10 +25,13 @@ class Config:
         'files',
         'pg_dump_binary',
         'pg_restore_binary',
+        'transfer_backend',
+        'transfer_backend_init_kwargs',
+        'transfer_path',
     ]
 
     def __init__(self, **kwargs):
-        self.restore = kwargs.pop('restore', False)
+        self.use_parent_dir = kwargs.pop('use_parent_dir', False)
 
         for key, value in kwargs.items():
             setattr(self, key, value)
@@ -54,14 +57,14 @@ def from_file(cls, config_file: str, **overrides):
 
     def write_to(self, path: str):
         as_dict = {key: getattr(self, key) for key in self.__slots__}
-        if not self.restore:
+        if not self.use_parent_dir:
             as_dict['base_dir'] = os.path.dirname(as_dict['base_dir'])
         as_dict['retention_policy'] = as_dict['retention_policy'].serialize()
         with open(path, 'w') as stream:
             yaml.dump(as_dict, stream=stream)
 
     def set_base_dir(self):
-        if self.restore:  # should be set via overrides
+        if self.use_parent_dir:  # should be set via overrides
             return
 
         self.base_dir = self.retention_policy.get_base_dir(self.base_dir)

ctrl_z/constants.py

@@ -0,0 +1,5 @@
+import re
+
+DATE_FORMAT = "%Y-%m-%d"
+
+BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)')

ctrl_z/retention.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import re
 import shutil
 from datetime import date, datetime
 from itertools import chain
@@ -9,33 +8,22 @@
 from dateutil.relativedelta import relativedelta
 from dateutil.rrule import DAILY, WEEKLY, rrule
 
+from .constants import DATE_FORMAT
+from .utils import is_backup_dir
+
 logger = logging.getLogger(__name__)
 
 
 class RetentionPolicy:
     __slots__ = ['day_of_week', 'days_to_keep', 'weeks_to_keep']
 
-    DATE_FORMAT = "%Y-%m-%d"
-
-    BACKUP_DIR_PATTERN = re.compile(r'^2[0-9]{3}-[0-1][0-9]-[0-3][0-9]-(daily|weekly)')
-
     def __init__(self, **config):
         for key, value in config.items():
             setattr(self, key, value)
 
     def serialize(self):
         return {key: getattr(self, key) for key in self.__slots__}
 
-    def is_backup_dir(self, dir_name: str) -> bool:
-        """
-        Test if a directory name fits the pattern of backup folder names.
-
-        The pattern is YYYY-MM-DD-suffix, where suffix is either 'daily' or 'weekly'.
-        """
-        if self.BACKUP_DIR_PATTERN.match(dir_name):
-            return True
-        return False
-
     def get_suffix(self, dt: Union[date, datetime]) -> str:
         return 'weekly' if dt.weekday() == self.day_of_week else 'daily'
 
@@ -44,7 +32,7 @@ def get_base_dir(self, base: str) -> str:
         Figure out the folder name for the current backup.
         """
         now = datetime.utcnow()
-        datestamp = now.strftime(self.DATE_FORMAT)
+        datestamp = now.strftime(DATE_FORMAT)
         suffix = self.get_suffix(now)
         return os.path.join(base, f"{datestamp}-{suffix}")
 
@@ -67,14 +55,14 @@ def rotate(self, base: str):
         weeklies = rrule(WEEKLY, dtstart=weekly_start, count=self.weeks_to_keep)
 
         to_keep = sorted({
-            "{}-{}".format(dt.strftime(self.DATE_FORMAT), self.get_suffix(dt))
+            "{}-{}".format(dt.strftime(DATE_FORMAT), self.get_suffix(dt))
             for dt in chain(dailies, weeklies)
         })
         logger.debug("Keeping backups from: %r", to_keep)
 
         to_delete = []
         for dir_name in os.listdir(base):
-            if not self.is_backup_dir(dir_name):
+            if not is_backup_dir(dir_name):
                 logger.debug("%s doesn't look like a backup directory, keeping it.", dir_name)
                 continue
 

ctrl_z/transfer/__init__.py

@@ -0,0 +1,2 @@
+from .archive import BackupArchive  # noqa
+from .transfer import BackupTransfer, UploadError  # noqa

ctrl_z/transfer/archive.py

@@ -0,0 +1,26 @@
+import hashlib
+import os
+
+
+class BackupArchive:
+    def __init__(self, archive: str):
+        """
+        A single backup as an archive.
+
+        The archive format depends on the backend, it could be tar.gz, zip...
+
+        :param archive: full path to the archive file
+        """
+        self.archive = archive
+
+    @property
+    def md5_checksum(self):
+        """
+        Calculate the md5 checksum of the archive
+        """
+        assert os.path.isfile(self.archive), "%s is not a file"
+        hash_md5 = hashlib.md5()
+        with open(self.archive, "rb") as archive:
+            for chunk in iter(lambda: archive.read(4096), b""):
+                hash_md5.update(chunk)
+        return hash_md5.hexdigest()

ctrl_z/transfer/backends/__init__.py

@@ -0,0 +1 @@
+from .base import Base  # noqa

ctrl_z/transfer/backends/base.py

@@ -0,0 +1,74 @@
+import argparse
+
+from .. import BackupArchive, BackupTransfer
+from ..._cli import CLI
+
+
+class Base:
+    """
+    Define the public API that a backend must implement.
+    """
+
+    def show_quota(self):
+        """
+        Show information about remote storage quota.
+        """
+        raise NotImplementedError("Transfer backends must implement a show_quota() method")
+
+    def ensure_dirs(self, path: str):
+        """
+        Ensure the folders in path exist on the remote.
+
+        :param path: the path to the base directory on the remote. Folder names
+          are separated by forward slashes, e.g. /var/backups/staging
+        """
+        pass
+
+    def prepare(self, full_path: str) -> BackupArchive:
+        """
+        Prepare the archive on the local drive for upload/check.
+
+        :param full_path: full path of the directory to archive
+        :return: BackupArchive instance
+        """
+        raise NotImplementedError("Transfer backends must implement a prepare() method")
+
+    def exists(self, backup_archive: BackupArchive) -> bool:
+        """
+        Check if the file exists on the remote
+        """
+        raise NotImplementedError("Transfer backends must implement a exists() method")
+
+    def upload(self, backup_archive):
+        """
+        Upload the prepared archive to the remote
+
+        :raises UploadError: if the integrity check failed, a
+          :class:`ctrl_z.transfer.UploadError` should be raised.
+        """
+        raise NotImplementedError("Transfer backends must implement a upload() method")
+
+    @classmethod
+    def add_arguments(cls, parser):
+        """
+        Add optional command line arguments to extend the CLI.
+
+        You may want to facilitate interacting with the remote through the CLI.
+
+        :param parser: the ``transfer`` subparser of the main CLI, this is an
+          argparse parser.
+        """
+        pass
+
+    def handle_command(self, cli: CLI, transfer: BackupTransfer, options: argparse.Namespace) -> bool:
+        """
+        Handle backend specific commands
+
+        :param cli: the CLI instance asking to handle the command
+        :param transfer: the :class:`ctrl_z.transfer.BackupTransfer` instance
+          holding the config.
+        :param options: the CLI arguments parsed into argparse.Namespace
+        :return: boolean indicating if a command was handled or not, if not,
+          the default behaviour is to transfer the backup.
+        """
+        return False