diff --git a/.gitignore b/.gitignore index 4a53c2e..80cf9e5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ __pycache__/ /build/ /*.egg-info /*.egg +env/ +venv/ +pyznap.local.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index e0c5a19..ad80607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.6.0+local-b2] - 2021-07-10 +### Added +- Finely grained processing via ZFS user properties in the "pyznap:" domain + + pyznap:exclude=[true|false] + use pyznap:exclude to skip ZFS *send* processing of tagged datasets + + pyznap:max_size=[size ] units=B|KB|MB|TB|PB + use pyznap:max_size=500M to skip processing of tagged datasets + + +Note that the default has changed to NO +## [1.6.0+local-b1] - 2021-05-15 +### Added +- Allow in-line comments in configuration file +- Added command line `dry-run` and config `dry_run` options +Use the `dry-run` option to see what changes would be made, but not make them +- Added option `prune_sanoid` +Use the `prune_sanoid` option to control if pyznap prunes existing sanoid snapshots +Note that the default has changed to NO + + ## [1.6.0] - 2020-09-22 ### Added - Added resumable send/receive. diff --git a/README.md b/README.md index 2c00e22..3219ada 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Before you can use pyznap, you will need to create a config file. For initial se pyznap setup [-p PATH] This will create a directory `PATH` (default is `/etc/pyznap/`) and copy a sample config there. A -config for your system might look like this (remove the comments): +config for your system might look like this: [rpool/filesystem] frequent = 4 # Keep 4 frequent snapshots @@ -67,6 +67,8 @@ config for your system might look like this (remove the comments): yearly = 1 # Keep 1 yearly snapshot snap = yes # Take snapshots on this filesystem clean = yes # Delete old snapshots on this filesystem + prune_sanoid = no # Don't delete old sanoid snapshots on this filesystem + dry_run = no # Run in normal mode, don't use dry_run dest = backup/filesystem # Backup this filesystem on this location exclude = rpool/filesystem/data/* # Exclude these datasets for pyznap send @@ -76,7 +78,7 @@ Then set up a cronjob by creating a file under `/etc/cron.d/` and let pyznap run regularly by adding the following lines - SHELL=/bin/sh + SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin */15 * * * * root /path/to/pyznap snap >> /var/log/pyznap.log 2>&1 @@ -84,12 +86,20 @@ and let pyznap run regularly by adding the following lines This will run pyznap every quarter hour to take and delete snapshots. 'frequent' snapshots can be taken up to once per minute, so adjust your cronjob accordingly. +For installation in a python virtual environment (venv) you can replace the line above with: + + */15 * * * * root /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1 + If you also want to send your filesystems to another location you can add a line 0 0 * * * root /path/to/pyznap send >> /var/log/pyznap.log 2>&1 This will backup your data once per day at 12am. +Or for a venv: + + 0 0 * * * root /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1 + You can also manage, send to and pull from remote ssh locations. Always specify ssh locations with ssh:port:user@host:rpool/data @@ -143,6 +153,8 @@ Here is a list of all options you can set in the config fie: | `yearly` | Integer | Number of yearly snapshots | | `snap` | yes/no | Should snapshots be taken | | `clean` | yes/no | Should snapshots be cleaned | +| `prune_sanoid` | yes/no | Should sanoid snapshots be cleaned | +| `dry_run` | yes/no | Display commands/changes, but don't update filesystem | | `dest` | List of string | Comma-separated list of destinations where to send source filesystem | | `dest_key` | List of string | Path to ssh keyfile for dest. Comma-separated list for multiple dest | | `compress` | List of string | Compression to use over ssh, supported are gzip, lzop, bzip2, pigz, xz & lz4. Default is lzop. Comma-separated list for multiple dest | @@ -166,6 +178,10 @@ Run `pyznap -h` to see all available options. Print more verbose output. ++ -n, --dry-run + + Use dry-run mode. + + setup [-p PATH] Initial setup. Creates a config dir and puts a sample config file there. You can specify the path @@ -211,6 +227,25 @@ Run `pyznap -h` to see all available options. but you can set the `--dest-auto-create` flag to automatically create it. +#### ZFS User Properties #### +pyznap now supports finely grained settings via ZFS user properties in the "pyznap:" domain. The following properties are in use: + ++ pyznap:exclude [true|false] + Setting this property to true will caused the dateset to be excluded during *send*. Any other value, including the default, is assume to be "false" + + + Example: Set the dataset to be excluded by pyznap send + + `ZFS set pyznap:exclude=true tank/some_dataset` + ++ pyznap:max_size [size] + Sets the maximum dataset size to be processed during this run. The following suffixes are supported: + + `B, KB, MB, GB, TB, PB` + + + Example: Set the maximum dataset processing size to 60GB + + `ZFS set pyznap:max_size=60G tank/some_other_dataset` + #### Usage examples #### + Take snapshots according to policy in default config file: @@ -233,6 +268,10 @@ Run `pyznap -h` to see all available options. `pyznap send -s tank/data -d backup/data` ++ Backup a single filesystem locally, display ZFS commands, and run in dry-run (dummy) mode: + + `pyznap --verbose --dry-run send -s tank/data -d backup/data` + + Send a single filesystem to a remote location, using `pigz` compression: `pyznap send -s tank/data -d ssh:20022:root@example.com:backup/data -i /root/.ssh/id_rsa -c pigz` diff --git a/pyznap/__init__.py b/pyznap/__init__.py index 56fa5c2..b2507b1 100644 --- a/pyznap/__init__.py +++ b/pyznap/__init__.py @@ -9,4 +9,4 @@ """ -__version__ = '1.6.0' +__version__ = 'v1.6.0+local-b2' diff --git a/pyznap/clean.py b/pyznap/clean.py index 7061869..e96cf5a 100644 --- a/pyznap/clean.py +++ b/pyznap/clean.py @@ -27,10 +27,11 @@ def clean_snap(snap): """ logger = logging.getLogger(__name__) - - logger.info('Deleting snapshot {}...'.format(snap)) + dry_run = snap.dry_run == True + dry_msg = '*** DRY RUN ***' if dry_run else '' + logger.info('Deleting snapshot {}... {}'.format(snap, dry_msg)) try: - snap.destroy() + snap.destroy(dry_run=dry_run) except DatasetBusyError as err: logger.error(err) except CalledProcessError as err: @@ -54,7 +55,8 @@ def clean_filesystem(filesystem, conf): """ logger = logging.getLogger(__name__) - logger.debug('Cleaning snapshots on {}...'.format(filesystem)) + prunes = 'pyznap' if (conf.get('prune_sanoid', None) == False) else ('autosnap', 'pyznap') + logger.debug("Cleaning snapshots on {}... prunes={}".format(filesystem, prunes ) ) snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []} # catch exception if dataset was destroyed since pyznap was started @@ -65,11 +67,12 @@ def clean_filesystem(filesystem, conf): return 1 # categorize snapshots for snap in fs_snapshots: - # Ignore snapshots not taken with pyznap or sanoid - if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')): + # Ignore snapshots not taken with pyznap or sanoid, depending on configuration + if not snap.name.split('@')[1].startswith(prunes): continue try: snap_type = snap.name.split('_')[-1] + snap.dry_run = conf.get('dry_run', None) snapshots[snap_type].append(snap) except (ValueError, KeyError): continue diff --git a/pyznap/config/etc/cron.d/pyznap b/pyznap/config/etc/cron.d/pyznap new file mode 100644 index 0000000..5ffbc42 --- /dev/null +++ b/pyznap/config/etc/cron.d/pyznap @@ -0,0 +1,9 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +# Take pyznap snapshots every 15 minutes +*/15 * * * * root /root/pyznap/bin/python /root/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1 + +# Copy to pyznap backup at midnight +0 0 * * * root /root/pyznap/bin/python /root/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1 + diff --git a/pyznap/config/etc/logrotate.d/pyznap b/pyznap/config/etc/logrotate.d/pyznap new file mode 100644 index 0000000..7b3d08f --- /dev/null +++ b/pyznap/config/etc/logrotate.d/pyznap @@ -0,0 +1,7 @@ +/var/log/pyznap.log { + daily + rotate 14 + compress + missingok + notifempty +} diff --git a/pyznap/config/pyznap.conf b/pyznap/config/etc/pyznap.conf similarity index 94% rename from pyznap/config/pyznap.conf rename to pyznap/config/etc/pyznap.conf index 22e0f4b..551bf3c 100644 --- a/pyznap/config/pyznap.conf +++ b/pyznap/config/etc/pyznap.conf @@ -4,8 +4,7 @@ ## filesystem. For remote syncronisation always keep enough snapshots on the destination. If there ## are no common snapshots the destination has to be destroyed and a full stream has to be sent. ## ssh locations are always specified with 'ssh:port:user@host:poolname/filesystem'. -## Remove the comments at the end of the lines in your config, as they will not be ignored. Only -## lines starting with '#' will be ignored. +## Lines starting with '#' will be ignored. # # # @@ -21,6 +20,8 @@ # yearly = 1 # Keep 1 yearly snapshot # snap = yes # Take snapshots on this filesystem # clean = yes # Delete old snapshots on this filesystem +# prune_sanoid = no # Prune sanoid snapshots yes|no +# dry_run = yes # Use dry-run mode? yes|no # dest = backup/filesystem # Backup this filesystem on this location # exclude = rpool/filesystem/data/* # Exclude these datasets for pyznap send # diff --git a/pyznap/main.py b/pyznap/main.py index 74945af..eabfad1 100644 --- a/pyznap/main.py +++ b/pyznap/main.py @@ -19,6 +19,7 @@ from .clean import clean_config from .take import take_config from .send import send_config +from pyznap import __version__ DIRNAME = os.path.dirname(os.path.abspath(__file__)) @@ -33,7 +34,10 @@ def _main(): Exit code """ - parser = ArgumentParser(prog='pyznap', description='ZFS snapshot tool written in python') + parser = ArgumentParser(prog='pyznap', + description='ZFS snapshot tool written in python {}'.format(__version__)) + parser.add_argument('-n', '--dry-run', action="store_true", + dest="dry_run", help="Dry-run, don't execute commands") parser.add_argument('-v', '--verbose', action="store_true", dest="verbose", help='print more verbose output') parser.add_argument('--config', action="store", @@ -80,6 +84,8 @@ def _main(): parser_send.add_argument('--retry-interval', action="store", type=int, dest='retry_interval', default=10, help='interval in seconds between retries. default is 10') + + parser.epilog = "ZFS properties: [pyznap:exclude, pyznap:max_size]" if len(sys.argv)==1: parser.print_help(sys.stderr) @@ -91,7 +97,7 @@ def _main(): datefmt='%b %d %H:%M:%S', stream=sys.stdout) logger = logging.getLogger(__name__) - logger.info('Starting pyznap...') + logger.info('Starting pyznap {}...'.format(__version__)) if args.command in ('snap', 'send'): config_path = args.config if args.config else os.path.join(CONFIG_DIR, 'pyznap.conf') @@ -99,6 +105,13 @@ def _main(): if config == None: return 1 + # Append global dry_run flag don't override existing dry-run = yes + try: + for conf in config: + conf['dry_run'] = True if (args.dry_run or conf.get('dry_run', None)) else False + except UnboundLocalError: + pass + if args.command == 'setup': path = args.path if args.path else CONFIG_DIR create_config(path) @@ -143,7 +156,8 @@ def _main(): send_config([{'name': args.source, 'dest': [args.dest], 'key': source_key, 'dest_keys': dest_key, 'compress': compress, 'exclude': exclude, 'raw_send': raw, 'resume': resume, 'dest_auto_create': dest_auto_create, - 'retries': retries, 'retry_interval': retry_interval}]) + 'retries': retries, 'retry_interval': retry_interval, + 'dry_run': dry_run}]) elif args.source and not args.dest: logger.error('Missing dest...') diff --git a/pyznap/process.py b/pyznap/process.py index 7bb8fe7..54a3a63 100644 --- a/pyznap/process.py +++ b/pyznap/process.py @@ -10,6 +10,7 @@ import re import errno as _errno +import logging import subprocess as sp import socket @@ -93,6 +94,7 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs): List of all lines from the output, seperated at '\t' into lists """ + logger = logging.getLogger(__name__) if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') if 'universal_newlines' in kwargs: @@ -100,6 +102,7 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs): if 'input' in kwargs: raise ValueError('input argument not allowed, it will be overridden.') + logger.debug("'{}'...".format(' '.join(*popenargs))) ret = run(*popenargs, stdout=PIPE, stderr=PIPE, timeout=timeout, universal_newlines=True, ssh=ssh, **kwargs) ret.check_returncode() diff --git a/pyznap/pyzfs.py b/pyznap/pyzfs.py index 31d9c05..99a61db 100644 --- a/pyznap/pyzfs.py +++ b/pyznap/pyzfs.py @@ -119,7 +119,7 @@ def roots(ssh=None): return find(ssh=ssh, max_depth=0) # note: force means create missing parent filesystems -def create(name, ssh=None, type='filesystem', props={}, force=False): +def create(name, ssh=None, type='filesystem', props={}, force=False, dry_run=False): cmd = ['zfs', 'create'] if type == 'volume': @@ -130,6 +130,9 @@ def create(name, ssh=None, type='filesystem', props={}, force=False): if force: cmd.append('-p') + if dry_run: + cmd.append('-n') + for prop, value in props.items(): cmd.append('-o') cmd.append(prop + '=' + str(value)) @@ -142,7 +145,7 @@ def create(name, ssh=None, type='filesystem', props={}, force=False): def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_path=False, - force=False, nomount=False, stream_size=0, raw=False, resume=False): + force=False, nomount=False, stream_size=0, raw=False, resume=False, dry_run=False): """Returns Popen instance for zfs receive""" logger = logging.getLogger(__name__) @@ -181,6 +184,8 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa cmd.append('-u') if resume: cmd.append('-s') + if dry_run: + cmd.append('-n') cmd.append(quote(name)) # use shlex to quote the name @@ -196,6 +201,7 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa # execute command with shell (sh or ssh) cmd = shell + [' '.join(cmd)] + logger.debug("'{}'...".format(' '.join(cmd))) return sp.Popen(cmd, stdin=stdin, stderr=sp.PIPE) # zfs receive process @@ -232,7 +238,7 @@ def dependents(self): # TODO: split force to allow -f, -r and -R to be specified individually # TODO: remove or ignore defer option for non-snapshot datasets - def destroy(self, defer=False, force=False): + def destroy(self, defer=False, force=False, dry_run=False): cmd = ['zfs', 'destroy'] cmd.append('-v') @@ -244,6 +250,9 @@ def destroy(self, defer=False, force=False): cmd.append('-f') cmd.append('-R') + if dry_run: + cmd.append('-n') + cmd.append(self.name) check_output(cmd, ssh=self.ssh) diff --git a/pyznap/send.py b/pyznap/send.py index 0e909e4..da55f3b 100644 --- a/pyznap/send.py +++ b/pyznap/send.py @@ -18,11 +18,12 @@ from time import sleep from .ssh import SSH, SSHException from .utils import parse_name, exists, check_recv, bytes_fmt +from .utils import parse_size import pyznap.pyzfs as zfs from .process import DatasetBusyError, DatasetNotFoundError, DatasetExistsError -def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=False, resume_token=None): +def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=False, resume_token=None, dry_run=False): """Sends snapshot to destination, incrementally and over ssh if specified. Parameters: @@ -35,6 +36,8 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F Base snapshot for incremental stream (the default is None, meaning a full stream) ssh_dest : {ssh.SSH}, optional Open ssh connection for remote backup (the default is None, meaning local backup) + dry_run : {boolean}, optional + Don't change filesystem Returns ------- @@ -51,7 +54,7 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F send = snapshot.send(ssh_dest=ssh_dest, base=base, intermediates=True, raw=raw, resume_token=resume_token) recv = zfs.receive(name=dest_name, stdin=send.stdout, ssh=ssh_dest, ssh_source=ssh_source, - force=True, nomount=True, stream_size=stream_size, raw=raw, resume=resume) + force=True, nomount=True, stream_size=stream_size, raw=raw, resume=resume, dry_run=dry_run) send.stdout.close() # write pv output to stderr / stdout @@ -83,7 +86,7 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F return 0 -def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False): +def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False, dry_run=False): """Checks for common snapshots between source and dest. If none are found, send the oldest snapshot, then update with the most recent one. If there are common snaps, update destination with the most recent one. @@ -96,6 +99,8 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False Name of the location to send to ssh_dest : {ssh.SSH}, optional Open ssh connection for remote backup (the default is None, meaning local backup) + dry_run : {boolean}, optional + Don't change filesystem Returns ------- @@ -106,7 +111,8 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False logger = logging.getLogger(__name__) dest_name_log = '{:s}@{:s}:{:s}'.format(ssh_dest.user, ssh_dest.host, dest_name) if ssh_dest else dest_name - logger.debug('Sending {} to {:s}...'.format(source_fs, dest_name_log)) + dry_msg = '*** DRY RUN ***' if dry_run else '' + logger.debug('Sending {} to {:s}... {}'.format(source_fs, dest_name_log, dry_msg)) resume_token = None # Check if dest already has a 'zfs receive' ongoing @@ -176,7 +182,7 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False if resume_token is not None: logger.info('Found resume token. Resuming last transfer of {:s} (~{:s})...' .format(dest_name_log, bytes_fmt(base.stream_size(raw=raw, resume_token=resume_token)))) - rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=True, resume_token=resume_token) + rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=True, resume_token=resume_token, dry_run=dry_run) if rc: return rc # we need to update common snapshots after finishing the resumable send @@ -191,7 +197,7 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False else: logger.info('No common snapshots on {:s}, sending oldest snapshot {} (~{:s})...' .format(dest_name_log, base, bytes_fmt(base.stream_size(raw=raw)))) - rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=resume) + rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run) if rc: return rc else: @@ -199,9 +205,9 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False base = next(filter(lambda x: x.name.split('@')[1] in common, snapshots), None) if base.name != snapshot.name: - logger.info('Updating {:s} with recent snapshot {} (~{:s})...' - .format(dest_name_log, snapshot, bytes_fmt(snapshot.stream_size(base, raw=raw)))) - rc = send_snap(snapshot, dest_name, base=base, ssh_dest=ssh_dest, raw=raw, resume=resume) + logger.info('Updating {:s} with recent snapshot {} (~{:s})... {}' + .format(dest_name_log, snapshot, bytes_fmt(snapshot.stream_size(base, raw=raw)), dry_msg)) + rc = send_snap(snapshot, dest_name, base=base, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run) if rc: return rc @@ -225,7 +231,9 @@ def send_config(config): for conf in config: if not conf.get('dest', None): continue - + + dry_run = conf.get('dry_run', None) + dry_msg = '*** DRY RUN ***' if dry_run else '' backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) @@ -301,8 +309,8 @@ def send_config(config): zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: if dest_auto_create: - logger.info('Destination {:s} does not exist, will create it...'.format(dest_name_log)) - if create_dataset(dest_name, dest_name_log, ssh=ssh_dest): + logger.info('Destination {:s} does not exist, will create it... {}'.format(dest_name_log, dry_msg)) + if create_dataset(dest_name, dest_name_log, ssh=ssh_dest, dry_run=dry_run): continue else: logger.error('Destination {:s} does not exist, manually create it or use "dest-auto-create" option...' @@ -325,9 +333,36 @@ def send_config(config): if any(fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug('Matched {} in exclude rules, not sending...'.format(source_fs)) continue + + # Check for ZFS user property to bypass filesystem + fs_props = source_fs.getprops() + + exclude_prop='pyznap:exclude' + ignore_me = fs_props.get(exclude_prop, ('false', 'false'))[0].lower() + logger.debug("Property {}={} for {}".format(exclude_prop, ignore_me, source_fs)) + if ignore_me == 'true': + logger.info('Matched {}={} for {}, not sending...' + .format(exclude_prop, ignore_me, source_fs)) + continue + + # Check for max size + used_prop='used' + fs_used_bytes = int(fs_props.get(used_prop, ('0', '0'))[0]) # Bytes + fs_used_fmt = bytes_fmt(fs_used_bytes) # MB + logger.debug("Property {}={} ({}) for {}".format(used_prop, fs_used_fmt, fs_used_bytes, source_fs)) + + max_prop='pyznap:max_size' + fs_max_fmt = fs_props.get(max_prop, ('0', '0'))[0] # String + fs_max_bytes = parse_size(fs_max_fmt) # Bytes + logger.debug("Property {}={} ({}) for {}".format(max_prop, fs_max_fmt, fs_max_bytes, source_fs)) + if fs_max_bytes > 0 and fs_used_bytes > fs_max_bytes: + logger.info('Filesystem size {} exceeds {}={} for {}, not sending...' + .format(fs_used_fmt, max_prop, fs_max_fmt, source_fs)) + continue + # send not excluded filesystems for retry in range(1,retries+2): - rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume) + rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run) if rc == 2 and retry <= retries: logger.info('Retrying send in {:d}s (retry {:d} of {:d})...'.format(retry_interval, retry, retries)) sleep(retry_interval) @@ -341,7 +376,7 @@ def send_config(config): ssh_source.close() -def create_dataset(name, name_log, ssh=None): +def create_dataset(name, name_log, ssh=None, dry_run=False): """Creates a dataset and logs success/fail Parameters @@ -352,6 +387,8 @@ def create_dataset(name, name_log, ssh=None): Name used for logging ssh : {SSH}, optional Open ssh connection, by default None + dry_run : {boolean}, optional + Dry run, don't change ZFS pool Returns ------- @@ -360,7 +397,7 @@ def create_dataset(name, name_log, ssh=None): """ logger = logging.getLogger(__name__) try: - zfs.create(name, ssh=ssh, force=True) + zfs.create(name, ssh=ssh, force=True, dry_run=dry_run) except CalledProcessError as err: message = err.stderr.rstrip() if message == "filesystem successfully created, but it may only be mounted by root": diff --git a/pyznap/take.py b/pyznap/take.py index 4ac1a5f..7514c6e 100644 --- a/pyznap/take.py +++ b/pyznap/take.py @@ -33,9 +33,12 @@ def take_snap(filesystem, _type): snapname = lambda _type: 'pyznap_{:s}_{:s}'.format(now().strftime('%Y-%m-%d_%H:%M:%S'), _type) - logger.info('Taking snapshot {}@{:s}...'.format(filesystem, snapname(_type))) + dry_run = filesystem.dry_run == True + dry_msg = '*** DRY RUN ***' if dry_run else '' + logger.info('Taking snapshot {}@{:s}... {}'.format(filesystem, snapname(_type), dry_msg)) try: - filesystem.snapshot(snapname=snapname(_type), recursive=True) + if not dry_run: + filesystem.snapshot(snapname=snapname(_type), recursive=True) except (DatasetBusyError, DatasetExistsError) as err: logger.error(err) except CalledProcessError as err: @@ -59,9 +62,11 @@ def take_filesystem(filesystem, conf): """ logger = logging.getLogger(__name__) - logger.debug('Taking snapshots on {}...'.format(filesystem)) + prunes = 'pyznap' if (conf.get('prune_sanoid', None) == False) else ('autosnap', 'pyznap') + logger.debug("Taking snapshots on {}... prunes={}".format(filesystem, prunes ) ) now = datetime.now + filesystem.dry_run = conf.get('dry_run', None) snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []} # catch exception if dataset was destroyed since pyznap was started try: @@ -71,8 +76,8 @@ def take_filesystem(filesystem, conf): return 1 # categorize snapshots for snap in fs_snapshots: - # Ignore snapshots not taken with pyznap or sanoid - if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')): + # Ignore snapshots not taken with pyznap or sanoid, depending on configuration + if not snap.name.split('@')[1].startswith(prunes): continue try: _date, _time, snap_type = snap.name.split('_')[-3:] diff --git a/pyznap/utils.py b/pyznap/utils.py index 1182419..e1508f6 100644 --- a/pyznap/utils.py +++ b/pyznap/utils.py @@ -76,7 +76,7 @@ def read_config(path): logger.error('Error while loading config: File {:s} does not exist.'.format(path)) return None - parser = ConfigParser() + parser = ConfigParser(inline_comment_prefixes="#") try: parser.read(path) except (MissingSectionHeaderError, DuplicateSectionError, DuplicateOptionError) as e: @@ -86,7 +86,8 @@ def read_config(path): config = [] options = ['key', 'frequent', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'snap', 'clean', 'dest', 'dest_keys', 'compress', 'exclude', 'raw_send', 'resume', 'dest_auto_create', - 'retries', 'retry_interval'] + 'retries', 'retry_interval', + 'dry_run', 'prune_sanoid'] for section in parser.sections(): dic = {} @@ -118,6 +119,11 @@ def read_config(path): for i in value.split(',')] elif option in ['retries', 'retry_interval']: dic[option] = [int(i) for i in value.split(',')] + elif option in ['dry_run']: + dic[option] = {'yes': True, 'no': False}.get(value.lower(), None) + elif option in ['prune_sanoid']: + dic[option] = {'yes': True, 'no': False}.get(value.lower(), None) + # Pass through values recursively for parent in config: for child in config: @@ -171,7 +177,7 @@ def create_config(path): logger = logging.getLogger(__name__) CONFIG_FILE = os.path.join(path, 'pyznap.conf') - config = resource_string(__name__, 'config/pyznap.conf').decode("utf-8") + config = resource_string(__name__, 'config/etc/pyznap.conf').decode("utf-8") logger.info('Initial setup...') @@ -266,3 +272,40 @@ def bytes_fmt(num): num /= 1024 else: return "{:3.1f}{:s}".format(num, 'Y') + + +# based on https://stackoverflow.com/a/42865957/2002471 +def parse_size(size): + """Converts human readable format to bytes + + Parameters + ---------- + str : size + formatted size + + Returns + ------- + float + actual bytes + """ + + units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40, "PB": 2**50} + + logger = logging.getLogger(__name__) + + size = str(size).upper() + #print("parsing size ", size) + if size[-1].strip() != "B": + size = size + "B" + if not re.match(r' ', size): + size = re.sub(r'([KMGT]?B)', r' \1', size) + + result = 0 + try: + number, unit = [string.strip() for string in size.split()] + result = int(float(number)*units[unit]) + except: + logger.error("Could not convert {} to a size in bytes, using default={}".format(size, result)) + + return result +