From 8dfcc63ecf01fd757e1adb7fe93e44b03e7c5bda Mon Sep 17 00:00:00 2001
From: ajay <84351018+ajayjay0@users.noreply.github.com>
Date: Wed, 12 May 2021 11:30:13 -0700
Subject: [PATCH 1/5] Enable comments in configuration file

---
 pyznap/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyznap/utils.py b/pyznap/utils.py
index 1182419..116f23a 100644
--- a/pyznap/utils.py
+++ b/pyznap/utils.py
@@ -76,7 +76,7 @@ def read_config(path):
         logger.error('Error while loading config: File {:s} does not exist.'.format(path))
         return None
 
-    parser = ConfigParser()
+    parser = ConfigParser(inline_comment_prefixes="#")
     try:
         parser.read(path)
     except (MissingSectionHeaderError, DuplicateSectionError, DuplicateOptionError) as e:

From 093b97c05c5cfbd74e337da8f0d74d3643e6de1f Mon Sep 17 00:00:00 2001
From: ajay <84351018+ajayjay0@users.noreply.github.com>
Date: Wed, 12 May 2021 15:08:43 -0700
Subject: [PATCH 2/5] Log ZFS commands

---
 pyznap/process.py | 3 +++
 pyznap/pyzfs.py   | 1 +
 2 files changed, 4 insertions(+)

diff --git a/pyznap/process.py b/pyznap/process.py
index 7bb8fe7..5b80ebf 100644
--- a/pyznap/process.py
+++ b/pyznap/process.py
@@ -10,6 +10,7 @@
 
 import re
 import errno as _errno
+import logging
 import subprocess as sp
 import socket
 
@@ -93,6 +94,7 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs):
         List of all lines from the output, seperated at '\t' into lists
     """
 
+    logger = logging.getLogger(__name__)
     if 'stdout' in kwargs:
         raise ValueError('stdout argument not allowed, it will be overridden.')
     if 'universal_newlines' in kwargs:
@@ -100,6 +102,7 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs):
     if 'input' in kwargs:
         raise ValueError('input argument not allowed, it will be overridden.')
 
+    logger.debug('cmd="{}"'.format(' '.join(*popenargs)))
     ret = run(*popenargs, stdout=PIPE, stderr=PIPE, timeout=timeout,
               universal_newlines=True, ssh=ssh, **kwargs)
     ret.check_returncode()
diff --git a/pyznap/pyzfs.py b/pyznap/pyzfs.py
index 31d9c05..9cebb40 100644
--- a/pyznap/pyzfs.py
+++ b/pyznap/pyzfs.py
@@ -196,6 +196,7 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa
     # execute command with shell (sh or ssh)
     cmd = shell + [' '.join(cmd)]
 
+    logger.debug('cmd="{}"'.format(' '.join(cmd)))
     return sp.Popen(cmd, stdin=stdin, stderr=sp.PIPE) # zfs receive process
 
 

From 5951044f3f36f8234aa805d6e42cf2735958f4db Mon Sep 17 00:00:00 2001
From: ajay <84351018+ajayjay0@users.noreply.github.com>
Date: Thu, 13 May 2021 11:50:16 -0700
Subject: [PATCH 3/5] Add dry_run mode and update doc

---
 CHANGELOG.md              | 10 ++++++++++
 README.md                 | 24 ++++++++++++++++++++++--
 pyznap/clean.py           |  8 +++++---
 pyznap/config/pyznap.conf |  5 +++--
 pyznap/main.py            | 12 +++++++++++-
 pyznap/process.py         |  2 +-
 pyznap/pyzfs.py           | 16 ++++++++++++----
 pyznap/send.py            | 39 ++++++++++++++++++++++++---------------
 pyznap/take.py            |  8 ++++++--
 pyznap/utils.py           |  5 ++++-
 10 files changed, 98 insertions(+), 31 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0c5a19..b164097 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.6.0+local-b1] - 2021-05-15
+### Added
+- Allow in-line comments in configuration file
+- Added command line `dry-run` and config `dry_run` options
+Use the `dry-run` option to see what changes would be made, but not make them
+- Added option `prune_sanoid`
+Use the `prune_sanoid` option to control if pyznap prunes existing sanoid snapshots
+Note that the default has changed to NO
+
+
 ## [1.6.0] - 2020-09-22
 ### Added
 - Added resumable send/receive.
diff --git a/README.md b/README.md
index 2c00e22..4ae24e4 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,7 @@ Before you can use pyznap, you will need to create a config file. For initial se
     pyznap setup [-p PATH]
 
 This will create a directory `PATH` (default is `/etc/pyznap/`) and copy a sample config there. A
-config for your system might look like this (remove the comments):
+config for your system might look like this:
 
     [rpool/filesystem]
     frequent = 4                          # Keep 4 frequent snapshots
@@ -67,6 +67,8 @@ config for your system might look like this (remove the comments):
     yearly = 1                            # Keep 1 yearly snapshot
     snap = yes                            # Take snapshots on this filesystem
     clean = yes                           # Delete old snapshots on this filesystem
+    prune_sanoid = no                     # Don't delete old sanoid snapshots on this filesystem
+    dry_run = no                          # Run in normal mode, don't use dry_run
     dest = backup/filesystem              # Backup this filesystem on this location
     exclude = rpool/filesystem/data/*     # Exclude these datasets for pyznap send
 
@@ -76,7 +78,7 @@ Then set up a cronjob by creating a file under `/etc/cron.d/`
 
 and let pyznap run regularly by adding the following lines
 
-    SHELL=/bin/sh
+    SHELL=/bin/bash
     PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 
     */15 * * * *   root    /path/to/pyznap snap >> /var/log/pyznap.log 2>&1
@@ -84,12 +86,20 @@ and let pyznap run regularly by adding the following lines
 This will run pyznap every quarter hour to take and delete snapshots. 'frequent' snapshots can be
 taken up to once per minute, so adjust your cronjob accordingly.
 
+For installation in a python virtual environment (venv) you can replace the line above with:
+
+    */15 * * * *   root   /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1
+
 If you also want to send your filesystems to another location you can add a line
 
     0 0 * * *   root    /path/to/pyznap send >> /var/log/pyznap.log 2>&1
 
 This will backup your data once per day at 12am.
 
+Or for a venv:
+
+    0 0 * * *   root    /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1
+
 You can also manage, send to and pull from remote ssh locations. Always specify ssh locations with
 
     ssh:port:user@host:rpool/data
@@ -143,6 +153,8 @@ Here is a list of all options you can set in the config fie:
 | `yearly`           | Integer         | Number of yearly snapshots |
 | `snap`             | yes/no          | Should snapshots be taken |
 | `clean`            | yes/no          | Should snapshots be cleaned |
+| `prune_sanoid`     | yes/no          | Should sanoid snapshots be cleaned |
+| `dry_run`          | yes/no          | Display commands/changes, but don't update filesystem |
 | `dest`             | List of string  | Comma-separated list of destinations where to send source filesystem |
 | `dest_key`         | List of string  | Path to ssh keyfile for dest. Comma-separated list for multiple dest |
 | `compress`         | List of string  | Compression to use over ssh, supported are gzip, lzop, bzip2, pigz, xz & lz4. Default is lzop. Comma-separated list for multiple dest |
@@ -166,6 +178,10 @@ Run `pyznap -h` to see all available options.
 
   Print more verbose output.
 
++ -n, --dry-run
+
+  Use dry-run mode.
+
 + setup [-p PATH]
 
   Initial setup. Creates a config dir and puts a sample config file there. You can specify the path
@@ -233,6 +249,10 @@ Run `pyznap -h` to see all available options.
 
     `pyznap send -s tank/data -d backup/data`
 
++ Backup a single filesystem locally, display ZFS commands, and run in dry-run (dummy) mode:
+
+    `pyznap --verbose --dry-run send -s tank/data -d backup/data`
+
 + Send a single filesystem to a remote location, using `pigz` compression:
 
     `pyznap send -s tank/data -d ssh:20022:root@example.com:backup/data -i /root/.ssh/id_rsa -c pigz`
diff --git a/pyznap/clean.py b/pyznap/clean.py
index 7061869..302fa3e 100644
--- a/pyznap/clean.py
+++ b/pyznap/clean.py
@@ -27,10 +27,11 @@ def clean_snap(snap):
     """
 
     logger = logging.getLogger(__name__)
-
-    logger.info('Deleting snapshot {}...'.format(snap))
+    dry_run = snap.dry_run == True
+    dry_msg = '*** DRY RUN ***' if dry_run else ''
+    logger.info('Deleting snapshot {}... {}'.format(snap, dry_msg))
     try:
-        snap.destroy()
+      snap.destroy(dry_run=dry_run)
     except DatasetBusyError as err:
         logger.error(err)
     except CalledProcessError as err:
@@ -70,6 +71,7 @@ def clean_filesystem(filesystem, conf):
             continue
         try:
             snap_type = snap.name.split('_')[-1]
+            snap.dry_run = conf.get('dry_run', None)
             snapshots[snap_type].append(snap)
         except (ValueError, KeyError):
             continue
diff --git a/pyznap/config/pyznap.conf b/pyznap/config/pyznap.conf
index 22e0f4b..551bf3c 100644
--- a/pyznap/config/pyznap.conf
+++ b/pyznap/config/pyznap.conf
@@ -4,8 +4,7 @@
 ## filesystem. For remote syncronisation always keep enough snapshots on the destination. If there
 ## are no common snapshots the destination has to be destroyed and a full stream has to be sent.
 ## ssh locations are always specified with 'ssh:port:user@host:poolname/filesystem'.
-## Remove the comments at the end of the lines in your config, as they will not be ignored. Only
-## lines starting with '#' will be ignored.
+## Lines starting with '#' will be ignored.
 #
 #
 #
@@ -21,6 +20,8 @@
 # yearly = 1                            # Keep 1 yearly snapshot
 # snap = yes                            # Take snapshots on this filesystem
 # clean = yes                           # Delete old snapshots on this filesystem
+# prune_sanoid = no                     # Prune sanoid snapshots yes|no
+# dry_run = yes                         # Use dry-run mode? yes|no
 # dest = backup/filesystem              # Backup this filesystem on this location
 # exclude = rpool/filesystem/data/*     # Exclude these datasets for pyznap send
 #
diff --git a/pyznap/main.py b/pyznap/main.py
index 74945af..86c8725 100644
--- a/pyznap/main.py
+++ b/pyznap/main.py
@@ -34,6 +34,8 @@ def _main():
     """
 
     parser = ArgumentParser(prog='pyznap', description='ZFS snapshot tool written in python')
+    parser.add_argument('-n', '--dry-run', action="store_true",
+                        dest="dry_run", help="Dry-run, don't execute commands")
     parser.add_argument('-v', '--verbose', action="store_true",
                         dest="verbose", help='print more verbose output')
     parser.add_argument('--config', action="store",
@@ -99,6 +101,13 @@ def _main():
         if config == None:
             return 1
 
+    # Append global dry_run flag don't override existing dry-run = yes
+    try:
+      for conf in config: 
+        conf['dry_run'] = True if (args.dry_run or conf.get('dry_run', None)) else False
+    except UnboundLocalError:
+      pass
+
     if args.command == 'setup':
         path = args.path if args.path else CONFIG_DIR
         create_config(path)
@@ -143,7 +152,8 @@ def _main():
             send_config([{'name': args.source, 'dest': [args.dest], 'key': source_key,
                           'dest_keys': dest_key, 'compress': compress, 'exclude': exclude,
                           'raw_send': raw, 'resume': resume, 'dest_auto_create': dest_auto_create,
-                          'retries': retries, 'retry_interval': retry_interval}])
+                          'retries': retries, 'retry_interval': retry_interval,
+                          'dry_run': dry_run}])
 
         elif args.source and not args.dest:
             logger.error('Missing dest...')
diff --git a/pyznap/process.py b/pyznap/process.py
index 5b80ebf..54a3a63 100644
--- a/pyznap/process.py
+++ b/pyznap/process.py
@@ -102,7 +102,7 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs):
     if 'input' in kwargs:
         raise ValueError('input argument not allowed, it will be overridden.')
 
-    logger.debug('cmd="{}"'.format(' '.join(*popenargs)))
+    logger.debug("'{}'...".format(' '.join(*popenargs)))
     ret = run(*popenargs, stdout=PIPE, stderr=PIPE, timeout=timeout,
               universal_newlines=True, ssh=ssh, **kwargs)
     ret.check_returncode()
diff --git a/pyznap/pyzfs.py b/pyznap/pyzfs.py
index 9cebb40..99a61db 100644
--- a/pyznap/pyzfs.py
+++ b/pyznap/pyzfs.py
@@ -119,7 +119,7 @@ def roots(ssh=None):
     return find(ssh=ssh, max_depth=0)
 
 # note: force means create missing parent filesystems
-def create(name, ssh=None, type='filesystem', props={}, force=False):
+def create(name, ssh=None, type='filesystem', props={}, force=False, dry_run=False):
     cmd = ['zfs', 'create']
 
     if type == 'volume':
@@ -130,6 +130,9 @@ def create(name, ssh=None, type='filesystem', props={}, force=False):
     if force:
         cmd.append('-p')
 
+    if dry_run:
+        cmd.append('-n')
+
     for prop, value in props.items():
         cmd.append('-o')
         cmd.append(prop + '=' + str(value))
@@ -142,7 +145,7 @@ def create(name, ssh=None, type='filesystem', props={}, force=False):
 
 
 def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_path=False,
-            force=False, nomount=False, stream_size=0, raw=False, resume=False):
+            force=False, nomount=False, stream_size=0, raw=False, resume=False, dry_run=False):
     """Returns Popen instance for zfs receive"""
     logger = logging.getLogger(__name__)
 
@@ -181,6 +184,8 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa
         cmd.append('-u')
     if resume:
         cmd.append('-s')
+    if dry_run:
+        cmd.append('-n')
 
     cmd.append(quote(name)) # use shlex to quote the name
 
@@ -196,7 +201,7 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa
     # execute command with shell (sh or ssh)
     cmd = shell + [' '.join(cmd)]
 
-    logger.debug('cmd="{}"'.format(' '.join(cmd)))
+    logger.debug("'{}'...".format(' '.join(cmd)))
     return sp.Popen(cmd, stdin=stdin, stderr=sp.PIPE) # zfs receive process
 
 
@@ -233,7 +238,7 @@ def dependents(self):
 
     # TODO: split force to allow -f, -r and -R to be specified individually
     # TODO: remove or ignore defer option for non-snapshot datasets
-    def destroy(self, defer=False, force=False):
+    def destroy(self, defer=False, force=False, dry_run=False):
         cmd = ['zfs', 'destroy']
 
         cmd.append('-v')
@@ -245,6 +250,9 @@ def destroy(self, defer=False, force=False):
             cmd.append('-f')
             cmd.append('-R')
 
+        if dry_run:
+            cmd.append('-n')
+
         cmd.append(self.name)
 
         check_output(cmd, ssh=self.ssh)
diff --git a/pyznap/send.py b/pyznap/send.py
index 0e909e4..cceaf20 100644
--- a/pyznap/send.py
+++ b/pyznap/send.py
@@ -22,7 +22,7 @@
 from .process import DatasetBusyError, DatasetNotFoundError, DatasetExistsError
 
 
-def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=False, resume_token=None):
+def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=False, resume_token=None, dry_run=False):
     """Sends snapshot to destination, incrementally and over ssh if specified.
 
     Parameters:
@@ -35,6 +35,8 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F
         Base snapshot for incremental stream (the default is None, meaning a full stream)
     ssh_dest : {ssh.SSH}, optional
         Open ssh connection for remote backup (the default is None, meaning local backup)
+    dry_run : {boolean}, optional
+        Don't change filesystem
 
     Returns
     -------
@@ -51,7 +53,7 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F
 
         send = snapshot.send(ssh_dest=ssh_dest, base=base, intermediates=True, raw=raw, resume_token=resume_token)
         recv = zfs.receive(name=dest_name, stdin=send.stdout, ssh=ssh_dest, ssh_source=ssh_source,
-                           force=True, nomount=True, stream_size=stream_size, raw=raw, resume=resume)
+                           force=True, nomount=True, stream_size=stream_size, raw=raw, resume=resume, dry_run=dry_run)
         send.stdout.close()
 
         # write pv output to stderr / stdout
@@ -83,7 +85,7 @@ def send_snap(snapshot, dest_name, base=None, ssh_dest=None, raw=False, resume=F
         return 0
 
 
-def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False):
+def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False, dry_run=False):
     """Checks for common snapshots between source and dest.
     If none are found, send the oldest snapshot, then update with the most recent one.
     If there are common snaps, update destination with the most recent one.
@@ -96,6 +98,8 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False
         Name of the location to send to
     ssh_dest : {ssh.SSH}, optional
         Open ssh connection for remote backup (the default is None, meaning local backup)
+    dry_run : {boolean}, optional
+        Don't change filesystem
 
     Returns
     -------
@@ -106,7 +110,8 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False
     logger = logging.getLogger(__name__)
     dest_name_log = '{:s}@{:s}:{:s}'.format(ssh_dest.user, ssh_dest.host, dest_name) if ssh_dest else dest_name
 
-    logger.debug('Sending {} to {:s}...'.format(source_fs, dest_name_log))
+    dry_msg = '*** DRY RUN ***' if dry_run else ''
+    logger.debug('Sending {} to {:s}... {}'.format(source_fs, dest_name_log, dry_msg))
 
     resume_token = None
     # Check if dest already has a 'zfs receive' ongoing
@@ -176,7 +181,7 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False
     if resume_token is not None:
         logger.info('Found resume token. Resuming last transfer of {:s} (~{:s})...'
                     .format(dest_name_log, bytes_fmt(base.stream_size(raw=raw, resume_token=resume_token))))
-        rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=True, resume_token=resume_token)
+        rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=True, resume_token=resume_token, dry_run=dry_run)
         if rc:
             return rc
         # we need to update common snapshots after finishing the resumable send
@@ -191,7 +196,7 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False
         else:
             logger.info('No common snapshots on {:s}, sending oldest snapshot {} (~{:s})...'
                         .format(dest_name_log, base, bytes_fmt(base.stream_size(raw=raw))))
-            rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=resume)
+            rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run)
             if rc:
                 return rc
     else:
@@ -199,9 +204,9 @@ def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False
         base = next(filter(lambda x: x.name.split('@')[1] in common, snapshots), None)
 
     if base.name != snapshot.name:
-        logger.info('Updating {:s} with recent snapshot {} (~{:s})...'
-                    .format(dest_name_log, snapshot, bytes_fmt(snapshot.stream_size(base, raw=raw))))
-        rc = send_snap(snapshot, dest_name, base=base, ssh_dest=ssh_dest, raw=raw, resume=resume)
+        logger.info('Updating {:s} with recent snapshot {} (~{:s})... {}'
+                    .format(dest_name_log, snapshot, bytes_fmt(snapshot.stream_size(base, raw=raw)), dry_msg))
+        rc = send_snap(snapshot, dest_name, base=base, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run)
         if rc:
             return rc
 
@@ -225,7 +230,9 @@ def send_config(config):
     for conf in config:
         if not conf.get('dest', None):
             continue
-
+        
+        dry_run = conf.get('dry_run', None)
+        dry_msg = '*** DRY RUN ***' if dry_run else ''
         backup_source = conf['name']
         try:
             _type, source_name, user, host, port = parse_name(backup_source)
@@ -301,8 +308,8 @@ def send_config(config):
                 zfs.open(dest_name, ssh=ssh_dest)
             except DatasetNotFoundError:
                 if dest_auto_create:
-                    logger.info('Destination {:s} does not exist, will create it...'.format(dest_name_log))
-                    if create_dataset(dest_name, dest_name_log, ssh=ssh_dest):
+                    logger.info('Destination {:s} does not exist, will create it... {}'.format(dest_name_log, dry_msg))
+                    if create_dataset(dest_name, dest_name_log, ssh=ssh_dest, dry_run=dry_run):
                         continue
                 else:
                     logger.error('Destination {:s} does not exist, manually create it or use "dest-auto-create" option...'
@@ -327,7 +334,7 @@ def send_config(config):
                     continue
                 # send not excluded filesystems
                 for retry in range(1,retries+2):
-                    rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume)
+                    rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run)
                     if rc == 2 and retry <= retries:
                         logger.info('Retrying send in {:d}s (retry {:d} of {:d})...'.format(retry_interval, retry, retries))
                         sleep(retry_interval)
@@ -341,7 +348,7 @@ def send_config(config):
             ssh_source.close()
 
 
-def create_dataset(name, name_log, ssh=None):
+def create_dataset(name, name_log, ssh=None, dry_run=False):
     """Creates a dataset and logs success/fail
 
     Parameters
@@ -352,6 +359,8 @@ def create_dataset(name, name_log, ssh=None):
         Name used for logging
     ssh : {SSH}, optional
         Open ssh connection, by default None
+    dry_run : {boolean}, optional
+        Dry run, don't change ZFS pool
 
     Returns
     -------
@@ -360,7 +369,7 @@ def create_dataset(name, name_log, ssh=None):
     """
     logger = logging.getLogger(__name__)
     try:
-        zfs.create(name, ssh=ssh, force=True)
+        zfs.create(name, ssh=ssh, force=True, dry_run=dry_run)
     except CalledProcessError as err:
         message = err.stderr.rstrip()
         if message == "filesystem successfully created, but it may only be mounted by root":
diff --git a/pyznap/take.py b/pyznap/take.py
index 4ac1a5f..1ebadc4 100644
--- a/pyznap/take.py
+++ b/pyznap/take.py
@@ -33,9 +33,12 @@ def take_snap(filesystem, _type):
 
     snapname = lambda _type: 'pyznap_{:s}_{:s}'.format(now().strftime('%Y-%m-%d_%H:%M:%S'), _type)
 
-    logger.info('Taking snapshot {}@{:s}...'.format(filesystem, snapname(_type)))
+    dry_run = filesystem.dry_run == True
+    dry_msg = '*** DRY RUN ***' if dry_run else ''
+    logger.info('Taking snapshot {}@{:s}... {}'.format(filesystem, snapname(_type), dry_msg))
     try:
-        filesystem.snapshot(snapname=snapname(_type), recursive=True)
+        if not dry_run:
+          filesystem.snapshot(snapname=snapname(_type), recursive=True)
     except (DatasetBusyError, DatasetExistsError) as err:
         logger.error(err)
     except CalledProcessError as err:
@@ -62,6 +65,7 @@ def take_filesystem(filesystem, conf):
     logger.debug('Taking snapshots on {}...'.format(filesystem))
     now = datetime.now
 
+    filesystem.dry_run = conf.get('dry_run', None)
     snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
     # catch exception if dataset was destroyed since pyznap was started
     try:
diff --git a/pyznap/utils.py b/pyznap/utils.py
index 116f23a..c37701c 100644
--- a/pyznap/utils.py
+++ b/pyznap/utils.py
@@ -86,7 +86,8 @@ def read_config(path):
     config = []
     options = ['key', 'frequent', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'snap', 'clean',
                'dest', 'dest_keys', 'compress', 'exclude', 'raw_send', 'resume', 'dest_auto_create',
-               'retries', 'retry_interval']
+               'retries', 'retry_interval', 
+               'dry_run']
 
     for section in parser.sections():
         dic = {}
@@ -118,6 +119,8 @@ def read_config(path):
                                    for i in value.split(',')]
                 elif option in ['retries', 'retry_interval']:
                     dic[option] = [int(i) for i in value.split(',')]
+                elif option in ['dry_run']:
+                    dic[option] = {'yes': True, 'no': False}.get(value.lower(), None)
     # Pass through values recursively
     for parent in config:
         for child in config:

From e6d8cc48a4122e26228f63ffd8fe518015a412b5 Mon Sep 17 00:00:00 2001
From: ajay <84351018+ajayjay0@users.noreply.github.com>
Date: Thu, 13 May 2021 14:28:31 -0700
Subject: [PATCH 4/5] Add prune_sanoid option

---
 pyznap/clean.py | 7 ++++---
 pyznap/take.py  | 7 ++++---
 pyznap/utils.py | 5 ++++-
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/pyznap/clean.py b/pyznap/clean.py
index 302fa3e..e96cf5a 100644
--- a/pyznap/clean.py
+++ b/pyznap/clean.py
@@ -55,7 +55,8 @@ def clean_filesystem(filesystem, conf):
     """
 
     logger = logging.getLogger(__name__)
-    logger.debug('Cleaning snapshots on {}...'.format(filesystem))
+    prunes = 'pyznap' if (conf.get('prune_sanoid', None) == False) else ('autosnap', 'pyznap')
+    logger.debug("Cleaning snapshots on {}... prunes={}".format(filesystem, prunes ) )
 
     snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
     # catch exception if dataset was destroyed since pyznap was started
@@ -66,8 +67,8 @@ def clean_filesystem(filesystem, conf):
         return 1
     # categorize snapshots
     for snap in fs_snapshots:
-        # Ignore snapshots not taken with pyznap or sanoid
-        if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')):
+        # Ignore snapshots not taken with pyznap or sanoid, depending on configuration
+        if not snap.name.split('@')[1].startswith(prunes):
             continue
         try:
             snap_type = snap.name.split('_')[-1]
diff --git a/pyznap/take.py b/pyznap/take.py
index 1ebadc4..7514c6e 100644
--- a/pyznap/take.py
+++ b/pyznap/take.py
@@ -62,7 +62,8 @@ def take_filesystem(filesystem, conf):
     """
 
     logger = logging.getLogger(__name__)
-    logger.debug('Taking snapshots on {}...'.format(filesystem))
+    prunes = 'pyznap' if (conf.get('prune_sanoid', None) == False) else ('autosnap', 'pyznap')
+    logger.debug("Taking snapshots on {}... prunes={}".format(filesystem, prunes ) )
     now = datetime.now
 
     filesystem.dry_run = conf.get('dry_run', None)
@@ -75,8 +76,8 @@ def take_filesystem(filesystem, conf):
         return 1
     # categorize snapshots
     for snap in fs_snapshots:
-        # Ignore snapshots not taken with pyznap or sanoid
-        if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')):
+        # Ignore snapshots not taken with pyznap or sanoid, depending on configuration
+        if not snap.name.split('@')[1].startswith(prunes):
             continue
         try:
             _date, _time, snap_type = snap.name.split('_')[-3:]
diff --git a/pyznap/utils.py b/pyznap/utils.py
index c37701c..26ec8e3 100644
--- a/pyznap/utils.py
+++ b/pyznap/utils.py
@@ -87,7 +87,7 @@ def read_config(path):
     options = ['key', 'frequent', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'snap', 'clean',
                'dest', 'dest_keys', 'compress', 'exclude', 'raw_send', 'resume', 'dest_auto_create',
                'retries', 'retry_interval', 
-               'dry_run']
+               'dry_run', 'prune_sanoid']
 
     for section in parser.sections():
         dic = {}
@@ -121,6 +121,9 @@ def read_config(path):
                     dic[option] = [int(i) for i in value.split(',')]
                 elif option in ['dry_run']:
                     dic[option] = {'yes': True, 'no': False}.get(value.lower(), None)
+                elif option in ['prune_sanoid']:
+                    dic[option] = {'yes': True, 'no': False}.get(value.lower(), None)
+
     # Pass through values recursively
     for parent in config:
         for child in config:

From 6c5d53cb889121d62718b69f45355d0585b6e510 Mon Sep 17 00:00:00 2001
From: ajay <84351018+ajayjay0@users.noreply.github.com>
Date: Sun, 11 Jul 2021 16:02:32 -0700
Subject: [PATCH 5/5] Add functions via user properties

---
 .gitignore                           |  3 +++
 CHANGELOG.md                         | 10 +++++++
 README.md                            | 19 ++++++++++++++
 pyznap/__init__.py                   |  2 +-
 pyznap/config/etc/cron.d/pyznap      |  9 +++++++
 pyznap/config/etc/logrotate.d/pyznap |  7 +++++
 pyznap/config/{ => etc}/pyznap.conf  |  0
 pyznap/main.py                       |  8 ++++--
 pyznap/send.py                       | 28 ++++++++++++++++++++
 pyznap/utils.py                      | 39 +++++++++++++++++++++++++++-
 10 files changed, 121 insertions(+), 4 deletions(-)
 create mode 100644 pyznap/config/etc/cron.d/pyznap
 create mode 100644 pyznap/config/etc/logrotate.d/pyznap
 rename pyznap/config/{ => etc}/pyznap.conf (100%)

diff --git a/.gitignore b/.gitignore
index 4a53c2e..80cf9e5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,6 @@ __pycache__/
 /build/
 /*.egg-info
 /*.egg
+env/
+venv/
+pyznap.local.conf
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b164097..ad80607 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.6.0+local-b2] - 2021-07-10
+### Added
+- Finely grained processing via ZFS user properties in the "pyznap:" domain
+  + pyznap:exclude=[true|false]
+    use pyznap:exclude to skip ZFS *send* processing of tagged datasets
+  + pyznap:max_size=[size <units>] units=B|KB|MB|TB|PB
+    use pyznap:max_size=500M to skip processing of tagged datasets
+
+
+Note that the default has changed to NO
 ## [1.6.0+local-b1] - 2021-05-15
 ### Added
 - Allow in-line comments in configuration file
diff --git a/README.md b/README.md
index 4ae24e4..3219ada 100644
--- a/README.md
+++ b/README.md
@@ -227,6 +227,25 @@ Run `pyznap -h` to see all available options.
     but you can set the `--dest-auto-create` flag to automatically create it.
 
 
+#### ZFS User Properties ####
+pyznap now supports finely grained settings via ZFS user properties in the "pyznap:" domain. The following properties are in use:
+
++ pyznap:exclude [true|false]
+  Setting this property to true will caused the dateset to be excluded during *send*. Any other value, including the default, is assume to be "false"
+    
+    + Example: Set the dataset to be excluded by pyznap send
+  
+      `ZFS set pyznap:exclude=true tank/some_dataset`
+
++ pyznap:max_size [size]
+  Sets the maximum dataset size to be processed during this run. The following suffixes are supported:
+    
+    `B, KB, MB, GB, TB, PB`
+    
+    + Example: Set the maximum dataset processing size to 60GB
+      
+      `ZFS set pyznap:max_size=60G tank/some_other_dataset`
+
 #### Usage examples ####
 
 + Take snapshots according to policy in default config file:
diff --git a/pyznap/__init__.py b/pyznap/__init__.py
index 56fa5c2..b2507b1 100644
--- a/pyznap/__init__.py
+++ b/pyznap/__init__.py
@@ -9,4 +9,4 @@
 """
 
 
-__version__ = '1.6.0'
+__version__ = 'v1.6.0+local-b2'
diff --git a/pyznap/config/etc/cron.d/pyznap b/pyznap/config/etc/cron.d/pyznap
new file mode 100644
index 0000000..5ffbc42
--- /dev/null
+++ b/pyznap/config/etc/cron.d/pyznap
@@ -0,0 +1,9 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+# Take pyznap snapshots every 15 minutes
+*/15 * * * *  root  /root/pyznap/bin/python /root/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1
+
+# Copy to pyznap backup at midnight
+0 0  * * *    root  /root/pyznap/bin/python /root/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1
+
diff --git a/pyznap/config/etc/logrotate.d/pyznap b/pyznap/config/etc/logrotate.d/pyznap
new file mode 100644
index 0000000..7b3d08f
--- /dev/null
+++ b/pyznap/config/etc/logrotate.d/pyznap
@@ -0,0 +1,7 @@
+/var/log/pyznap.log {
+	daily
+	rotate 14
+	compress
+	missingok
+	notifempty
+}
diff --git a/pyznap/config/pyznap.conf b/pyznap/config/etc/pyznap.conf
similarity index 100%
rename from pyznap/config/pyznap.conf
rename to pyznap/config/etc/pyznap.conf
diff --git a/pyznap/main.py b/pyznap/main.py
index 86c8725..eabfad1 100644
--- a/pyznap/main.py
+++ b/pyznap/main.py
@@ -19,6 +19,7 @@
 from .clean import clean_config
 from .take import take_config
 from .send import send_config
+from pyznap import __version__
 
 
 DIRNAME = os.path.dirname(os.path.abspath(__file__))
@@ -33,7 +34,8 @@ def _main():
         Exit code
     """
 
-    parser = ArgumentParser(prog='pyznap', description='ZFS snapshot tool written in python')
+    parser = ArgumentParser(prog='pyznap', 
+                            description='ZFS snapshot tool written in python {}'.format(__version__))
     parser.add_argument('-n', '--dry-run', action="store_true",
                         dest="dry_run", help="Dry-run, don't execute commands")
     parser.add_argument('-v', '--verbose', action="store_true",
@@ -82,6 +84,8 @@ def _main():
     parser_send.add_argument('--retry-interval', action="store", type=int,
                              dest='retry_interval', default=10,
                              help='interval in seconds between retries. default is 10')
+    
+    parser.epilog = "ZFS properties: [pyznap:exclude, pyznap:max_size]"
 
     if len(sys.argv)==1:
         parser.print_help(sys.stderr)
@@ -93,7 +97,7 @@ def _main():
                         datefmt='%b %d %H:%M:%S', stream=sys.stdout)
     logger = logging.getLogger(__name__)
 
-    logger.info('Starting pyznap...')
+    logger.info('Starting pyznap {}...'.format(__version__))
 
     if args.command in ('snap', 'send'):
         config_path = args.config if args.config else os.path.join(CONFIG_DIR, 'pyznap.conf')
diff --git a/pyznap/send.py b/pyznap/send.py
index cceaf20..da55f3b 100644
--- a/pyznap/send.py
+++ b/pyznap/send.py
@@ -18,6 +18,7 @@
 from time import sleep
 from .ssh import SSH, SSHException
 from .utils import parse_name, exists, check_recv, bytes_fmt
+from .utils import parse_size
 import pyznap.pyzfs as zfs
 from .process import DatasetBusyError, DatasetNotFoundError, DatasetExistsError
 
@@ -332,6 +333,33 @@ def send_config(config):
                 if any(fnmatch(source_fs.name, pattern) for pattern in exclude):
                     logger.debug('Matched {} in exclude rules, not sending...'.format(source_fs))
                     continue
+                
+                # Check for ZFS user property to bypass filesystem
+                fs_props = source_fs.getprops()
+                
+                exclude_prop='pyznap:exclude'
+                ignore_me = fs_props.get(exclude_prop, ('false', 'false'))[0].lower()
+                logger.debug("Property {}={} for {}".format(exclude_prop, ignore_me, source_fs))
+                if ignore_me == 'true':
+                    logger.info('Matched {}={} for {}, not sending...'
+                      .format(exclude_prop, ignore_me, source_fs))
+                    continue
+                
+                # Check for max size
+                used_prop='used'
+                fs_used_bytes = int(fs_props.get(used_prop, ('0', '0'))[0])   # Bytes
+                fs_used_fmt   = bytes_fmt(fs_used_bytes) # MB
+                logger.debug("Property {}={} ({}) for {}".format(used_prop, fs_used_fmt, fs_used_bytes, source_fs))
+                
+                max_prop='pyznap:max_size'
+                fs_max_fmt   = fs_props.get(max_prop, ('0', '0'))[0] # String
+                fs_max_bytes = parse_size(fs_max_fmt)   # Bytes  
+                logger.debug("Property {}={} ({}) for {}".format(max_prop, fs_max_fmt, fs_max_bytes, source_fs))
+                if fs_max_bytes > 0 and fs_used_bytes > fs_max_bytes:
+                    logger.info('Filesystem size {} exceeds {}={} for {}, not sending...'
+                      .format(fs_used_fmt, max_prop, fs_max_fmt, source_fs))
+                    continue
+
                 # send not excluded filesystems
                 for retry in range(1,retries+2):
                     rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run)
diff --git a/pyznap/utils.py b/pyznap/utils.py
index 26ec8e3..e1508f6 100644
--- a/pyznap/utils.py
+++ b/pyznap/utils.py
@@ -177,7 +177,7 @@ def create_config(path):
     logger = logging.getLogger(__name__)
 
     CONFIG_FILE = os.path.join(path, 'pyznap.conf')
-    config = resource_string(__name__, 'config/pyznap.conf').decode("utf-8")
+    config = resource_string(__name__, 'config/etc/pyznap.conf').decode("utf-8")
 
     logger.info('Initial setup...')
 
@@ -272,3 +272,40 @@ def bytes_fmt(num):
         num /= 1024
     else:
         return "{:3.1f}{:s}".format(num, 'Y')
+
+
+# based on https://stackoverflow.com/a/42865957/2002471
+def parse_size(size):
+    """Converts human readable format to bytes
+
+    Parameters
+    ----------
+    str : size
+        formatted size
+
+    Returns
+    -------
+    float
+        actual bytes
+    """
+
+    units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40, "PB": 2**50}
+    
+    logger = logging.getLogger(__name__)
+    
+    size = str(size).upper()
+    #print("parsing size ", size)
+    if size[-1].strip() != "B":
+      size = size + "B"
+    if not re.match(r' ', size):
+        size = re.sub(r'([KMGT]?B)', r' \1', size)
+    
+    result = 0
+    try:
+      number, unit = [string.strip() for string in size.split()]
+      result = int(float(number)*units[unit])
+    except:
+      logger.error("Could not convert {} to a size in bytes, using default={}".format(size, result))
+    
+    return result
+