Update to use Pyrax

.gitignore

@@ -1 +1,134 @@
 *.pyc
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+myvenv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VSCode
+.vscode/

README.md

@@ -0,0 +1,33 @@
+# multi-cloud-mirror
+A Github copy of Joe Masters Emison's multi-cloud-mirror script, updated to
+use Pyrax, Rackspace's official SDK for Python.
+
+Forked from https://github.com/boxuk/multi-cloud-mirror.
+See https://www.boxuk.com/insight/synchronising-assets-between-rackspace-and-s3/
+for an article on the original work.
+
+Also see commit (https://github.com/boxuk/multi-cloud-mirror/pull/1/commits/139d6310758067e3dcc991e92c52ff751c7a4c15)
+for the README from the original Google Code repo.
+
+The code has been updated to use `pyrax` instead of `python-cloudfiles` to support
+Cloud Files regions in Rackspace. While some commit remarks may
+suggest support for Python 3.x, this is not the case, as Pyrax requires
+Python 2.7.x (https://developer.rackspace.com/sdks/python/).
+
+To copy files in a region other than your Rackspace account's default Cloud Files region,
+set the `region` attribute in the `/etc/cloudfiles.cfg` config file. Example to copy files
+to/from the `DFW` region:
+
+```
+[Credentials]
+username=your_username
+api_key=your_api_key
+region=DFW
+```
+
+Other than the Pyrax migration, The original work has been preserved as much as possible.
+There is room for future improvement, such as:
+
+- Support for objects greater than 5 GB.
+- Network bandwith (and cost) optimization by skipping retrieval of object attributes, if possible.
+- Support for Python 3 (one possible approach: replacing Pyrax with our own library using REST API calls - example: https://gist.github.com/bahostetterlewis/c84a08cd1f8dad74853c)

multi_cloud_mirror.py

@@ -7,6 +7,8 @@
 Licensed under the Mozilla Public License (MPL) v1.1
 Full license terms available at http://www.mozilla.org/MPL/MPL-1.1.html
 
+Updated to use Pyrax by Juey C. Ong.
+
 multi_cloud_mirror provides an easy, multi-processing way of synchronizing
 a bucket at Amazon S3 to a container at Rackspace Cloud Files, or vice
 versa.
@@ -17,21 +19,18 @@
 ### Imports
 #######################################################################
 import boto
-import cloudfiles
+import pyrax
 import smtplib
 import os
 import sys
 import time
 import datetime
 import argparse
-import ConfigParser
+import configparser
 import multiprocessing
 from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError
-from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl,
-                               ContainerNotPublic, AuthenticationFailed, AuthenticationError,
-                               NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue,
-                               InvalidObjectSize, IncompleteSend)
-from ConfigParser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError
+from pyrax.exceptions import (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject, InvalidSize)
+from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError
 from email.mime.text import MIMEText
 from subprocess import Popen, PIPE
 
@@ -48,15 +47,18 @@ def connectToClouds():
       ## boto reads from /etc/boto.cfg (or ~/boto.cfg)
       s3Conn = boto.connect_s3()
       ## the cloud files library doesn't automatically read from a file, so we handle that here:
-      cfConfig = ConfigParser.ConfigParser()
+      cfConfig = configparser.ConfigParser()
       cfConfig.read('/etc/cloudfiles.cfg')
-      cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key'))
-   except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError), err:
+      pyrax.set_setting("identity_type", "rackspace")
+      pyrax.set_default_region(cfConfig.get('Credentials','region'))
+      pyrax.set_credentials(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key'))
+      cfConn = pyrax.connect_to_cloudfiles(cfConfig.get('Credentials','region'))
+   except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err:
       raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err))
-   except (S3ResponseError, S3PermissionsError), err:
+   except (S3ResponseError, S3PermissionsError) as err:
       raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason))
-   except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError), err:
-      raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err))
+   except (ClientException, AuthenticationFailed) as err:
+      raise MultiCloudMirrorException("Error in connecting to CF: %s" % str(err))
    return (s3Conn, cfConn)
 
 
@@ -72,7 +74,7 @@ def copyToS3(srcBucketName, myKeyName, destBucketName,tmpDir):
    # note that maximum file size (as of this writing) for Cloud Files is 5GB, and we expect 6+GB free on the drive
    (s3Conn, cfConn) = connectToClouds()
    tmpFile = str(tmpDir + '/' + myKeyName)
-   cfConn.get_container(srcBucketName).get_object(myKeyName).save_to_filename(tmpFile)
+   pyrax.cloudfiles.download_object(cfConn.get_container(srcBucketName), myKeyName, tmpDir)
    destBucket = s3Conn.get_bucket(destBucketName)
    newObj = None
    try:
@@ -95,12 +97,9 @@ def copyToCF(srcBucketName, myKeyName, destBucketName):
    destBucket = cfConn.get_container(destBucketName)
    #with S3, we must request the key singly to get its metadata:
    fullKey = srcBucket.get_key(myKeyName)
-   #initialize new object at Cloud Files
-   newObj = destBucket.create_object(myKeyName)
-   newObj.content_type = fullKey.content_type or "application/octet-stream"
-   newObj.size = fullKey.size
-   #stream the file from S3 to Cloud Files
-   newObj.send(fullKey)
+   # create new Cloud Files object and stream the file from S3 to Cloud Files
+   content_type = fullKey.content_type or fullKey.DefaultContentType
+   newObj = destBucket.create(obj_name=myKeyName, data=fullKey.get_contents_as_string(), content_type=content_type)
 
 #######################################################################
 ### Delete functions
@@ -169,7 +168,9 @@ def logItem(self, msg, level):
       Log function for MultiCloudMirror class: email and printing to screen
       """
       if level >= 0:
-         if self.debug: print msg
+         if self.debug: 
+            print (msg)
+            sys.stdout.flush()
          if level >= 1:
             self.emailMsg += msg + "\n"
 
@@ -275,21 +276,37 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN
       """
       Check to see if this file should be copied, and, if so, queue it
       """
-      myKeyName = getattr(sKey, 'key', sKey.name)
-      self.filesAtSource[myKeyName] = sKey.etag.replace('"','')
+      try:
+         myKeyName = getattr(sKey, 'key', sKey.name)
+         self.filesAtSource[myKeyName] = sKey.etag.replace('"','')
+      except ClientException as err:
+         self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, str(err)), self.LOG_DEBUG)
+         return
+      except Exception as err:
+         raise Exception ("Error in getting key attribute: %s" % str(err))
+         return
 
       # skip S3 "folders", since Cloud Files doesn't support them, and skip files that are too large
       self.logItem("Found %s at source" % (myKeyName), self.LOG_DEBUG)
       if myKeyName[-1] == '/':
          self.logItem("Skipping %s because it is a 'folder'" % (myKeyName), self.LOG_DEBUG)
          return
+      if srcService == "cf":
+         cfBucketName = srcBucketName
+         s3BucketName = destBucketName
+         sKey.size = sKey.total_bytes
+      else:
+         s3BucketName = srcBucketName
+         cfBucketName = destBucketName
       if (sKey.size > self.maxFileSize):
          self.logItem("Skipping %s because it is too large (%d bytes)" % (myKeyName, sKey.size), self.LOG_WARN)
          return
       # Copy if MD5 (etag) values are different, or if file does not exist at destination
       doCopy = False;
       try:
-         if self.filesAtDestination[myKeyName] != sKey.etag.replace('"',''):
+         self.logItem("Source md5: %s" % self.filesAtSource[myKeyName], self.LOG_DEBUG)
+         self.logItem("Dest   md5: %s" % self.filesAtDestination[myKeyName], self.LOG_DEBUG)
+         if self.filesAtDestination[myKeyName] != self.filesAtSource[myKeyName]:
             # the file is at the destination, but the md5sums do not match, so overwrite
             doCopy = True
             self.logItem("...Found at destination, but md5sums did not match, so it will be copied", self.LOG_DEBUG)
@@ -304,7 +321,7 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN
             job = self.pool.apply_async(copyToCF, (srcBucketName, myKeyName, destBucketName))
          elif srcService == "cf":
             job = self.pool.apply_async(copyToS3, (srcBucketName, myKeyName, destBucketName, self.tmpDir))
-         job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService)
+         job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService, cfBucketName=cfBucketName, s3BucketName=s3BucketName)
          self.jobs.append(job_dict)
          self.copyCount = self.copyCount + 1
       else:
@@ -331,9 +348,8 @@ def waitForJobstoFinish(self):
                      except (S3ResponseError, S3PermissionsError, S3CopyError) as err:
                         self.logItem("Error in %s %s to/from S3 bucket %s: [%d] %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['s3BucketName'], err.status, err.reason), self.LOG_WARN)
                         self.jobs.remove(job_dict)
-                     except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError,
-                             NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend), err:
-                        self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], err), self.LOG_WARN)
+                     except (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject) as err:
+                        self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], str(err)), self.LOG_WARN)
                         self.jobs.remove(job_dict)
                      except MultiCloudMirrorException as err:
                         self.logItem("MultiCloudMirror error in %s %s: %s" % (job_dict['task'], job_dict['myKeyName'], str(err)), self.LOG_WARN)
@@ -393,11 +409,11 @@ def run(self):
          # Connect to the proper buckets and retrieve file lists
          try:
             self.connectToBuckets(srcService, srcBucketName, destBucketName)
-         except (S3ResponseError, S3PermissionsError), err:
+         except (S3ResponseError, S3PermissionsError) as err:
             self.logItem("Error in connecting to S3 bucket: [%d] %s" % (err.status, err.reason), self.LOG_WARN)
             continue
-         except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError), err:
-            self.logItem("Error in connecting to CF container: %s" % (err), self.LOG_WARN)
+         except (ClientException, NoSuchContainer, AuthenticationFailed) as err:
+            self.logItem("Error in connecting to CF container: %s" % str(err), self.LOG_WARN)
             continue
          # Iterate through files at the source to see which ones to copy, and put them on the multiprocessing queue:
          for sKey in self.srcList:
@@ -427,7 +443,7 @@ def run(self):
       parser = argparse.ArgumentParser(description='Multi-Cloud Mirror Script')
       parser.add_argument('--process', dest='numProcesses',type=int, default=4,
                           help='number of simultaneous file upload threads to run')
-      parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=5368709120,
+      parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=pyrax.object_storage.MAX_FILE_SIZE,
                           help='maximium file size to sync, in bytes (files larger than this size will be skipped)')
       parser.add_argument('--from', help='email address from which to send the status email; must be specified to receive message', dest='emailSrc')
       parser.add_argument('--to', dest='emailDest',
@@ -447,4 +463,4 @@ def run(self):
       mcm = MultiCloudMirror(args.sync, args.numProcesses, args.maxFileSize, args.emailDest, args.emailSrc, args.emailSubj, args.tmpDir, args.debug, args.sendmail, args.delete, args.maxFileDeletion, args.minFileSync)
       mcm.run()
    except MultiCloudMirrorException as err:
-      print "Error from MultiCloudMirror: %s" % (str(err))
+      print ("Error from MultiCloudMirror: %s" % (str(err)))

requirements.txt

@@ -1,2 +1,3 @@
-boto==2.6.0
-python-cloudfiles==1.7.10
+boto==2.49.0
+configparser==4.0.2
+pyrax==1.9.8