diff --git a/.gitignore b/.gitignore index 0d20b64..fd14c18 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,134 @@ *.pyc +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +myvenv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..90198f6 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# multi-cloud-mirror +A Github copy of Joe Masters Emison's multi-cloud-mirror script, updated to +use Pyrax, Rackspace's official SDK for Python. + +Forked from https://github.com/boxuk/multi-cloud-mirror. +See https://www.boxuk.com/insight/synchronising-assets-between-rackspace-and-s3/ +for an article on the original work. + +Also see commit (https://github.com/boxuk/multi-cloud-mirror/pull/1/commits/139d6310758067e3dcc991e92c52ff751c7a4c15) +for the README from the original Google Code repo. + +The code has been updated to use `pyrax` instead of `python-cloudfiles` to support +Cloud Files regions in Rackspace. While some commit remarks may +suggest support for Python 3.x, this is not the case, as Pyrax requires +Python 2.7.x (https://developer.rackspace.com/sdks/python/). + +To copy files in a region other than your Rackspace account's default Cloud Files region, +set the `region` attribute in the `/etc/cloudfiles.cfg` config file. Example to copy files +to/from the `DFW` region: + +``` +[Credentials] +username=your_username +api_key=your_api_key +region=DFW +``` + +Other than the Pyrax migration, The original work has been preserved as much as possible. +There is room for future improvement, such as: + +- Support for objects greater than 5 GB. +- Network bandwith (and cost) optimization by skipping retrieval of object attributes, if possible. +- Support for Python 3 (one possible approach: replacing Pyrax with our own library using REST API calls - example: https://gist.github.com/bahostetterlewis/c84a08cd1f8dad74853c) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index d29740d..d7ddb2a 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -7,6 +7,8 @@ Licensed under the Mozilla Public License (MPL) v1.1 Full license terms available at http://www.mozilla.org/MPL/MPL-1.1.html +Updated to use Pyrax by Juey C. Ong. + multi_cloud_mirror provides an easy, multi-processing way of synchronizing a bucket at Amazon S3 to a container at Rackspace Cloud Files, or vice versa. @@ -17,21 +19,18 @@ ### Imports ####################################################################### import boto -import cloudfiles +import pyrax import smtplib import os import sys import time import datetime import argparse -import ConfigParser +import configparser import multiprocessing from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError -from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, - ContainerNotPublic, AuthenticationFailed, AuthenticationError, - NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, - InvalidObjectSize, IncompleteSend) -from ConfigParser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError +from pyrax.exceptions import (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject, InvalidSize) +from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError from email.mime.text import MIMEText from subprocess import Popen, PIPE @@ -48,15 +47,18 @@ def connectToClouds(): ## boto reads from /etc/boto.cfg (or ~/boto.cfg) s3Conn = boto.connect_s3() ## the cloud files library doesn't automatically read from a file, so we handle that here: - cfConfig = ConfigParser.ConfigParser() + cfConfig = configparser.ConfigParser() cfConfig.read('/etc/cloudfiles.cfg') - cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) - except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError), err: + pyrax.set_setting("identity_type", "rackspace") + pyrax.set_default_region(cfConfig.get('Credentials','region')) + pyrax.set_credentials(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) + cfConn = pyrax.connect_to_cloudfiles(cfConfig.get('Credentials','region')) + except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) - except (S3ResponseError, S3PermissionsError), err: + except (S3ResponseError, S3PermissionsError) as err: raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) - except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError), err: - raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) + except (ClientException, AuthenticationFailed) as err: + raise MultiCloudMirrorException("Error in connecting to CF: %s" % str(err)) return (s3Conn, cfConn) @@ -72,7 +74,7 @@ def copyToS3(srcBucketName, myKeyName, destBucketName,tmpDir): # note that maximum file size (as of this writing) for Cloud Files is 5GB, and we expect 6+GB free on the drive (s3Conn, cfConn) = connectToClouds() tmpFile = str(tmpDir + '/' + myKeyName) - cfConn.get_container(srcBucketName).get_object(myKeyName).save_to_filename(tmpFile) + pyrax.cloudfiles.download_object(cfConn.get_container(srcBucketName), myKeyName, tmpDir) destBucket = s3Conn.get_bucket(destBucketName) newObj = None try: @@ -95,12 +97,9 @@ def copyToCF(srcBucketName, myKeyName, destBucketName): destBucket = cfConn.get_container(destBucketName) #with S3, we must request the key singly to get its metadata: fullKey = srcBucket.get_key(myKeyName) - #initialize new object at Cloud Files - newObj = destBucket.create_object(myKeyName) - newObj.content_type = fullKey.content_type or "application/octet-stream" - newObj.size = fullKey.size - #stream the file from S3 to Cloud Files - newObj.send(fullKey) + # create new Cloud Files object and stream the file from S3 to Cloud Files + content_type = fullKey.content_type or fullKey.DefaultContentType + newObj = destBucket.create(obj_name=myKeyName, data=fullKey.get_contents_as_string(), content_type=content_type) ####################################################################### ### Delete functions @@ -169,7 +168,9 @@ def logItem(self, msg, level): Log function for MultiCloudMirror class: email and printing to screen """ if level >= 0: - if self.debug: print msg + if self.debug: + print (msg) + sys.stdout.flush() if level >= 1: self.emailMsg += msg + "\n" @@ -275,21 +276,37 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN """ Check to see if this file should be copied, and, if so, queue it """ - myKeyName = getattr(sKey, 'key', sKey.name) - self.filesAtSource[myKeyName] = sKey.etag.replace('"','') + try: + myKeyName = getattr(sKey, 'key', sKey.name) + self.filesAtSource[myKeyName] = sKey.etag.replace('"','') + except ClientException as err: + self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, str(err)), self.LOG_DEBUG) + return + except Exception as err: + raise Exception ("Error in getting key attribute: %s" % str(err)) + return # skip S3 "folders", since Cloud Files doesn't support them, and skip files that are too large self.logItem("Found %s at source" % (myKeyName), self.LOG_DEBUG) if myKeyName[-1] == '/': self.logItem("Skipping %s because it is a 'folder'" % (myKeyName), self.LOG_DEBUG) return + if srcService == "cf": + cfBucketName = srcBucketName + s3BucketName = destBucketName + sKey.size = sKey.total_bytes + else: + s3BucketName = srcBucketName + cfBucketName = destBucketName if (sKey.size > self.maxFileSize): self.logItem("Skipping %s because it is too large (%d bytes)" % (myKeyName, sKey.size), self.LOG_WARN) return # Copy if MD5 (etag) values are different, or if file does not exist at destination doCopy = False; try: - if self.filesAtDestination[myKeyName] != sKey.etag.replace('"',''): + self.logItem("Source md5: %s" % self.filesAtSource[myKeyName], self.LOG_DEBUG) + self.logItem("Dest md5: %s" % self.filesAtDestination[myKeyName], self.LOG_DEBUG) + if self.filesAtDestination[myKeyName] != self.filesAtSource[myKeyName]: # the file is at the destination, but the md5sums do not match, so overwrite doCopy = True self.logItem("...Found at destination, but md5sums did not match, so it will be copied", self.LOG_DEBUG) @@ -304,7 +321,7 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN job = self.pool.apply_async(copyToCF, (srcBucketName, myKeyName, destBucketName)) elif srcService == "cf": job = self.pool.apply_async(copyToS3, (srcBucketName, myKeyName, destBucketName, self.tmpDir)) - job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService) + job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService, cfBucketName=cfBucketName, s3BucketName=s3BucketName) self.jobs.append(job_dict) self.copyCount = self.copyCount + 1 else: @@ -331,9 +348,8 @@ def waitForJobstoFinish(self): except (S3ResponseError, S3PermissionsError, S3CopyError) as err: self.logItem("Error in %s %s to/from S3 bucket %s: [%d] %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['s3BucketName'], err.status, err.reason), self.LOG_WARN) self.jobs.remove(job_dict) - except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, - NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend), err: - self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], err), self.LOG_WARN) + except (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject) as err: + self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], str(err)), self.LOG_WARN) self.jobs.remove(job_dict) except MultiCloudMirrorException as err: self.logItem("MultiCloudMirror error in %s %s: %s" % (job_dict['task'], job_dict['myKeyName'], str(err)), self.LOG_WARN) @@ -393,11 +409,11 @@ def run(self): # Connect to the proper buckets and retrieve file lists try: self.connectToBuckets(srcService, srcBucketName, destBucketName) - except (S3ResponseError, S3PermissionsError), err: + except (S3ResponseError, S3PermissionsError) as err: self.logItem("Error in connecting to S3 bucket: [%d] %s" % (err.status, err.reason), self.LOG_WARN) continue - except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError), err: - self.logItem("Error in connecting to CF container: %s" % (err), self.LOG_WARN) + except (ClientException, NoSuchContainer, AuthenticationFailed) as err: + self.logItem("Error in connecting to CF container: %s" % str(err), self.LOG_WARN) continue # Iterate through files at the source to see which ones to copy, and put them on the multiprocessing queue: for sKey in self.srcList: @@ -427,7 +443,7 @@ def run(self): parser = argparse.ArgumentParser(description='Multi-Cloud Mirror Script') parser.add_argument('--process', dest='numProcesses',type=int, default=4, help='number of simultaneous file upload threads to run') - parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=5368709120, + parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=pyrax.object_storage.MAX_FILE_SIZE, help='maximium file size to sync, in bytes (files larger than this size will be skipped)') parser.add_argument('--from', help='email address from which to send the status email; must be specified to receive message', dest='emailSrc') parser.add_argument('--to', dest='emailDest', @@ -447,4 +463,4 @@ def run(self): mcm = MultiCloudMirror(args.sync, args.numProcesses, args.maxFileSize, args.emailDest, args.emailSrc, args.emailSubj, args.tmpDir, args.debug, args.sendmail, args.delete, args.maxFileDeletion, args.minFileSync) mcm.run() except MultiCloudMirrorException as err: - print "Error from MultiCloudMirror: %s" % (str(err)) + print ("Error from MultiCloudMirror: %s" % (str(err))) diff --git a/requirements.txt b/requirements.txt index c4cf63f..f59c4b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -boto==2.6.0 -python-cloudfiles==1.7.10 +boto==2.49.0 +configparser==4.0.2 +pyrax==1.9.8 \ No newline at end of file