From 680917f86c87dd0f510bbc76a04991034b188569 Mon Sep 17 00:00:00 2001 From: Juey Ong Date: Wed, 3 Jun 2020 17:01:17 -0400 Subject: [PATCH 01/18] Create README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..4f85a64 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# multi-cloud-mirror +A Github copy of Joe Masters Emison's multi-cloud-mirror script. + +See https://www.boxuk.com/insight/synchronising-assets-between-rackspace-and-s3/ +for an article on the original work. From 174c301b7158e1a86b5cf61f1098c47026d506fb Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Fri, 5 Jun 2020 15:29:00 -0400 Subject: [PATCH 02/18] Add GitHub's Python boilerplate and more. --- .gitignore | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/.gitignore b/.gitignore index 0d20b64..fd14c18 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,134 @@ *.pyc +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +myvenv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode +.vscode/ \ No newline at end of file From b64c37f12fb20149a59d8333ed6c3ddf5a33ecad Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Fri, 5 Jun 2020 15:30:33 -0400 Subject: [PATCH 03/18] Update to run under Python 3.6 --- multi_cloud_mirror.py | 22 +++++++++++----------- requirements.txt | 5 +++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index d29740d..2e89a0a 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -24,14 +24,14 @@ import time import datetime import argparse -import ConfigParser +import configparser import multiprocessing from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) -from ConfigParser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError +from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError from email.mime.text import MIMEText from subprocess import Popen, PIPE @@ -48,14 +48,14 @@ def connectToClouds(): ## boto reads from /etc/boto.cfg (or ~/boto.cfg) s3Conn = boto.connect_s3() ## the cloud files library doesn't automatically read from a file, so we handle that here: - cfConfig = ConfigParser.ConfigParser() + cfConfig = configparser.ConfigParser() cfConfig.read('/etc/cloudfiles.cfg') cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) - except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError), err: + except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) - except (S3ResponseError, S3PermissionsError), err: + except (S3ResponseError, S3PermissionsError) as err: raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) - except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError), err: + except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError) as err: raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) return (s3Conn, cfConn) @@ -169,7 +169,7 @@ def logItem(self, msg, level): Log function for MultiCloudMirror class: email and printing to screen """ if level >= 0: - if self.debug: print msg + if self.debug: print (msg) if level >= 1: self.emailMsg += msg + "\n" @@ -332,7 +332,7 @@ def waitForJobstoFinish(self): self.logItem("Error in %s %s to/from S3 bucket %s: [%d] %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['s3BucketName'], err.status, err.reason), self.LOG_WARN) self.jobs.remove(job_dict) except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, - NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend), err: + NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) as err: self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], err), self.LOG_WARN) self.jobs.remove(job_dict) except MultiCloudMirrorException as err: @@ -393,10 +393,10 @@ def run(self): # Connect to the proper buckets and retrieve file lists try: self.connectToBuckets(srcService, srcBucketName, destBucketName) - except (S3ResponseError, S3PermissionsError), err: + except (S3ResponseError, S3PermissionsError) as err: self.logItem("Error in connecting to S3 bucket: [%d] %s" % (err.status, err.reason), self.LOG_WARN) continue - except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError), err: + except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError) as err: self.logItem("Error in connecting to CF container: %s" % (err), self.LOG_WARN) continue # Iterate through files at the source to see which ones to copy, and put them on the multiprocessing queue: @@ -447,4 +447,4 @@ def run(self): mcm = MultiCloudMirror(args.sync, args.numProcesses, args.maxFileSize, args.emailDest, args.emailSrc, args.emailSubj, args.tmpDir, args.debug, args.sendmail, args.delete, args.maxFileDeletion, args.minFileSync) mcm.run() except MultiCloudMirrorException as err: - print "Error from MultiCloudMirror: %s" % (str(err)) + print ("Error from MultiCloudMirror: %s" % (str(err))) diff --git a/requirements.txt b/requirements.txt index c4cf63f..25878bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -boto==2.6.0 -python-cloudfiles==1.7.10 +boto==2.49.0 +python-cloudfiles==1.7.11 +configparser==5.0.0 \ No newline at end of file From 8fd0efa7edfa818487fbf64e678e4802fa6685ea Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Fri, 12 Jun 2020 17:24:43 -0400 Subject: [PATCH 04/18] downgrade configparser to 4.0.2 for Python 2.7. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25878bd..eaf70de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ boto==2.49.0 python-cloudfiles==1.7.11 -configparser==5.0.0 \ No newline at end of file +configparser==4.0.2 \ No newline at end of file From 8eb47b74387964ed735ba2c5fa9d0c61e6af0a25 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Sun, 21 Jun 2020 00:29:03 -0400 Subject: [PATCH 05/18] Convert to use pyrax instead of cloudfiles. --- multi_cloud_mirror.py | 58 ++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 2e89a0a..a39280e 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -17,7 +17,8 @@ ### Imports ####################################################################### import boto -import cloudfiles +# import cloudfiles +import pyrax import smtplib import os import sys @@ -27,10 +28,10 @@ import configparser import multiprocessing from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError -from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, - ContainerNotPublic, AuthenticationFailed, AuthenticationError, - NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, - InvalidObjectSize, IncompleteSend) +# from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, +# ContainerNotPublic, AuthenticationFailed, AuthenticationError, +# NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, +# InvalidObjectSize, IncompleteSend) from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError from email.mime.text import MIMEText from subprocess import Popen, PIPE @@ -50,13 +51,20 @@ def connectToClouds(): ## the cloud files library doesn't automatically read from a file, so we handle that here: cfConfig = configparser.ConfigParser() cfConfig.read('/etc/cloudfiles.cfg') - cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) - except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: - raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) - except (S3ResponseError, S3PermissionsError) as err: - raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) - except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError) as err: - raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) + pyrax.set_setting("identity_type", "rackspace") + pyrax.set_default_region(cfConfig.get('Credentials','region')) + pyrax.set_credentials(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) + cfConn = pyrax.connect_to_cloudfiles(cfConfig.get('Credentials','region')) + # cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) + # except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: + # raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) + # except (S3ResponseError, S3PermissionsError) as err: + # raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) + # except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError) as err: + # raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) + except Exception as err: + raise Exception ("Error in connecting to CF: %s" % (err)) + return (s3Conn, cfConn) @@ -72,7 +80,7 @@ def copyToS3(srcBucketName, myKeyName, destBucketName,tmpDir): # note that maximum file size (as of this writing) for Cloud Files is 5GB, and we expect 6+GB free on the drive (s3Conn, cfConn) = connectToClouds() tmpFile = str(tmpDir + '/' + myKeyName) - cfConn.get_container(srcBucketName).get_object(myKeyName).save_to_filename(tmpFile) + pyrax.cloudfiles.download_object(cfConn.get_container(srcBucketName), myKeyName, tmpDir) destBucket = s3Conn.get_bucket(destBucketName) newObj = None try: @@ -96,11 +104,13 @@ def copyToCF(srcBucketName, myKeyName, destBucketName): #with S3, we must request the key singly to get its metadata: fullKey = srcBucket.get_key(myKeyName) #initialize new object at Cloud Files - newObj = destBucket.create_object(myKeyName) - newObj.content_type = fullKey.content_type or "application/octet-stream" - newObj.size = fullKey.size + # newObj = destBucket.create_object(myKeyName) + # newObj.content_type = fullKey.content_type or "application/octet-stream" + # newObj.size = fullKey.size #stream the file from S3 to Cloud Files - newObj.send(fullKey) + # newObj.send(fullKey) + content_type = fullKey.content_type or fullKey.DefaultContentType + newObj = destBucket.create(obj_name=myKeyName, data=fullKey.get_contents_as_string(), content_type=content_type) ####################################################################### ### Delete functions @@ -283,6 +293,13 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN if myKeyName[-1] == '/': self.logItem("Skipping %s because it is a 'folder'" % (myKeyName), self.LOG_DEBUG) return + if srcService == "cf": + cfBucketName = srcBucketName + s3BucketName = destBucketName + sKey.size = sKey.total_bytes + else: + s3BucketName = srcBucketName + cfBucketName = destBucketName if (sKey.size > self.maxFileSize): self.logItem("Skipping %s because it is too large (%d bytes)" % (myKeyName, sKey.size), self.LOG_WARN) return @@ -304,7 +321,7 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN job = self.pool.apply_async(copyToCF, (srcBucketName, myKeyName, destBucketName)) elif srcService == "cf": job = self.pool.apply_async(copyToS3, (srcBucketName, myKeyName, destBucketName, self.tmpDir)) - job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService) + job_dict = dict(job=job, task="copy", myKeyName=myKeyName, srcService=srcService, srcBucketName=srcBucketName, destBucketName=destBucketName, destService=destService, cfBucketName=cfBucketName, s3BucketName=s3BucketName) self.jobs.append(job_dict) self.copyCount = self.copyCount + 1 else: @@ -331,8 +348,9 @@ def waitForJobstoFinish(self): except (S3ResponseError, S3PermissionsError, S3CopyError) as err: self.logItem("Error in %s %s to/from S3 bucket %s: [%d] %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['s3BucketName'], err.status, err.reason), self.LOG_WARN) self.jobs.remove(job_dict) - except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, - NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) as err: + # except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, + # NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) as err: + except Exception as err: self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], err), self.LOG_WARN) self.jobs.remove(job_dict) except MultiCloudMirrorException as err: From 411b641ae44b1a790ac88af7e4f4ad0bcfc79eb5 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Mon, 22 Jun 2020 11:13:19 -0400 Subject: [PATCH 06/18] Replace cloudfiles with pyrax. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index eaf70de..f59c4b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ boto==2.49.0 -python-cloudfiles==1.7.11 -configparser==4.0.2 \ No newline at end of file +configparser==4.0.2 +pyrax==1.9.8 \ No newline at end of file From 94c47625e14be9f8183245d8d0fda32d56909b40 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Tue, 23 Jun 2020 11:54:29 -0400 Subject: [PATCH 07/18] Flush messages for easier debugging. --- multi_cloud_mirror.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index a39280e..8903fe3 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -179,7 +179,9 @@ def logItem(self, msg, level): Log function for MultiCloudMirror class: email and printing to screen """ if level >= 0: - if self.debug: print (msg) + if self.debug: + print (msg) + sys.stdout.flush() if level >= 1: self.emailMsg += msg + "\n" From 67d00676c70876278c560994a629ac594b3b90f4 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Tue, 23 Jun 2020 16:42:19 -0400 Subject: [PATCH 08/18] Handle pyrax ClientException --- multi_cloud_mirror.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 8903fe3..1e98e85 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -32,6 +32,7 @@ # ContainerNotPublic, AuthenticationFailed, AuthenticationError, # NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, # InvalidObjectSize, IncompleteSend) +from pyrax.exceptions import ClientException from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError from email.mime.text import MIMEText from subprocess import Popen, PIPE @@ -287,8 +288,12 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN """ Check to see if this file should be copied, and, if so, queue it """ - myKeyName = getattr(sKey, 'key', sKey.name) - self.filesAtSource[myKeyName] = sKey.etag.replace('"','') + try: + myKeyName = getattr(sKey, 'key', sKey.name) + self.filesAtSource[myKeyName] = sKey.etag.replace('"','') + except ClientException as err: + self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, err), self.LOG_DEBUG) + return # skip S3 "folders", since Cloud Files doesn't support them, and skip files that are too large self.logItem("Found %s at source" % (myKeyName), self.LOG_DEBUG) From 9789426a923dc55db7d912d0c1075eeb00edd306 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Tue, 23 Jun 2020 16:44:49 -0400 Subject: [PATCH 09/18] Get maxFileSize from pyrax MAX_FILE_SIZE --- multi_cloud_mirror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 1e98e85..7a73500 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -452,7 +452,7 @@ def run(self): parser = argparse.ArgumentParser(description='Multi-Cloud Mirror Script') parser.add_argument('--process', dest='numProcesses',type=int, default=4, help='number of simultaneous file upload threads to run') - parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=5368709120, + parser.add_argument('--maxsize', dest='maxFileSize',type=int, default=pyrax.object_storage.MAX_FILE_SIZE, help='maximium file size to sync, in bytes (files larger than this size will be skipped)') parser.add_argument('--from', help='email address from which to send the status email; must be specified to receive message', dest='emailSrc') parser.add_argument('--to', dest='emailDest', From e7d6844409433b03edc8195b3847c6d3c2ae6975 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Thu, 25 Jun 2020 13:05:48 -0400 Subject: [PATCH 10/18] Catch other exceptions. --- multi_cloud_mirror.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 7a73500..c4f54f4 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -294,6 +294,9 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN except ClientException as err: self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, err), self.LOG_DEBUG) return + except Exception as err: + raise Exception ("Error in getting key attribute: %s" % str(err)) + return # skip S3 "folders", since Cloud Files doesn't support them, and skip files that are too large self.logItem("Found %s at source" % (myKeyName), self.LOG_DEBUG) From 4c61ed1b4af3e54f5191524dd7b8629de627ed5e Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Thu, 25 Jun 2020 13:08:56 -0400 Subject: [PATCH 11/18] Stringify exceptions in logging items. --- multi_cloud_mirror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index c4f54f4..c4950e8 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -64,7 +64,7 @@ def connectToClouds(): # except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError) as err: # raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) except Exception as err: - raise Exception ("Error in connecting to CF: %s" % (err)) + raise Exception ("Error in connecting to CF: %s" % str(err)) return (s3Conn, cfConn) @@ -292,7 +292,7 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN myKeyName = getattr(sKey, 'key', sKey.name) self.filesAtSource[myKeyName] = sKey.etag.replace('"','') except ClientException as err: - self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, err), self.LOG_DEBUG) + self.logItem("Skipping %s because Pyrax ClientException: %s" % (myKeyName, str(err)), self.LOG_DEBUG) return except Exception as err: raise Exception ("Error in getting key attribute: %s" % str(err)) @@ -361,7 +361,7 @@ def waitForJobstoFinish(self): # except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, # NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) as err: except Exception as err: - self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], err), self.LOG_WARN) + self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], str(err)), self.LOG_WARN) self.jobs.remove(job_dict) except MultiCloudMirrorException as err: self.logItem("MultiCloudMirror error in %s %s: %s" % (job_dict['task'], job_dict['myKeyName'], str(err)), self.LOG_WARN) From 2ad5afbd458fe15797966c332d3d5184dd1eaf21 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Tue, 10 Nov 2020 16:09:38 -0500 Subject: [PATCH 12/18] Stringify exception in logging. --- multi_cloud_mirror.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index c4950e8..97b036a 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -316,7 +316,9 @@ def checkAndCopy(self, sKey, srcService, srcBucketName, destService, destBucketN # Copy if MD5 (etag) values are different, or if file does not exist at destination doCopy = False; try: - if self.filesAtDestination[myKeyName] != sKey.etag.replace('"',''): + self.logItem("Source md5: %s" % self.filesAtSource[myKeyName], self.LOG_DEBUG) + self.logItem("Dest md5: %s" % self.filesAtDestination[myKeyName], self.LOG_DEBUG) + if self.filesAtDestination[myKeyName] != self.filesAtSource[myKeyName]: # the file is at the destination, but the md5sums do not match, so overwrite doCopy = True self.logItem("...Found at destination, but md5sums did not match, so it will be copied", self.LOG_DEBUG) @@ -425,7 +427,7 @@ def run(self): self.logItem("Error in connecting to S3 bucket: [%d] %s" % (err.status, err.reason), self.LOG_WARN) continue except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError) as err: - self.logItem("Error in connecting to CF container: %s" % (err), self.LOG_WARN) + self.logItem("Error in connecting to CF container: %s" % str(err), self.LOG_WARN) continue # Iterate through files at the source to see which ones to copy, and put them on the multiprocessing queue: for sKey in self.srcList: From a8bf1f2ba3fc0489578b3a918d22dd53723b5960 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Wed, 11 Nov 2020 14:17:55 -0500 Subject: [PATCH 13/18] Clean up exception handling Migrate Cloudfiles exception handling to handle Pyrax exceptions. --- multi_cloud_mirror.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 97b036a..f3fc735 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -17,7 +17,6 @@ ### Imports ####################################################################### import boto -# import cloudfiles import pyrax import smtplib import os @@ -28,11 +27,7 @@ import configparser import multiprocessing from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError -# from cloudfiles.errors import (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, -# ContainerNotPublic, AuthenticationFailed, AuthenticationError, -# NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, -# InvalidObjectSize, IncompleteSend) -from pyrax.exceptions import ClientException +from pyrax.exceptions import (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject, InvalidSize) from configparser import NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError from email.mime.text import MIMEText from subprocess import Popen, PIPE @@ -56,15 +51,12 @@ def connectToClouds(): pyrax.set_default_region(cfConfig.get('Credentials','region')) pyrax.set_credentials(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) cfConn = pyrax.connect_to_cloudfiles(cfConfig.get('Credentials','region')) - # cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) - # except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: - # raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) - # except (S3ResponseError, S3PermissionsError) as err: - # raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) - # except (ResponseError, InvalidUrl, AuthenticationFailed, AuthenticationError) as err: - # raise MultiCloudMirrorException("Error in connecting to CF: %s" % (err)) - except Exception as err: - raise Exception ("Error in connecting to CF: %s" % str(err)) + except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError) as err: + raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err)) + except (S3ResponseError, S3PermissionsError) as err: + raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) + except (ClientException, AuthenticationFailed) as err: + raise MultiCloudMirrorException("Error in connecting to CF: %s" % str(err)) return (s3Conn, cfConn) @@ -426,7 +418,7 @@ def run(self): except (S3ResponseError, S3PermissionsError) as err: self.logItem("Error in connecting to S3 bucket: [%d] %s" % (err.status, err.reason), self.LOG_WARN) continue - except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError) as err: + except (ClientException, NoSuchContainer, AuthenticationFailed) as err: self.logItem("Error in connecting to CF container: %s" % str(err), self.LOG_WARN) continue # Iterate through files at the source to see which ones to copy, and put them on the multiprocessing queue: From 11e91d719cafb69b908a55acf45b9d2b6ebe9db2 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Wed, 11 Nov 2020 14:30:04 -0500 Subject: [PATCH 14/18] Continue clean up exception handling --- multi_cloud_mirror.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index f3fc735..cfffc57 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -352,9 +352,7 @@ def waitForJobstoFinish(self): except (S3ResponseError, S3PermissionsError, S3CopyError) as err: self.logItem("Error in %s %s to/from S3 bucket %s: [%d] %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['s3BucketName'], err.status, err.reason), self.LOG_WARN) self.jobs.remove(job_dict) - # except (ResponseError, NoSuchContainer, InvalidContainerName, InvalidUrl, ContainerNotPublic, AuthenticationFailed, AuthenticationError, - # NoSuchObject, InvalidObjectName, InvalidMetaName, InvalidMetaValue, InvalidObjectSize, IncompleteSend) as err: - except Exception as err: + except (ClientException, NoSuchContainer, AuthenticationFailed, NoSuchObject) as err: self.logItem("Error in %s %s to/from to CF container %s: %s" % (job_dict['task'], job_dict['myKeyName'], job_dict['cfBucketName'], str(err)), self.LOG_WARN) self.jobs.remove(job_dict) except MultiCloudMirrorException as err: From 2f463a40703a3eb831f7afac43a6fcfb2cab065f Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Mon, 16 Nov 2020 14:19:51 -0500 Subject: [PATCH 15/18] Remove whitespace. --- multi_cloud_mirror.py | 1 - 1 file changed, 1 deletion(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index cfffc57..93d4400 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -57,7 +57,6 @@ def connectToClouds(): raise MultiCloudMirrorException("Error in connecting to S3: [%d] %s" % (err.status, err.reason)) except (ClientException, AuthenticationFailed) as err: raise MultiCloudMirrorException("Error in connecting to CF: %s" % str(err)) - return (s3Conn, cfConn) From 0a384cb90fcf77d78002c5f19e45642026323cd0 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Mon, 16 Nov 2020 14:26:25 -0500 Subject: [PATCH 16/18] Comment cleanup Refine comments and remove commented-out code. --- multi_cloud_mirror.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 93d4400..5d7f0bb 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -95,12 +95,7 @@ def copyToCF(srcBucketName, myKeyName, destBucketName): destBucket = cfConn.get_container(destBucketName) #with S3, we must request the key singly to get its metadata: fullKey = srcBucket.get_key(myKeyName) - #initialize new object at Cloud Files - # newObj = destBucket.create_object(myKeyName) - # newObj.content_type = fullKey.content_type or "application/octet-stream" - # newObj.size = fullKey.size - #stream the file from S3 to Cloud Files - # newObj.send(fullKey) + # create new Cloud Files object and stream the file from S3 to Cloud Files content_type = fullKey.content_type or fullKey.DefaultContentType newObj = destBucket.create(obj_name=myKeyName, data=fullKey.get_contents_as_string(), content_type=content_type) From 12c38d554f668841491d1ceed099212e5d42a585 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Wed, 18 Nov 2020 12:28:48 -0500 Subject: [PATCH 17/18] Fleshed out README. --- README.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f85a64..90198f6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,33 @@ # multi-cloud-mirror -A Github copy of Joe Masters Emison's multi-cloud-mirror script. +A Github copy of Joe Masters Emison's multi-cloud-mirror script, updated to +use Pyrax, Rackspace's official SDK for Python. +Forked from https://github.com/boxuk/multi-cloud-mirror. See https://www.boxuk.com/insight/synchronising-assets-between-rackspace-and-s3/ for an article on the original work. + +Also see commit (https://github.com/boxuk/multi-cloud-mirror/pull/1/commits/139d6310758067e3dcc991e92c52ff751c7a4c15) +for the README from the original Google Code repo. + +The code has been updated to use `pyrax` instead of `python-cloudfiles` to support +Cloud Files regions in Rackspace. While some commit remarks may +suggest support for Python 3.x, this is not the case, as Pyrax requires +Python 2.7.x (https://developer.rackspace.com/sdks/python/). + +To copy files in a region other than your Rackspace account's default Cloud Files region, +set the `region` attribute in the `/etc/cloudfiles.cfg` config file. Example to copy files +to/from the `DFW` region: + +``` +[Credentials] +username=your_username +api_key=your_api_key +region=DFW +``` + +Other than the Pyrax migration, The original work has been preserved as much as possible. +There is room for future improvement, such as: + +- Support for objects greater than 5 GB. +- Network bandwith (and cost) optimization by skipping retrieval of object attributes, if possible. +- Support for Python 3 (one possible approach: replacing Pyrax with our own library using REST API calls - example: https://gist.github.com/bahostetterlewis/c84a08cd1f8dad74853c) From 3af251e7256aae6b846c722a5c33cc6e12872a90 Mon Sep 17 00:00:00 2001 From: Juey Chong Ong Date: Wed, 18 Nov 2020 12:30:27 -0500 Subject: [PATCH 18/18] Updated header. --- multi_cloud_mirror.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multi_cloud_mirror.py b/multi_cloud_mirror.py index 5d7f0bb..d7ddb2a 100755 --- a/multi_cloud_mirror.py +++ b/multi_cloud_mirror.py @@ -7,6 +7,8 @@ Licensed under the Mozilla Public License (MPL) v1.1 Full license terms available at http://www.mozilla.org/MPL/MPL-1.1.html +Updated to use Pyrax by Juey C. Ong. + multi_cloud_mirror provides an easy, multi-processing way of synchronizing a bucket at Amazon S3 to a container at Rackspace Cloud Files, or vice versa.