From 378f8f3d4a25ada01746916cf2d5b0f3fb5a2868 Mon Sep 17 00:00:00 2001 From: roulance Date: Mon, 24 Nov 2014 15:31:22 +0100 Subject: [PATCH 1/5] #2 - add parmeters model in init --- emailvision/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emailvision/__init__.py b/emailvision/__init__.py index ff9774c..0f827bc 100644 --- a/emailvision/__init__.py +++ b/emailvision/__init__.py @@ -1,4 +1,5 @@ from member_api import MemberAPI +from member_api import EMVAPIMergeUploadParams from utils import BadResponse -__all__ = ['MemeberAPI', 'BadResponse'] +__all__ = ['MemeberAPI', 'BadResponse', 'EMVAPIMergeUploadParams'] From 314d88e72fa7e8484f8b2711784e00fb37cec03b Mon Sep 17 00:00:00 2001 From: roulance Date: Mon, 24 Nov 2014 15:32:55 +0100 Subject: [PATCH 2/5] #2 - add data mass upload n parameters class --- emailvision/member_api.py | 227 +++++++++++++++++++++++++++++++++----- 1 file changed, 197 insertions(+), 30 deletions(-) diff --git a/emailvision/member_api.py b/emailvision/member_api.py index 7a859ba..22a108e 100644 --- a/emailvision/member_api.py +++ b/emailvision/member_api.py @@ -1,44 +1,54 @@ # -*- coding: utf-8 -*- -from django.utils.encoding import smart_unicode +import codecs +import requests +import base64 from utils import Client, NotConnected, FailedApiCall, Error class MemberAPI (object): + response_debug = None + def __init__(self, login, password, key, server_name): self.login = login self.password = password self.key = key self.client = Client(server_name) self.token = None - self.connected = False def get(self, action, *values): if self.token is None: raise NotConnected() - return self.client.get(action, - self.token, - *values) + return self.client.get(action, *values) def post(self, data): path = '/apimember/services/MemberService?wsdl' return self.client.post(path, data) + def put(self, url, xml_data=None, csv_data=None): + if self.token is None: + raise NotConnected() + return self.client.put(url, xml_data, csv_data) + def __enter__(self): self.__connect() return self def __exit__(self, exc_type, exc_val, exc_tb): # FIXME: manage exceptions - if self.connected: + if self.token: self.__disconnect() def __connect(self): - response = self.client.get('/apimember/services/rest/connect/open/', - self.login, - self.password, - self.key) + url = self.get_member_url( + 'connect/open/{0}/{1}/{2}'.format( + self.login, + self.password, + self.key + ) + ) + response = self.client.get(url) self.token = response.find('result').text def __disconnect(self): @@ -47,6 +57,18 @@ def __disconnect(self): self.client.get('/apimember/services/rest/connect/close/', self.token) self.token = None + def get_batch_url(self, remote_method): + output = "/apibatchmember/services/rest/{0}".format( + remote_method + ) + return output + + def get_member_url(self, remote_method): + output = "/apimember/services/rest/{0}".format( + remote_method + ) + return output + def insert_member_by_email(self, email): """returns a job id""" response = self.get('/apimember/services/rest/member/insert/', email) @@ -85,7 +107,10 @@ def insert_or_update_member_by_object(self, email, values): def update_member_by_email(self, email, key, value): """returns a job id""" if not value: - raise FailedApiCall(Error('NO_VALUE_SET', 'No value provided'), self.client.server_name + '/apimember/services/MemberService?wsdl') + raise FailedApiCall( + Error('NO_VALUE_SET', 'No value provided'), + self.client.server_name + '/apimember/services/MemberService?wsdl' + ) data = u""" @@ -108,7 +133,13 @@ def update_member_by_email(self, email, key, value): def retrieve_insert_member_job_status(self, job_id): # This does not work, it is always answering a 404. - response = self.get('/apimember/services/rest/member/getInsertMemberStatus/', str(job_id)) + url = self.get_member_url( + 'member/getInsertMemberStatus/{0}/{1}'.format( + self.token, + job_id + ) + ) + response = self.get(url) return response def __build_member_attributes(self, attributes_node): @@ -121,23 +152,31 @@ def __build_member_attributes(self, attributes_node): attributes[key] = value return attributes - def iter_over_members_by_email(self, email): - response = self.get('/apimember/services/rest/member/getMemberByEmail/', email) + def iter_over_members_by_email(self, user_email): + url = self.get_member_url( + 'member/getMemberByEmail/{0}/{1}'.format( + self.token, + user_email + ) + ) + response = self.get(url) for member in response.find('members').iterfind('member'): yield self.__build_member_attributes(member.find('attributes')) - def retrieve_member_by_id(self, id): - response = self.get('/apimember/services/rest/member/getMemberById/', str(id)) + def retrieve_member_by_id(self, user_id): + url = self.get_member_url( + 'member/getMemberById/{0}/{1}'.format( + self.token, + user_id + ) + ) + response = self.get(url) attributes_nodes = response.find('apiMember').find('attributes') return self.__build_member_attributes(attributes_nodes) - def iter_over_members_by_page(self, page): - response = self.get('/apimember/services/rest/getListMembersByPage/', str(page)) - for member in response.find('result').iterfind('list'): - yield self.__build_member_attributes(member.find('attributes')) - def member_table_column_names(self): - response = self.get('/apimember/services/rest/member/descMemberTable/') + url = self.get_member_url('member/descMemberTable/') + response = self.get(url) entries = response.find('memberTable').find('fields').iterfind('entry') columns = dict() for entry in entries: @@ -146,23 +185,151 @@ def member_table_column_names(self): columns[key] = value return columns - def unsubscribe_members_by_email(self, email): + def unsubscribe_members_by_email(self, user_email): """returns a job id""" - response = self.get('/apimember/services/rest/member/unjoinByEmail/', email) + url = self.get_member_url( + 'member/unjoinByEmail/{0}/{1}'.format( + self.token, + user_email + ) + ) + response = self.get(url) return response.result - def unsubscribe_members_by_id(self, id): + def unsubscribe_members_by_id(self, user_id): """returns a job id""" - response = self.get('/apimember/services/rest/member/unjoinByMemberId/', str(id)) + url = self.get_member_url( + 'member/unjoinByMemberId/{0}/{1}'.format( + self.token, + user_id + ) + ) + response = self.get(url) return response.result - def resubscribe_members_by_email(self, email): + def resubscribe_members_by_email(self, user_email): """returns a job id""" - response = self.get('/apimember/services/rest/member/rejoinByEmail/', str(id)) + url = self.get_member_url( + 'member/rejoinByEmail/{0}/{1}'.format( + self.token, + user_email + ) + ) + response = self.get(url) return response.result - def resubscribe_members_by_id(self, id): + def resubscribe_members_by_id(self, user_id): """returns a job id""" - response = self.get('/apimember/services/rest/member/rejoinByMemberId/', str(id)) + url = self.get_member_url( + 'member/rejoinByMemberId/{0}/{1}'.format( + self.token, + user_id + ) + ) + response = self.client.get(url) return response.result + def assemble_upstream_body(self, parameters): + data = u""" + +{0} +{1} +{2} +{3} +{4} +""".format( + parameters.criteria, + parameters.path.split('/')[-1], + parameters.separator, + parameters.file_encoding, + parameters.skip_first_line, + ) + + for param in parameters.mapping: + data += u""" +{0} +{1} +{2} +""".format(param['colNum'], param['fieldName'], param['toReplace']) + data += u""" +""" + return data + + def upload_file_merge(self, parameters): + if not self.token: + return "You should try to connect first!" + url = self.get_batch_url( + 'batchmemberservice/{0}/batchmember/mergeUpload'.format(self.token) + ) + + data = self.assemble_upstream_body(parameters) + + response = self.put(url, data, parameters.file_content) + + self.response_debug = response + + return u"Upload status : {0}".format( + self.get_last_upload_status() + ) + + def get_log_file(self, upload_id=None): + if not upload_id: + upload_id = self.get_last_upload_id() + + url = self.get_batch_url( + 'batchmemberservice/{0}/batchmember/{1}/getLogFile'.format( + self.token, + upload_id + ) + ) + response = self.get(url) + return response.result + + def get_bad_file(self, upload_id=None): + if not upload_id: + upload_id = self.get_last_upload_id() + + url = self.get_batch_url( + 'batchmemberservice/{0}/batchmember/{1}/getBadFile'.format( + self.token, + upload_id + ) + ) + response = self.get(url) + return response.result + + def get_last_upload_status(self): + url = self.get_batch_url( + 'batchmemberservice/{0}/batchmember/getLastUpload'.format(self.token) + ) + response = self.client.get(url) + return response.find('lastUploads[1]/status').text + + def get_last_upload_id(self): + url = self.get_batch_url( + 'batchmemberservice/{0}/batchmember/getLastUpload'.format(self.token) + ) + response = self.client.get(url) + return response.find('lastUploads[1]/id').text + + +class EMVAPIMergeUploadParams(object): + + def __init__( + self, + path, + file_encoding, + separator, + criteria, + skip_first_line, + mapping + ): + self.path = path + self.file_encoding = file_encoding + self.separator = separator + self.criteria = criteria + self.skip_first_line = skip_first_line + self.mapping = mapping + self.file_content = base64.b64encode( + codecs.open(self.path, 'r', 'utf-8').read() + ) From d163123d91b1153101d5c7ebfa26a90ba63aa34a Mon Sep 17 00:00:00 2001 From: roulance Date: Mon, 24 Nov 2014 15:34:20 +0100 Subject: [PATCH 3/5] #2 - add put method n change get method --- emailvision/utils.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/emailvision/utils.py b/emailvision/utils.py index 62ac1f9..fcccf4c 100644 --- a/emailvision/utils.py +++ b/emailvision/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- +import requests + from collections import namedtuple -from urllib import quote try: from lxml import etree @@ -47,8 +48,7 @@ def __init__(self, server_name): self.server_name = 'https://' + server_name def get(self, action_path, *values): - path = action_path + '/'.join([quote(unicode(value)) for value in values]) - url = ''.join((self.server_name, path)) + url = ''.join((self.server_name, action_path)) response, content = self.http.request(url) if response['status'] == '500': @@ -73,6 +73,30 @@ def post(self, path, data): response, content = self.http.request(url, 'POST', data.encode('latin1', 'ignore')) return PostResponse(content) + def put(self, url, xml_data=None, csv_data=None): + url = self.server_name + url + files = { + 'mergeUpload': ('export_user.xml', xml_data, 'text/xml'), + 'inputStream': ( + 'dump.csv', + csv_data, + 'application/octet-stream\r\nContent-Transfer-Encoding: base64' + ) + } + response = requests.put(url, files=files) + if response.status_code == 500: + raise FailedApiCall(GetResponse(response.content).error, url) + + if response.status_code != 200: + msg = 'Server `%s` answered %s status for `%s`.\n%s' + raise BadResponse(msg % (self.server_name, + response.status_code, + url, + response.content)) + + response = GetResponse(response.content) + return response + class BaseResponse (object): @@ -94,8 +118,6 @@ def iterfind(self, xpath_query): return self.root.iterfind(xpath_query) - - class GetResponse (BaseResponse): def __init__(self, f): @@ -136,4 +158,3 @@ def error(self): def soap_find(self, xpath): return self.find('{%s}%s' % ('http://schemas.xmlsoap.org/soap/envelope/', xpath)) - From aeff4517c34ee1181de5c4a0e9ee94f1e62f4d1e Mon Sep 17 00:00:00 2001 From: roulance Date: Wed, 26 Nov 2014 17:49:54 +0100 Subject: [PATCH 4/5] #2 - get rid of urllib2 and optis --- emailvision/member_api.py | 10 +++++----- emailvision/utils.py | 30 ++++++++++++++---------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/emailvision/member_api.py b/emailvision/member_api.py index 22a108e..7976792 100644 --- a/emailvision/member_api.py +++ b/emailvision/member_api.py @@ -54,7 +54,11 @@ def __connect(self): def __disconnect(self): if self.token is None: raise NotConnected() - self.client.get('/apimember/services/rest/connect/close/', self.token) + url = self.get_member_url( + 'connect/close/{0}'.format(self.token) + ) + response = self.get(url) + return response self.token = None def get_batch_url(self, remote_method): @@ -263,11 +267,7 @@ def upload_file_merge(self, parameters): ) data = self.assemble_upstream_body(parameters) - response = self.put(url, data, parameters.file_content) - - self.response_debug = response - return u"Upload status : {0}".format( self.get_last_upload_status() ) diff --git a/emailvision/utils.py b/emailvision/utils.py index fcccf4c..dd118c9 100644 --- a/emailvision/utils.py +++ b/emailvision/utils.py @@ -21,8 +21,6 @@ # normal ElementTree install import elementtree.ElementTree as etree -from httplib2 import Http - Error = namedtuple('Error', ['status', 'description']) @@ -32,11 +30,13 @@ class BadResponse(Exception): def __init__(self, message): super(Exception, self).__init__(message) + class FailedApiCall(Exception): """The API call is not legit""" def __init__(self, error, url): super(Exception, self).__init__('%s: %s. url is %s ' % (error.status, error.description, url)) + class NotConnected(Exception): pass @@ -44,34 +44,32 @@ class NotConnected(Exception): class Client(object): def __init__(self, server_name): - self.http = Http() self.server_name = 'https://' + server_name def get(self, action_path, *values): url = ''.join((self.server_name, action_path)) - response, content = self.http.request(url) - - if response['status'] == '500': - raise FailedApiCall(GetResponse(content).error, url) - if response['status'] != '200': + response = requests.get(url) + if response.status_code == 500: + raise FailedApiCall(GetResponse(response.content).error, url) + if response.status_code != 200: msg = 'Server `%s` answered %s status for `%s`.\n%s' raise BadResponse(msg % (self.server_name, - response['status'], + response.status_code, url, - content)) - if response['content-type'] == 'text/xml': + response.content)) + if response.headers['content-type'] == 'text/xml': msg = 'Server `%s` answered %s content-type for `%s`.\n%s' raise BaseResponse(msg % (self.server_name, - response['content-type'], + response.headers['content-type'], url, - content)) - response = GetResponse(content) + response.content)) + response = GetResponse(response.content) return response def post(self, path, data): url = self.server_name + path - response, content = self.http.request(url, 'POST', data.encode('latin1', 'ignore')) - return PostResponse(content) + response = requests.post(url, data.encode('latin1', 'ignore')) + return PostResponse(response.content) def put(self, url, xml_data=None, csv_data=None): url = self.server_name + url From dc4b75076a38926be3c9a7293615eb0d9a0aac3a Mon Sep 17 00:00:00 2001 From: roulance Date: Thu, 4 Dec 2014 16:40:52 +0100 Subject: [PATCH 5/5] #2 - generate xml with lxml --- emailvision/member_api.py | 48 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/emailvision/member_api.py b/emailvision/member_api.py index 7976792..f42fa1e 100644 --- a/emailvision/member_api.py +++ b/emailvision/member_api.py @@ -3,6 +3,8 @@ import requests import base64 +from lxml import etree + from utils import Client, NotConnected, FailedApiCall, Error @@ -234,29 +236,33 @@ def resubscribe_members_by_id(self, user_id): return response.result def assemble_upstream_body(self, parameters): - data = u""" - -{0} -{1} -{2} -{3} -{4} -""".format( - parameters.criteria, - parameters.path.split('/')[-1], - parameters.separator, - parameters.file_encoding, - parameters.skip_first_line, + if not isinstance(parameters, EMVAPIMergeUploadParams): + return "something went wrong check your parameters" + mergeUpload = etree.Element('mergeUpload') + criteria = etree.SubElement(mergeUpload, 'criteria') + criteria.text = parameters.criteria + file_name = etree.SubElement(mergeUpload, 'fileName') + file_name.text = parameters.path.split('/')[-1] + separator = etree.SubElement(mergeUpload, 'separator') + separator.text = parameters.separator + file_encoding = etree.SubElement(mergeUpload, 'fileEncoding') + file_encoding.text = parameters.file_encoding + skip_first_line = etree.SubElement(mergeUpload, 'skipFirstLine') + skip_first_line.text = parameters.skip_first_line + mapping = etree.SubElement(mergeUpload, 'mapping') + for param in parameters.mapping: + column = etree.SubElement(mapping, 'column') + for key, value in param.iteritems(): + node = etree.SubElement(column, key) + node.text = str(value) + + data = etree.tostring( + mergeUpload, + xml_declaration=True, + encoding="UTF-8", + pretty_print=True ) - for param in parameters.mapping: - data += u""" -{0} -{1} -{2} -""".format(param['colNum'], param['fieldName'], param['toReplace']) - data += u""" -""" return data def upload_file_merge(self, parameters):