diff --git a/splackt_actions/splackt.py b/splackt_actions/splackt.py index 71e9445..a223764 100644 --- a/splackt_actions/splackt.py +++ b/splackt_actions/splackt.py @@ -18,6 +18,7 @@ #import pandas g = py2neo.Graph('http://neo4j:skynet@178.62.55.202:7474/db/data') +print('[splackt] loaded') # standard word lists product_types = "(book|ebook|video|subscription)" @@ -33,170 +34,6 @@ def joke(message, incoming_message): message.send_webapi(joke_response.json()['joke']) -@respond_to("(.*)") -#@listen_to("(.*)") -def testLooking(message, incoming_message): - - incoming_message = incoming_message.replace('?', '') - - #data search variables - search_type = "" - search_content = "" - search_period = "" - - response = "You're looking for" - - if re.search(sales_keywords, incoming_message, re.IGNORECASE): - response += " sales (value)" - search_type = "sales-value" - - if re.search(quantity_keywords, incoming_message, re.IGNORECASE): - response += " sales (volume)" - search_type = "sales-volume" - - m = re.search(engagement_keywords, incoming_message, re.IGNORECASE) - if m: - response += " " + m.group(0) - search_type = "engagement" - - - response += " info" - - m = re.search(product_types, incoming_message, re.IGNORECASE) - if m: - response += " for " + m.group(0) - search_content = m.group(0) - - tagged = timex.tag(incoming_message) - print (tagged) - base_date = datetime.datetime.now() - grounded = timex.ground(tagged, base_date) - - print(grounded) - - m = re.search("TIMEX2 val=\"(.*)\"", grounded) - if m: - response += " over the period " + m.group(1) - search_period = m.group(1) - - - if response == "You're looking for info": - response = "I don't understand you.... try it again in Klingon" - return - - message.reply(response) - getSimpleData(message, incoming_message, search_type, search_content, search_period) - -def getSimpleData(message, incoming_message, search_type, search_content, search_period): - print("searching:" + search_type) - print("for: " + search_content) - - sql = "" - date_value = "" - table_name = "" - metric = "" - - if search_type == "sales-volume" or search_type == "sales-value": - if search_type == "sales-volume": - date_value = "order_transaction_date" - table_name = "home.fact_direct_sales" - metric = "COUNT(*) as `count`" - - if search_type == "sales-value": - date_value = "order_transaction_date" - table_name = "home.fact_direct_sales" - #metric = "CONCAT('£',CAST(SUM(order_gbp_paid) as char(10))) as `sum`" - metric = "SUM(order_gbp_paid) as `sum`" - - group_by = check_each(incoming_message, date_value) - sql = "SELECT " + group_by - if group_by != "": - sql += ", " - sql += metric + " FROM " + table_name + " WHERE 1=1 " - - - if search_content != "": - sql += " AND order_sale_type = '" + search_content + "'" - - if search_period != "": - print("search_period: " + search_period) - if len(search_period) == 4: - sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_year(period_to_date(search_period))) + "'" - if search_period.count('-') == 1: - sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_month(period_to_date(search_period))) + "'" - if search_period.count('-') == 2: - sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_day(period_to_date(search_period))) + "'" - if 'W' in search_period: - sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_week(period_to_date(search_period))) + "'" - else: - today = datetime.datetime.now() - this_month = period_to_date(str(today.year) + "-" + str(today.month)) - sql += " AND " + date_value + " BETWEEN '" + date_to_string(this_month) +"' AND '" + date_to_string(date_add_month(this_month)) + "'" - message.reply("No date found so looking for this month only...") - - if group_by != "": - sql += " GROUP BY " + group_by + " ORDER BY 1 " - - if sql != "": - print(sql) - message.reply("Looking...") - res = titlator.db.p.execute(sql) - ox = res.fetchall() - - - print(ox) - if group_by: - plot_results(message, ox, "", re.search("(by|each) (day|week|month)", incoming_message.lower(), re.IGNORECASE).group(2), search_type) - message.reply('\n'.join(map(lambda x: '%s - %s' %x, ox))) - else: - message.reply("Answer: " + str(ox[0][0])) - else: - message.reply("I can't figure that out yet, ask someone in the data team...") - -def check_each(incoming_message, date_value): - m = re.search("(by|each) (day|week|month)", incoming_message.lower(), re.IGNORECASE) - if m: - if m.group(2) == "day": - return "DATE_FORMAT(" + date_value + ", '%Y-%m-%d')" - if m.group(2) == "week": - return "DATE_FORMAT(" + date_value + ", '%Y-%U')" - if m.group(2) == "month": - return "DATE_FORMAT(" + date_value + ", '%Y-%m-%b')" - return "" - -def period_to_date(search_period): - if 'W' in search_period: - search_period = str(Week.fromstring(search_period).monday()) - if search_period.count('-') < 2: - search_period += "-01" - if search_period.count('-') < 2: - search_period += "-01" - - return datetime.datetime.strptime(search_period, '%Y-%m-%d') - -def date_to_string(date_in): - return date_in.strftime('%Y-%m-%d') - -def date_add_month(date_in, months=1): - month = date_in.month - 1 + months - year = int(date_in.year + month / 12 ) - month = month % 12 + 1 - day = min(date_in.day,calendar.monthrange(year,month)[1]) - return datetime.date(year,month,day) - -def date_add_year(date_in, years=1): - year = date_in.year + years - month = date_in.month - day = min(date_in.day,calendar.monthrange(year,month)[1]) - return datetime.date(year,month,day) - -def date_add_day(date_in, days=1): - return date_in + datetime.timedelta(days=days) - -def date_add_week(date_in, weeks=1): - return date_in + datetime.timedelta(weeks=weeks) - - @respond_to('hello') def hello(message): message.reply('yo') @@ -273,49 +110,3 @@ def getme(message, tags =''): message.reply('from book: %s' % i.bname) message.reply('has score %.2f' % i.aboutness) message.reply('========') - -def plot_results(message, result, title, x_label, y_label): - data = [] - xTickMarks = [] - plt.style.use('fivethirtyeight') - fig = plt.figure() - ax = fig.add_subplot(111) - - for row in result: - data.append(int(row[1])) - xTickMarks.append(str(row[0])) - - - ## necessary variables - ind = np.arange(len(data)) # the x locations for the groups - width = 0.35 # the width of the bars - - ## the bars - rects1 = ax.bar(ind, data, width, - color='black', - error_kw=dict(elinewidth=2,ecolor='red')) - - - # axes and labels - ax.set_xlim(-width,len(ind)+width) - ax.set_ylim(0,max(data)) - - - ax.set_ylabel(y_label) - ax.set_xlabel(x_label) - ax.set_title(title) - - ax.set_xticks(ind+width) - xtickNames = ax.set_xticklabels(xTickMarks) - - plt.setp(xtickNames, rotation=45, fontsize=10) - fig.subplots_adjust(bottom=0.2,left=0.15) - filename = str(uuid.uuid1()) + '.png' - plt.savefig(filename) - post_image(filename, slackbot_settings.API_TOKEN, message.channel._body['id']) - - -def post_image(filename, token, channels): - f = {'file': (filename, open(filename, 'rb'), 'image/png', {'Expires':'0'})} - response = requests.post(url='https://slack.com/api/files.upload', data={'token': token, 'channels': channels, 'media': f}, headers={'Accept': 'application/json'}, files=f) - print(response.text) diff --git a/splackt_actions/splackt_engagement.py b/splackt_actions/splackt_engagement.py new file mode 100644 index 0000000..bbf6672 --- /dev/null +++ b/splackt_actions/splackt_engagement.py @@ -0,0 +1,75 @@ +# coding=utf-8 + +from slackbot.bot import respond_to, listen_to +import io +import re +import uuid +import py2neo +import dbapi as pdbc +import titlator, json +import requests +import timex +import datetime +import calendar +import traceback +from isoweek import Week +import numpy as np +import slackbot_settings +import splackt_helper + +#import pandas + +g = py2neo.Graph('http://neo4j:skynet@178.62.55.202:7474/db/data') +print('[splackt_engagement] loaded') + + +# standard word lists +engagement_keywords = "(page ?views|titles? read|books? read|users|pages? read|pages? viewed)" + +input_check = "(.*" + engagement_keywords + ".*)" +@respond_to(input_check) +#@listen_to(input_check) +def testLooking(message, incoming_message, type): + try: + print('[splackt_engagement] verifying') + + incoming_message = incoming_message.replace('?', '') + + #data search variables + search_type = "" + search_content = "" + search_period = "" + + response = "You're looking for" + + m = re.search(engagement_keywords, incoming_message, re.IGNORECASE) + if m: + response += " " + m.group(0) + search_type = "engagement" + + response += " info" + + print(response) + + if response == "You're looking for info": + response = "I don't understand you.... try it again in Klingon" + return + + tagged = timex.tag(incoming_message) + print (tagged) + base_date = datetime.datetime.now() + grounded = timex.ground(tagged, base_date) + + print(grounded) + + m = re.search("TIMEX2 val=\"(.*)\"", grounded) + if m: + response += " over the period " + m.group(1) + search_period = m.group(1) + + message.reply(response) + splackt_helper.getSimpleEngagementData(message, incoming_message, search_type, search_content, search_period) + + except: + print('something went wrong....') + print(traceback.format_exc()) \ No newline at end of file diff --git a/splackt_actions/splackt_jira.py b/splackt_actions/splackt_jira.py index ed1c91f..235d7d1 100644 --- a/splackt_actions/splackt_jira.py +++ b/splackt_actions/splackt_jira.py @@ -4,44 +4,51 @@ import json import requests import slackbot_settings +import traceback jira_reg = '(\w+-\d+)' @listen_to(jira_reg) @respond_to(jira_reg) def check_slack(message, incoming_message): - jira_url = 'https://packtpub.atlassian.net/rest/api/2/search?jql=id=' + incoming_message - print('checking JIRA for ' + incoming_message) - r = requests.get(url=jira_url, headers={'Authorization': splackt_settings.JIRA_AUTH, 'Content-Type': 'application/json', 'accept':'application/json'}) - j = r.json() - if r.status_code == 200 and j['total'] == 1: - title = j['issues'][0]['fields']['summary'] - status = j['issues'][0]['fields']['status']['name'] - key = j['issues'][0]['key'] - url = 'https://packtpub.atlassian.net/browse/' + key - description = j['issues'][0]['fields']['description'] - creator = j['issues'][0]['fields']['creator']['displayName'] - assignee = '' - if j['issues'][0]['fields']['assignee']: - assignee = j['issues'][0]['fields']['assignee']['displayName'] - reply = key + ': `' + status + '`' - att = [ - { - 'fallback': description, - 'title': title, - 'title_link': url, - 'fields': [ - { - "title": "Creator", - "value": creator, - "short": True - }, - { - "title": "Assignee", - "value": assignee, - "short": True - } - ], - 'text': description - } - ] - message.send_webapi(reply, json.dumps(att)) + try: + jira_url = 'https://packtpub.atlassian.net/rest/api/2/search?jql=id=' + incoming_message + print('checking JIRA for ' + incoming_message) + r = requests.get(url=jira_url, headers={'Authorization': slackbot_settings.JIRA_AUTH, 'Content-Type': 'application/json', 'accept':'application/json'}) + #print (r) + j = r.json() + #print(j) + if r.status_code == 200 and j['total'] == 1: + title = j['issues'][0]['fields']['summary'] + status = j['issues'][0]['fields']['status']['name'] + key = j['issues'][0]['key'] + url = 'https://packtpub.atlassian.net/browse/' + key + description = j['issues'][0]['fields']['description'] + creator = j['issues'][0]['fields']['creator']['displayName'] + assignee = '' + if j['issues'][0]['fields']['assignee']: + assignee = j['issues'][0]['fields']['assignee']['displayName'] + reply = key + ': `' + status + '`' + att = [ + { + 'fallback': description, + 'title': title, + 'title_link': url, + 'fields': [ + { + "title": "Creator", + "value": creator, + "short": True + }, + { + "title": "Assignee", + "value": assignee, + "short": True + } + ], + 'text': description + } + ] + message.send_webapi(reply, json.dumps(att)) + except: + print('something went wrong....') + print(traceback.format_exc()) \ No newline at end of file diff --git a/splackt_actions/splackt_sales.py b/splackt_actions/splackt_sales.py index 98afe41..d13965b 100644 --- a/splackt_actions/splackt_sales.py +++ b/splackt_actions/splackt_sales.py @@ -10,6 +10,7 @@ import requests import timex import datetime +import traceback import calendar from isoweek import Week import numpy as np @@ -19,6 +20,7 @@ #import pandas g = py2neo.Graph('http://neo4j:skynet@178.62.55.202:7474/db/data') +print('[splackt_sales] loaded') # standard word lists @@ -31,53 +33,57 @@ @respond_to("(.*)") #@listen_to("(.*)") def testLooking(message, incoming_message): - - incoming_message = incoming_message.replace('?', '') - - #data search variables - search_type = "" - search_content = "" - search_period = "" + try: + incoming_message = incoming_message.replace('?', '') + + #data search variables + search_type = "" + search_content = "" + search_period = "" - response = "You're looking for" - - if re.search(sales_keywords, incoming_message, re.IGNORECASE): - response += " sales (value)" - search_type = "sales-value" - - if re.search(quantity_keywords, incoming_message, re.IGNORECASE): - response += " sales (volume)" - search_type = "sales-volume" - - m = re.search(engagement_keywords, incoming_message, re.IGNORECASE) - if m: - response += " " + m.group(0) - search_type = "engagement" - - - response += " info" - - m = re.search(product_types, incoming_message, re.IGNORECASE) - if m: - response += " for " + m.group(0) - search_content = m.group(0) - - tagged = timex.tag(incoming_message) - print (tagged) - base_date = datetime.datetime.now() - grounded = timex.ground(tagged, base_date) - - print(grounded) - - m = re.search("TIMEX2 val=\"(.*)\"", grounded) - if m: - response += " over the period " + m.group(1) - search_period = m.group(1) - - - if response == "You're looking for info": - response = "I don't understand you.... try it again in Klingon" - return - - message.reply(response) - splackt_helper.getSimpleSalesData(message, incoming_message, search_type, search_content, search_period) + response = "You're looking for" + + if re.search(sales_keywords, incoming_message, re.IGNORECASE): + response += " sales (value)" + search_type = "sales-value" + + if re.search(quantity_keywords, incoming_message, re.IGNORECASE): + response += " sales (volume)" + search_type = "sales-volume" + + m = re.search(engagement_keywords, incoming_message, re.IGNORECASE) + if m: + response += " " + m.group(0) + search_type = "engagement" + + + response += " info" + + + if response == "You're looking for info": + response = "I don't understand you.... try it again in Klingon" + return + + m = re.search(product_types, incoming_message, re.IGNORECASE) + if m: + response += " for " + m.group(0) + search_content = m.group(0) + + tagged = timex.tag(incoming_message) + print (tagged) + base_date = datetime.datetime.now() + grounded = timex.ground(tagged, base_date) + + print(grounded) + + m = re.search("TIMEX2 val=\"(.*)\"", grounded) + if m: + response += " over the period " + m.group(1) + search_period = m.group(1) + + + message.reply(response) + splackt_helper.getSimpleSalesData(message, incoming_message, search_type, search_content, search_period) + except: + print('something went wrong....') + print(traceback.format_exc()) \ No newline at end of file diff --git a/splackt_helper.py b/splackt_helper.py index df5aa47..9ea11e4 100644 --- a/splackt_helper.py +++ b/splackt_helper.py @@ -72,7 +72,7 @@ def getSimpleSalesData(message, incoming_message, search_type, search_content, s if sql != "": print(sql) - message.reply("Looking...") + #message.reply("Looking...") res = titlator.db.p.execute(sql) ox = res.fetchall() @@ -80,12 +80,115 @@ def getSimpleSalesData(message, incoming_message, search_type, search_content, s print(ox) if group_by: plot_results(message, ox, "", re.search("(by|each) (day|week|month)", incoming_message.lower(), re.IGNORECASE).group(2), search_type) - message.reply('\n'.join(map(lambda x: '{0} - {1}'.format(x[0],x[1]), ox))) + att = [ + { + 'fallback':'search results', + 'text': '\n'.join(map(lambda x: '{0} - {1}'.format(x[0],x[1]), ox)) + } + ] + + message.send_webapi('', json.dumps(att)) else: message.reply("Answer: " + str(ox[0][0])) else: message.reply("I can't figure that out yet, ask someone in the data team...") + +def getSimpleEngagementData(message, incoming_message, search_type, search_content, search_period): + print("searching:" + search_type) + print("for: " + search_content) + + sql = "" + date_value = "" + table_name = "" + metric = [] + + if search_type == "engagement": + date_value = "read_date" + table_name = "home.fact_views" + if "page" in incoming_message: + metric.append("SUM(read_pageviews) as `pages`") + + if "title" in incoming_message or "book" in incoming_message: + metric.append("COUNT(DISTINCT title_product_nid) as `books`") + + if "user" in incoming_message: + metric.append("COUNT(DISTINCT user_id) as `users`") + + if metric.count == 0: + return + + group_by = check_each(incoming_message, date_value) + sql = "SELECT " + group_by + if group_by != "": + sql += ", " + sql += ", ".join(metric) + sql += " FROM " + table_name + " WHERE 1=1 " + + if "mapt pro" in incoming_message.lower(): + sql += " AND subscription_title LIKE 'Mapt - Pro%%'" + + if "mapt basic" in incoming_message.lower(): + sql += " AND subscription_title LIKE 'Mapt - Basic%%'" + + if "paid" in incoming_message.lower(): + sql += " AND subscription_title NOT LIKE '%%Trial'" + + if "monthly" in incoming_message.lower(): + sql += " AND subscription_title LIKE '%%month%%'" + + if "annual" in incoming_message.lower(): + sql += " AND subscription_title NOT LIKE '%%month%%'" + + + + + if search_period != "": + print("search_period: " + search_period) + if len(search_period) == 4: + sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_year(period_to_date(search_period))) + "'" + if search_period.count('-') == 1: + sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_month(period_to_date(search_period))) + "'" + if search_period.count('-') == 2: + sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_day(period_to_date(search_period))) + "'" + if 'W' in search_period: + sql += " AND " + date_value + " BETWEEN '" + date_to_string(period_to_date(search_period)) + "' AND '" + date_to_string(date_add_week(period_to_date(search_period))) + "'" + else: + today = datetime.datetime.now() + this_month = period_to_date(str(today.year) + "-" + str(today.month)) + sql += " AND " + date_value + " BETWEEN '" + date_to_string(this_month) +"' AND '" + date_to_string(date_add_month(this_month)) + "'" + message.reply("No date found so looking for this month only...") + + if group_by != "": + sql += " GROUP BY " + group_by + " ORDER BY 1 " + + if sql != "": + print(sql) + #message.reply("Looking...") + res = titlator.db.p.execute(sql) + ox = res.fetchall() + + + print(ox) + if group_by: + plot_results(message, ox, "", re.search("(by|each) (day|week|month)", incoming_message.lower(), re.IGNORECASE).group(2), search_type) + + att = [ + { + 'fallback':'search results', + 'text': '\n'.join(map(lambda x: '{0} - {1}'.format(x[0],x[1]), ox)) + } + ] + + message.send_webapi('', json.dumps(att)) + else: + message.reply("Answer: " + str(ox[0][0])) + else: + message.reply("I can't figure that out yet, ask someone in the data team...") + + + + def check_each(incoming_message, date_value): m = re.search("(by|each) (day|week|month)", incoming_message.lower(), re.IGNORECASE) if m: