parser.py

# -*- coding: utf-8 -*-
"""parser.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FphVO9q_YybsvC5dWKZMPFVs-ei2fqx4

# Importing Data
"""

import pandas as pd
import io
from google.colab import files
uploaded = files.upload()

############## Cleaning the HTML
df = pd.read_csv(io.BytesIO(uploaded['data.csv']))
#get rid of ID and CreatedDate

features = df.iloc[:, 1:6]
#features.drop('CreatedDate', axis = 1, inplace=True)
common_words = ['please', 'unit', 'represented','hover', 'text', 'br','said', 'office','new','million','costar', 'square', 'building', 'market', 'space', 'year', 'estate', 'real', 'align', 'development', 'feet', 'property', 'percent', 'company','foot', 'investment', 'based', 'retail', 'years', 'according']
import re

def cleanhtml(raw_html):
  cleanr = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')
  cleantext = re.sub(cleanr, '', raw_html)
  return cleantext

for i in range(0, 1933):
  body_text = features.iloc[i , 2]
  body_text = cleanhtml(body_text)
  body_text = body_text.replace('\n', ' ')
  body_text  = [word for word in re.split("\W+",body_text) if word.lower() not in common_words]
  body_text = ' '.join(body_text)
  features.loc[i, 'Body'] = body_text
print(features.shape)

Location = []
import spacy
nlp = spacy.load("en_core_web_sm")
for i in range(0,1933):
  title_text = features.iloc[i,0]
  doc = nlp(title_text)
  locations = []
  for ent in doc.ents:
    if ent.label_ == "GPE":
      locations.append(ent.text)    
  if not locations:
    summary_text = features.iloc[i,1]
    if(pd.isnull(summary_text)== False):
      doc = nlp(summary_text)
      locations = []
      for ent in doc.ents:
        if ent.label_ == "GPE":
          locations.append(ent.text)
    else:
      body_text = features.iloc[i,2]
      doc = nlp(body_text)
      locations = []
      for ent in doc.ents:
        if ent.label_ == "GPE":
          locations.append(ent.text)
    if not locations:
      body_text = features.iloc[i,2]
      doc = nlp(body_text)
      locations = []
      for ent in doc.ents:
        if ent.label_ == "GPE":
          locations.append(ent.text)
      Location.append(locations)        
    else:
      Location.append(locations)      
  else:
    Location.append(locations) 
features['GPE'] = Location
print(features.shape)

# Python3 program to find the most  
# frequent element in an array. 
  
def mostFrequent(arr, n): 
  
    # Sort the array 
    arr.sort() 
  
    # find the max frequency using 
    # linear traversal 
    max_count = 1; res = arr[0]; curr_count = 1
      
    for i in range(1, n):  
        if (arr[i] == arr[i - 1]): 
            curr_count += 1
              
        else : 
            if (curr_count > max_count):  
                max_count = curr_count 
                res = arr[i - 1] 
              
            curr_count = 1
      
    # If last element is most frequent 
    if (curr_count > max_count): 
      
        max_count = curr_count 
        res = arr[n - 1] 
      
    return res 
features = features[(features['GPE'].str.len() != 0) | (features['GPE'].str.len() != 0)]
cities_list = []
for i in range(0, 1872):
    cities = features.iloc[i,5]
    n = len(cities)
    most = mostFrequent(cities, n)
    cities_list.append(most)
features['final_city'] = cities_list

features['final_city'] = features['final_city'].replace('SF', 'San Francisco')

final_cities = ['Atlanta', 
                'Chicago', 
                'Houston', 
                'Dallas', 
                'Denver', 
                'Seattle', 
                'Phoenix', 
                'San Francisco', 
                'Los Angeles', 
                'Baltimore', 
                'Sacramento', 
                'Birmingham',
                'Nashville', 
                'San Diego',
                'Minneapolis',
                'Boston']
##### final features
features =  features.loc[features['final_city'].isin(final_cities)]
print(features.shape)

print(features.columns)
print(features.shape)

import nltk
from sklearn.feature_extraction import text
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer("english")

eng_contractions = ["ain't", "amn't", "aren't", "can't", "could've", "couldn't",
                    "daresn't", "didn't", "doesn't", "don't", "gonna", "gotta", 
                    "hadn't", "hasn't", "haven't", "he'd", "he'll", "he's", "how'd",
                    "how'll", "how's", "I'd", "I'll", "I'm", "I've", "isn't", "it'd",
                    "it'll", "it's", "let's", "mayn't", "may've", "mightn't", 
                    "might've", "mustn't", "must've", "needn't", "o'clock", "ol'",
                    "oughtn't", "shan't", "she'd", "she'll", "she's", "should've",
                    "shouldn't", "somebody's", "someone's", "something's", "that'll",
                    "that're", "that's", "that'd", "there'd", "there're", "there's", 
                    "these're", "they'd", "they'll", "they're", "they've", "this's",
                    "those're", "tis", "twas", "twasn't", "wasn't", "we'd", "we'd've",
                    "we'll", "we're", "we've", "weren't", "what'd", "what'll", 
                    "what're", "what's", "what've", "when's", "where'd", "where're",
                    "where's", "where've", "which's", "who'd", "who'd've", "who'll",
                    "who're", "who's", "who've", "why'd", "why're", "why's", "won't",
                    "would've", "wouldn't", "y'all", "you'd", "you'll", "you're", 
                    "you've", "'s", "s"
                     ]

nltk.download('stopwords')
nltk.download('punkt')

custom_stopwords = text.ENGLISH_STOP_WORDS.union(eng_contractions)

import nltk
import re

def tokenize_and_stem(text, do_stem=True):
    # first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
    tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    filtered_tokens = []
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
            
    # stem filtered tokens
    stems = [stemmer.stem(t) for t in filtered_tokens]
    
    if do_stem:
        return stems
    else:
        return filtered_tokens

# not super pythonic, no, not at all.
# use extend so it's a big flat list of vocab
totalvocab_stemmed = []
totalvocab_tokenized = []
for i in range(0, 391):
    body_text = features.iloc[i , 2]
    allwords_stemmed = tokenize_and_stem(body_text)
    totalvocab_stemmed.extend(allwords_stemmed)
    
    allwords_tokenized = tokenize_and_stem(body_text, False)
    totalvocab_tokenized.extend(allwords_tokenized)  
  
vocab_frame = pd.DataFrame({'words': totalvocab_tokenized}, index = totalvocab_stemmed)

from sklearn.feature_extraction.text import TfidfVectorizer

#define vectorizer parameters
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,
                                 min_df=0.2, stop_words=custom_stopwords,
                                 use_idf=True, tokenizer=tokenize_and_stem, ngram_range=(1,3))

tfidf_matrix = tfidf_vectorizer.fit_transform(features['Body']) #fit the vectorizer to synopses

print(tfidf_matrix.shape)

terms = tfidf_vectorizer.get_feature_names()

from sklearn.cluster import KMeans
import math

num_clusters = 5

km = KMeans(n_clusters=num_clusters)

km.fit(tfidf_matrix)

clusters = km.labels_.tolist()
final_data = features
final_data['cluster'] = clusters

print("Top terms per cluster:")
print()

#sort cluster centers by proximity to centroid
order_centroids = km.cluster_centers_.argsort()[:, ::-1] 


for i in range(num_clusters):
    print("Cluster %d words:" % i, end='')
    
    for ind in order_centroids[i, :6]: #replace 6 with n words per cluster
        print(' %s' % vocab_frame.loc[terms[ind].split(' ')].values.tolist()[0][0], end=',')
    print() #add whitespace
    print() #add whitespace
    
    print("Cluster %d titles:" % i, end='')
    print()
    for title in features[features['cluster'] == i]['Title'].values.tolist():
        print(' - %s' % title)
    print() #add whitespace
    print() #add whitespace
    
print()
print()

# from google.colab import files

# final_data.to_csv('final_data.csv')
# files.download('final_data.csv')

from textblob import TextBlob

sentiments = []
for i in range(0, 391):
  blob = final_data.iloc[i, 2]
  blob = TextBlob(blob)
  feels = blob.sentiment.polarity
  sentiments.append(feels)
  
final_data['sentiments'] = sentiments

# from google.colab import files

# final_data.to_csv('final_data_sentiments.csv')
# files.download('final_data_sentiments.csv')