Main

# Packages used in this notebook
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.model_selection import train_test_split, cross_val_score, cross_validate
import os
import re
from numpy.lib.function_base import corrcoef
from sklearn.dummy import DummyClassifier
from sklearn.metrics import mean_squared_error, accuracy_score
from sklearn.linear_model import SGDClassifier,SGDRegressor, LinearRegression, RidgeClassifier, LogisticRegressionCV, LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer, HashingVectorizer
from sklearn.preprocessing import StandardScaler, OrdinalEncoder, LabelEncoder, Normalizer, MinMaxScaler
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from scipy.sparse import csc_matrix, csr_matrix
from sklearn.decomposition import TruncatedSVD
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier, VotingClassifier, BaggingClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB, ComplementNB
from sklearn.svm import LinearSVC, SVC
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report,confusion_matrix, accuracy_score, precision_score, recall_score, f1_score
import seaborn as sns
import matplotlib.pyplot as plt
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest, chi2, f_regression
import spacy
from spacy.lang.en.stop_words import STOP_WORDS



# Loading the 4 files : Movies, Train (containing review text) , Test(sentiment variable absent) and Sample output
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

movies = pd.read_csv("/kaggle/input/sentiment-prediction-on-movie-reviews/movies.csv")
train = pd.read_csv("/kaggle/input/sentiment-prediction-on-movie-reviews/train.csv")
sample = pd.read_csv("/kaggle/input/sentiment-prediction-on-movie-reviews/sample.csv")
test = pd.read_csv("/kaggle/input/sentiment-prediction-on-movie-reviews/test.csv")

# Converting all files into type - dataframe
movies_df = pd.DataFrame(movies)
train_df = pd.DataFrame(train)
test_df = pd.DataFrame(test)
sample_df = pd.DataFrame(sample)

# Seperating target variable from feature matrix 
x_train = train_df.drop('sentiment', axis = 1)
y_train = train_df['sentiment']
    
'''
# Trying out all strategies of Dummy Regressor as the baseline model
 for strategy in ['stratified','prior','uniform','most_frequent']:
    dummy = DummyClassifier(strategy=strategy)
    outfilename = 'dummy_' + strategy
    dummy.fit(x_train, y_train)
    y_test = dummy.predict(test)
    ids = list(range(55315))
    out = pd.DataFrame(list(zip(y_test)),columns = ['sentiment'])  
    out.to_csv("/kaggle/working/"+outfilename+".csv")

dummy = DummyClassifier(strategy='prior')
outfilename = 'submission'
dummy.fit(x_train, y_train)
y_test = dummy.predict(test)
ids = list(range(55315))
out = pd.DataFrame({'id':ids, 'sentiment': y_test})
out.to_csv("/kaggle/working/"+outfilename+".csv", header=True, index=False)
'''



# Does frequency of reviewing have an impact on sentiment of movies watched
l = LabelEncoder()
print("Correlation between Sentiment and Frequency of Reviewing")
print(corrcoef(l.fit_transform(train_df['isFrequentReviewer']),
               l.fit_transform(train_df['sentiment'])))
print(" ")
movies_df['movieid'].nunique()
movies_df['title'].nunique()
train_df['movieid'].nunique()

# Choosing only those movieids which are common in both movies and train dataset and making a subset of the movies dataset
reviewed_movies = np.intersect1d(movies_df['movieid'], train_df['movieid'])
reviewed_movies_df = movies_df[movies_df['movieid'].isin(reviewed_movies)]
reviewed_movies_df.info()

theatres = reviewed_movies_df['releaseDateTheaters']
stream = reviewed_movies_df['releaseDateStreaming']
ratings = reviewed_movies_df['audienceScore']
box = reviewed_movies_df['boxOffice'].str[1:-2]

# Checking if difference between date of release in theatres and OTTs indicate high audience score
df2 = pd.DataFrame({'Theaters' : theatres, 'streaming': stream, 'AudienceScore': ratings})
df2 = df2.dropna()
df2['Theaters'] = (pd.to_datetime(df2['Theaters']).astype(int))
df2['streaming'] = (pd.to_datetime(df2['streaming']).astype(int))
df2['DateDiff'] = (df2['streaming']) - (df2['Theaters'])
print(" ")
print("Correlation between difference between release in theatre and OTT and Audience score")
print(corrcoef(df2['DateDiff'], df2['AudienceScore']))

# Checking if high budget in boxoffice has influence on audience score
df = pd.DataFrame({'boxOffice' : box, 'AudienceScore' : ratings})
df['boxOffice'] = pd.to_numeric(df['boxOffice'], errors='coerce')
df = df.dropna()
print(" ")
print("Correlation between budget and audience score")
print(corrcoef(df['boxOffice'], df['AudienceScore']))
print(" ")
# Which combination of genres get highest average audience score and least audience score
reviewed_movies_df = reviewed_movies_df.groupby('genre')
mean = reviewed_movies_df['audienceScore'].mean()
mean1 = mean[mean > 95]
plt.show()
mean1.plot(kind = 'bar')
plt.xlabel("Genre")
plt.ylabel("Audience Score")
plt.title("Genres with Average Score over 95", fontdict={'fontsize': 10, 'fontweight': 'bold', 'color': 'red'})

mean2 = mean[mean < 20]
plt.show()
mean2.plot(kind = 'bar')
plt.title("Genres with Average Score below 20", fontdict={'fontsize': 10, 'fontweight': 'bold', 'color': 'red'})
plt.xlabel("Genre")
plt.ylabel("Audience Score")




def reviews_handling(x) :
  x['reviewText'].fillna('', inplace=True)
  x['reviewText'] = x['reviewText'].str.replace('[^\w\s]', '',regex=True).str.lower()
  rules = [(r"ing\b", ""), (r"ed\b", ""),(r"s\b", ""),
   (r"\b\w{1,2}\b", ""),(r"[^a-zA-Z\s]", "")]
  for rule in rules:
    x['reviewText'] = x['reviewText'].apply(lambda x: re.sub(rule[0], rule[1], x))
  stop_words = list(STOP_WORDS)
  x['reviewText'] = x['reviewText'].apply(lambda y: y.split())
  x['reviewText'] = x['reviewText'].apply(lambda y: [word for word in y if word.casefold() not in stop_words])
  x['reviewText'] = x['reviewText'].apply(lambda y: ' '.join(y))
  return(x)

# Removing columns that dont influence sentiment and seperating features from target
x_train = train_df.drop(['movieid', 'reviewerName','sentiment'], axis = 1)
x_test = test_df.drop(['movieid', 'reviewerName'], axis = 1)



# How many sentiments in the train dataset are positive and negative
train_df['sentiment'] = train_df['sentiment'].str.replace('POSITIVE', 'POSITIV')
train_df['sentiment'] = train_df['sentiment'].str.replace('POSITIV', 'POSITIVE')
y_train = train_df['sentiment']
plt.figure(figsize=(5,6))
plt.title("Distribution of Sentiments", 
          fontdict={'fontsize': 10, 'fontweight': 'bold', 'color': 'red'})
print(sns.countplot(x = 'sentiment',data=train_df))
lenc = LabelEncoder()
y_train = np.array(lenc.fit_transform(y_train))



plt.figure(figsize=(5,6))
plt.title("Distribution of Frequency of Reviewing", 
          fontdict={'fontsize': 10, 'fontweight': 'bold', 'color': 'red'})
print(sns.countplot(x = 'isFrequentReviewer',data=train_df))





# Encoding the target variable and plot
plt.figure(figsize=(5,6))
pd.DataFrame(y_train).value_counts().plot.bar(rot=0)
plt.title('Encoded Sentiments', fontdict={'fontsize': 10, 'fontweight': 'bold', 'color': 'red'})
plt.xlabel('Sentiments')
plt.ylabel('Frequency')





# Applying the reviews handling function on x_train and x_test dataframes

x_train = reviews_handling(x_train)
x_test = reviews_handling(x_test)

# Splitting the train dataframe into train and validation sets
X_train, X_val, Y_train, Y_val  = train_test_split(
        x_train,
        y_train,
        train_size=0.75,
        random_state=1234)

# Define the pipeline
# Using feature extraction methods - Vectorizer to convert text to for classification
pipe = Pipeline([('Tfidf', TfidfVectorizer(use_idf = True, stop_words='english', norm = 'l2',
                                           smooth_idf = True, max_features = 8500))])
pipe2 = Pipeline([('Tfidf', TfidfVectorizer(use_idf = True, stop_words='english', norm = 'l2',
                                           smooth_idf = True, max_features = 150))])

#pipe = Pipeline([('vector', HashingVectorizer(stop_words='english')),('Tfidf', TfidfTransformer())])
#pipe = Pipeline([('vector', CountVectorizer(stop_words='english')),('Tfidf', TfidfTransformer( use_idf = True, norm = 'l2', smooth_idf = True))])
#pipe = Pipeline([('vector', CountVectorizer(stop_words='english'))])

# Making converted feature matrix as Data Frames
xtraindf = pd.DataFrame(pipe.fit_transform(X_train['reviewText']).toarray())
xgcv = pd.DataFrame(pipe2.fit_transform(X_train['reviewText']).toarray())
xvaldf = pd.DataFrame(pipe.transform(X_val['reviewText']).toarray())
xtestdf = pd.DataFrame(pipe.transform(x_test['reviewText']).toarray())

xtraindf.columns = xtraindf.columns.astype(str)
xvaldf.columns = xvaldf.columns.astype(str)
xtestdf.columns = xtestdf.columns.astype(str)

print(xtraindf.shape, xvaldf.shape, xtestdf.shape, xgcv.shape)
print(Y_train.shape)



def tuning(gcv):

  gcv.fit(xgcv ,Y_train)
  print("Best Parameters:", gcv.best_params_)
  print("Best Score :", gcv.best_score_)


clf = LinearSVC(max_iter = 3000)
regularizations = [0.1,1,15]
loss = ['squared_hinge']

gcv = GridSearchCV(clf, param_grid={'C': regularizations, 'loss': loss},cv=5, scoring='accuracy')
tuning(gcv)





clf = SGDClassifier(learning_rate = 'constant',
                    eta0 = 0.1,random_state=42, warm_start =True,max_iter=1000)

gcv = GridSearchCV(clf, param_grid={'loss': ['perceptron', 'hinge'],'penalty':['l1','l2'],
                                    'alpha': [0.001,0.01,0.1]},cv=4, scoring='accuracy')
tuning(gcv)




cnb = ComplementNB()
paramgrid = {'alpha': [0.1, 1, 10], 'fit_prior': [True, False], 'norm': [True, False]}
gcv = GridSearchCV(estimator=cnb, param_grid=paramgrid, scoring='accuracy', cv =4)
tuning(gcv)



param_grid = {'alpha': [0.1, 0.3, 0.4, 0.5], 'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg']}
rc = RidgeClassifier()
grid_cv = GridSearchCV(estimator=rc, param_grid=param_grid, scoring='accuracy', cv =4)
tuning(grid_cv)





lr = LogisticRegression()
param_grid = {'solver': [ 'lbfgs', 'liblinear'], 'C': [0.1, 1, 10]}
gcv = GridSearchCV(estimator=lr, param_grid=param_grid, scoring='accuracy', cv = 4)
tuning(gcv)



def classifier(clf):

  clf.fit(xtraindf,Y_train)
  y_pred = clf.predict(xvaldf)
  y_pred2 = clf.predict(xtraindf)
  
  accuracy = accuracy_score(Y_train, y_pred2)
  precision = precision_score(Y_train, y_pred2, zero_division=1)
  recall = recall_score(Y_train, y_pred2)
  f1 = f1_score(Y_train, y_pred2, zero_division=1)

  print("Metrics for train data")  
  print("Accuracy:", accuracy)
  print("Precision:", precision)
  print("Recall:", recall)
  print("F1-score:", f1)

  accuracy = accuracy_score(Y_val, y_pred)
  precision = precision_score(Y_val, y_pred,zero_division=1)
  recall = recall_score(Y_val, y_pred)
  f1 = f1_score(Y_val, y_pred, zero_division=1)
  print(" ")  
    
  print("Metrics for validation data")
  print("Accuracy:", accuracy)
  print("Precision:", precision)
  print("Recall:", recall)
  print("F1-score:", f1)

  print(" ")  
  print("Classification for validation data")
  print(classification_report(Y_val, y_pred))
  print(" ")  
  print("Classification for train data")
  print(classification_report(Y_train, y_pred2))

  cf_matrix_val = confusion_matrix(Y_val, y_pred)
  cf_matrix_train = confusion_matrix(Y_train, y_pred2)
  
  print(" ")  
  print("Heatmap for validation data")
  plt.figure(figsize=(6,6))
  print(sns.heatmap(cf_matrix_val, xticklabels = ['Negative', 'Positive'], 
                    yticklabels = [ 'Negative', 'Positive'], annot=True, 
                    fmt='d', cmap = 'Greens'))
  
  plt.xlabel('Predicted')
  plt.ylabel('Actual')
  plt.title('Confusion Matrix')
  print(" ")  
  print("Heatmap for train data")
  plt.figure(figsize=(6,6))
  print(sns.heatmap(cf_matrix_train, xticklabels = ['Negative', 'Positive'], 
                    yticklabels = [ 'Negative', 'Positive'],annot=True,
                    fmt='d', cmap = 'Greens'))
  
  plt.xlabel('Predicted')
  plt.ylabel('Actual')
  plt.title('Confusion Matrix')
  
  predicted_test = clf.predict(xtestdf)
  predicted_test = lenc.inverse_transform(predicted_test)
  ids = list(range(55315))
  out = pd.DataFrame({'id':ids, 'sentiment': predicted_test})
  out.to_csv("/kaggle/working/submission.csv", header=True, index=False)
  print(out.head())  



#lin_svc = LinearSVC(C=0.1,loss = 'squared_hinge',max_iter=1000)
#classifier(lin_svc)




#sgd_clf = SGDClassifier(penalty='l2', learning_rate = 'constant',alpha= 0.001, loss = 'hinge', 
                     #   eta0 = 0.1,random_state=42, warm_start =True,max_iter=1000)
#classifier(sgd_clf)

#kneighbor_classifier = KNeighborsClassifier(n_neighbors = 3)
#classifier(kneighbor_classifier)


#CNB = ComplementNB(alpha = 1.0, fit_prior = True, norm = False)
#classifier(CNB)


#log_model = LogisticRegression(C = 1,penalty = "l1", fit_intercept = False, solver = 'liblinear',
                              # warm_start = True, max_iter = 5000, class_weight = 'balanced')

#classifier(log_model)


#ridge = RidgeClassifier(solver = 'auto', alpha = 0.1)
#classifier(ridge)



#log_reg = LogisticRegression(max_iter = 1000)
#svc = SVC(max_iter = 1000)

#voting_clf = VotingClassifier(estimators=[('lr', log_reg), ('svc', svc)], voting='hard')
#classifier(voting_clf)