Can machines pick up sarcasm?

import pandas as pd
import numpy as np
import re
import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder, StandardScaler, LabelBinarizer
from sklearn_pandas import DataFrameMapper
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import GridSearchCV 
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.exceptions import DataConversionWarning
import warnings
warnings.filterwarnings(action='ignore', category=DataConversionWarning)
warnings.filterwarnings(action='ignore', category=FutureWarning) 
np.set_printoptions(suppress=True)
pd.set_option('display.max_colwidth', -1)
from IPython.display import HTML

# encoding utf-8 for special characters
raw_lpt = pd.read_csv("./data/lpt.csv", encoding='utf-8')
raw_ulpt = pd.read_csv("./data/ulpt.csv", encoding='utf-8')

df = pd.merge(raw_lpt, raw_ulpt, how='outer')
HTML(df.sample(2).to_html(classes="table table-responsive table-striped table-bordered"))

y = df.subreddit
X = df.drop(["subreddit",'name'],axis=1) # drop name, it's a unique identifier, not predictive
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)

def post_to_words(raw_post):
    '''Returns a list of words ready for classification, by tokenizing,
    removing punctuation, setting to lower case and removing stop words.'''
    tokenizer = RegexpTokenizer(r'[a-z]+')
    words = tokenizer.tokenize(raw_post.lower())
    meaningful_words = [w for w in words if not w in set(stopwords.words('english'))]
    return(" ".join(meaningful_words))

X_train.loc[:,"title_clean"] = X_train["title"].apply(lambda row : re.sub(r"[uU]*[lL][pP][tT]\s*:*", '', row)).apply(lambda row: post_to_words(row))

# for humans
X_test.loc[:,"title_read"] = X_test["title"].apply(lambda row : re.sub(r"[uU]*[lL][pP][tT]\s*:*", '', row))

# for model
X_test.loc[:,"title_clean"] = X_test["title_read"].apply(lambda row: post_to_words(row))

def my_vectorizer(vectorizer,X_train,X_test,y_train,y_test,stop=None):
    '''Takes a vectorizer, fits the model, learns the vocabulary,
    transforms data and returns the transformed matrices'''
    # transform text
    vect = vectorizer(stop_words=stop) 
    train_data_features = vect.fit_transform(X_train.title_clean)
    test_data_features = vect.transform(X_test.title_clean)
    le = LabelEncoder()
    target_train = le.fit_transform(y_train)
    target_test = le.transform(y_test)

    # transform non text
    mapper = DataFrameMapper([
    ("author", LabelBinarizer()),
    (["num_comments"], StandardScaler()),
    (["score"], StandardScaler())], df_out=True)
    Z_train = mapper.fit_transform(X_train)
    Z_test = mapper.transform(X_test)
    print(f' Training set learned {train_data_features.shape[1]} distinct vocabulary')
    print(f' Remember: 0 -> {le.classes_[0]}, 1 -> {le.classes_[1]}')

    # Baseline model
    print(f' Baseline model that guessed all LPT -> {round(1-sum(target_test)/len(target_test),2)} accurate')

    # Combine both df columns together
    a = pd.DataFrame(train_data_features.todense())
    b = Z_train
    c = pd.DataFrame(test_data_features.todense())
    d = Z_test

    # reset indices in order to merge
    a = a.reset_index().drop("index",axis=1)
    b = b.reset_index().drop("index",axis=1)
    c = c.reset_index().drop("index",axis=1)
    d = d.reset_index().drop("index",axis=1)

    Z_train = pd.merge(a,b, left_index=True, right_index=True)
    Z_test = pd.merge(c,d, left_index=True, right_index=True)
    return (Z_train, Z_test, target_train, target_test)

my_tuple = my_vectorizer(CountVectorizer,X_train,X_test,y_train,y_test,stop='english')
Z_train = my_tuple[0]
Z_test = my_tuple[1]
target_train = my_tuple[2]
target_test = my_tuple[3]

 Training set learned 5104 distinct vocabulary
 Remember: 0 -> LifeProTips, 1 -> UnethicalLifeProTips
 Baseline model that guessed all LPT -> 0.5 accurate

def results(model):
    '''Return a sample of 5 wrong predictions'''
    model.fit(Z_train, target_train)
    print(f' Training accuracy: {model.score(Z_train, target_train)}')
    print(f' Test accuracy: {model.score(Z_test, target_test)}')
    predictions = model.predict(Z_test)
    predictions = np.where(predictions==0,"LifeProTips","UnethicalLifeProTips")
    proba_lpt = model.predict_proba(Z_test)[:,0]
    final = pd.DataFrame(list(zip(predictions, y_test, proba_lpt, X_test.title_read, X_test.num_comments, X_test.score)), columns=['prediction', 'label', 'proba_lpt','title', "num_comments", "score"])   
    wrong = final[final.prediction!=final.label] 
    return HTML(wrong.sample(2).to_html(classes="table table-responsive table-striped table-bordered"))

results(LogisticRegression())

 Training accuracy: 0.9993021632937893
 Test accuracy: 0.895397489539749

log_model = LogisticRegression()
log_model.fit(Z_train, target_train);

pd.options.display.float_format = '{:.2f}'.format
my_coef = pd.DataFrame(list(zip(Z_train.columns,abs(log_model.coef_[0]))),columns=["x","coef"]).sort_values(by="coef",ascending=False)
my_coef.head()

my_coef = pd.DataFrame(list(zip(Z_train.columns,log_model.coef_[0])),columns=["x","coef"])
large_coef = my_coef[(my_coef["x"]=="num_comments") | (my_coef["x"]=="score") | (my_coef["x"]==576) | (my_coef["x"]==4945) | (my_coef["x"]==3125)] 
HTML(large_coef.to_html(classes="table table-responsive table-striped table-bordered"))

cvect = CountVectorizer(stop_words='english') 
train_data_features = cvect.fit_transform(X_train.title_clean)

print(f' {cvect.get_feature_names()[576]}: {round(np.exp(1.11),2)}x')
print(f' {cvect.get_feature_names()[4945]}: {round(np.exp(1.03),2)}x')
print(f' {cvect.get_feature_names()[3215]}: {round(np.exp(1.05),2)}x')

 business: 3.03x
 want: 2.8x
 pitched: 2.86x

f'If the number of comments increases by 1, the likelihood of being an Unethical Life Pro Tip is {round(np.exp(-4.33),2)}x more likely.'

'If the number of comments increases by 1, the likelihood of being an Unethical Life Pro Tip is 0.01x more likely.'

f'If the score (upvotes - downvotes) increases by 1, the likelihood of being an Unethical Life Pro Tip is {round(np.exp(-1.83),2)}x more likely.'

'If the score (upvotes - downvotes) increases by 1, the likelihood of being an Unethical Life Pro Tip is 0.16x more likely.'

my_tuple = my_vectorizer(TfidfVectorizer,X_train,X_test,y_train,y_test,stop='english')
Z_train = my_tuple[0]
Z_test = my_tuple[1]
target_train = my_tuple[2]
target_test = my_tuple[3]

 Training set learned 5104 distinct vocabulary
 Remember: 0 -> LifeProTips, 1 -> UnethicalLifeProTips
 Baseline model that guessed all LPT -> 0.5 accurate

results(LogisticRegression())

 Training accuracy: 0.9567341242149338
 Test accuracy: 0.8535564853556485

model = MultinomialNB()
Z_train = my_tuple[0]
Z_test = my_tuple[1]
target_train = my_tuple[2]
target_test = my_tuple[3]

Z_train.num_comments = abs(Z_train.num_comments)
Z_train.score = abs(Z_train.score)
model.fit(Z_train, target_train)

print(f' Training accuracy: {model.score(Z_train, target_train)}')
print(f' Test accuracy: {model.score(Z_test, target_test)}')

predictions = model.predict(Z_test)
predictions = np.where(predictions==0,"LifeProTips","UnethicalLifeProTips")
final = pd.DataFrame(list(zip(predictions, y_test, X_test.title_read, X_test.num_comments, X_test.score)), columns=['prediction', 'label', 'title', "num_comments", "score"])
wrong = final[final.prediction!=final.label] 
HTML(wrong.sample(2).to_html(classes="table table-responsive table-striped table-bordered"))

 Training accuracy: 0.9951151430565248
 Test accuracy: 0.805439330543933

from sklearn.metrics import confusion_matrix
predictions = model.predict(my_tuple[1])
print(confusion_matrix(my_tuple[3], predictions))
tn, fp, fn, tp = confusion_matrix(my_tuple[3], predictions).ravel()
print("True Negatives: %s" % tn)
print("False Positives: %s" % fp)
print("False Negatives: %s" % fn)
print("True Positives: %s" % tp)

[[167  74]
 [ 19 218]]
True Negatives: 167
False Positives: 74
False Negatives: 19
True Positives: 218

my_tuple = my_vectorizer(CountVectorizer,X_train,X_test,y_train,y_test,stop='english')
Z_train = my_tuple[0]
Z_test = my_tuple[1]
target_train = my_tuple[2]
target_test = my_tuple[3]

results(svm.SVC(probability=True))

 Training set learned 5104 distinct vocabulary
 Remember: 0 -> LifeProTips, 1 -> UnethicalLifeProTips
 Baseline model that guessed all LPT -> 0.5 accurate
 Training accuracy: 0.6189811584089323
 Test accuracy: 0.5962343096234309

params = {'C': [1,3],'gamma': ["scale"]}

grid_search = GridSearchCV(svm.SVC(), param_grid=params, cv=5)
grid_search.fit(Z_train, target_train)

print(grid_search.best_score_)
print(grid_search.best_params_)
print(grid_search.score(Z_test,target_test))

0.9204466154919749
{'C': 3, 'gamma': 'scale'}
0.9037656903765691

results(svm.SVC(3,gamma="scale",probability=True))

 Training accuracy: 1.0
 Test accuracy: 0.9037656903765691