# Load in cleaned .csv files
text = pd.read_csv('./datasets/text2.csv').drop('Unnamed: 0', axis=1)
df = pd.read_csv('./datasets/df2.csv').drop('Unnamed: 0', axis=1)

# For text data only
X = text['combined']
y = text['yes_right']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)

cvec = CountVectorizer(preprocessor=my_preprocessor,
                       strip_accents='unicode',
                       ngram_range=(1,4),
                       stop_words=cust_stop_words,
                       min_df=5)

cvec.fit(X_train, y_train);

X_feats = pd.DataFrame(cvec.transform(X_train).todense(),
                       columns=cvec.get_feature_names()).sum(axis=0)

X_feats.sort_values(ascending = False).head(15)
# Most common n-grams

cv_train = cvec.transform(X_train).todense()
cv_test = cvec.transform(X_test).todense()

lr = LogisticRegression(random_state=42)
lr.fit(cv_train, y_train);

lr.score(cv_train, y_train), lr.score(cv_test, y_test)

cross_val_score(lr, cv_test, y_test, scoring='accuracy', cv=7).mean()

print(classification_report(y_test, lr.predict(cv_test)))

print(confusion_matrix(y_test, lr.predict(cv_test)))
# Getting a lot of false negatives.
# Model is predicting article is not right wing when it actually is

cfs = lr.coef_[0]
fts = cvec.get_feature_names()

pd.DataFrame(
    list(zip(fts, np.abs(cfs), cfs)),
    columns=['feat','abs','coef']).sort_values('abs',ascending=False).head(15)
# Still some noise to correct, but looking better than previous dead giveaways of source

pac = PassiveAggressiveClassifier(C=0.5, random_state=42)
pac.fit(cv_train, y_train)
pac.score(cv_train, y_train)

pac.score(cv_test, y_test)

cross_val_score(pac, cv_test, y_test, scoring='accuracy', cv=7).mean()

tf = TfidfVectorizer(strip_accents='unicode', preprocessor=my_preprocessor,
                     ngram_range=(2,4), stop_words=cust_stop_words, min_df=2)

tf.fit(X_train, y_train)
tf_train = tf.transform(X_train)
tf_test = tf.transform(X_test)

lr = LogisticRegression(random_state=42)
parameters = {
    'penalty': ['l2','l1'],
    'C': [1.0,0.6]}

grid = GridSearchCV(lr, parameters, scoring='accuracy')

grid.fit(tf_train, y_train)
grid.score(tf_train, y_train), grid.score(tf_test, y_test)

grid.best_params_

print(classification_report(y_test, grid.predict(tf_test)))

print(confusion_matrix(y_test, grid.predict(tf_test)))
# More false negatives but fewer false positives

fts = tf.get_feature_names()
cfs = grid.best_estimator_.coef_[0]

pd.DataFrame(list(zip(fts, np.abs(cfs), cfs)),
             columns=['feat','abs','coef']).sort_values('abs',ascending=False).head(15)
# Much, much less noise than CountVectorizer

text[text['combined'].str.contains('IG report')]['source'].unique()

X = df[[col for col in df.columns if col != 'yes_right']]
y = df['yes_right']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)

pipe = Pipeline([
#     ('poly', PolynomialFeatures(interaction_only=True)),
    ('model', LogisticRegression())
])

pipe.fit(X_train, y_train);

pipe.score(X_train, y_train), pipe.score(X_test, y_test)

pd.DataFrame(list(zip(list(pipe.steps[0][1].coef_[0]),
                      np.abs(list(pipe.steps[0][1].coef_[0])), X.columns)), 
             columns=['coef','coef_abs','feat']).sort_values('coef_abs', ascending=False).head(10)

# For Pipeline:
X = pd.concat([text['combined'], df[[col for col in df.columns if col != 'yes_right']]], axis=1)
y = df['yes_right']
num_cols = [col for col in X.columns if col != 'combined']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)

# Function for FeatureUnion to extract data
class DfExtract(BaseEstimator, TransformerMixin):
    def __init__(self, column):
        self.column = column
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        if len(self.column) > 1:
            return pd.DataFrame(X[self.column])
        else:
            return pd.Series(X[self.column[0]]) 
    def get_feature_names(self):
        return X.columns.tolist()

text_pipe = Pipeline([
            ('ext', DfExtract(['combined'])),
            ('tf', TfidfVectorizer(preprocessor=my_preprocessor,
                                   strip_accents='unicode',
                                   stop_words=cust_stop_words, min_df=2,
                                   ngram_range=(2,4), use_idf=True, binary=False)
            ),
#             ('lda', TruncatedSVD(n_components=500))
        ])

num_pipe = Pipeline([
            ('ext', DfExtract(num_cols)),
#             ('poly', PolynomialFeatures(interaction_only=True))
            ])

feat_union = FeatureUnion([('text', text_pipe),('numerical', num_pipe)],
                          transformer_weights={'text': 2, 'numerical': 1})

pipe = Pipeline([
    ('features', feat_union),
#     ('lda', TruncatedSVD(n_components=100)),
#     ('lr', LogisticRegression(random_state=42)),
    ('pac', PassiveAggressiveClassifier(fit_intercept=True, random_state=42))
#     ('mnb', MultinomialNB())
#     ('sgdc', SGDClassifier(random_state=42))
])

params = {
#     'lr__penalty': ['l1','l2'],
#     'lr__C': [0.33, 0.5, 0.75, 1.0],
    'pac__C': [0.25, 0.35, 0.45, 0.5, 0.65, 1],
    'pac__loss': ['hinge','squared_hinge'],
    'pac__average': [True, False]
#     'mnb__alpha': [0.001, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25]
#     'sgdc__loss': ['log','perceptron'],
#     'sgdc__penalty': ['l2','elasticnet'],
#     'sgdc__alpha': [0.00001, 0.0001, 0.001]
}

gs = GridSearchCV(pipe, params, scoring='accuracy')

gs.fit(X_train, y_train);

gs.score(X_train, y_train)

gs.score(X_test, y_test)

gs.best_params_

y_counts = pd.DataFrame(y_test)['yes_right'].value_counts()
yhat_counts = pd.DataFrame(gs.predict(X_test))[0].value_counts()
pred_df = pd.DataFrame(pd.concat([y_counts, yhat_counts], axis=0))
pred_df['index']=['Actual Not Right','Actual Right','Predicted Not Right', 'Predicted Right']
pred_df.set_index('index', inplace=True)
pred_df.plot(kind='bar', rot=45, fontsize=12, figsize=(8,5), legend=False, use_index=True)
plt.title("Predicted Labels and Actual Labels", fontsize=16)
plt.ylabel('Counts');

print(classification_report(y_test, gs.predict(X_test)))

import itertools
# From ScikitLearn's documentation:
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap="Blues"):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title, fontsize=18)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label', fontsize=16)
    plt.xlabel('Predicted label', fontsize=16)

np.set_printoptions(precision=2)
plt.figure(figsize=(8,8))
plot_confusion_matrix(confusion_matrix(y_test, gs.predict(X_test)),
                      classes=['Not Right Wing','Right Wing'],
                      title='Confusion matrix')
plt.show()

# ROC AUC Curve looks a bit silly without predict_proba
y_hat = gs.predict(X_test) # Model's predictions

# For class 1, find the area under the curve
false_pos, true_pos, _ = roc_curve(y_test, y_hat)
ROC_AUC = auc(false_pos, true_pos)
prec, rec, _ = precision_recall_curve(y_test, y_hat)
PR_AUC = auc(rec, prec)

# Plot of a ROC curve for class 1 (has_cancer)
plt.figure(figsize=[12,10])
plt.plot(false_pos, true_pos, label='ROC curve (area = %0.2f)' % ROC_AUC, linewidth=4)
plt.plot(rec, prec, label='PR curve (area = %0.2f)' % PR_AUC, linewidth=4)
plt.plot([0, 1], [0, 1], 'k--', linewidth=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate', fontsize=18)
plt.ylabel('True Positive Rate', fontsize=18)
plt.title('ROC AUC and Precision Recall Curve', fontsize=16)
plt.legend(loc="lower right")
plt.show()

feature_names = text_pipe.fit(X_train, y_train).steps[1][1].get_feature_names()
feature_names.extend(num_pipe.fit(X_train, y_train).steps[0][1].get_feature_names())

pipe_coefs = list(gs.best_estimator_.steps[1][1].coef_[0])

feature_names.remove(feature_names[0])

importances = pd.DataFrame(list(zip(feature_names, pipe_coefs, np.abs(pipe_coefs))),
                          columns=['feature','coef','abs_coef'])

importances.sort_values('abs_coef', ascending=False).head(15)

text[text['combined'].str.contains('Gerber baby')]['source'].unique()

feat_imports = importances.sort_values('abs_coef', ascending=True).tail(15)
feat_imports['sign'] = feat_imports['coef'].map(lambda x: 'Negative' if x < 0 else 'Positive')
feat_imports.set_index('feature', inplace=True)

feat_imports['coef'].plot(kind='barh', figsize=(12,6),
                  title='Strongest Coefficients (Absolute Value)',
                  fontsize=14, legend=False, color='navy')
plt.xlabel('Coefficient')
plt.title('Top 15 Strongest Coefficients', fontsize=18);

# Plot words ignored by model
text_pipe.fit(X_train, y_train)
ignored_text = text_pipe.steps[1][1].stop_words_

plt.figure(figsize=(15,8))
wordcloud = WordCloud(font_path='/Library/Fonts/Verdana.ttf',
                      width=2000, height=1000,
                      relative_scaling = 1.0,
                      background_color='white',
                      colormap='Dark2'
                  
).generate(' '.join(ignored_text))
plt.imshow(wordcloud)
plt.title("Words Ignored by Model", fontsize=28)
plt.axis("off")
plt.show()